0: ch1=8074 mean# 8074 dev# 0 part# 0
1: ch1=8074 ch2=8074 mean# 8074 dev# 0 part# 0
2: ch1=13338 ch2=10884 mean# 12111 dev# 1227 part# 0.1013128561
3: ch1=18148 ch2=14148 mean# 16148 dev# 2000 part# 0.1238543473
4: ch1=20188 ch2=20182 mean# 20185 dev# 3 part# 0.0001486252167
5: ch1=27764 ch2=20680 mean# 24222 dev# 3542 part# 0.1462306994
6: ch1=28222 ch2=28296 mean# 28259 dev# 37 part# 0.001309317386
7: ch1=34120 ch2=30472 mean# 32296 dev# 1824 part# 0.05647758236
8: ch1=37842 ch2=34824 mean# 36333 dev# 1509 part# 0.04153249112
9: ch1=40380 ch2=40360 mean# 40370 dev# 10 part# 0.0002477086946
10: ch1=47458 ch2=41356 mean# 44407 dev# 3051 part# 0.06870538429
11: ch1=49004 ch2=47884 mean# 48444 dev# 560 part# 0.01155973908
12: ch1=53814 ch2=51148 mean# 52481 dev# 1333 part# 0.02539966845
13: ch1=58624 ch2=54412 mean# 56518 dev# 2106 part# 0.03726246506
14: ch1=60572 ch2=60538 mean# 60555 dev# 17 part# 0.0002807365205
15: ch1=68240 ch2=60944 mean# 64592 dev# 3648 part# 0.05647758236
16: ch1=68698 ch2=68560 mean# 68629 dev# 69 part# 0.001005405878
17: ch1=74596 ch2=70736 mean# 72666 dev# 1930 part# 0.0265598767
18: ch1=78318 ch2=75088 mean# 76703 dev# 1615 part# 0.02105523904
19: ch1=80764 ch2=80716 mean# 80740 dev# 24 part# 0.0002972504335
20: ch1=87934 ch2=81620 mean# 84777 dev# 3157 part# 0.03723887375
21: ch1=89480 ch2=88148 mean# 88814 dev# 666 part# 0.007498817754
22: ch1=94290 ch2=91412 mean# 92851 dev# 1439 part# 0.01549794833
23: ch1=99100 ch2=94676 mean# 96888 dev# 2212 part# 0.02283048468
24: ch1=100956 ch2=100894 mean# 100925 dev# 31 part# 0.0003071587813
25: ch1=108716 ch2=101208 mean# 104962 dev# 3754 part# 0.03576532459
26: ch1=109174 ch2=108824 mean# 108999 dev# 175 part# 0.001605519317
27: ch1=113984 ch2=112088 mean# 113036 dev# 948 part# 0.008386708659
28: ch1=119882 ch2=114264 mean# 117073 dev# 2809 part# 0.02399357666
29: ch1=121148 ch2=121072 mean# 121110 dev# 38 part# 0.0003137643465
30: ch1=128410 ch2=121884 mean# 125147 dev# 3263 part# 0.02607333775
31: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
32: ch1=126692 ch2=123602 mean# 125147 dev# 1545 part# 0.01234548171
33: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
34: ch1=123192 ch2=127102 mean# 125147 dev# 1955 part# 0.01562162896
35: ch1=129004 ch2=121290 mean# 125147 dev# 3857 part# 0.03081975597
36: ch1=121428 ch2=128866 mean# 125147 dev# 3719 part# 0.02971705275
37: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
38: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
39: ch1=123686 ch2=126608 mean# 125147 dev# 1461 part# 0.01167427106
40: ch1=128502 ch2=121792 mean# 125147 dev# 3355 part# 0.02680847324
41: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
42: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
43: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
44: ch1=123096 ch2=127198 mean# 125147 dev# 2051 part# 0.01638872686
45: ch1=129092 ch2=121202 mean# 125147 dev# 3945 part# 0.03152292904
46: ch1=121424 ch2=128870 mean# 125147 dev# 3723 part# 0.02974901516
47: ch1=126230 ch2=124064 mean# 125147 dev# 1083 part# 0.008653823104
48: ch1=125142 ch2=125152 mean# 125147 dev# 5 part# 3.995301525e-05
49: ch1=123590 ch2=126704 mean# 125147 dev# 1557 part# 0.01244136895
50: ch1=128590 ch2=121704 mean# 125147 dev# 3443 part# 0.0275116463
51: ch1=121878 ch2=128416 mean# 125147 dev# 3269 part# 0.02612128137
52: ch1=126230 ch2=124064 mean# 125147 dev# 1083 part# 0.008653823104
53: ch1=125142 ch2=125152 mean# 125147 dev# 5 part# 3.995301525e-05
54: ch1=123000 ch2=127244 mean# 125122 dev# 2122 part# 0.01695944758
55: ch1=128046 ch2=122198 mean# 125122 dev# 2924 part# 0.02336919167
56: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
57: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
58: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
59: ch1=122410 ch2=127834 mean# 125122 dev# 2712 part# 0.02167484535
60: ch1=128636 ch2=121608 mean# 125122 dev# 3514 part# 0.02808458944
61: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
62: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
63: ch1=125096 ch2=125148 mean# 125122 dev# 26 part# 0.0002077971899
64: ch1=122908 ch2=127336 mean# 125122 dev# 2214 part# 0.01769472994
65: ch1=128138 ch2=122106 mean# 125122 dev# 3016 part# 0.02410447403
66: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
67: ch1=126642 ch2=123602 mean# 125122 dev# 1520 part# 0.01214814341
68: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
69: ch1=122322 ch2=127922 mean# 125122 dev# 2800 part# 0.02237815892
70: ch1=128732 ch2=121512 mean# 125122 dev# 3610 part# 0.0288518406
71: ch1=121836 ch2=128408 mean# 125122 dev# 3286 part# 0.02626236793
72: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
73: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
74: ch1=123908 ch2=126336 mean# 125122 dev# 1214 part# 0.00970253033
75: ch1=128234 ch2=122010 mean# 125122 dev# 3112 part# 0.0248717252
76: ch1=121836 ch2=128408 mean# 125122 dev# 3286 part# 0.02626236793
77: ch1=126646 ch2=123598 mean# 125122 dev# 1524 part# 0.01218011221
78: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
79: ch1=123322 ch2=126922 mean# 125122 dev# 1800 part# 0.0143859593
80: ch1=128828 ch2=121416 mean# 125122 dev# 3706 part# 0.02961909177
81: ch1=121840 ch2=128404 mean# 125122 dev# 3282 part# 0.02623039913
82: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
83: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
84: ch1=123820 ch2=126424 mean# 125122 dev# 1302 part# 0.0104058439
85: ch1=128330 ch2=121964 mean# 125147 dev# 3183 part# 0.02543408951
86: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
87: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
88: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
89: ch1=123276 ch2=127018 mean# 125147 dev# 1871 part# 0.01495041831
90: ch1=128920 ch2=121374 mean# 125147 dev# 3773 part# 0.03014854531
91: ch1=121428 ch2=128866 mean# 125147 dev# 3719 part# 0.02971705275
92: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
93: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
94: ch1=123770 ch2=126524 mean# 125147 dev# 1377 part# 0.0110030604
95: ch1=128418 ch2=121876 mean# 125147 dev# 3271 part# 0.02613726258
96: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
97: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
98: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
99: ch1=123180 ch2=127114 mean# 125147 dev# 1967 part# 0.0157175162
100: ch1=126832 ch2=122374 ch3=1088 mean# 83431.33333 dev# 58253.9661 part# 0.6982264788
101: ch1=119248 ch2=125606 ch3=5440 mean# 83431.33333 dev# 55209.25102 part# 0.6617328145
102: ch1=123198 ch2=119712 ch3=7384 mean# 83431.33333 dev# 53792.41415 part# 0.6447507429
103: ch1=118610 ch2=120800 ch3=10884 mean# 83431.33333 dev# 51306.5019 part# 0.6149548359
104: ch1=120786 ch2=118624 ch3=10884 mean# 83431.33333 dev# 51306.30397 part# 0.6149524636
105: ch1=119344 ch2=115714 ch3=15236 mean# 83431.33333 dev# 48244.14881 part# 0.578249764
106: ch1=112258 ch2=118624 ch3=19412 mean# 83431.33333 dev# 45343.04628 part# 0.5434774259
107: ch1=115342 ch2=114272 ch3=20680 mean# 83431.33333 dev# 44374.04347 part# 0.5318630507
108: ch1=112078 ch2=114272 ch3=23944 mean# 83431.33333 dev# 42073.43207 part# 0.504288142
109: ch1=112988 ch2=112096 ch3=25210 mean# 83431.33333 dev# 41170.31015 part# 0.4934634089
110: ch1=111048 ch2=109858 ch3=29388 mean# 83431.33333 dev# 38217.49541 part# 0.4580712531
111: ch1=107722 ch2=111008 ch3=31564 mean# 83431.33333 dev# 36700.26929 part# 0.4398859257
112: ch1=107264 ch2=107744 ch3=35286 mean# 83431.33333 dev# 34044.45566 part# 0.4080535968
113: ch1=108196 ch2=105568 ch3=36530 mean# 83431.33333 dev# 33181.6003 part# 0.3977114948
114: ch1=102298 ch2=107660 ch3=40336 mean# 83431.33333 dev# 30551.52563 part# 0.3661876708
115: ch1=106818 ch2=102108 ch3=41368 mean# 83431.33333 dev# 29805.35783 part# 0.3572441748
116: ch1=99640 ch2=104934 ch3=45720 mean# 83431.33333 dev# 26753.38116 part# 0.3206634737
117: ch1=103404 ch2=99078 ch3=47812 mean# 83431.33333 dev# 25248.5148 part# 0.3026262891
118: ch1=97914 ch2=101220 ch3=51160 mean# 83431.33333 dev# 22859.15754 part# 0.273987681
119: ch1=100090 ch2=97956 ch3=52248 mean# 83431.33333 dev# 22067.15051 part# 0.2644947603
120: ch1=98242 ch2=96082 ch3=55970 mean# 83431.33333 dev# 19438.10727 part# 0.2329832989
121: ch1=92474 ch2=99040 ch3=58780 mean# 83431.33333 dev# 17636.02875 part# 0.2113837578
122: ch1=96196 ch2=93142 ch3=60956 mean# 83431.33333 dev# 15941.29199 part# 0.1910708046
123: ch1=92478 ch2=93596 ch3=64220 mean# 83431.33333 dev# 13592.12952 part# 0.1629139674
124: ch1=91390 ch2=92508 ch3=66396 mean# 83431.33333 dev# 12054.44364 part# 0.1444834112
125: ch1=90132 ch2=90044 ch3=70118 mean# 83431.33333 dev# 9414.016831 part# 0.1128355074
126: ch1=87038 ch2=91416 ch3=71840 mean# 83431.33333 dev# 8388.920365 part# 0.100548799
127: ch1=88126 ch2=87064 ch3=75104 mean# 83431.33333 dev# 5904.253909 part# 0.07076782395
128: ch1=87330 ch2=85976 ch3=76988 mean# 83431.33333 dev# 4589.534278 part# 0.05500971991
129: ch1=83608 ch2=85976 ch3=80710 mean# 83431.33333 dev# 2153.461916 part# 0.02581118903
130: ch1=86820 ch2=81834 ch3=81640 mean# 83431.33333 dev# 2397.457728 part# 0.02873569955
131: ch1=81136 ch2=84254 ch3=84904 mean# 83431.33333 dev# 1644.595459 part# 0.01971196423
132: ch1=85172 ch2=82394 ch3=82728 mean# 83431.33333 dev# 1238.367025 part# 0.01484294899
133: ch1=81358 ch2=84888 ch3=84048 mean# 83431.33333 dev# 1505.641244 part# 0.01804647228
134: ch1=87034 ch2=81624 ch3=81636 mean# 83431.33333 dev# 2547.474741 part# 0.03053378915
135: ch1=78960 ch2=85346 ch3=85988 mean# 83431.33333 dev# 3172.554947 part# 0.03802594085
136: ch1=82564 ch2=83918 ch3=83812 mean# 83431.33333 dev# 614.8221062 part# 0.007369199096
137: ch1=84678 ch2=82716 ch3=82900 mean# 83431.33333 dev# 884.7211739 part# 0.01060418357
138: ch1=84858 ch2=82716 ch3=82720 mean# 83431.33333 dev# 1008.806996 part# 0.01209146439
139: ch1=83312 ch2=83174 ch3=83808 mean# 83431.33333 dev# 272.236825 part# 0.003263004607
140: ch1=80764 ch2=84812 ch3=84718 mean# 83431.33333 dev# 1886.479849 part# 0.02261116746
141: ch1=84402 ch2=83176 ch3=82716 mean# 83431.33333 dev# 711.5922678 part# 0.008529077019
142: ch1=81598 ch2=84892 ch3=83804 mean# 83431.33333 dev# 1370.345293 part# 0.01642482792
143: ch1=86408 ch2=81628 ch3=82258 mean# 83431.33333 dev# 2120.476886 part# 0.02541583361
144: ch1=80666 ch2=84892 ch3=84736 mean# 83431.33333 dev# 1956.422813 part# 0.02344949714
145: ch1=87018 ch2=81712 ch3=81564 mean# 83431.33333 dev# 2536.875944 part# 0.03040675299
146: ch1=81148 ch2=84262 ch3=84884 mean# 83431.33333 dev# 1634.406994 part# 0.01958984627
147: ch1=84236 ch2=83350 ch3=82708 mean# 83431.33333 dev# 626.4488983 part# 0.007508556717
148: ch1=81560 ch2=84854 ch3=83880 mean# 83431.33333 dev# 1381.686248 part# 0.01656075953
149: ch1=87050 ch2=81624 ch3=81620 mean# 83431.33333 dev# 2558.78426 part# 0.03066934397
150: ch1=80064 ch2=85346 ch3=84884 mean# 83431.33333 dev# 2388.522742 part# 0.02862860566
151: ch1=83478 ch2=83478 ch3=83338 mean# 83431.33333 dev# 65.99663291 part# 0.0007910293444
152: ch1=83782 ch2=82720 ch3=83792 mean# 83431.33333 dev# 503.005191 part# 0.006028972221
153: ch1=82694 ch2=83808 ch3=83792 mean# 83431.33333 dev# 521.414316 part# 0.006249622236
154: ch1=84870 ch2=82720 ch3=82704 mean# 83431.33333 dev# 1017.311926 part# 0.01219340367
155: ch1=80518 ch2=84896 ch3=84880 mean# 83431.33333 dev# 2060.048112 part# 0.02469154009
156: ch1=85328 ch2=82720 ch3=82246 mean# 83431.33333 dev# 1355.034399 part# 0.016241313
157: ch1=82698 ch2=83808 ch3=83788 mean# 83431.33333 dev# 518.6092513 part# 0.006216000999
158: ch1=84874 ch2=82720 ch3=82700 mean# 83431.33333 dev# 1020.152058 part# 0.01222744523
159: ch1=80780 ch2=84896 ch3=84618 mean# 83431.33333 dev# 1878.207893 part# 0.02251202058
160: ch1=87212 ch2=81632 ch3=81450 mean# 83431.33333 dev# 2674.367381 part# 0.03205471223
161: ch1=81156 ch2=84266 ch3=84872 mean# 83431.33333 dev# 1627.81353 part# 0.01951081764
162: ch1=84244 ch2=83354 ch3=82696 mean# 83431.33333 dev# 634.3297425 part# 0.007603015763
163: ch1=81754 ch2=84756 ch3=83784 mean# 83431.33333 dev# 1250.67484 part# 0.01499046929
164: ch1=87058 ch2=81628 ch3=81608 mean# 83431.33333 dev# 2564.453591 part# 0.03073729604
165: ch1=81160 ch2=84262 ch3=84872 mean# 83431.33333 dev# 1625.267431 part# 0.01948030034
166: ch1=84248 ch2=83350 ch3=82696 mean# 83431.33333 dev# 636.2061161 part# 0.007625505799
167: ch1=82190 ch2=84320 ch3=83784 mean# 83431.33333 dev# 904.6197482 part# 0.01084268598
168: ch1=85974 ch2=82712 ch3=81608 mean# 83431.33333 dev# 1853.567611 part# 0.02221668451
169: ch1=81164 ch2=84258 ch3=84872 mean# 83431.33333 dev# 1622.72391 part# 0.01944981394
170: ch1=83960 ch2=83346 ch3=82988 mean# 83431.33333 dev# 401.378735 part# 0.004810887216
171: ch1=84798 ch2=82796 ch3=82700 mean# 83431.33333 dev# 967.1736602 part# 0.0115924512
172: ch1=81160 ch2=84258 ch3=84876 mean# 83431.33333 dev# 1625.771065 part# 0.01948633685
173: ch1=85696 ch2=81898 ch3=82700 mean# 83431.33333 dev# 1634.490203 part# 0.0195908436
174: ch1=80886 ch2=84896 ch3=84512 mean# 83431.33333 dev# 1806.6369 part# 0.02165417749
175: ch1=87504 ch2=81632 ch3=81158 mean# 83431.33333 dev# 2886.304365 part# 0.0345949687
176: ch1=81152 ch2=84266 ch3=84876 mean# 83431.33333 dev# 1630.857716 part# 0.01954730496
177: ch1=84240 ch2=83354 ch3=82700 mean# 83431.33333 dev# 631.0759771 part# 0.007564016442
178: ch1=81610 ch2=83808 ch3=84876 mean# 83431.33333 dev# 1359.680681 part# 0.0162970029
179: ch1=88138 ch2=80544 ch3=81612 mean# 83431.33333 dev# 3356.554715 part# 0.04023134452
180: ch1=82240 ch2=83808 ch3=84246 mean# 83431.33333 dev# 861.1687149 part# 0.01032188604
181: ch1=84194 ch2=85572 ch3=80528 mean# 83431.33333 dev# 2128.650485 part# 0.02551380159
182: ch1=87808 ch2=84134 ch3=78352 mean# 83431.33333 dev# 3892.23923 part# 0.046652008
183: ch1=87050 ch2=88156 ch3=75088 mean# 83431.33333 dev# 5916.880785 part# 0.07091916848
184: ch1=89226 ch2=88156 ch3=72912 mean# 83431.33333 dev# 7451.107539 part# 0.08930826395
185: ch1=90314 ch2=89244 ch3=70736 mean# 83431.33333 dev# 8987.578144 part# 0.107724254
186: ch1=91854 ch2=92056 ch3=66384 mean# 83431.33333 dev# 12054.56708 part# 0.1444848908
187: ch1=94662 ch2=90336 ch3=65296 mean# 83431.33333 dev# 12944.65927 part# 0.1551534508
188: ch1=93574 ch2=94688 ch3=62032 mean# 83431.33333 dev# 15138.44663 part# 0.1814479767
189: ch1=95750 ch2=94688 ch3=59856 mean# 83431.33333 dev# 16675.91511 part# 0.1998759273
190: ch1=98464 ch2=95776 ch3=56054 mean# 83431.33333 dev# 19389.77602 part# 0.2324040052
191: ch1=94528 ch2=100266 ch3=55500 mean# 83431.33333 dev# 19888.86954 part# 0.2383860924
192: ch1=101648 ch2=97498 ch3=51148 mean# 83431.33333 dev# 22890.54875 part# 0.2743639331
193: ch1=100106 ch2=101216 ch3=48972 mean# 83431.33333 dev# 24370.6417 part# 0.2921041859
194: ch1=103370 ch2=101216 ch3=45708 mean# 83431.33333 dev# 26688.91577 part# 0.3198907976
195: ch1=102282 ch2=103392 ch3=44620 mean# 83431.33333 dev# 27447.49802 part# 0.3289830921
196: ch1=104188 ch2=105838 ch3=40268 mean# 83431.33333 dev# 30528.51818 part# 0.3659119058
197: ch1=108810 ch2=103392 ch3=38092 mean# 83431.33333 dev# 32135.96159 part# 0.3851785691
198: ch1=107722 ch2=107744 ch3=34828 mean# 83431.33333 dev# 34367.74776 part# 0.4119285452
199: ch1=106634 ch2=109920 ch3=33740 mean# 83431.33333 dev# 35162.67818 part# 0.4214565053
200: ch1=112532 ch2=108374 ch3=29388 mean# 83431.33333 dev# 38252.09057 part# 0.4584859074
201: ch1=111052 ch2=112322 ch3=26920 mean# 83431.33333 dev# 39962.91048 part# 0.4789916316
202: ch1=114254 ch2=111008 ch3=25032 mean# 83431.33333 dev# 41315.82206 part# 0.4952075008
203: ch1=114712 ch2=114902 ch3=20680 mean# 83431.33333 dev# 44371.96113 part# 0.5318380919
204: ch1=114528 ch2=116174 ch3=19592 mean# 83431.33333 dev# 45146.22678 part# 0.5411183662
205: ch1=119338 ch2=115352 ch3=15604 mean# 83431.33333 dev# 47988.76542 part# 0.575188763
206: ch1=114534 ch2=120524 ch3=15236 mean# 83431.33333 dev# 48283.34869 part# 0.57871961
207: ch1=122336 ch2=117074 ch3=10884 mean# 83431.33333 dev# 51343.67109 part# 0.6154003423
208: ch1=119248 ch2=122338 ch3=8708 mean# 83431.33333 dev# 52852.43251 part# 0.6334842127
209: ch1=124054 ch2=121884 ch3=4356 mean# 83431.33333 dev# 55921.72197 part# 0.6702724232
210: ch1=121878 ch2=124060 ch3=4356 mean# 83431.33333 dev# 55921.79979 part# 0.670273356
211: ch1=123102 ch2=126558 ch3=634 mean# 83431.33333 dev# 58563.55402 part# 0.7019371701
212: ch1=128452 ch2=121842 mean# 125147 dev# 3305 part# 0.02640894308
213: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
214: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
215: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
216: ch1=123146 ch2=127148 mean# 125147 dev# 2001 part# 0.0159891967
217: ch1=127954 ch2=122340 mean# 125147 dev# 2807 part# 0.02242962276
218: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
219: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
220: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
221: ch1=123644 ch2=126650 mean# 125147 dev# 1503 part# 0.01200987639
222: ch1=128544 ch2=121750 mean# 125147 dev# 3397 part# 0.02714407856
223: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
224: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
225: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
226: ch1=123054 ch2=127240 mean# 125147 dev# 2093 part# 0.01672433219
227: ch1=128050 ch2=122244 mean# 125147 dev# 2903 part# 0.02319672066
228: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
229: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
230: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
231: ch1=122464 ch2=127830 mean# 125147 dev# 2683 part# 0.02143878799
232: ch1=128640 ch2=121654 mean# 125147 dev# 3493 part# 0.02791117646
233: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
234: ch1=127326 ch2=122968 mean# 125147 dev# 2179 part# 0.01741152405
235: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
236: ch1=122962 ch2=127332 mean# 125147 dev# 2185 part# 0.01745946767
237: ch1=128142 ch2=122152 mean# 125147 dev# 2995 part# 0.02393185614
238: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
239: ch1=126696 ch2=123598 mean# 125147 dev# 1549 part# 0.01237744413
240: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
241: ch1=122376 ch2=127918 mean# 125147 dev# 2771 part# 0.02214196105
242: ch1=128736 ch2=121558 mean# 125147 dev# 3589 part# 0.02867827435
243: ch1=121890 ch2=128404 mean# 125147 dev# 3257 part# 0.02602539414
244: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
245: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
246: ch1=123962 ch2=126332 mean# 125147 dev# 1185 part# 0.009468864615
247: ch1=128238 ch2=122056 mean# 125147 dev# 3091 part# 0.02469895403
248: ch1=122978 ch2=127316 mean# 125147 dev# 2169 part# 0.01733161802
249: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
250: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
251: ch1=123372 ch2=126922 mean# 125147 dev# 1775 part# 0.01418332042
252: ch1=128828 ch2=121466 mean# 125147 dev# 3681 part# 0.02941340983
253: ch1=121890 ch2=128404 mean# 125147 dev# 3257 part# 0.02602539414
254: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
255: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
256: ch1=123870 ch2=126424 mean# 125147 dev# 1277 part# 0.0102040001
257: ch1=128330 ch2=121964 mean# 125147 dev# 3183 part# 0.02543408951
258: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
259: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
260: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
261: ch1=123276 ch2=127018 mean# 125147 dev# 1871 part# 0.01495041831
262: ch1=127832 ch2=122462 mean# 125147 dev# 2685 part# 0.02145476919
263: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
264: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
265: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
266: ch1=123774 ch2=126520 mean# 125147 dev# 1373 part# 0.01097109799
267: ch1=128422 ch2=121872 mean# 125147 dev# 3275 part# 0.02616922499
268: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
269: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
270: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
271: ch1=123184 ch2=127110 mean# 125147 dev# 1963 part# 0.01568555379
272: ch1=127924 ch2=122370 mean# 125147 dev# 2777 part# 0.02218990467
273: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
274: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
275: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
276: ch1=122594 ch2=127700 mean# 125147 dev# 2553 part# 0.02040000959
277: ch1=121152 ch2=129142 mean# 125147 dev# 3995 part# 0.03192245919
278: ch1=114066 ch2=136228 mean# 125147 dev# 11081 part# 0.08854387241
279: ch1=112520 ch2=137774 mean# 125147 dev# 12627 part# 0.1008973447
280: ch1=107710 ch2=142584 mean# 125147 dev# 17437 part# 0.1393321454
281: ch1=102900 ch2=147394 mean# 125147 dev# 22247 part# 0.1777669461
282: ch1=100960 ch2=149334 mean# 125147 dev# 24187 part# 0.193268716
283: ch1=93284 ch2=157010 mean# 125147 dev# 31863 part# 0.254604585
284: ch1=92826 ch2=157468 mean# 125147 dev# 32321 part# 0.2582642812
285: ch1=88016 ch2=162278 mean# 125147 dev# 37131 part# 0.2966990819
286: ch1=82118 ch2=166520 mean# 124319 dev# 42201 part# 0.3394573637
2025-11-27T18:16:57.865652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470859160768357:2048] [node 2] ICP01 ready to work
2025-11-27T18:16:57.891352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470859344130649:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:57.930387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470859160768367:2057] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:57.932454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470859344130660:2057] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:57.931942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577470859160768357:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:57.948813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470859160768367:2057] [node 2] ICH04 handshake succeeded
2025-11-27T18:16:57.949241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577470859160768357:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:16:57.949392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470859344130660:2057] [node 1] ICH04 handshake succeeded
2025-11-27T18:16:57.949453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:16:57.949500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470859160768357:2048] [node 2] ICP22 created new session: [1:7577470859160768370:2048]
2025-11-27T18:16:57.949712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470859344130649:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:16:57.949778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:16:57.949800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470859344130649:2048] [node 1] ICP22 created new session: [2:7577470859344130663:2048]
2025-11-27T18:16:57.950526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470859160768370:2048] [node 2] ICS09 handshake done sender: [1:7577470859160768367:2057] self: [1:7577470859160768368:0] peer: [2:7577470859344130659:0] socket: 18 qp: 24
2025-11-27T18:16:57.953365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470859344130663:2048] [node 1] ICS09 handshake done sender: [2:7577470859344130660:2057] self: [2:7577470859344130659:0] peer: [1:7577470859160768368:0] socket: 16 qp: 23
2025-11-27T18:16:57.953432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470859344130663:2048] [node 1] ICS10 traffic start
2025-11-27T18:16:57.953641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470859160768370:2048] [node 2] ICS10 traffic start
2025-11-27T18:16:57.954360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:16:57.954224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:16:58.004815Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:16:58.005123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470859344130663:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:16:58.005333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470859344130649:2048] [node 1] ICP30 unregister session Session# [2:7577470859344130663:2048] VirtualId# [2:7577470859344130659:0]
2025-11-27T18:16:58.005392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470859344130663:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:58.005475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:16:58.005526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:16:58.065679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470865901864165:2048] [node 2] ICP01 ready to work
2025-11-27T18:16:58.088623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470864374593399:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:58.094730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470865901864175:2057] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.096243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577470865901864165:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:58.096579Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1583: Handshake [1:7577470865901864175:2057] [node 2] ICH03 handshake failed, explanation# incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected
2025-11-27T18:16:58.096742Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1583: Handshake [2:7577470864374593410:2057] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:11096) error from peer: connection checker refused to proceed with incoming connection: rejected
2025-11-27T18:16:58.097721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:404: Proxy [1:7577470865901864165:2048] [node 2] ICP24 incoming handshake failed, temporary: 1 explanation: incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected outgoing: [0:0:0]
2025-11-27T18:16:58.097792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [1:7577470865901864165:2048] [node 2] ICP32 transit to hold-by-error state Explanation# incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:16:58.097829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:16:58.098102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470864374593399:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# ::1(::1:11096) error from peer: connection checker refused to proceed with incoming connection: rejected incoming: [0:0:0] held: no
2025-11-27T18:16:58.098168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470864374593399:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# ::1(::1:11096) error from peer: connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:16:58.098193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# ::1(::1:11096) error from peer: connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:16:58.146017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470865307569205:2048] [node 2] ICP01 ready to work
2025-11-27T18:16:58.146064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470865307569206:2049] [node 3] ICP01 ready to work
2025-11-27T18:16:58.168528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470862396649096:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:58.168600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470862396649097:2049] [node 3] ICP01 ready to work
2025-11-27T18:16:58.193376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470864338204265:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:58.193425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470864338204266:2049] [node 2] ICP01 ready to work
2025-11-27T18:16:58.215611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470864941259201:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:58.215662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470864941259202:2049] [node 2] ICP01 ready to work
2025-11-27T18:16:58.215692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470864941259203:2050] [node 3] ICP01 ready to work
2025-11-27T18:16:58.240687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470863468925705:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:58.240720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470863468925706:2049] [node 2] ICP01 ready to work
2025-11-27T18:16:58.240737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470863468925707:2050] [node 3] ICP01 ready to work
2025-11-27T18:16:58.274500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470864159319580:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:58.274557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470864159319581:2049] [node 2] ICP01 ready to work
2025-11-27T18:16:58.274579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470864159319582:2050] [node 3] ICP01 ready to work
2025-11-27T18:16:58.444203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470864159319629:2096] [node 5] ICP01 ready to work
2025-11-27T18:16:58.487771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470864159319629:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(42763325ns) {Configure:104(42758104ns) {ProcessConfigured:135(1392274ns) {StartInitialHandshake:157(1378267ns) {DropHandshakes:470(1795ns) {DropIncomingHandshake:442(434ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(493ns) PrepareNewSessionHandshake:477(3060ns) {SwitchToState:179(1875ns)}}}}}
2025-11-27T18:16:58.488165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470864159319646:2097] [node 4] ICP01 ready to work
2025-11-27T18:16:58.487877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470865307569268:2107] [node 4] ICP01 ready to work
2025-11-27T18:16:58.488249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470865307569269:2108] [node 5] ICP01 ready to work
2025-11-27T18:16:58.504046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470862396649159:2107] [node 5] ICP01 ready to work
2025-11-27T18:16:58.511921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470862396649160:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.518248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470863468925766:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28921
2025-11-27T18:16:58.519512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470862396649161:2109] [node 6] ICP01 ready to work
2025-11-27T18:16:58.519510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470865307569274:2109] [node 6] ICP01 ready to work
2025-11-27T18:16:58.523866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470863468925766:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.524480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470863468925768:2109] [node 6] ICP01 ready to work
2025-11-27T18:16:58.544912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470863468925768:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:58.545234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470863468925768:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20418596ns) {Configure:104(20416058ns) {ProcessConfigured:135(126762ns) {IncomingHandshake:261(106369ns) {DropIncomingHandshake:442(431ns) PrepareNewSessionHandshake:477(2420ns) {SwitchToState:179(1177ns)} GenerateSessionVirtualId:988(3904ns) IssueIncomingHandshakeReply:193(74214ns) {SwitchToState:179(323ns)}}}}}
2025-11-27T18:16:58.621758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470864338204328:2107] [node 4] ICP01 ready to work
2025-11-27T18:16:58.602051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470865307569268:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(81965121ns) {Configure:104(81957896ns) {ProcessConfigured:135(28349575ns) {StartInitialHandshake:157(28344466ns) {DropHandshakes:470(2517ns) {DropIncomingHandshake:442(545ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(1665ns) PrepareNewSessionHandshake:477(4676ns) {SwitchToState:179(2177ns)}}}}}
2025-11-27T18:16:58.650358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470865307569269:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(48077498ns) {Configure:104(48071468ns) {ProcessConfigured:135(85670ns) {StartInitialHandshake:157(82605ns) {DropHandshakes:470(3004ns) {DropIncomingHandshake:442(574ns) DropOutgoingHandshake:457(338ns)} GenerateSessionVirtualId:988(1179ns) PrepareNewSessionHandshake:477(4592ns) {SwitchToState:179(492ns)}}}}}
2025-11-27T18:16:58.658178Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470863468925705:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(95521528ns) {Configure:104(95514840ns) {ProcessConfigured:135(80726ns) {StartInitialHandshake:157(78378ns) {DropHandshakes:470(3001ns) {DropIncomingHandshake:442(756ns) DropOutgoingHandshake:457(702ns)} GenerateSessionVirtualId:988(753ns) PrepareNewSessionHandshake:477(2047ns) {SwitchToState:179(467ns)}}}}}
2025-11-27T18:16:58.661233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470864941259264:2108] [node 5] ICP01 ready to work
2025-11-27T18:16:58.666255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470864159319582:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(116637442ns) {Configure:104(116633220ns) {ProcessConfigured:135(55229ns) {StartInitialHandshake:157(52988ns) {DropHandshakes:470(2166ns) {DropIncomingHandshake:442(509ns) DropOutgoingHandshake:457(317ns)} GenerateSessionVirtualId:988(852ns) PrepareNewSessionHandshake:477(1690ns) {SwitchToState:179(566ns)}}}}}
2025-11-27T18:16:58.666403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470864159319645:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.666438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7577470864159319645:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:16:58.680350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470863468925775:2110] [node 4] ICP01 ready to work
2025-11-27T18:16:58.681298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470864159319645:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:16:58.681099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470864941259265:2109] [node 6] ICP01 ready to work
2025-11-27T18:16:58.696693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7577470864159319629:2096] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:16:58.696898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:16:58.696812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470862396649159:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(159009406ns) {Configure:104(159002648ns) {ProcessConfigured:135(5383435ns) {StartInitialHandshake:157(5380929ns) {DropHandshakes:470(3719435ns) {DropIncomingHandshake:442(2109ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(2092ns) PrepareNewSessionHandshake:477(3421ns) {SwitchToState:179(1658ns)}}}}}
2025-11-27T18:16:58.704372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470864338204266:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(68478225ns) {Configure:104(68469703ns) {ProcessConfigured:135(5734415ns) {StartInitialHandshake:157(5732226ns) {DropHandshakes:470(2247ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(899ns) PrepareNewSessionHandshake:477(2339ns) {SwitchToState:179(191ns)}}}}}
2025-11-27T18:16:58.705770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470865307569274:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27135662ns) {Configure:104(25451742ns) {ProcessConfigured:135(59062ns) {StartInitialHandshake:157(57054ns) {DropHandshakes:470(764ns) {DropIncomingHandshake:442(150ns) DropOutgoingHandshake:457(147ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:477(1802ns) {SwitchToState:179(607ns)}}}}}
2025-11-27T18:16:58.704839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470864338204331:2108] [node 5] ICP01 ready to work
2025-11-27T18:16:58.705159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470864338204332:2109] [node 6] ICP01 ready to work
2025-11-27T18:16:58.705608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470864159319629:2096] [node 5] ICP22 created new session: [6:7577470864159319659:2096]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.722205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470864159319659:2096] [node 5] ICS09 handshake done sender: [6:7577470864159319645:2110] self: [6:7577470864159319643:0] peer: [5:7577470863468925769:0] socket: 36 qp: -1
2025-11-27T18:16:58.732147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470863468925776:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28921
2025-11-27T18:16:58.732717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470864159319659:2096] [node 5] ICS10 traffic start
2025-11-27T18:16:58.741095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.755047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470863468925777:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28921
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.773622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470864338204332:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20381217ns) {Configure:104(20378192ns) {ProcessConfigured:135(4073467ns) {StartInitialHandshake:157(4059270ns) {DropHandshakes:470(3565ns) {DropIncomingHandshake:442(569ns) DropOutgoingHandshake:457(697ns)} GenerateSessionVirtualId:988(1384ns) PrepareNewSessionHandshake:477(3545ns) {SwitchToState:179(1524ns)}}}}}
2025-11-27T18:16:58.755675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470863468925766:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.756370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470864159319629:2096] [node 5] ICP35 event processing took too much time PendingConnection:279(60762108ns) {HandleHandshakeStatus:329(60758992ns) {DropIncomingHandshake:442(1846ns) DropOutgoingHandshake:457(2240ns) SwitchToState:179(984ns) ProcessPendingSessionEvents:495(140266ns) {ForwardSessionEventToSession:612(77024ns) ForwardSessionEventToSession:612(3057ns) ForwardSessionEventToSession:612(8691ns) ForwardSessionEventToSession:612(1581ns) ForwardSessionEventToSession:612(4723ns) ForwardSessionEventToSession:612(2504ns) ForwardSessionEventToSession:612(2051ns) ForwardSessionEventToSession:612(4689ns) ForwardSessionEventToSession:612(2658ns) ForwardSessionEventToSession:612(23338ns)}}}
2025-11-27T18:16:58.774221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470864338204333:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46929
2025-11-27T18:16:58.774275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470864338204334:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46929
2025-11-27T18:16:58.774799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470864338204343:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46929
2025-11-27T18:16:58.788796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470865307569284:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2369
2025-11-27T18:16:58.788831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470864941259264:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(72779752ns) {Configure:104(72772586ns) {ProcessConfigured:135(13768057ns) {StartInitialHandshake:157(13765239ns) {DropHandshakes:470(1661ns) {DropIncomingHandshake:442(317ns) DropOutgoingHandshake:457(454ns)} GenerateSessionVirtualId:988(492ns) PrepareNewSessionHandshake:477(4206ns) {SwitchToState:179(2030ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.825751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470863468925777:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.825807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470863468925768:2109] [node 6] ICP19 incoming handshake succeeded
2025-11-27T18:16:58.825939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:16:58.824741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470862396649175:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
2025-11-27T18:16:58.825516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470862396649176:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
2025-11-27T18:16:58.824631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470864338204334:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.825544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470864338204266:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:58.826556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470862396649177:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.826548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470863468925768:2109] [node 6] ICP22 created new session: [5:7577470863468925785:2109]
2025-11-27T18:16:58.828049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470864941259266:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33364
2025-11-27T18:16:58.828347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470864941259277:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33364
2025-11-27T18:16:58.829167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470864338204333:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.830190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470862396649167:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.826834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470863468925785:2109] [node 6] ICS09 handshake done sender: [5:7577470863468925766:2108] self: [5:7577470863468925769:0] peer: [6:7577470864159319643:0] socket: 39 qp: -1
2025-11-27T18:16:58.828439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470864941259278:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33364
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.830222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470862396649167:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:16:58.867625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470864159319662:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3388
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.828506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470864941259279:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33364
2025-11-27T18:16:58.837443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470862396649167:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:16:58.837728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470862396649097:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:16:58.837799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:16:58.829292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470863468925785:2109] [node 6] ICS10 traffic start
2025-11-27T18:16:58.837844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470862396649097:2049] [node 3] ICP22 created new session: [2:7577470862396649179:2049]
2025-11-27T18:16:58.837878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470862396649179:2049] [node 3] ICS09 handshake done sender: [2:7577470862396649167:2112] self: [2:7577470862396649166:0] peer: [3:7577470864338204347:0] socket: 46 qp: -1
2025-11-27T18:16:58.837906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470862396649179:2049] [node 3] ICS10 traffic start
2025-11-27T18:16:58.837945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:16:58.867693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470864159319663:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3388
2025-11-27T18:16:58.873594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470864159319663:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.829345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:16:58.854591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470863468925706:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:58.887992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470863468925787:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28921
2025-11-27T18:16:58.828594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470864941259280:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33364
2025-11-27T18:16:58.854455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470864941259266:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.867961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470864941259277:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.894323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470864159319581:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:58.894614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470862396649163:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.894664Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470862396649163:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.907968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470864338204348:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46929
2025-11-27T18:16:58.908080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470864338204349:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46929
2025-11-27T18:16:58.908310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470862396649169:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.908350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470864941259202:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:58.917459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470864941259279:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.922435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470864338204334:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:16:58.908380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470862396649169:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:16:58.929643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470864338204266:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:16:58.929756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:16:58.929825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470864338204266:2049] [node 2] ICP22 created new session: [3:7577470864338204351:2049]
2025-11-27T18:16:58.929882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470864338204351:2049] [node 2] ICS09 handshake done sender: [3:7577470864338204334:2112] self: [3:7577470864338204347:0] peer: [2:7577470862396649166:0] socket: 50 qp: -1
2025-11-27T18:16:58.929921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470864338204351:2049] [node 2] ICS10 traffic start
2025-11-27T18:16:58.929971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.932451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470864159319665:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3388
2025-11-27T18:16:58.931059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470862396649177:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.944834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470864159319663:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:16:58.933766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470862396649097:2049] [node 3] ICP18 handshake (actor [2:7577470862396649177:2117]) is too old
2025-11-27T18:16:58.940416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470862396649163:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:16:58.940800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470862396649169:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:16:58.944910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470862396649159:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:16:58.944994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:16:58.945050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470862396649159:2107] [node 5] ICP22 created new session: [2:7577470862396649184:2107]
2025-11-27T18:16:58.945091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470862396649184:2107] [node 5] ICS09 handshake done sender: [2:7577470862396649163:2110] self: [2:7577470862396649162:0] peer: [5:7577470863468925788:0] socket: 44 qp: -1
2025-11-27T18:16:58.945583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470862396649184:2107] [node 5] ICS10 traffic start
2025-11-27T18:16:58.958509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:16:58.958602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470864159319581:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:16:58.961815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470862396649159:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(17108852ns) {HandleHandshakeStatus:329(17100683ns) {DropIncomingHandshake:442(1054ns) DropOutgoingHandshake:457(716ns) SwitchToState:179(998ns) ProcessPendingSessionEvents:495(2846973ns) {ForwardSessionEventToSession:612(2781953ns) ForwardSessionEventToSession:612(8919ns) ForwardSessionEventToSession:612(2708ns) ForwardSessionEventToSession:612(10223ns) ForwardSessionEventToSession:612(2753ns) ForwardSessionEventToSession:612(2632ns) ForwardSessionEventToSession:612(5037ns) ForwardSessionEventToSession:612(4649ns) ForwardSessionEventToSession:612(4603ns) ForwardSessionEventToSession:612(5799ns)}}}
2025-11-27T18:16:58.961926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470862396649161:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:16:58.962009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:16:58.962057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470862396649161:2109] [node 6] ICP22 created new session: [2:7577470862396649186:2109]
2025-11-27T18:16:58.962132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470862396649186:2109] [node 6] ICS09 handshake done sender: [2:7577470862396649169:2113] self: [2:7577470862396649168:0] peer: [6:7577470864159319664:0] socket: 79 qp: -1
2025-11-27T18:16:58.962172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470862396649186:2109] [node 6] ICS10 traffic start
2025-11-27T18:16:58.962229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:16:58.958716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:16:58.958774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470864159319581:2049] [node 2] ICP22 created new session: [6:7577470864159319666:2049]
2025-11-27T18:16:58.962969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470864159319666:2049] [node 2] ICS09 handshake done sender: [6:7577470864159319663:2116] self: [6:7577470864159319664:0] peer: [2:7577470862396649168:0] socket: 82 qp: -1
2025-11-27T18:16:58.963007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470864159319666:2049] [node 2] ICS10 traffic start
2025-11-27T18:16:58.963046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.965504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470865307569285:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2369
2025-11-27T18:16:58.965863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470864338204343:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.966105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470864338204265:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.969703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470864338204355:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46929
2025-11-27T18:16:58.967183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470865307569286:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2369
2025-11-27T18:16:58.967401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470865307569285:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.967523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470865307569273:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.967541Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470865307569273:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:16:58.972401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470862396649165:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.972478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470862396649165:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:16:58.974108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470864338204343:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:16:58.974427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470864338204265:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:16:58.974514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:16:58.974570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470864338204265:2048] [node 1] ICP22 created new session: [3:7577470864338204356:2048]
2025-11-27T18:16:58.974633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470864338204356:2048] [node 1] ICS09 handshake done sender: [3:7577470864338204343:2117] self: [3:7577470864338204354:0] peer: [1:7577470865307569272:0] socket: 59 qp: -1
2025-11-27T18:16:58.974671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470864338204356:2048] [node 1] ICS10 traffic start
2025-11-27T18:16:58.974702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:16:58.984282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470862396649175:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.984952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470862396649096:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:58.975638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470865307569273:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:16:58.992475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470862396649165:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:16:58.991191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470864941259278:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.991391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470864941259203:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:58.991751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470864941259201:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:58.991838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470864941259286:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33364
2025-11-27T18:16:58.994770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470864941259277:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:16:58.995179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470864941259202:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:16:58.995269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:16:58.995314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470864941259202:2049] [node 2] ICP22 created new session: [4:7577470864941259289:2049]
2025-11-27T18:16:58.995364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470864941259289:2049] [node 2] ICS09 handshake done sender: [4:7577470864941259277:2116] self: [4:7577470864941259285:0] peer: [2:7577470862396649164:0] socket: 68 qp: -1
2025-11-27T18:16:58.995396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470864941259289:2049] [node 2] ICS10 traffic start
2025-11-27T18:16:59.000916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470862396649160:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:16:58.993090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470864159319662:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:58.995443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:16:59.010532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470864941259270:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.010570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470864941259270:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.011603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470865307569286:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.011835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470865307569271:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.011855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470865307569271:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.012518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470865307569276:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.012535Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470865307569276:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.010832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:16:59.010018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470864159319646:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.020613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470865307569206:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:16:59.022004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:16:59.022061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470865307569206:2049] [node 3] ICP22 created new session: [1:7577470869602536587:2049]
2025-11-27T18:16:59.022101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470869602536587:2049] [node 3] ICS09 handshake done sender: [1:7577470865307569273:2111] self: [1:7577470865307569272:0] peer: [3:7577470864338204354:0] socket: 54 qp: -1
2025-11-27T18:16:59.022131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470869602536587:2049] [node 3] ICS10 traffic start
2025-11-27T18:16:59.022177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:16:59.022735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470862396649160:2108] [node 4] ICP22 created new session: [2:7577470866691616487:2108]
2025-11-27T18:16:59.022795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470866691616487:2108] [node 4] ICS09 handshake done sender: [2:7577470862396649165:2111] self: [2:7577470862396649164:0] peer: [4:7577470864941259285:0] socket: 45 qp: -1
2025-11-27T18:16:59.022821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470866691616487:2108] [node 4] ICS10 traffic start
2025-11-27T18:16:59.022876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:16:59.023372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470862396649160:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(22310707ns) {HandleHandshakeStatus:329(22306523ns) {DropIncomingHandshake:442(702ns) DropOutgoingHandshake:457(715ns) SwitchToState:179(808ns) ProcessPendingSessionEvents:495(110732ns) {ForwardSessionEventToSession:612(42034ns) ForwardSessionEventToSession:612(2211ns) ForwardSessionEventToSession:612(30927ns) ForwardSessionEventToSession:612(3562ns) ForwardSessionEventToSession:612(8573ns) ForwardSessionEventToSession:612(2143ns) ForwardSessionEventToSession:612(1703ns) ForwardSessionEventToSession:612(4527ns) ForwardSessionEventToSession:612(2002ns) ForwardSessionEventToSession:612(2323ns)}}}
2025-11-27T18:16:59.023723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470862396649176:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.026080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470862396649159:2107] [node 5] ICP18 handshake (actor [2:7577470862396649176:2116]) is too old
2025-11-27T18:16:59.033212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470864941259280:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.032648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470864338204348:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.032784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470864338204336:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.032804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470864338204336:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.041866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470865307569284:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.043707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470866691616489:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53767
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.045033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470869602536585:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2369
2025-11-27T18:16:59.045539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470864338204336:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.045940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470863468925706:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2002) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470863468925777:2114] held: no
2025-11-27T18:16:59.046132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470863468925787:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.046212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470867763893085:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28921
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.047033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470868454286965:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3388
2025-11-27T18:16:59.047196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470869602536586:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2369
2025-11-27T18:16:59.047757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470869602536585:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.049330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470869236226589:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33364
2025-11-27T18:16:59.049400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470869236226590:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33364
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.049886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470868454286966:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3388
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.050535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470867763893086:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28921
2025-11-27T18:16:59.051023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470863468925777:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:16:59.051332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470865307569271:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:16:59.051866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470862396649175:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:16:59.052562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470862396649096:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:16:59.052666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:16:59.052723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470862396649096:2048] [node 1] ICP22 created new session: [2:7577470866691616490:2048]
2025-11-27T18:16:59.052782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470866691616490:2048] [node 1] ICS09 handshake done sender: [2:7577470862396649175:2115] self: [2:7577470862396649190:0] peer: [1:7577470865307569270:0] socket: 57 qp: -1
2025-11-27T18:16:59.052823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470866691616490:2048] [node 1] ICS10 traffic start
2025-11-27T18:16:59.052868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:16:59.053718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470864338204328:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:16:59.053797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:16:59.053844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470864338204328:2107] [node 4] ICP22 created new session: [3:7577470868633171657:2107]
2025-11-27T18:16:59.053910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470868633171657:2107] [node 4] ICS09 handshake done sender: [3:7577470864338204336:2113] self: [3:7577470864338204335:0] peer: [4:7577470864941259287:0] socket: 61 qp: -1
2025-11-27T18:16:59.053939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470868633171657:2107] [node 4] ICS10 traffic start
2025-11-27T18:16:59.053984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:16:59.054657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470865307569276:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:16:59.055493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470863468925775:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.055789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470867763893086:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.055827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470863468925706:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:16:59.055892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:16:59.055936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470863468925706:2049] [node 2] ICP22 created new session: [5:7577470867763893090:2049]
2025-11-27T18:16:59.055972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470867763893090:2049] [node 2] ICS09 handshake done sender: [5:7577470863468925777:2114] self: [5:7577470863468925788:0] peer: [2:7577470862396649162:0] socket: 49 qp: -1
2025-11-27T18:16:59.056021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470867763893090:2049] [node 2] ICS10 traffic start
2025-11-27T18:16:59.056054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:16:59.057628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470864941259270:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:16:59.057240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470867763893087:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28921
2025-11-27T18:16:59.061082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470864159319662:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:16:59.061428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470864159319646:2097] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:16:59.061510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:16:59.061576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470864159319646:2097] [node 4] ICP22 created new session: [6:7577470868454286967:2097]
2025-11-27T18:16:59.061616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470868454286967:2097] [node 4] ICS09 handshake done sender: [6:7577470864159319662:2115] self: [6:7577470868454286964:0] peer: [4:7577470864941259269:0] socket: 77 qp: -1
2025-11-27T18:16:59.061642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470868454286967:2097] [node 4] ICS10 traffic start
2025-11-27T18:16:59.061704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.064109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470868633171659:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46929
2025-11-27T18:16:59.057494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470869602536586:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.059490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470865307569205:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:16:59.067418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470863468925707:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.067026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:16:59.067688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470868633171659:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.065119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470864941259268:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.067877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470865307569205:2048] [node 2] ICP22 created new session: [1:7577470869602536595:2048]
2025-11-27T18:16:59.065153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470864941259268:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.065900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470864941259265:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:16:59.065976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:16:59.068018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470869602536595:2048] [node 2] ICS09 handshake done sender: [1:7577470865307569271:2110] self: [1:7577470865307569270:0] peer: [2:7577470862396649190:0] socket: 52 qp: -1
2025-11-27T18:16:59.068018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470864338204340:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.068087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470869602536595:2048] [node 2] ICS10 traffic start
2025-11-27T18:16:59.066040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470864941259265:2109] [node 6] ICP22 created new session: [4:7577470869236226592:2109]
2025-11-27T18:16:59.068057Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470864338204340:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.066101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470869236226592:2109] [node 6] ICS09 handshake done sender: [4:7577470864941259270:2112] self: [4:7577470864941259269:0] peer: [6:7577470868454286964:0] socket: 75 qp: -1
2025-11-27T18:16:59.066141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470869236226592:2109] [node 6] ICS10 traffic start
2025-11-27T18:16:59.068193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:16:59.066189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:16:59.069416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470864941259279:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:16:59.069410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470868454286965:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.069660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470864159319582:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.069925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470864338204340:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:16:59.069549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470865307569268:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:16:59.069647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:16:59.069727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470865307569268:2107] [node 4] ICP22 created new session: [1:7577470869602536597:2107]
2025-11-27T18:16:59.069774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470869602536597:2107] [node 4] ICS09 handshake done sender: [1:7577470865307569276:2112] self: [1:7577470865307569275:0] peer: [4:7577470864941259288:0] socket: 55 qp: -1
2025-11-27T18:16:59.069803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470869602536597:2107] [node 4] ICS10 traffic start
2025-11-27T18:16:59.069835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.070962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470864338204342:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.071008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470864338204342:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.071625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470864338204331:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:16:59.071827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:16:59.071872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470864338204331:2108] [node 5] ICP22 created new session: [3:7577470868633171662:2108]
2025-11-27T18:16:59.071912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470868633171662:2108] [node 5] ICS09 handshake done sender: [3:7577470864338204340:2115] self: [3:7577470864338204339:0] peer: [5:7577470867763893092:0] socket: 60 qp: -1
2025-11-27T18:16:59.071938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470868633171662:2108] [node 5] ICS10 traffic start
2025-11-27T18:16:59.071972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:16:59.075359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470864941259203:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20919) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470864941259279:2118] held: no
2025-11-27T18:16:59.075464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470864941259203:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:16:59.075530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:16:59.075594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470864941259203:2050] [node 3] ICP22 created new session: [4:7577470869236226594:2050]
2025-11-27T18:16:59.075632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470869236226594:2050] [node 3] ICS09 handshake done sender: [4:7577470864941259279:2118] self: [4:7577470864941259287:0] peer: [3:7577470864338204335:0] socket: 72 qp: -1
2025-11-27T18:16:59.075663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470869236226594:2050] [node 3] ICS10 traffic start
2025-11-27T18:16:59.075703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:16:59.074604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470867763893093:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28921
2025-11-27T18:16:59.077550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470864941259278:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:16:59.078129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470864941259201:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:16:59.078207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:16:59.078263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470864941259201:2048] [node 1] ICP22 created new session: [4:7577470869236226597:2048]
2025-11-27T18:16:59.078299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470869236226597:2048] [node 1] ICS09 handshake done sender: [4:7577470864941259278:2117] self: [4:7577470864941259288:0] peer: [1:7577470865307569275:0] socket: 70 qp: -1
2025-11-27T18:16:59.078322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470869236226597:2048] [node 1] ICS10 traffic start
2025-11-27T18:16:59.078361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.083995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470868454286970:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3388
2025-11-27T18:16:59.084392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470864338204342:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:16:59.085366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470868454286965:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:16:59.085510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470865307569268:2107] [node 4] ICP18 handshake (actor [1:7577470869602536586:2119]) is too old
2025-11-27T18:16:59.089717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470867763893086:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:16:59.089950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470863468925707:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20919) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470867763893086:2119] held: no
2025-11-27T18:16:59.088733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470864159319582:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:16:59.090033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470863468925707:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:16:59.092007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470864941259268:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:16:59.090100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:16:59.090137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470863468925707:2050] [node 3] ICP22 created new session: [5:7577470867763893095:2050]
2025-11-27T18:16:59.084761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470864338204332:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:16:59.088820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:16:59.084858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:16:59.084905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470864338204332:2109] [node 6] ICP22 created new session: [3:7577470868633171665:2109]
2025-11-27T18:16:59.084952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470868633171665:2109] [node 6] ICS09 handshake done sender: [3:7577470864338204342:2116] self: [3:7577470864338204341:0] peer: [6:7577470868454286969:0] socket: 98 qp: -1
2025-11-27T18:16:59.084988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470868633171665:2109] [node 6] ICS10 traffic start
2025-11-27T18:16:59.085060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:16:59.088979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470864159319582:2050] [node 3] ICP22 created new session: [6:7577470868454286971:2050]
2025-11-27T18:16:59.090181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470867763893095:2050] [node 3] ICS09 handshake done sender: [5:7577470867763893086:2119] self: [5:7577470867763893092:0] peer: [3:7577470864338204339:0] socket: 62 qp: -1
2025-11-27T18:16:59.099165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470868454286971:2050] [node 3] ICS09 handshake done sender: [6:7577470868454286965:2118] self: [6:7577470868454286969:0] peer: [3:7577470864338204341:0] socket: 99 qp: -1
2025-11-27T18:16:59.103423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470868454286971:2050] [node 3] ICS10 traffic start
2025-11-27T18:16:59.105638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:16:59.125463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470864159319582:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(21147814ns) {HandleHandshakeStatus:329(21144610ns) {DropIncomingHandshake:442(1424ns) DropOutgoingHandshake:457(5535ns) SwitchToState:179(640ns) ProcessPendingSessionEvents:495(3945457ns) {ForwardSessionEventToSession:612(3880628ns) ForwardSessionEventToSession:612(6794ns) ForwardSessionEventToSession:612(2360ns) ForwardSessionEventToSession:612(8023ns) ForwardSessionEventToSession:612(2258ns) ForwardSessionEventToSession:612(8389ns) ForwardSessionEventToSession:612(6867ns) ForwardSessionEventToSession:612(5190ns) ForwardSessionEventToSession:612(7267ns) ForwardSessionEventToSession:612(1621ns)}}}
2025-11-27T18:16:59.090226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470867763893095:2050] [node 3] ICS10 traffic start
2025-11-27T18:16:59.090255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.129477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470867763893094:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28921
2025-11-27T18:16:59.130146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470867763893087:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.130437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470863468925787:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:16:59.130808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470863468925775:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:16:59.130911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:16:59.130955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470863468925775:2110] [node 4] ICP22 created new session: [5:7577470867763893097:2110]
2025-11-27T18:16:59.130996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470867763893097:2110] [node 4] ICS09 handshake done sender: [5:7577470863468925787:2117] self: [5:7577470867763893089:0] peer: [4:7577470864941259267:0] socket: 80 qp: -1
2025-11-27T18:16:59.131028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470867763893097:2110] [node 4] ICS10 traffic start
2025-11-27T18:16:59.131087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:16:59.126995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470864941259264:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:16:59.127118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:16:59.127168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470864941259264:2108] [node 5] ICP22 created new session: [4:7577470869236226599:2108]
2025-11-27T18:16:59.127211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470869236226599:2108] [node 5] ICS09 handshake done sender: [4:7577470864941259268:2111] self: [4:7577470864941259267:0] peer: [5:7577470867763893089:0] socket: 74 qp: -1
2025-11-27T18:16:59.127245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470869236226599:2108] [node 5] ICS10 traffic start
2025-11-27T18:16:59.127283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.135372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470868454286973:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3388
2025-11-27T18:16:59.138028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470863468925705:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.141803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470868454286973:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.141961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470864159319580:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.142274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470865307569278:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.142329Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470865307569278:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.143311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470865307569280:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.143336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470865307569280:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.144310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470865307569278:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:16:59.144616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470865307569269:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.145694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470867763893100:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28921
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.146031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470868454286975:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3388
2025-11-27T18:16:59.144700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:16:59.144767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470865307569269:2108] [node 5] ICP22 created new session: [1:7577470869602536602:2108]
2025-11-27T18:16:59.146235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470867763893087:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:16:59.144824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470869602536602:2108] [node 5] ICS09 handshake done sender: [1:7577470865307569278:2113] self: [1:7577470865307569277:0] peer: [5:7577470867763893099:0] socket: 100 qp: -1
2025-11-27T18:16:59.144856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470869602536602:2108] [node 5] ICS10 traffic start
2025-11-27T18:16:59.144884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:16:59.146598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470865307569280:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:16:59.146970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470868454286973:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:16:59.147135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470864159319580:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:16:59.147201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:16:59.147265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470864159319580:2048] [node 1] ICP22 created new session: [6:7577470868454286976:2048]
2025-11-27T18:16:59.147307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470868454286976:2048] [node 1] ICS09 handshake done sender: [6:7577470868454286973:2121] self: [6:7577470868454286974:0] peer: [1:7577470865307569279:0] socket: 41 qp: -1
2025-11-27T18:16:59.147338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470868454286976:2048] [node 1] ICS10 traffic start
2025-11-27T18:16:59.147393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:16:59.148574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470865307569274:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:16:59.148664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:16:59.148705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470865307569274:2109] [node 6] ICP22 created new session: [1:7577470869602536604:2109]
2025-11-27T18:16:59.148739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470869602536604:2109] [node 6] ICS09 handshake done sender: [1:7577470865307569280:2114] self: [1:7577470865307569279:0] peer: [6:7577470868454286974:0] socket: 48 qp: -1
2025-11-27T18:16:59.148769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470869602536604:2109] [node 6] ICS10 traffic start
2025-11-27T18:16:59.148800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:16:59.154861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470863468925705:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:16:59.154952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:16:59.155004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470863468925705:2048] [node 1] ICP22 created new session: [5:7577470867763893101:2048]
2025-11-27T18:16:59.155046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470867763893101:2048] [node 1] ICS09 handshake done sender: [5:7577470867763893087:2120] self: [5:7577470867763893099:0] peer: [1:7577470865307569277:0] socket: 105 qp: -1
2025-11-27T18:16:59.155079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470867763893101:2048] [node 1] ICS10 traffic start
2025-11-27T18:16:59.155111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:16:59.155343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470863468925705:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:16:59.173715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:16:59.174491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:16:59.174718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470864338204356:2048] [node 1] ICS01 socket: 59 reason# EndOfStream
2025-11-27T18:16:59.174758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470864338204265:2048] [node 1] ICP30 unregister session Session# [3:7577470864338204356:2048] VirtualId# [3:7577470864338204354:0]
2025-11-27T18:16:59.174808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470864338204356:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.174898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:16:59.175953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:16:59.176535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470866691616490:2048] [node 1] ICS01 socket: 57 reason# EndOfStream
2025-11-27T18:16:59.176594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470862396649096:2048] [node 1] ICP30 unregister session Session# [2:7577470866691616490:2048] VirtualId# [2:7577470862396649190:0]
2025-11-27T18:16:59.176633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470866691616490:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.176719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:16:59.176758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:16:59.185391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:16:59.185601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:16:59.185784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470868454286976:2048] [node 1] ICS01 socket: 41 reason# EndOfStream
2025-11-27T18:16:59.185815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470864159319580:2048] [node 1] ICP30 unregister session Session# [6:7577470868454286976:2048] VirtualId# [6:7577470868454286974:0]
2025-11-27T18:16:59.185850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470868454286976:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.185391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:16:59.185909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:16:59.185946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:16:59.185628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470867763893101:2048] [node 1] ICS01 socket: 105 reason# EndOfStream
2025-11-27T18:16:59.185673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470863468925705:2048] [node 1] ICP30 unregister session Session# [5:7577470867763893101:2048] VirtualId# [5:7577470867763893099:0]
2025-11-27T18:16:59.185723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470867763893101:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.185806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:16:59.185853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:16:59.189533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470869236226597:2048] [node 1] ICS01 socket: 70 reason# EndOfStream
2025-11-27T18:16:59.198136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470864941259201:2048] [node 1] ICP30 unregister session Session# [4:7577470869236226597:2048] VirtualId# [4:7577470864941259288:0]
2025-11-27T18:16:59.198205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470869236226597:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.198321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:16:59.198369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:16:59.213448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:16:59.213696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470864338204351:2049] [node 2] ICS01 socket: 50 reason# EndOfStream
2025-11-27T18:16:59.213738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470864338204266:2049] [node 2] ICP30 unregister session Session# [3:7577470864338204351:2049] VirtualId# [3:7577470864338204347:0]
2025-11-27T18:16:59.213771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470864338204351:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.213848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:16:59.213886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:16:59.220103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:16:59.220173Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:16:59.220338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470864941259289:2049] [node 2] ICS01 socket: 68 reason# EndOfStream
2025-11-27T18:16:59.220422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470864159319666:2049] [node 2] ICS01 socket: 82 reason# EndOfStream
2025-11-27T18:16:59.220377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470864941259202:2049] [node 2] ICP30 unregister session Session# [4:7577470864941259289:2049] VirtualId# [4:7577470864941259285:0]
2025-11-27T18:16:59.220404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470864941259289:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.220478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470864159319581:2049] [node 2] ICP30 unregister session Session# [6:7577470864159319666:2049] VirtualId# [6:7577470864159319664:0]
2025-11-27T18:16:59.220569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:16:59.220634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:16:59.222183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470864159319666:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.222307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:16:59.222357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:16:59.218456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:16:59.218707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470867763893090:2049] [node 2] ICS01 socket: 49 reason# EndOfStream
2025-11-27T18:16:59.218743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470863468925706:2049] [node 2] ICP30 unregister session Session# [5:7577470867763893090:2049] VirtualId# [5:7577470863468925788:0]
2025-11-27T18:16:59.218771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470867763893090:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.218858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:16:59.218927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:16:59.240349Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:16:59.240607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470869236226594:2050] [node 3] ICS01 socket: 72 reason# EndOfStream
2025-11-27T18:16:59.240649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470864941259203:2050] [node 3] ICP30 unregister session Session# [4:7577470869236226594:2050] VirtualId# [4:7577470864941259287:0]
2025-11-27T18:16:59.240681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470869236226594:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.240757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:16:59.241025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:16:59.241318Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:16:59.241585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470867763893095:2050] [node 3] ICS01 socket: 62 reason# EndOfStream
2025-11-27T18:16:59.241618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470863468925707:2050] [node 3] ICP30 unregister session Session# [5:7577470867763893095:2050] VirtualId# [5:7577470867763893092:0]
2025-11-27T18:16:59.241671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470867763893095:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.241794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:16:59.241845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:16:59.247223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:16:59.247452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470868454286971:2050] [node 3] ICS07 socket disconnect 99 reason# EndOfStream
2025-11-27T18:16:59.247491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470868454286971:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.247573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:16:59.247620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470868454286971:2050] [node 3] ICS15 start handshake
2025-11-27T18:16:59.249505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470864159319582:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20919) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:16:59.249565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470868454286971:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:16:59.249602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470864159319582:2050] [node 3] ICP30 unregister session Session# [6:7577470868454286971:2050] VirtualId# [6:7577470868454286969:0]
2025-11-27T18:16:59.249645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:16:59.249672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470864159319582:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20919) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:16:59.249685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20919) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:16:59.260284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470864159319582:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:16:59.264509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:16:59.264762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470867763893097:2110] [node 4] ICS01 socket: 80 reason# EndOfStream
2025-11-27T18:16:59.268379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:16:59.264800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470863468925775:2110] [node 4] ICP30 unregister session Session# [5:7577470867763893097:2110] VirtualId# [5:7577470867763893089:0]
2025-11-27T18:16:59.264866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470867763893097:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.268628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470868454286967:2097] [node 4] ICS01 socket: 77 reason# EndOfStream
2025-11-27T18:16:59.264991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:16:59.268669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470864159319646:2097] [node 4] ICP30 unregister session Session# [6:7577470868454286967:2097] VirtualId# [6:7577470868454286964:0]
2025-11-27T18:16:59.265039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:16:59.268759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470868454286967:2097] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.268934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:16:59.269032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:16:59.289552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:16:59.289973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470864159319659:2096] [node 5] ICS01 socket: 36 reason# EndOfStream
2025-11-27T18:16:59.290055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470864159319629:2096] [node 5] ICP30 unregister session Session# [6:7577470864159319659:2096] VirtualId# [6:7577470864159319643:0]
2025-11-27T18:16:59.290086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470864159319659:2096] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:16:59.290175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:16:59.290281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:16:59.349229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470868107628270:2048] [node 2] ICP01 ready to work
2025-11-27T18:16:59.349283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470868107628271:2049] [node 3] ICP01 ready to work
2025-11-27T18:16:59.369088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470869176158926:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:59.369430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470869176158927:2049] [node 3] ICP01 ready to work
2025-11-27T18:16:59.395117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470869789668539:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:59.395178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470869789668540:2049] [node 2] ICP01 ready to work
2025-11-27T18:16:59.421076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470869862173304:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:59.421136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470869862173306:2050] [node 3] ICP01 ready to work
2025-11-27T18:16:59.421630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470869862173305:2049] [node 2] ICP01 ready to work
2025-11-27T18:16:59.446331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470867433195749:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:59.446385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470867433195751:2050] [node 3] ICP01 ready to work
2025-11-27T18:16:59.446537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470867433195750:2049] [node 2] ICP01 ready to work
2025-11-27T18:16:59.472305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470869110414021:2049] [node 2] ICP01 ready to work
2025-11-27T18:16:59.472348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470869110414022:2050] [node 3] ICP01 ready to work
2025-11-27T18:16:59.473126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470869110414020:2048] [node 1] ICP01 ready to work
2025-11-27T18:16:59.561847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470868107628332:2107] [node 4] ICP01 ready to work
2025-11-27T18:16:59.562110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470868107628333:2108] [node 5] ICP01 ready to work
2025-11-27T18:16:59.723141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470869789668617:2107] [node 4] ICP01 ready to work
2025-11-27T18:16:59.733640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470869176158993:2108] [node 5] ICP01 ready to work
2025-11-27T18:16:59.734010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470869176158994:2109] [node 6] ICP01 ready to work
2025-11-27T18:16:59.748487Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470868107628333:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(156927872ns) {Configure:104(156921049ns) {ProcessConfigured:135(58146ns) {StartInitialHandshake:157(53183ns) {DropHandshakes:470(2090ns) {DropIncomingHandshake:442(667ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(752ns) PrepareNewSessionHandshake:477(938ns) {SwitchToState:179(287ns)}}}}}
2025-11-27T18:16:59.765850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470868107628332:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(185923648ns) {Configure:104(185912907ns) {ProcessConfigured:135(74831ns) {StartInitialHandshake:157(72192ns) {DropHandshakes:470(2597ns) {DropIncomingHandshake:442(586ns) DropOutgoingHandshake:457(352ns)} GenerateSessionVirtualId:988(922ns) PrepareNewSessionHandshake:477(1378ns) {SwitchToState:179(377ns)}}}}}
2025-11-27T18:16:59.757874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470869789668618:2112] [node 6] ICP01 ready to work
2025-11-27T18:16:59.770215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470868107628339:2109] [node 6] ICP01 ready to work
2025-11-27T18:16:59.768949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470869110414086:2103] [node 5] ICP01 ready to work
2025-11-27T18:16:59.769845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470869110414087:2102] [node 4] ICP01 ready to work
2025-11-27T18:16:59.769573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470869176159006:2107] [node 4] ICP01 ready to work
2025-11-27T18:16:59.779099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470869789668619:2124] [node 5] ICP01 ready to work
2025-11-27T18:16:59.779055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470869862173375:2109] [node 6] ICP01 ready to work
2025-11-27T18:16:59.781630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470869862173374:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.785618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470868107628271:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19560598ns) {Configure:104(19545859ns) {ProcessConfigured:135(56754ns) {StartInitialHandshake:157(54634ns) {DropHandshakes:470(3271ns) {DropIncomingHandshake:442(952ns) DropOutgoingHandshake:457(390ns)} GenerateSessionVirtualId:988(736ns) PrepareNewSessionHandshake:477(3928ns) {SwitchToState:179(249ns)}}}}}
2025-11-27T18:16:59.787414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470868107628342:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16189
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.796731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470869789668624:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28737
2025-11-27T18:16:59.797252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470869789668625:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28737
2025-11-27T18:16:59.788163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470867433195812:2109] [node 6] ICP01 ready to work
2025-11-27T18:16:59.797514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470867433195813:2108] [node 4] ICP01 ready to work
2025-11-27T18:16:59.819884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470869110414086:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(36941918ns) {Configure:104(36938513ns) {ProcessConfigured:135(53034ns) {StartInitialHandshake:157(51091ns) {DropHandshakes:470(2254ns) {DropIncomingHandshake:442(477ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(512ns) PrepareNewSessionHandshake:477(1570ns) {SwitchToState:179(470ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.826278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470868107628348:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16189
2025-11-27T18:16:59.826349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470868107628350:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16189
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.853603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470869110414099:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40734
2025-11-27T18:16:59.857061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470868107628270:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(61619291ns) {Configure:104(61615048ns) {ProcessConfigured:135(18103981ns) {StartInitialHandshake:157(18100695ns) {DropHandshakes:470(3068ns) {DropIncomingHandshake:442(675ns) DropOutgoingHandshake:457(373ns)} GenerateSessionVirtualId:988(1356ns) PrepareNewSessionHandshake:477(5859ns) {SwitchToState:179(2554ns)}}}}}
2025-11-27T18:16:59.857226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470868107628342:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.860686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470868107628348:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.857416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470869176159009:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30257
2025-11-27T18:16:59.861099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470868107628341:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.861118Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470868107628341:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.861969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470868107628341:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.858464Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470869789668617:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(79135110ns) {Configure:104(79131035ns) {ProcessConfigured:135(58781ns) {StartInitialHandshake:157(53280ns) {DropHandshakes:470(1055ns) {DropIncomingHandshake:442(226ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(485ns) PrepareNewSessionHandshake:477(983ns) {SwitchToState:179(380ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.861056Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470867433195749:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53533399ns) {Configure:104(53527500ns) {ProcessConfigured:135(2900884ns) {StartInitialHandshake:157(2899037ns) {DropHandshakes:470(2033ns) {DropIncomingHandshake:442(397ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(661ns) PrepareNewSessionHandshake:477(2099ns) {SwitchToState:179(1129ns)}}}}}
2025-11-27T18:16:59.874273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470867433195750:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(66938252ns) {Configure:104(66931959ns) {ProcessConfigured:135(68083ns) {StartInitialHandshake:157(66495ns) {DropHandshakes:470(4172ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(449ns) PrepareNewSessionHandshake:477(2531ns) {SwitchToState:179(1093ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.901327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470868107628350:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.876135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470867433195816:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41496
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.909847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470868107628271:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.884988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470867433195817:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41496
2025-11-27T18:16:59.859888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470869789668624:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.860009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470869789668625:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.860804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470869789668540:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.860846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470869789668539:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.883940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470869789668637:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28737
2025-11-27T18:16:59.909948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:16:59.918001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470869862173305:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(136088744ns) {Configure:104(136084463ns) {ProcessConfigured:135(64716ns) {StartInitialHandshake:157(62528ns) {DropHandshakes:470(2393ns) {DropIncomingHandshake:442(490ns) DropOutgoingHandshake:457(406ns)} GenerateSessionVirtualId:988(886ns) PrepareNewSessionHandshake:477(2453ns) {SwitchToState:179(965ns)}}}}}
2025-11-27T18:16:59.944664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470869862173384:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12862
2025-11-27T18:16:59.944860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470869862173385:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12862
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.889661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470869110414099:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.867867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470869176158998:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.885155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470867433195818:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41496
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.867906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470869176158998:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.890163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470869110414022:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.909995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470868107628271:2049] [node 3] ICP22 created new session: [1:7577470868107628356:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.969084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470869862173304:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(189656128ns) {Configure:104(189652965ns) {ProcessConfigured:135(14971758ns) {StartInitialHandshake:157(14969206ns) {DropHandshakes:470(2681ns) {DropIncomingHandshake:442(492ns) DropOutgoingHandshake:457(362ns)} GenerateSessionVirtualId:988(870ns) PrepareNewSessionHandshake:477(1761ns) {SwitchToState:179(442ns)}}}}}
2025-11-27T18:16:59.957881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470869789668539:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15679) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470869789668625:2128] held: no
2025-11-27T18:16:59.958054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470869789668627:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.958073Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470869789668627:2129] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.958770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470869789668625:2128] [node 1] ICH04 handshake succeeded
2025-11-27T18:16:59.912532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470867433195818:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.910048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470868107628356:2049] [node 3] ICS09 handshake done sender: [1:7577470868107628341:2112] self: [1:7577470868107628340:0] peer: [3:7577470869789668636:0] socket: 49 qp: -1
2025-11-27T18:16:59.910083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470868107628356:2049] [node 3] ICS10 traffic start
2025-11-27T18:16:59.910114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:16:59.926391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470868107628355:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16189
2025-11-27T18:16:59.965705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470868107628355:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.965805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470868107628336:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.965834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470868107628336:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.969990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470868107628336:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:16:59.970308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470868107628333:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:16:59.974691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:16:59.995944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470868107628344:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.995963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470868107628344:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:16:59.997498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470868107628344:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:00.005537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470868107628333:2108] [node 5] ICP22 created new session: [1:7577470868107628362:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.026021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470869789668639:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28737
2025-11-27T18:17:00.026145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470869789668539:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.026256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:00.026336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470869789668539:2048] [node 1] ICP22 created new session: [3:7577470874084635938:2048]
2025-11-27T18:17:00.026394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470874084635938:2048] [node 1] ICS09 handshake done sender: [3:7577470869789668625:2128] self: [3:7577470869789668636:0] peer: [1:7577470868107628340:0] socket: 51 qp: -1
2025-11-27T18:17:00.026425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470874084635938:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:00.026460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.914189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470869110414100:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40734
2025-11-27T18:16:59.878142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470869176159009:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.029630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470869789668638:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28737
2025-11-27T18:16:59.934465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470867433195816:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.959055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470867433195817:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.963224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470867433195750:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.963444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470867433195749:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.969787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470867433195833:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41496
2025-11-27T18:17:00.002577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470867433195749:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15679) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470867433195817:2112] held: no
2025-11-27T18:17:00.002761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470867433195820:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.002780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470867433195820:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:00.003260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470867433195817:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:00.003562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470867433195749:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.009604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:00.009661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470867433195749:2048] [node 1] ICP22 created new session: [5:7577470871728163131:2048]
2025-11-27T18:17:00.009700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470871728163131:2048] [node 1] ICS09 handshake done sender: [5:7577470867433195817:2112] self: [5:7577470867433195832:0] peer: [1:7577470868107628335:0] socket: 66 qp: -1
2025-11-27T18:17:00.009733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470871728163131:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:00.009779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:00.011144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470867433195820:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:00.011399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470867433195812:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.011473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:00.011519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470867433195812:2109] [node 6] ICP22 created new session: [5:7577470871728163134:2109]
2025-11-27T18:17:00.011561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470871728163134:2109] [node 6] ICS09 handshake done sender: [5:7577470867433195820:2114] self: [5:7577470867433195819:0] peer: [6:7577470869110414107:0] socket: 71 qp: -1
2025-11-27T18:17:00.011586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470871728163134:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:00.011612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.906017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470869176159011:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30257
2025-11-27T18:16:59.974580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470869862173384:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.009513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470869862173304:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:00.038009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470869862173304:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(34874264ns) {IncomingHandshake:261(24684889ns) {DropIncomingHandshake:442(1470ns) PrepareNewSessionHandshake:477(1369ns) {SwitchToState:179(719ns)} GenerateSessionVirtualId:988(1016ns) IssueIncomingHandshakeReply:193(21454233ns) {SwitchToState:179(2364ns)}}}
2025-11-27T18:17:00.029492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470868107628362:2108] [node 5] ICS09 handshake done sender: [1:7577470868107628336:2110] self: [1:7577470868107628335:0] peer: [5:7577470867433195832:0] socket: 47 qp: -1
2025-11-27T18:16:59.953626Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470869110414021:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(145799988ns) {Configure:104(145794604ns) {ProcessConfigured:135(9194636ns) {StartInitialHandshake:157(9192559ns) {DropHandshakes:470(2605ns) {DropIncomingHandshake:442(771ns) DropOutgoingHandshake:457(381ns)} GenerateSessionVirtualId:988(732ns) PrepareNewSessionHandshake:477(2199ns) {SwitchToState:179(1047ns)}}}}}
2025-11-27T18:17:00.029543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470868107628362:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:00.038429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:16:59.993798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470869110414100:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.052382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470871728163137:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41496
2025-11-27T18:16:59.921518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470869176158998:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:00.054341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470869789668627:2129] [node 6] ICH04 handshake succeeded
2025-11-27T18:16:59.938587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470869176158927:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.067921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470869789668540:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12662) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470869789668624:2127] held: no
2025-11-27T18:16:59.994161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470869110414104:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40734
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:16:59.994327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470869110414105:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40734
2025-11-27T18:17:00.111559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470868107628338:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.127052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470867433195816:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:00.068341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470869789668618:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:16:59.989368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:16:59.994493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470869110414086:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:16:59.994569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470869110414106:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40734
2025-11-27T18:16:59.995549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470869110414104:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:16:59.995659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470869110414020:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:00.029113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470869110414086:2103] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470869110414100:2118] held: no
2025-11-27T18:17:00.029239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470869110414109:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40734
2025-11-27T18:17:00.043244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470873405381406:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40734
2025-11-27T18:17:00.062169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470869110414104:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:00.137518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470869110414020:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.137599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:00.137645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470869110414020:2048] [node 1] ICP22 created new session: [6:7577470873405381409:2048]
2025-11-27T18:17:00.137694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470873405381409:2048] [node 1] ICS09 handshake done sender: [6:7577470869110414104:2120] self: [6:7577470869110414108:0] peer: [1:7577470868107628343:0] socket: 83 qp: -1
2025-11-27T18:17:00.137731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470873405381409:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:00.137775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:00.138536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470869110414100:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:00.111600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470868107628338:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:00.116594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470868107628339:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.116666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:00.116711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470868107628339:2109] [node 6] ICP22 created new session: [1:7577470872402595662:2109]
2025-11-27T18:17:00.116767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470872402595662:2109] [node 6] ICS09 handshake done sender: [1:7577470868107628344:2114] self: [1:7577470868107628343:0] peer: [6:7577470869110414108:0] socket: 79 qp: -1
2025-11-27T18:17:00.116792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470872402595662:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:00.116841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:00.125473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470868107628333:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(151198475ns) {HandleHandshakeStatus:329(151182150ns) {DropIncomingHandshake:442(15854ns) DropOutgoingHandshake:457(587ns) SwitchToState:179(441ns) ProcessPendingSessionEvents:495(9750166ns) {ForwardSessionEventToSession:612(9706923ns) ForwardSessionEventToSession:612(4924ns) ForwardSessionEventToSession:612(2145ns) ForwardSessionEventToSession:612(5910ns) ForwardSessionEventToSession:612(4472ns) ForwardSessionEventToSession:612(6746ns) ForwardSessionEventToSession:612(1407ns) ForwardSessionEventToSession:612(1639ns) ForwardSessionEventToSession:612(4085ns) ForwardSessionEventToSession:612(2004ns)}}}
2025-11-27T18:17:00.125859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470868107628351:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.125883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470868107628351:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:00.149771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470868107628338:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.005503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470869176158927:2049] [node 3] ICP22 created new session: [2:7577470869176159013:2049]
2025-11-27T18:17:00.154507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470868107628351:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:00.068397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.068432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470869789668618:2112] [node 6] ICP22 created new session: [3:7577470874084635941:2112]
2025-11-27T18:17:00.068470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470874084635941:2112] [node 6] ICS09 handshake done sender: [3:7577470869789668627:2129] self: [3:7577470869789668626:0] peer: [6:7577470869110414101:0] socket: 57 qp: -1
2025-11-27T18:17:00.068497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470874084635941:2112] [node 6] ICS10 traffic start
2025-11-27T18:17:00.068562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:00.069013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470869789668624:2127] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:00.117965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470869789668540:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.121960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:00.122010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470869789668540:2049] [node 2] ICP22 created new session: [3:7577470874084635943:2049]
2025-11-27T18:17:00.122049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470874084635943:2049] [node 2] ICS09 handshake done sender: [3:7577470869789668624:2127] self: [3:7577470869789668635:0] peer: [2:7577470869176158997:0] socket: 50 qp: -1
2025-11-27T18:17:00.122079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470874084635943:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:00.122118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:00.122637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470874084635940:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28737
2025-11-27T18:17:00.154228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470874084635946:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28737
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.182449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470869110414086:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.161537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470868107628332:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.193912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470867433195750:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.194007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:00.194044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470867433195750:2049] [node 2] ICP22 created new session: [5:7577470871728163139:2049]
2025-11-27T18:17:00.194081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470871728163139:2049] [node 2] ICS09 handshake done sender: [5:7577470867433195816:2111] self: [5:7577470867433195831:0] peer: [2:7577470869176158999:0] socket: 65 qp: -1
2025-11-27T18:17:00.194113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470871728163139:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:00.194141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:00.195054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470871728163138:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41496
2025-11-27T18:17:00.161609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:00.161663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470868107628332:2107] [node 4] ICP22 created new session: [1:7577470872402595666:2107]
2025-11-27T18:17:00.161706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470872402595666:2107] [node 4] ICS09 handshake done sender: [1:7577470868107628338:2111] self: [1:7577470868107628337:0] peer: [4:7577470869862173391:0] socket: 48 qp: -1
2025-11-27T18:17:00.161745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470872402595666:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:00.161798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:00.171395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470868107628270:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.189592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:00.189662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470868107628270:2048] [node 2] ICP22 created new session: [1:7577470872402595668:2048]
2025-11-27T18:17:00.189705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470872402595668:2048] [node 2] ICS09 handshake done sender: [1:7577470868107628351:2117] self: [1:7577470868107628349:0] peer: [2:7577470873471126318:0] socket: 76 qp: -1
2025-11-27T18:17:00.189752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470872402595668:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:00.189801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:00.190199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470868107628270:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(18689931ns) {HandleHandshakeStatus:329(18687043ns) {DropIncomingHandshake:442(16948ns) DropOutgoingHandshake:457(433ns) SwitchToState:179(522ns) ProcessPendingSessionEvents:495(108969ns) {ForwardSessionEventToSession:612(57127ns) ForwardSessionEventToSession:612(2436ns) ForwardSessionEventToSession:612(1913ns) ForwardSessionEventToSession:612(13317ns) ForwardSessionEventToSession:612(2237ns) ForwardSessionEventToSession:612(13446ns) ForwardSessionEventToSession:612(1465ns) ForwardSessionEventToSession:612(1666ns) ForwardSessionEventToSession:612(6109ns) ForwardSessionEventToSession:612(1485ns)}}}
2025-11-27T18:17:00.182538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:00.182611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470869110414086:2103] [node 5] ICP22 created new session: [6:7577470873405381411:2103]
2025-11-27T18:17:00.182679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470873405381411:2103] [node 5] ICS09 handshake done sender: [6:7577470869110414100:2118] self: [6:7577470869110414107:0] peer: [5:7577470867433195819:0] socket: 73 qp: -1
2025-11-27T18:17:00.182720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470873405381411:2103] [node 5] ICS10 traffic start
2025-11-27T18:17:00.182759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:00.183361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470869110414099:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:00.212984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470874084635940:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.225475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470871728163138:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.233864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470867433195751:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:00.038482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470869176159013:2049] [node 3] ICS09 handshake done sender: [2:7577470869176158998:2115] self: [2:7577470869176158997:0] peer: [3:7577470869789668635:0] socket: 43 qp: -1
2025-11-27T18:17:00.252121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577470869110414020:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:00.252418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470873405381408:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40734
2025-11-27T18:17:00.272856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470869789668629:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.272898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470869789668629:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:00.252596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470869110414022:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.280736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470869862173385:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.038517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470869176159013:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.289877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470874157140688:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12862
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.294241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470869789668629:2130] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:00.252659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:00.043568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470869176159000:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.252693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470869110414022:2050] [node 3] ICP22 created new session: [6:7577470873405381413:2050]
2025-11-27T18:17:00.043588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470869176159000:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:00.294552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470869789668619:2124] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.290120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470874157140689:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12862
2025-11-27T18:17:00.306875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470867433195751:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470871728163138:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.252736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470873405381413:2050] [node 3] ICS09 handshake done sender: [6:7577470869110414099:2117] self: [6:7577470869110414101:0] peer: [3:7577470869789668626:0] socket: 62 qp: -1
2025-11-27T18:17:00.252767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470873405381413:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:00.252832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:00.294646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:00.263352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470873405381408:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.267964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470869110414021:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:00.306170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470869110414105:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.294705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470869789668619:2124] [node 5] ICP22 created new session: [3:7577470874084635949:2124]
2025-11-27T18:17:00.307731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470871728163142:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41496
2025-11-27T18:17:00.307391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470869110414087:2102] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:00.044582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470873471126310:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30257
2025-11-27T18:17:00.290224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470874157140690:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12862
2025-11-27T18:17:00.290339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470874157140691:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12862
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.294751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470874084635949:2124] [node 5] ICS09 handshake done sender: [3:7577470869789668629:2130] self: [3:7577470869789668628:0] peer: [5:7577470871728163141:0] socket: 99 qp: -1
2025-11-27T18:17:00.044691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470873471126311:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30257
2025-11-27T18:17:00.309777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470874157140689:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.294773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470874084635949:2124] [node 5] ICS10 traffic start
2025-11-27T18:17:00.294795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:00.049494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:00.309900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470874157140688:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.310026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470874157140690:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.310124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470869862173379:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.310162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470869862173379:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:00.310825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470869862173384:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:00.315253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470869862173304:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.316373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:00.316411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470869862173304:2048] [node 1] ICP22 created new session: [4:7577470874157140694:2048]
2025-11-27T18:17:00.313729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470871728163138:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:00.050001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470869176158927:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(111203265ns) {HandleHandshakeStatus:329(111199226ns) {DropIncomingHandshake:442(12157ns) DropOutgoingHandshake:457(529ns) SwitchToState:179(821ns) ProcessPendingSessionEvents:495(98860ns) {ForwardSessionEventToSession:612(56549ns) ForwardSessionEventToSession:612(6037ns) ForwardSessionEventToSession:612(5137ns) ForwardSessionEventToSession:612(5025ns) ForwardSessionEventToSession:612(2323ns) ForwardSessionEventToSession:612(1914ns) ForwardSessionEventToSession:612(4073ns) ForwardSessionEventToSession:612(1757ns) ForwardSessionEventToSession:612(1787ns) ForwardSessionEventToSession:612(4254ns)}}}
2025-11-27T18:17:00.093434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470869176159000:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:00.110291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470869176158993:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.110359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:00.110405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470869176158993:2108] [node 5] ICP22 created new session: [2:7577470873471126315:2108]
2025-11-27T18:17:00.110446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470873471126315:2108] [node 5] ICS09 handshake done sender: [2:7577470869176159000:2116] self: [2:7577470869176158999:0] peer: [5:7577470867433195831:0] socket: 44 qp: -1
2025-11-27T18:17:00.110478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470873471126315:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:00.110505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:00.124535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470873471126311:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.124798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470869176158926:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:00.173801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470873471126319:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30257
2025-11-27T18:17:00.217439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470869176158926:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15679) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577470873471126311:2122] held: no
2025-11-27T18:17:00.227186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470869176159011:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.227311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470869176158993:2108] [node 5] ICP18 handshake (actor [2:7577470869176159011:2120]) is too old
2025-11-27T18:17:00.315079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470873405381418:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40734
2025-11-27T18:17:00.316453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470874157140694:2048] [node 1] ICS09 handshake done sender: [4:7577470869862173384:2121] self: [4:7577470869862173391:0] peer: [1:7577470868107628337:0] socket: 77 qp: -1
2025-11-27T18:17:00.316626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470874157140694:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:00.316670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:00.236847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470869176158926:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24626530ns) {HandleHandshakeStatus:395(24623682ns) {LogHandshakeFail:472(14652ns) {UpdateErrorStateLog:545(5576ns)} DropOutgoingHandshake:457(2474ns) ProcessPendingSessionEvents:495(19133038ns) {DropSessionEvent:512(13859ns) DropSessionEvent:512(2800ns) DropSessionEvent:512(6121ns) DropSessionEvent:512(11931468ns) DropSessionEvent:512(4117ns) DropSessionEvent:512(7139555ns) DropSessionEvent:512(13801ns) DropSessionEvent:512(3299ns) DropSessionEvent:512(3219ns) DropSessionEvent:512(2858ns)}}}
2025-11-27T18:17:00.236966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470873471126311:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:00.262061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470869176158926:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.262124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:00.262179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470869176158926:2048] [node 1] ICP22 created new session: [2:7577470873471126320:2048]
2025-11-27T18:17:00.262222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470873471126320:2048] [node 1] ICS09 handshake done sender: [2:7577470873471126311:2122] self: [2:7577470873471126318:0] peer: [1:7577470868107628349:0] socket: 94 qp: -1
2025-11-27T18:17:00.317915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470869862173305:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:00.262260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470873471126320:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:00.262296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:00.272618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470869176159002:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.272649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470869176159002:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:00.290752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470873471126310:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.290995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470873471126322:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30257
2025-11-27T18:17:00.303034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470873471126322:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.308964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470869176159002:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:00.310786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470869176158994:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.312273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:00.312326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470869176158994:2109] [node 6] ICP22 created new session: [2:7577470873471126326:2109]
2025-11-27T18:17:00.314269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470873471126326:2109] [node 6] ICS09 handshake done sender: [2:7577470869176159002:2117] self: [2:7577470869176159001:0] peer: [6:7577470873405381416:0] socket: 96 qp: -1
2025-11-27T18:17:00.314314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470873471126326:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:00.314037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470867433195751:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.321573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470869789668639:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.321690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470869789668634:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.321707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470869789668634:2131] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:00.314377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:00.322871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470869176159008:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.322886Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470869176159008:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.315430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:00.318180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470869862173306:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:00.321238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470869862173379:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:00.326052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470874157140699:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12862
2025-11-27T18:17:00.326121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470874157140700:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12862
2025-11-27T18:17:00.315504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470867433195751:2050] [node 3] ICP22 created new session: [5:7577470871728163143:2050]
2025-11-27T18:17:00.327300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470869789668634:2131] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:00.315541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470871728163143:2050] [node 3] ICS09 handshake done sender: [5:7577470871728163138:2120] self: [5:7577470871728163141:0] peer: [3:7577470869789668628:0] socket: 100 qp: -1
2025-11-27T18:17:00.315564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470871728163143:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:00.315592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.326699Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470869110414021:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12662) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470873405381408:2125] held: no
2025-11-27T18:17:00.327137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470873405381408:2125] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:00.327722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470869110414021:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.327783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:00.327836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470869110414021:2049] [node 2] ICP22 created new session: [6:7577470873405381420:2049]
2025-11-27T18:17:00.327879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470873405381420:2049] [node 2] ICS09 handshake done sender: [6:7577470873405381408:2125] self: [6:7577470873405381416:0] peer: [2:7577470869176159001:0] socket: 101 qp: -1
2025-11-27T18:17:00.327911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470873405381420:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:00.327951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:00.328536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470873405381419:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40734
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.329147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470871728163145:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41496
2025-11-27T18:17:00.329500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470869789668617:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.329575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:00.329613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470869789668617:2107] [node 4] ICP22 created new session: [3:7577470874084635953:2107]
2025-11-27T18:17:00.329655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470874084635953:2107] [node 4] ICS09 handshake done sender: [3:7577470869789668634:2131] self: [3:7577470869789668633:0] peer: [4:7577470874157140698:0] socket: 61 qp: -1
2025-11-27T18:17:00.329714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470874084635953:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:00.329756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:00.331026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470869176159008:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:00.331622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470869176159006:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.332128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470869110414105:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:00.345740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470869110414087:2102] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.345850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:00.345897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470869110414087:2102] [node 4] ICP22 created new session: [6:7577470873405381422:2102]
2025-11-27T18:17:00.345940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470873405381422:2102] [node 4] ICS09 handshake done sender: [6:7577470869110414105:2121] self: [6:7577470873405381417:0] peer: [4:7577470869862173378:0] socket: 86 qp: -1
2025-11-27T18:17:00.345975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470873405381422:2102] [node 4] ICS10 traffic start
2025-11-27T18:17:00.346011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:00.332254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:00.349037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470869862173375:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.349131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:00.349174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470869862173375:2109] [node 6] ICP22 created new session: [4:7577470874157140702:2109]
2025-11-27T18:17:00.349229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470874157140702:2109] [node 6] ICS09 handshake done sender: [4:7577470869862173379:2118] self: [4:7577470869862173378:0] peer: [6:7577470873405381417:0] socket: 81 qp: -1
2025-11-27T18:17:00.349278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470874157140702:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:00.349307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:00.349762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470869862173306:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470874157140690:2126] held: no
2025-11-27T18:17:00.355299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470869862173305:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12662) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470874157140689:2125] held: no
2025-11-27T18:17:00.355422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470874157140690:2126] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:00.332298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470869176159006:2107] [node 4] ICP22 created new session: [2:7577470873471126329:2107]
2025-11-27T18:17:00.332530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470873471126329:2107] [node 4] ICS09 handshake done sender: [2:7577470869176159008:2118] self: [2:7577470869176159007:0] peer: [4:7577470874157140696:0] socket: 102 qp: -1
2025-11-27T18:17:00.332561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470873471126329:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:00.332595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:00.356220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470874157140689:2125] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:00.356481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470869862173306:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.356552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:00.356592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470869862173306:2050] [node 3] ICP22 created new session: [4:7577470874157140704:2050]
2025-11-27T18:17:00.356639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470874157140704:2050] [node 3] ICS09 handshake done sender: [4:7577470874157140690:2126] self: [4:7577470874157140698:0] peer: [3:7577470869789668633:0] socket: 72 qp: -1
2025-11-27T18:17:00.356668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470874157140704:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:00.356692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:00.357140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470869862173305:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.357198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:00.357248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470869862173305:2049] [node 2] ICP22 created new session: [4:7577470874157140706:2049]
2025-11-27T18:17:00.357297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470874157140706:2049] [node 2] ICS09 handshake done sender: [4:7577470874157140689:2125] self: [4:7577470874157140696:0] peer: [2:7577470869176159007:0] socket: 58 qp: -1
2025-11-27T18:17:00.357316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470874157140706:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:00.357362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:00.358533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470871728163145:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.358631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470867433195813:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:00.358841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470869862173383:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:00.358866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470869862173383:2120] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:00.359868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470869862173383:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:00.360537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470869862173374:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:00.360616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:00.360659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470869862173374:2108] [node 5] ICP22 created new session: [4:7577470874157140709:2108]
2025-11-27T18:17:00.360696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470874157140709:2108] [node 5] ICS09 handshake done sender: [4:7577470869862173383:2120] self: [4:7577470869862173382:0] peer: [5:7577470871728163146:0] socket: 112 qp: -1
2025-11-27T18:17:00.360718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470874157140709:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:00.360748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:00.360093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470871728163147:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41496
2025-11-27T18:17:00.360580Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470871728163145:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:00.361778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470867433195813:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:00.361874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:00.361913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470867433195813:2108] [node 4] ICP22 created new session: [5:7577470871728163148:2108]
2025-11-27T18:17:00.361981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470871728163148:2108] [node 4] ICS09 handshake done sender: [5:7577470871728163145:2122] self: [5:7577470871728163146:0] peer: [4:7577470869862173382:0] socket: 113 qp: -1
2025-11-27T18:17:00.362015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470871728163148:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:00.362045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:00.390075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:00.390317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470873471126320:2048] [node 1] ICS01 socket: 94 reason# EndOfStream
2025-11-27T18:17:00.390351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470869176158926:2048] [node 1] ICP30 unregister session Session# [2:7577470873471126320:2048] VirtualId# [2:7577470873471126318:0]
2025-11-27T18:17:00.390383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470873471126320:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.390464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:00.390507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:00.392881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:00.393088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470874157140694:2048] [node 1] ICS01 socket: 77 reason# EndOfStream
2025-11-27T18:17:00.393114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470869862173304:2048] [node 1] ICP30 unregister session Session# [4:7577470874157140694:2048] VirtualId# [4:7577470869862173391:0]
2025-11-27T18:17:00.393156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470874157140694:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.393235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:00.393295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:00.393392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:00.393760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470874084635938:2048] [node 1] ICS01 socket: 51 reason# EndOfStream
2025-11-27T18:17:00.393800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470869789668539:2048] [node 1] ICP30 unregister session Session# [3:7577470874084635938:2048] VirtualId# [3:7577470869789668636:0]
2025-11-27T18:17:00.393822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470874084635938:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.393884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:00.393922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:00.394911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:00.395083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470871728163131:2048] [node 1] ICS01 socket: 66 reason# EndOfStream
2025-11-27T18:17:00.395132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470867433195749:2048] [node 1] ICP30 unregister session Session# [5:7577470871728163131:2048] VirtualId# [5:7577470867433195832:0]
2025-11-27T18:17:00.395157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470871728163131:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.395209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:00.395246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:00.395989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:00.396155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470873405381409:2048] [node 1] ICS01 socket: 83 reason# EndOfStream
2025-11-27T18:17:00.396183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470869110414020:2048] [node 1] ICP30 unregister session Session# [6:7577470873405381409:2048] VirtualId# [6:7577470869110414108:0]
2025-11-27T18:17:00.396201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470873405381409:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.396260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:00.396291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:00.426930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:00.427150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470874084635943:2049] [node 2] ICS01 socket: 50 reason# EndOfStream
2025-11-27T18:17:00.427188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470869789668540:2049] [node 2] ICP30 unregister session Session# [3:7577470874084635943:2049] VirtualId# [3:7577470869789668635:0]
2025-11-27T18:17:00.427215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470874084635943:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.427294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:00.427334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:00.433066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:00.433329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470874157140706:2049] [node 2] ICS01 socket: 58 reason# EndOfStream
2025-11-27T18:17:00.433373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470869862173305:2049] [node 2] ICP30 unregister session Session# [4:7577470874157140706:2049] VirtualId# [4:7577470874157140696:0]
2025-11-27T18:17:00.433399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470874157140706:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.433494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:00.433539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:00.433909Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:00.434107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470871728163139:2049] [node 2] ICS01 socket: 65 reason# EndOfStream
2025-11-27T18:17:00.434133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470867433195750:2049] [node 2] ICP30 unregister session Session# [5:7577470871728163139:2049] VirtualId# [5:7577470867433195831:0]
2025-11-27T18:17:00.434152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470871728163139:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.434208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:00.434239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:00.434306Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:00.434476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470873405381420:2049] [node 2] ICS01 socket: 101 reason# EndOfStream
2025-11-27T18:17:00.434510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470869110414021:2049] [node 2] ICP30 unregister session Session# [6:7577470873405381420:2049] VirtualId# [6:7577470873405381416:0]
2025-11-27T18:17:00.434529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470873405381420:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.434570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:00.434603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:00.457485Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:00.457708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470874157140704:2050] [node 3] ICS01 socket: 72 reason# EndOfStream
2025-11-27T18:17:00.457747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470869862173306:2050] [node 3] ICP30 unregister session Session# [4:7577470874157140704:2050] VirtualId# [4:7577470874157140698:0]
2025-11-27T18:17:00.457776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470874157140704:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.457848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:00.457885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:00.457973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:00.458179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470873405381413:2050] [node 3] ICS01 socket: 62 reason# EndOfStream
2025-11-27T18:17:00.458199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470869110414022:2050] [node 3] ICP30 unregister session Session# [6:7577470873405381413:2050] VirtualId# [6:7577470869110414101:0]
2025-11-27T18:17:00.458216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470873405381413:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.458262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:00.458295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:00.458315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:00.458519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470871728163143:2050] [node 3] ICS01 socket: 100 reason# EndOfStream
2025-11-27T18:17:00.458552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470867433195751:2050] [node 3] ICP30 unregister session Session# [5:7577470871728163143:2050] VirtualId# [5:7577470871728163141:0]
2025-11-27T18:17:00.458590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470871728163143:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.458682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:00.458743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:00.492691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:00.492842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470873405381422:2102] [node 4] ICS01 socket: 86 reason# EndOfStream
2025-11-27T18:17:00.492868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470869110414087:2102] [node 4] ICP30 unregister session Session# [6:7577470873405381422:2102] VirtualId# [6:7577470873405381417:0]
2025-11-27T18:17:00.492891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470873405381422:2102] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.492958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:00.493004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:00.497072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:00.497545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470871728163148:2108] [node 4] ICS07 socket disconnect 113 reason# EndOfStream
2025-11-27T18:17:00.497632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470871728163148:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.497839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:00.497946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470871728163148:2108] [node 4] ICS15 start handshake
2025-11-27T18:17:00.528338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:00.528500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470873405381411:2103] [node 5] ICS01 socket: 73 reason# EndOfStream
2025-11-27T18:17:00.528535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470869110414086:2103] [node 5] ICP30 unregister session Session# [6:7577470873405381411:2103] VirtualId# [6:7577470869110414107:0]
2025-11-27T18:17:00.528555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470873405381411:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:00.528607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:00.528632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:00.597391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470872252897076:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:00.597422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470872252897077:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:00.630214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470871962975249:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:00.630270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470871962975250:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:00.654210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470873191351406:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:00.656451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470873191351407:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:00.680811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470872921679627:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:00.680848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470872921679628:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:00.680867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470872921679629:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:00.710792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470872675701347:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:00.710825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470872675701348:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:00.710844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470872675701349:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:00.737425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470872212170273:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:00.737456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470872212170274:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:00.737476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470872212170275:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:00.926518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470872212170295:2068] [node 5] ICP01 ready to work
2025-11-27T18:17:00.986049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470873191351467:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:01.025717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470873191351469:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:01.149867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470876547864435:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:01.219856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470877216646991:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:01.187484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470872212170274:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(116987078ns) {Configure:104(116981209ns) {ProcessConfigured:135(66553ns) {StartInitialHandshake:157(64109ns) {DropHandshakes:470(2743ns) {DropIncomingHandshake:442(486ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(1089ns) PrepareNewSessionHandshake:477(2010ns) {SwitchToState:179(952ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.187990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470876507137621:2076] [node 4] ICP01 ready to work
2025-11-27T18:17:01.233780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470877216646993:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:01.236411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470877216646992:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58420
2025-11-27T18:17:01.229685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470876257942608:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.285478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470876547864436:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:01.261684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470876970668706:2105] [node 6] ICP01 ready to work
2025-11-27T18:17:01.237446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470876257942609:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:01.314666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470873191351406:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(255830183ns) {Configure:104(241137670ns) {ProcessConfigured:135(70899ns) {StartInitialHandshake:157(68250ns) {DropHandshakes:470(3330ns) {DropIncomingHandshake:442(724ns) DropOutgoingHandshake:457(433ns)} GenerateSessionVirtualId:988(812ns) PrepareNewSessionHandshake:477(1678ns) {SwitchToState:179(498ns)}}}}}
2025-11-27T18:17:01.316773Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470872212170295:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(346798672ns) {Configure:104(346793216ns) {ProcessConfigured:135(18303035ns) {StartInitialHandshake:157(18300119ns) {DropHandshakes:470(2558ns) {DropIncomingHandshake:442(719ns) DropOutgoingHandshake:457(265ns)} GenerateSessionVirtualId:988(760ns) PrepareNewSessionHandshake:477(4771ns) {SwitchToState:179(2805ns)}}}}}
2025-11-27T18:17:01.274134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470871962975250:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(44142378ns) {Configure:104(44137684ns) {ProcessConfigured:135(70535ns) {StartInitialHandshake:157(68596ns) {DropHandshakes:470(2999ns) {DropIncomingHandshake:442(939ns) DropOutgoingHandshake:457(475ns)} GenerateSessionVirtualId:988(1186ns) PrepareNewSessionHandshake:477(1384ns) {SwitchToState:179(460ns)}}}}}
2025-11-27T18:17:01.280430Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470871962975249:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42745647ns) {Configure:104(42741887ns) {ProcessConfigured:135(74709ns) {StartInitialHandshake:157(65809ns) {DropHandshakes:470(2785ns) {DropIncomingHandshake:442(501ns) DropOutgoingHandshake:457(344ns)} GenerateSessionVirtualId:988(1011ns) PrepareNewSessionHandshake:477(1972ns) {SwitchToState:179(581ns)}}}}}
2025-11-27T18:17:01.281030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470876257942613:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26863
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.321756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470872675701348:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21668694ns) {Configure:104(21659357ns) {ProcessConfigured:135(83687ns) {StartInitialHandshake:157(82106ns) {DropHandshakes:470(4111ns) {DropIncomingHandshake:442(754ns) DropOutgoingHandshake:457(458ns)} GenerateSessionVirtualId:988(1137ns) PrepareNewSessionHandshake:477(2247ns) {SwitchToState:179(640ns)}}}}}
2025-11-27T18:17:01.324653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470876970668713:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4129
2025-11-27T18:17:01.277425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470877216646992:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.329451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470877486318770:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.344521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470877486318773:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12325
2025-11-27T18:17:01.344717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470877486318774:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12325
2025-11-27T18:17:01.344822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470877486318775:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12325
2025-11-27T18:17:01.345181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470876257942612:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26863
2025-11-27T18:17:01.329554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470876547864445:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.329807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470876970668713:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.357683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470876507137645:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53255
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.370045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470872212170273:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(427609890ns) {Configure:104(427605377ns) {ProcessConfigured:135(80469ns) {StartInitialHandshake:157(77564ns) {DropHandshakes:470(3598ns) {DropIncomingHandshake:442(1169ns) DropOutgoingHandshake:457(640ns)} GenerateSessionVirtualId:988(897ns) PrepareNewSessionHandshake:477(2106ns) {SwitchToState:179(807ns)}}}}}
2025-11-27T18:17:01.377760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470876507137649:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53255
2025-11-27T18:17:01.350878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470876970668714:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4129
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.337389Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470872252897077:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(160023071ns) {Configure:104(160018127ns) {ProcessConfigured:135(46036141ns) {StartInitialHandshake:157(46032405ns) {DropHandshakes:470(1635ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(183ns)} GenerateSessionVirtualId:988(922ns) PrepareNewSessionHandshake:477(7553ns) {SwitchToState:179(3137ns)}}}}}
2025-11-27T18:17:01.349588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470872921679627:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(130489612ns) {Configure:104(130485528ns) {ProcessConfigured:135(11072328ns) {StartInitialHandshake:157(11067494ns) {DropHandshakes:470(2328ns) {DropIncomingHandshake:442(661ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(1480ns) PrepareNewSessionHandshake:477(4711ns) {SwitchToState:179(2093ns)}}}}}
2025-11-27T18:17:01.384770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470877486318773:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.389512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470877486318774:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.391498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470873191351469:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(76523791ns) {Configure:104(76519480ns) {ProcessConfigured:135(77799ns) {StartInitialHandshake:157(74563ns) {DropHandshakes:470(4015ns) {DropIncomingHandshake:442(678ns) DropOutgoingHandshake:457(521ns)} GenerateSessionVirtualId:988(978ns) PrepareNewSessionHandshake:477(1497ns) {SwitchToState:179(614ns)}}}}}
2025-11-27T18:17:01.392357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470877486318783:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12325
2025-11-27T18:17:01.397322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470877486318775:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.409386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470873191351407:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.393587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470876507137645:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.354157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470876257942613:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.386254Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470872675701347:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(92401596ns) {Configure:104(92391348ns) {ProcessConfigured:135(57846ns) {StartInitialHandshake:157(55631ns) {DropHandshakes:470(2317ns) {DropIncomingHandshake:442(542ns) DropOutgoingHandshake:457(291ns)} GenerateSessionVirtualId:988(1147ns) PrepareNewSessionHandshake:477(1594ns) {SwitchToState:179(543ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.441507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470877486318788:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12325
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.376285Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470876547864445:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18522730ns) {Configure:104(18516313ns) {ProcessConfigured:135(53992ns) {StartInitialHandshake:157(52176ns) {DropHandshakes:470(2322ns) {DropIncomingHandshake:442(546ns) DropOutgoingHandshake:457(371ns)} GenerateSessionVirtualId:988(1161ns) PrepareNewSessionHandshake:477(1926ns) {SwitchToState:179(788ns)}}}}}
2025-11-27T18:17:01.393743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470872212170274:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.386388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470876970668714:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.366955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470876257942612:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.386859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470872675701348:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.398360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470876507137651:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53255
2025-11-27T18:17:01.373909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470876257942625:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26863
2025-11-27T18:17:01.445519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470873191351467:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(246609407ns) {Configure:104(232586446ns) {ProcessConfigured:135(12510697ns) {StartInitialHandshake:157(12507517ns) {DropHandshakes:470(1489ns) {DropIncomingHandshake:442(225ns) DropOutgoingHandshake:457(339ns)} GenerateSessionVirtualId:988(1062ns) PrepareNewSessionHandshake:477(3212ns) {SwitchToState:179(1046ns)}}}}}
2025-11-27T18:17:01.473878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470877216646995:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.473951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470877216646995:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.377093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470876547864448:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26219
2025-11-27T18:17:01.467582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470876507137648:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53255
2025-11-27T18:17:01.405571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470876970668718:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:01.485730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470876507137649:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.373972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470876257942626:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26863
2025-11-27T18:17:01.414159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470876970668721:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4129
2025-11-27T18:17:01.442859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470876547864448:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.394199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470876257942624:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:01.499982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470872212170273:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.437624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470876970668723:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4129
2025-11-27T18:17:01.405974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470876257942617:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.509399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470876507137651:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.515334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470872212170275:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.509624Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470877216646995:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:01.510110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470877486318775:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:01.510108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470876547864447:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.510154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470876547864447:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.525645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470876507137648:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.525784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470872212170295:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.529355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470873191351407:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.529454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:01.529509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470873191351407:2049] [node 2] ICP22 created new session: [3:7577470877486318790:2049]
2025-11-27T18:17:01.529559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470877486318790:2049] [node 2] ICS09 handshake done sender: [3:7577470877486318775:2115] self: [3:7577470877486318786:0] peer: [2:7577470876257942610:0] socket: 68 qp: -1
2025-11-27T18:17:01.529587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470877486318790:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:01.529636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:01.454154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470876970668718:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.545427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470877486318777:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.545472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470877486318777:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.406010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470876257942617:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.545445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470876547864447:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:01.544606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470876507137656:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53255
2025-11-27T18:17:01.454393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470876970668718:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(48505192ns) {Configure:104(48503605ns) {ProcessConfigured:135(171042ns) {StartInitialHandshake:157(85298ns) {DropHandshakes:470(2358ns) {DropIncomingHandshake:442(505ns) DropOutgoingHandshake:457(451ns)} GenerateSessionVirtualId:988(4592ns) PrepareNewSessionHandshake:477(2559ns) {SwitchToState:179(1058ns)}} IncomingHandshake:261(64165ns) {DropIncomingHandshake:442(183ns) PrepareNewSessionHandshake:477(232ns) {SwitchToState:179(122ns)} GenerateSessionVirtualId:988(165ns) IssueIncomingHandshakeReply:193(46255ns) {SwitchToState:179(222ns)}}}}}
2025-11-27T18:17:01.412462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470876257942611:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.554038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470877486318777:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:01.536988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470876970668714:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.559554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470877216646991:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.554620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470877486318770:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.412482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470876257942611:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.584773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470876547864439:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.579889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470876970668721:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.596554Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470872212170274:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61288) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470876507137645:2114] held: no
2025-11-27T18:17:01.565378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:01.584809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470876547864439:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.430553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470876257942615:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.554681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:01.600643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470876507137654:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53255
2025-11-27T18:17:01.569419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470877216646991:2109] [node 5] ICP22 created new session: [4:7577470877216647004:2109]
No RDMA context found for GID: 2025-11-27T18:17:01.579986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470876970668710:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:17:01.430581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470876257942615:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.554737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470877486318770:2110] [node 6] ICP22 created new session: [3:7577470877486318793:2110]
2025-11-27T18:17:01.580004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470876970668710:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.596252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470876547864445:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.437222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470876257942617:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:01.554774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470877486318793:2110] [node 6] ICS09 handshake done sender: [3:7577470877486318777:2116] self: [3:7577470877486318776:0] peer: [6:7577470876507137653:0] socket: 86 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.569620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470877216647004:2109] [node 5] ICS09 handshake done sender: [4:7577470877216646995:2113] self: [4:7577470877216646994:0] peer: [5:7577470876970668726:0] socket: 56 qp: -1
2025-11-27T18:17:01.580566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470872675701348:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.596312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:01.613439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470876507137649:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:01.554800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470877486318793:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:01.598088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470876547864439:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:01.601004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470876547864445:2109] [node 6] ICP22 created new session: [1:7577470876547864455:2109]
2025-11-27T18:17:01.607861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470876547864455:2109] [node 6] ICS09 handshake done sender: [1:7577470876547864447:2114] self: [1:7577470876547864446:0] peer: [6:7577470876507137652:0] socket: 80 qp: -1
2025-11-27T18:17:01.607924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470876547864455:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:01.609678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470876547864454:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26219
2025-11-27T18:17:01.610635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470872252897076:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.612492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:01.618289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:01.624780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470872252897076:2048] [node 2] ICP22 created new session: [1:7577470876547864459:2048]
2025-11-27T18:17:01.625720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470876547864459:2048] [node 2] ICS09 handshake done sender: [1:7577470876547864439:2110] self: [1:7577470876547864438:0] peer: [2:7577470876257942638:0] socket: 79 qp: -1
2025-11-27T18:17:01.613718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470872212170275:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9520) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470876507137651:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.580607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:01.569662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470877216647004:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:01.457971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470876257942627:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26863
2025-11-27T18:17:01.469248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470876257942611:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:01.626924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470876547864459:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:01.569711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:01.613853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470872212170273:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.580654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470872675701348:2049] [node 2] ICP22 created new session: [5:7577470876970668729:2049]
2025-11-27T18:17:01.554834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.494299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470876257942626:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.621163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470876507137657:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53255
2025-11-27T18:17:01.578669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470877216647005:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58420
2025-11-27T18:17:01.628012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:01.580696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470876970668729:2049] [node 2] ICS09 handshake done sender: [5:7577470876970668714:2114] self: [5:7577470876970668722:0] peer: [2:7577470876257942616:0] socket: 64 qp: -1
2025-11-27T18:17:01.497620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470876257942627:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.580753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470876970668729:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:01.579771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470877216647005:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.625786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:01.580826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:01.634874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470876547864454:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.618255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470872675701348:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(29894999ns) {HandleHandshakeStatus:329(29892115ns) {DropIncomingHandshake:442(568ns) DropOutgoingHandshake:457(3435ns) SwitchToState:179(478ns) ProcessPendingSessionEvents:495(24352918ns) {ForwardSessionEventToSession:612(5852360ns) ForwardSessionEventToSession:612(10827138ns) ForwardSessionEventToSession:612(7629ns) ForwardSessionEventToSession:612(2117ns) ForwardSessionEventToSession:612(3032532ns) ForwardSessionEventToSession:612(2285ns) ForwardSessionEventToSession:612(3747277ns) ForwardSessionEventToSession:612(11772ns) ForwardSessionEventToSession:612(2087ns) ForwardSessionEventToSession:612(841735ns)}}}
2025-11-27T18:17:01.499697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470876257942609:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.607250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470873191351407:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(29013113ns) {HandleHandshakeStatus:329(29007963ns) {DropIncomingHandshake:442(981ns) DropOutgoingHandshake:457(7857ns) SwitchToState:179(508ns) ProcessPendingSessionEvents:495(28578117ns) {ForwardSessionEventToSession:612(8361383ns) ForwardSessionEventToSession:612(6446ns) ForwardSessionEventToSession:612(5518356ns) ForwardSessionEventToSession:612(3221ns) ForwardSessionEventToSession:612(1928690ns) ForwardSessionEventToSession:612(3938ns) ForwardSessionEventToSession:612(2554ns) ForwardSessionEventToSession:612(4206697ns) ForwardSessionEventToSession:612(980606ns) ForwardSessionEventToSession:612(5386ns)}}}
2025-11-27T18:17:01.635553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470877486318797:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12325
No RDMA context found for GID: 2025-11-27T18:17:01.647320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470877486318797:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:17:01.645444Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470876547864445:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(31268490ns) {HandleHandshakeStatus:329(31257552ns) {DropIncomingHandshake:442(977ns) DropOutgoingHandshake:457(810ns) SwitchToState:179(1130ns) ProcessPendingSessionEvents:495(137408ns) {ForwardSessionEventToSession:612(65565ns) ForwardSessionEventToSession:612(23555ns) ForwardSessionEventToSession:612(2831ns) ForwardSessionEventToSession:612(6550ns) ForwardSessionEventToSession:612(5838ns) ForwardSessionEventToSession:612(2078ns) ForwardSessionEventToSession:612(7364ns) ForwardSessionEventToSession:612(1993ns) ForwardSessionEventToSession:612(1693ns) ForwardSessionEventToSession:612(4138ns)}}}
2025-11-27T18:17:01.585876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470872921679628:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.505529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:01.626524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470876507137645:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:01.509419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470876257942609:2108] [node 5] ICP22 created new session: [2:7577470876257942634:2108]
2025-11-27T18:17:01.518064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470871962975250:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.525358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470876257942634:2108] [node 5] ICS09 handshake done sender: [2:7577470876257942617:2114] self: [2:7577470876257942616:0] peer: [5:7577470876970668722:0] socket: 61 qp: -1
2025-11-27T18:17:01.529431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470876257942634:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:01.529482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:01.649553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470873191351406:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.536910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:01.545230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470871962975250:2049] [node 3] ICP22 created new session: [2:7577470876257942636:2049]
2025-11-27T18:17:01.546103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470876257942609:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(48239793ns) {HandleHandshakeStatus:329(48236275ns) {DropIncomingHandshake:442(1488ns) DropOutgoingHandshake:457(577ns) SwitchToState:179(446ns) ProcessPendingSessionEvents:495(143853ns) {ForwardSessionEventToSession:612(73071ns) ForwardSessionEventToSession:612(6271ns) ForwardSessionEventToSession:612(2726ns) ForwardSessionEventToSession:612(4241ns) ForwardSessionEventToSession:612(2432ns) ForwardSessionEventToSession:612(2120ns) ForwardSessionEventToSession:612(10440ns) ForwardSessionEventToSession:612(20334ns) ForwardSessionEventToSession:612(6439ns) ForwardSessionEventToSession:612(4558ns)}}}
2025-11-27T18:17:01.546686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470876257942615:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:01.627122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470872212170273:2048] [node 1] ICP22 created new session: [6:7577470876507137659:2048]
2025-11-27T18:17:01.550477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470871962975249:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.550754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470876257942608:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.550831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:01.550876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470876257942608:2109] [node 6] ICP22 created new session: [2:7577470876257942639:2109]
2025-11-27T18:17:01.655956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470877486318787:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.655978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470877486318787:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.658031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470877486318787:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:01.658417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470873191351467:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.658501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:01.658577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470873191351467:2107] [node 4] ICP22 created new session: [3:7577470877486318801:2107]
2025-11-27T18:17:01.658617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470877486318801:2107] [node 4] ICS09 handshake done sender: [3:7577470877486318787:2119] self: [3:7577470877486318785:0] peer: [4:7577470877216647014:0] socket: 106 qp: -1
2025-11-27T18:17:01.658671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470877486318801:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:01.658711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:01.607311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470877216647010:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58420
2025-11-27T18:17:01.652217Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470872252897076:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(41055124ns) {HandleHandshakeStatus:329(41041157ns) {DropIncomingHandshake:442(395ns) DropOutgoingHandshake:457(1092ns) SwitchToState:179(780ns) ProcessPendingSessionEvents:495(4191302ns) {ForwardSessionEventToSession:612(3757042ns) ForwardSessionEventToSession:612(5860ns) ForwardSessionEventToSession:612(2593ns) ForwardSessionEventToSession:612(383511ns) ForwardSessionEventToSession:612(2489ns) ForwardSessionEventToSession:612(2621ns) ForwardSessionEventToSession:612(4963ns) ForwardSessionEventToSession:612(5394ns) ForwardSessionEventToSession:612(4682ns) ForwardSessionEventToSession:612(2132ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:17:01.550939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470876257942639:2109] [node 6] ICS09 handshake done sender: [2:7577470876257942615:2113] self: [2:7577470876257942614:0] peer: [6:7577470876507137650:0] socket: 60 qp: -1
2025-11-27T18:17:01.628036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470876507137659:2048] [node 1] ICS09 handshake done sender: [6:7577470876507137649:2117] self: [6:7577470876507137652:0] peer: [1:7577470876547864446:0] socket: 82 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.652289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470876547864444:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.673490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470877486318800:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12325
2025-11-27T18:17:01.674140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470877486318797:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:01.674377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470873191351406:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.674454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:01.674504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470873191351406:2048] [node 1] ICP22 created new session: [3:7577470877486318803:2048]
2025-11-27T18:17:01.674554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470877486318803:2048] [node 1] ICS09 handshake done sender: [3:7577470877486318797:2121] self: [3:7577470877486318798:0] peer: [1:7577470876547864437:0] socket: 57 qp: -1
2025-11-27T18:17:01.674588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470877486318803:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:01.674621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.550970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470876257942639:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:01.630036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470876507137659:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:01.652314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470876547864444:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.557370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470876257942636:2049] [node 3] ICS09 handshake done sender: [2:7577470876257942611:2110] self: [2:7577470876257942610:0] peer: [3:7577470877486318786:0] socket: 59 qp: -1
2025-11-27T18:17:01.630090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:01.695118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470876970668727:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4129
2025-11-27T18:17:01.614624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470877216646991:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(34649316ns) {HandleHandshakeStatus:329(34646333ns) {DropIncomingHandshake:442(723ns) DropOutgoingHandshake:457(684ns) SwitchToState:179(372ns) ProcessPendingSessionEvents:495(24307617ns) {ForwardSessionEventToSession:612(444483ns) ForwardSessionEventToSession:612(14940ns) ForwardSessionEventToSession:612(1773ns) ForwardSessionEventToSession:612(4000010ns) ForwardSessionEventToSession:612(4966ns) ForwardSessionEventToSession:612(6946969ns) ForwardSessionEventToSession:612(5736ns) ForwardSessionEventToSession:612(2904ns) ForwardSessionEventToSession:612(1966657ns) ForwardSessionEventToSession:612(3589ns)}}}
2025-11-27T18:17:01.617925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470877216647005:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:01.620995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470872921679628:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.621900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:01.627091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470872921679628:2049] [node 2] ICP22 created new session: [4:7577470877216647012:2049]
2025-11-27T18:17:01.628002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470877216647012:2049] [node 2] ICS09 handshake done sender: [4:7577470877216647005:2116] self: [4:7577470877216647008:0] peer: [2:7577470876257942628:0] socket: 66 qp: -1
2025-11-27T18:17:01.628038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470877216647012:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:01.629430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470877216647009:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58420
2025-11-27T18:17:01.633378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:01.651410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470877216647009:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.653669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470872921679629:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.653922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470877216647011:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58420
2025-11-27T18:17:01.659671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470877216647011:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.659801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470872921679627:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.690052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470872921679628:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(32516821ns) {HandleHandshakeStatus:329(32513268ns) {DropIncomingHandshake:442(769ns) DropOutgoingHandshake:457(4350ns) SwitchToState:179(503ns) ProcessPendingSessionEvents:495(1122799ns) {ForwardSessionEventToSession:612(1064430ns) ForwardSessionEventToSession:612(6533ns) ForwardSessionEventToSession:612(3129ns) ForwardSessionEventToSession:612(7195ns) ForwardSessionEventToSession:612(2296ns) ForwardSessionEventToSession:612(5646ns) ForwardSessionEventToSession:612(2288ns) ForwardSessionEventToSession:612(5983ns) ForwardSessionEventToSession:612(4747ns) ForwardSessionEventToSession:612(2671ns)}}}
2025-11-27T18:17:01.661146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470876547864441:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.557409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:01.631245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470872212170274:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.719431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470876970668728:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4129
2025-11-27T18:17:01.661173Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470876547864441:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.558402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470876257942636:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:01.631291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:01.725977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470872921679629:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9520) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470877216647009:2117] held: no
2025-11-27T18:17:01.664943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470872252897076:2048] [node 2] ICP18 handshake (actor [1:7577470876547864454:2116]) is too old
2025-11-27T18:17:01.729606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470872675701347:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.741997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470872921679629:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(43451363ns) {HandleHandshakeStatus:395(43446303ns) {LogHandshakeFail:472(12716034ns) {UpdateErrorStateLog:545(7314202ns)} DropOutgoingHandshake:457(3559ns) ProcessPendingSessionEvents:495(7201003ns) {DropSessionEvent:512(21181ns) DropSessionEvent:512(6770ns) DropSessionEvent:512(3732ns) DropSessionEvent:512(3485ns) DropSessionEvent:512(3864681ns) DropSessionEvent:512(6837ns) DropSessionEvent:512(4083ns) DropSessionEvent:512(25611ns) DropSessionEvent:512(4185ns) DropSessionEvent:512(3540ns)}}}
2025-11-27T18:17:01.631346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470872212170274:2049] [node 2] ICP22 created new session: [6:7577470876507137661:2049]
2025-11-27T18:17:01.566291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:01.583815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470876257942608:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(32837167ns) {HandleHandshakeStatus:329(32835014ns) {DropIncomingHandshake:442(11606ns) DropOutgoingHandshake:457(419ns) SwitchToState:179(186ns) ProcessPendingSessionEvents:495(10345005ns) {ForwardSessionEventToSession:612(10285351ns) ForwardSessionEventToSession:612(6420ns) ForwardSessionEventToSession:612(5234ns) ForwardSessionEventToSession:612(1921ns) ForwardSessionEventToSession:612(19945ns) ForwardSessionEventToSession:612(2053ns) ForwardSessionEventToSession:612(2085ns) ForwardSessionEventToSession:612(7633ns) ForwardSessionEventToSession:612(2181ns) ForwardSessionEventToSession:612(2038ns)}}}
2025-11-27T18:17:01.592861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470876257942629:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.592884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470876257942629:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.665646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470876547864444:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:01.672172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470876547864463:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26219
2025-11-27T18:17:01.631385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470876507137661:2049] [node 2] ICS09 handshake done sender: [6:7577470876507137645:2114] self: [6:7577470876507137650:0] peer: [2:7577470876257942614:0] socket: 65 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.745058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470877216646997:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.689445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470876547864441:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:17:01.744543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470876970668728:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.599658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470876257942629:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:01.631415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470876507137661:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:01.744727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470876970668713:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:01.744732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470876970668710:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:01.745102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470877216646997:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.748939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470876970668718:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.609167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470876257942624:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.631442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:01.752218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-11-27T18:17:01.652312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470876507137651:2118] [node 3] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:17:01.752481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470876970668732:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4129
2025-11-27T18:17:01.655211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470872212170275:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.691648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470872252897077:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.609775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:01.752579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470876970668718:2108] [node 4] ICP22 created new session: [5:7577470876970668734:2108]
2025-11-27T18:17:01.691738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:01.691789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470872252897077:2049] [node 3] ICP22 created new session: [1:7577470876547864465:2049]
2025-11-27T18:17:01.691838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470876547864465:2049] [node 3] ICS09 handshake done sender: [1:7577470876547864444:2113] self: [1:7577470876547864437:0] peer: [3:7577470877486318798:0] socket: 51 qp: -1
2025-11-27T18:17:01.691893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470876547864465:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:01.701379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:01.722846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470876547864463:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.726209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470876547864435:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.735695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:01.735781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470876547864435:2107] [node 4] ICP22 created new session: [1:7577470876547864467:2107]
2025-11-27T18:17:01.735835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470876547864467:2107] [node 4] ICS09 handshake done sender: [1:7577470876547864441:2111] self: [1:7577470876547864440:0] peer: [4:7577470877216647016:0] socket: 112 qp: -1
2025-11-27T18:17:01.735870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470876547864467:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:01.735917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:01.744821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470876547864443:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.744845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470876547864443:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.753060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470876547864443:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:01.754030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470876547864436:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.655277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:01.753422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470877216647018:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58420
2025-11-27T18:17:01.656071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470872212170275:2050] [node 3] ICP22 created new session: [6:7577470876507137665:2050]
2025-11-27T18:17:01.657624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470876507137665:2050] [node 3] ICS09 handshake done sender: [6:7577470876507137651:2118] self: [6:7577470876507137653:0] peer: [3:7577470877486318776:0] socket: 87 qp: -1
2025-11-27T18:17:01.755412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470876970668734:2108] [node 4] ICS09 handshake done sender: [5:7577470876970668713:2113] self: [5:7577470876970668726:0] peer: [4:7577470877216646994:0] socket: 63 qp: -1
2025-11-27T18:17:01.614257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470876257942624:2107] [node 4] ICP22 created new session: [2:7577470876257942645:2107]
2025-11-27T18:17:01.754966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:01.755706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470877216647017:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58420
2025-11-27T18:17:01.657672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470876507137665:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:01.755711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470876970668734:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:01.657735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:01.755854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:01.756214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470876547864436:2108] [node 5] ICP22 created new session: [1:7577470876547864470:2108]
2025-11-27T18:17:01.621490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470876257942645:2107] [node 4] ICS09 handshake done sender: [2:7577470876257942629:2119] self: [2:7577470876257942628:0] peer: [4:7577470877216647008:0] socket: 58 qp: -1
2025-11-27T18:17:01.756266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470877216647011:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:01.660771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470872212170273:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(38401274ns) {HandleHandshakeStatus:329(38399953ns) {DropIncomingHandshake:442(964ns) DropOutgoingHandshake:457(2846ns) SwitchToState:179(446ns) ProcessPendingSessionEvents:495(546951ns) {ForwardSessionEventToSession:612(487584ns) ForwardSessionEventToSession:612(8843ns) ForwardSessionEventToSession:612(2574ns) ForwardSessionEventToSession:612(9268ns) ForwardSessionEventToSession:612(1864ns) ForwardSessionEventToSession:612(4477ns) ForwardSessionEventToSession:612(8922ns) ForwardSessionEventToSession:612(2585ns) ForwardSessionEventToSession:612(1702ns) ForwardSessionEventToSession:612(5615ns)}}}
2025-11-27T18:17:01.621553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470876257942645:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:01.757470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470877216646997:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:01.756925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470876547864470:2108] [node 5] ICS09 handshake done sender: [1:7577470876547864443:2112] self: [1:7577470876547864442:0] peer: [5:7577470876970668733:0] socket: 78 qp: -1
2025-11-27T18:17:01.672811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470876507137658:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53255
2025-11-27T18:17:01.757064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470876547864470:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:01.625382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:01.760929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470876970668706:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.725783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470876507137667:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53255
2025-11-27T18:17:01.742441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470876507137658:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.742579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470876507137621:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.746236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470876507137648:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:01.748515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470872212170295:2068] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.748879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:01.752248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470872212170295:2068] [node 5] ICP22 created new session: [6:7577470876507137669:2068]
2025-11-27T18:17:01.753588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470876507137669:2068] [node 5] ICS09 handshake done sender: [6:7577470876507137648:2116] self: [6:7577470876507137655:0] peer: [5:7577470876970668709:0] socket: 74 qp: -1
2025-11-27T18:17:01.753847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470876507137669:2068] [node 5] ICS10 traffic start
2025-11-27T18:17:01.753942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:01.760990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470872921679627:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.638009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470876257942644:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26863
2025-11-27T18:17:01.761327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:01.761956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:01.762050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470876970668706:2105] [node 6] ICP22 created new session: [5:7577470876970668736:2105]
2025-11-27T18:17:01.638884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470876257942627:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:01.757106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:01.639113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470871962975249:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.761996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470872921679627:2048] [node 1] ICP22 created new session: [4:7577470877216647020:2048]
2025-11-27T18:17:01.639184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:01.762113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470876970668736:2105] [node 6] ICS09 handshake done sender: [5:7577470876970668710:2111] self: [5:7577470876970668709:0] peer: [6:7577470876507137655:0] socket: 71 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.639231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470871962975249:2048] [node 1] ICP22 created new session: [2:7577470876257942647:2048]
2025-11-27T18:17:01.762311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470877216646993:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.757956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470876547864435:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(30995285ns) {HandleHandshakeStatus:329(30991854ns) {DropIncomingHandshake:442(1210ns) DropOutgoingHandshake:457(613ns) SwitchToState:179(619ns) ProcessPendingSessionEvents:495(1501404ns) {ForwardSessionEventToSession:612(1007300ns) ForwardSessionEventToSession:612(416851ns) ForwardSessionEventToSession:612(9242ns) ForwardSessionEventToSession:612(17011ns) ForwardSessionEventToSession:612(2759ns) ForwardSessionEventToSession:612(1837ns) ForwardSessionEventToSession:612(21789ns) ForwardSessionEventToSession:612(2519ns) ForwardSessionEventToSession:612(1906ns) ForwardSessionEventToSession:612(4676ns)}}}
2025-11-27T18:17:01.762148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470876970668736:2105] [node 6] ICS10 traffic start
2025-11-27T18:17:01.762378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:01.639273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470876257942647:2048] [node 1] ICS09 handshake done sender: [2:7577470876257942627:2118] self: [2:7577470876257942638:0] peer: [1:7577470876547864438:0] socket: 81 qp: -1
2025-11-27T18:17:01.762196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:01.639335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470876257942647:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:01.639379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:01.760430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470876547864435:2107] [node 4] ICP18 handshake (actor [1:7577470876547864463:2117]) is too old
2025-11-27T18:17:01.764134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470876507137671:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53255
2025-11-27T18:17:01.762831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470877216646993:2110] [node 6] ICP22 created new session: [4:7577470877216647021:2110]
2025-11-27T18:17:01.765891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470872675701349:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:01.652218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470871962975250:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(107748042ns) {HandleHandshakeStatus:329(107732914ns) {DropIncomingHandshake:442(2448ns) DropOutgoingHandshake:457(2296ns) SwitchToState:179(578ns) ProcessPendingSessionEvents:495(19945944ns) {ForwardSessionEventToSession:612(6139374ns) ForwardSessionEventToSession:612(2385985ns) ForwardSessionEventToSession:612(6364372ns) ForwardSessionEventToSession:612(3514ns) ForwardSessionEventToSession:612(2263ns) ForwardSessionEventToSession:612(788618ns) ForwardSessionEventToSession:612(1825ns) ForwardSessionEventToSession:612(4224667ns) ForwardSessionEventToSession:612(6543ns) ForwardSessionEventToSession:612(2353ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.762908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470877216647021:2110] [node 6] ICS09 handshake done sender: [4:7577470877216646997:2114] self: [4:7577470877216646996:0] peer: [6:7577470876507137668:0] socket: 102 qp: -1
2025-11-27T18:17:01.767590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470877486318781:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.767032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470876507137658:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:01.767629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470877486318781:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:01.653953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470871962975250:2049] [node 3] ICP18 handshake (actor [2:7577470876257942626:2117]) is too old
2025-11-27T18:17:01.762944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470877216647020:2048] [node 1] ICS09 handshake done sender: [4:7577470877216647011:2119] self: [4:7577470877216647016:0] peer: [1:7577470876547864440:0] socket: 50 qp: -1
2025-11-27T18:17:01.767474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470876970668739:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4129
2025-11-27T18:17:01.773704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470876547864472:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26219
2025-11-27T18:17:01.763099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470877216647020:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:01.690624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470876257942624:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(81201333ns) {HandleHandshakeStatus:329(81198386ns) {DropIncomingHandshake:442(12160ns) DropOutgoingHandshake:457(530ns) SwitchToState:179(404ns) ProcessPendingSessionEvents:495(38470332ns) {ForwardSessionEventToSession:612(13949799ns) ForwardSessionEventToSession:612(7122253ns) ForwardSessionEventToSession:612(20947ns) ForwardSessionEventToSession:612(4801659ns) ForwardSessionEventToSession:612(6529ns) ForwardSessionEventToSession:612(2661ns) ForwardSessionEventToSession:612(2957858ns) ForwardSessionEventToSession:612(4450ns) ForwardSessionEventToSession:612(2472ns) ForwardSessionEventToSession:612(9570900ns)}}}
2025-11-27T18:17:01.768826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470876970668706:2105] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:01.783118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470876507137621:2076] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.763115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470877216647021:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:01.719545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470876257942625:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.789108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470877486318781:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:01.738139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470876257942609:2108] [node 5] ICP18 handshake (actor [2:7577470876257942625:2116]) is too old
2025-11-27T18:17:01.797076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470876970668721:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:01.763703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:01.797630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470872675701347:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.797706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:01.797754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470872675701347:2048] [node 1] ICP22 created new session: [5:7577470876970668742:2048]
2025-11-27T18:17:01.787700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:01.789074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470876507137621:2076] [node 4] ICP22 created new session: [6:7577470876507137672:2076]
2025-11-27T18:17:01.793168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470873191351469:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:01.798158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470876547864472:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.763801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:01.798452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470876970668742:2048] [node 1] ICS09 handshake done sender: [5:7577470876970668721:2116] self: [5:7577470876970668733:0] peer: [1:7577470876547864442:0] socket: 84 qp: -1
2025-11-27T18:17:01.793687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470876507137672:2076] [node 4] ICS09 handshake done sender: [6:7577470876507137658:2122] self: [6:7577470876507137668:0] peer: [4:7577470877216646996:0] socket: 108 qp: -1
2025-11-27T18:17:01.798780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470876970668742:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:01.793221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:01.799547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470876547864436:2108] [node 5] ICP18 handshake (actor [1:7577470876547864472:2118]) is too old
2025-11-27T18:17:01.766795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470872921679627:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:01.794764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470876507137672:2076] [node 4] ICS10 traffic start
2025-11-27T18:17:01.798832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:01.794849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470873191351469:2108] [node 5] ICP22 created new session: [3:7577470877486318806:2108]
2025-11-27T18:17:01.798670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470877486318806:2108] [node 5] ICS09 handshake done sender: [3:7577470877486318781:2117] self: [3:7577470877486318780:0] peer: [5:7577470876970668740:0] socket: 101 qp: -1
2025-11-27T18:17:01.799345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470877486318806:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:01.800497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:01.773519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470877216647009:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:01.784120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470872921679629:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.788284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:01.788340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470872921679629:2050] [node 3] ICP22 created new session: [4:7577470877216647024:2050]
2025-11-27T18:17:01.788394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470877216647024:2050] [node 3] ICS09 handshake done sender: [4:7577470877216647009:2117] self: [4:7577470877216647014:0] peer: [3:7577470877486318785:0] socket: 107 qp: -1
2025-11-27T18:17:01.788425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470877216647024:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:01.788460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:01.796718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:01.812376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470876970668741:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4129
2025-11-27T18:17:01.813025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470876970668728:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:01.806095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470876507137621:2076] [node 4] ICP35 event processing took too much time PendingConnection:279(20455987ns) {HandleHandshakeStatus:329(18922093ns) {DropIncomingHandshake:442(744ns) DropOutgoingHandshake:457(5950ns) SwitchToState:179(393ns) ProcessPendingSessionEvents:495(975970ns) {ForwardSessionEventToSession:612(860627ns) ForwardSessionEventToSession:612(2855ns) ForwardSessionEventToSession:612(2045ns) ForwardSessionEventToSession:612(88157ns) ForwardSessionEventToSession:612(2555ns) ForwardSessionEventToSession:612(1506ns) ForwardSessionEventToSession:612(3215ns) ForwardSessionEventToSession:612(3706ns) ForwardSessionEventToSession:612(3193ns) ForwardSessionEventToSession:612(1074ns)}}}
2025-11-27T18:17:01.814042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470872675701349:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:01.814094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:01.814136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470872675701349:2050] [node 3] ICP22 created new session: [5:7577470876970668744:2050]
2025-11-27T18:17:01.814176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470876970668744:2050] [node 3] ICS09 handshake done sender: [5:7577470876970668728:2120] self: [5:7577470876970668740:0] peer: [3:7577470877486318780:0] socket: 104 qp: -1
2025-11-27T18:17:01.814211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470876970668744:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:01.814259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:01.816316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470877486318783:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:01.817078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577470873191351469:2108] [node 5] ICP18 handshake (actor [3:7577470877486318783:2118]) is too old
2025-11-27T18:17:01.863218Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:01.863464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470876257942647:2048] [node 1] ICS01 socket: 81 reason# EndOfStream
2025-11-27T18:17:01.863511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470871962975249:2048] [node 1] ICP30 unregister session Session# [2:7577470876257942647:2048] VirtualId# [2:7577470876257942638:0]
2025-11-27T18:17:01.863550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470876257942647:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.863683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:01.863798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:01.864694Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:01.864866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470877486318803:2048] [node 1] ICS01 socket: 57 reason# EndOfStream
2025-11-27T18:17:01.864912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470873191351406:2048] [node 1] ICP30 unregister session Session# [3:7577470877486318803:2048] VirtualId# [3:7577470877486318798:0]
2025-11-27T18:17:01.864938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470877486318803:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.865082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:01.865151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:01.867675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:01.867948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470877216647020:2048] [node 1] ICS07 socket disconnect 50 reason# EndOfStream
2025-11-27T18:17:01.867982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470877216647020:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.868047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:01.868094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470877216647020:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:01.868836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:01.868995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470872921679627:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27494) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:01.869078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470877216647020:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:01.869120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470872921679627:2048] [node 1] ICP30 unregister session Session# [4:7577470877216647020:2048] VirtualId# [4:7577470877216647016:0]
2025-11-27T18:17:01.869187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:01.869107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470876970668742:2048] [node 1] ICS01 socket: 84 reason# EndOfStream
2025-11-27T18:17:01.869217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470872921679627:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27494) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:01.869228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27494) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:01.869224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470872675701347:2048] [node 1] ICP30 unregister session Session# [5:7577470876970668742:2048] VirtualId# [5:7577470876970668733:0]
2025-11-27T18:17:01.869275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470876970668742:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.869373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:01.869426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:01.869959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:01.870209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470876507137659:2048] [node 1] ICS07 socket disconnect 82 reason# EndOfStream
2025-11-27T18:17:01.870235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470876507137659:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.870321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:01.870379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470876507137659:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:01.871385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470872212170273:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27494) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:01.871448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470876507137659:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:01.871520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470872212170273:2048] [node 1] ICP30 unregister session Session# [6:7577470876507137659:2048] VirtualId# [6:7577470876507137652:0]
2025-11-27T18:17:01.871572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:01.871598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470872212170273:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27494) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:01.871613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27494) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:01.878635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470872921679627:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:01.881626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470872212170273:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:01.895605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:01.895880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470877486318790:2049] [node 2] ICS01 socket: 68 reason# EndOfStream
2025-11-27T18:17:01.895929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470873191351407:2049] [node 2] ICP30 unregister session Session# [3:7577470877486318790:2049] VirtualId# [3:7577470877486318786:0]
2025-11-27T18:17:01.895978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470877486318790:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.896099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:01.896164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:01.902249Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:01.902487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470877216647012:2049] [node 2] ICS07 socket disconnect 66 reason# EndOfStream
2025-11-27T18:17:01.902539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470877216647012:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.902635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:01.902681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470877216647012:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:01.910078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:01.910646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470872921679628:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61288) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:01.910302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470876970668729:2049] [node 2] ICS07 socket disconnect 64 reason# EndOfStream
2025-11-27T18:17:01.910325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470876970668729:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.910434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:01.919693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:01.910472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470876970668729:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:01.917474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470877216647012:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:01.919877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470876507137661:2049] [node 2] ICS01 socket: 65 reason# EndOfStream
2025-11-27T18:17:01.919727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470872921679628:2049] [node 2] ICP30 unregister session Session# [4:7577470877216647012:2049] VirtualId# [4:7577470877216647008:0]
2025-11-27T18:17:01.919975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:01.920041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470872212170274:2049] [node 2] ICP30 unregister session Session# [6:7577470876507137661:2049] VirtualId# [6:7577470876507137650:0]
2025-11-27T18:17:01.920052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470872921679628:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61288) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:01.920072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61288) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:01.920115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470876507137661:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.920273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:01.920377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:01.924218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470872675701348:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61288) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:01.924778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470876970668729:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:01.924833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470872675701348:2049] [node 2] ICP30 unregister session Session# [5:7577470876970668729:2049] VirtualId# [5:7577470876970668722:0]
2025-11-27T18:17:01.925005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:01.925153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470872675701348:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61288) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:01.925177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61288) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:01.929793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470872921679628:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:01.935970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470872675701348:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:01.967499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:01.967787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470876970668744:2050] [node 3] ICS01 socket: 104 reason# EndOfStream
2025-11-27T18:17:01.967838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470872675701349:2050] [node 3] ICP30 unregister session Session# [5:7577470876970668744:2050] VirtualId# [5:7577470876970668740:0]
2025-11-27T18:17:01.967879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470876970668744:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.967985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:01.968078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:01.969819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:01.970145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470876507137665:2050] [node 3] ICS01 socket: 87 reason# EndOfStream
2025-11-27T18:17:01.970237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470872212170275:2050] [node 3] ICP30 unregister session Session# [6:7577470876507137665:2050] VirtualId# [6:7577470876507137653:0]
2025-11-27T18:17:01.970308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470876507137665:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.970448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:01.970542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:01.973422Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:01.973692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470877216647024:2050] [node 3] ICS01 socket: 107 reason# EndOfStream
2025-11-27T18:17:01.973733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470872921679629:2050] [node 3] ICP30 unregister session Session# [4:7577470877216647024:2050] VirtualId# [4:7577470877216647014:0]
2025-11-27T18:17:01.973766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470877216647024:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:01.973867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:01.973942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:02.005377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:02.005650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470876970668734:2108] [node 4] ICS07 socket disconnect 63 reason# EndOfStream
2025-11-27T18:17:02.005687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470876970668734:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:02.005763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:02.005813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470876970668734:2108] [node 4] ICS15 start handshake
2025-11-27T18:17:02.007860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470876970668718:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13540) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:02.008380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470876970668734:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:02.008451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470876970668718:2108] [node 4] ICP30 unregister session Session# [5:7577470876970668734:2108] VirtualId# [5:7577470876970668726:0]
2025-11-27T18:17:02.008507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:02.008473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:02.008559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470876970668718:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:02.008684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470876507137672:2076] [node 4] ICS07 socket disconnect 108 reason# EndOfStream
2025-11-27T18:17:02.008582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:02.008853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470876507137672:2076] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:02.008941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:02.008991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470876507137672:2076] [node 4] ICS15 start handshake
2025-11-27T18:17:02.010392Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470876507137621:2076] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13540) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:02.013250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470876507137672:2076] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:02.014057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470876507137621:2076] [node 4] ICP30 unregister session Session# [6:7577470876507137672:2076] VirtualId# [6:7577470876507137668:0]
2025-11-27T18:17:02.014122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:02.014226Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470876507137621:2076] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:02.014241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:02.025271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470876507137621:2076] [node 4] ICP33 wake up from error state
2025-11-27T18:17:02.050866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:02.051155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470876507137669:2068] [node 5] ICS01 socket: 74 reason# EndOfStream
2025-11-27T18:17:02.051193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470872212170295:2068] [node 5] ICP30 unregister session Session# [6:7577470876507137669:2068] VirtualId# [6:7577470876507137655:0]
2025-11-27T18:17:02.051229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470876507137669:2068] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:02.051309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:02.051347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:02.118006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470881112265587:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:02.118048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470881112265588:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:02.144660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470881709599897:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:02.144764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470881709599898:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:02.169811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470879037182930:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:02.169853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470879037182931:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:02.193559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470882701197018:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:02.193586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470882701197019:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:02.194059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470882701197017:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:02.222777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470882220959642:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:02.222810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470882220959643:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:02.222829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470882220959644:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:02.251226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470882935911461:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:02.251857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470882935911463:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:02.252104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470882935911462:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:02.446367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470881112265649:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:02.593515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470882935911522:2095] [node 4] ICP01 ready to work
2025-11-27T18:17:02.649410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470881112265651:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:02.645837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470882701197083:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:02.645801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470881709599961:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:02.650510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470881709599962:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:02.653117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470879037183002:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:02.649696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470882935911525:2100] [node 5] ICP01 ready to work
2025-11-27T18:17:02.654341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470881112265652:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:02.655346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470881709599963:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:02.653134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470879037183003:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:02.656533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470882701197084:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:02.656355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470882220959706:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:02.657043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470882220959705:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:02.654839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470879037183004:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:02.670806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470882935911463:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(232819630ns) {Configure:104(232813797ns) {ProcessConfigured:135(75241ns) {StartInitialHandshake:157(72705ns) {DropHandshakes:470(3459ns) {DropIncomingHandshake:442(947ns) DropOutgoingHandshake:457(647ns)} GenerateSessionVirtualId:988(1215ns) PrepareNewSessionHandshake:477(1861ns) {SwitchToState:179(691ns)}}}}}
2025-11-27T18:17:02.694870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470882935911525:2100] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18334202ns) {Configure:104(18326660ns) {ProcessConfigured:135(81425ns) {StartInitialHandshake:157(79196ns) {DropHandshakes:470(3322ns) {DropIncomingHandshake:442(746ns) DropOutgoingHandshake:457(479ns)} GenerateSessionVirtualId:988(1039ns) PrepareNewSessionHandshake:477(1839ns) {SwitchToState:179(746ns)}}}}}
2025-11-27T18:17:02.693467Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470881112265649:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(219069820ns) {Configure:104(219065006ns) {ProcessConfigured:135(7450096ns) {StartInitialHandshake:157(7447364ns) {DropHandshakes:470(3244ns) {DropIncomingHandshake:442(1095ns) DropOutgoingHandshake:457(409ns)} GenerateSessionVirtualId:988(662ns) PrepareNewSessionHandshake:477(5511ns) {SwitchToState:179(2523ns)}}}}}
2025-11-27T18:17:02.689459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470882701197018:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47610866ns) {Configure:104(47604343ns) {ProcessConfigured:135(10609217ns) {StartInitialHandshake:157(10605414ns) {DropHandshakes:470(1650ns) {DropIncomingHandshake:442(581ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(501ns) PrepareNewSessionHandshake:477(4455ns) {SwitchToState:179(2094ns)}}}}}
2025-11-27T18:17:02.715812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470882935911462:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(65075413ns) {Configure:104(65067414ns) {ProcessConfigured:135(5148488ns) {StartInitialHandshake:157(5145681ns) {DropHandshakes:470(3370ns) {DropIncomingHandshake:442(743ns) DropOutgoingHandshake:457(522ns)} GenerateSessionVirtualId:988(716ns) PrepareNewSessionHandshake:477(4123ns) {SwitchToState:179(1356ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:02.697597Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470881112265587:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(247565979ns) {Configure:104(247562048ns) {ProcessConfigured:135(19684824ns) {StartInitialHandshake:157(19682154ns) {DropHandshakes:470(4808ns) {DropIncomingHandshake:442(681ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(968ns) PrepareNewSessionHandshake:477(4906ns) {SwitchToState:179(2839ns)}}}}}
2025-11-27T18:17:02.721455Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470882935911461:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(66467177ns) {Configure:104(66463339ns) {ProcessConfigured:135(2645538ns) {StartInitialHandshake:157(2643377ns) {DropHandshakes:470(1462ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(567ns) PrepareNewSessionHandshake:477(1452ns) {SwitchToState:179(138ns)}}}}}
2025-11-27T18:17:02.720367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470881709599961:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(60057817ns) {Configure:104(60052604ns) {ProcessConfigured:135(8829942ns) {StartInitialHandshake:157(8827203ns) {DropHandshakes:470(3465ns) {DropIncomingHandshake:442(731ns) DropOutgoingHandshake:457(519ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(3337ns) {SwitchToState:179(946ns)}}}}}
2025-11-27T18:17:02.711699Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470882701197083:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(53497305ns) {Configure:104(53493597ns) {ProcessConfigured:135(77253ns) {StartInitialHandshake:157(74898ns) {DropHandshakes:470(2430ns) {DropIncomingHandshake:442(736ns) DropOutgoingHandshake:457(315ns)} GenerateSessionVirtualId:988(1220ns) PrepareNewSessionHandshake:477(1965ns) {SwitchToState:179(787ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:02.720799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470882701197019:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(89890736ns) {Configure:104(88288199ns) {ProcessConfigured:135(5034443ns) {StartInitialHandshake:157(5032033ns) {DropHandshakes:470(2699ns) {DropIncomingHandshake:442(651ns) DropOutgoingHandshake:457(365ns)} GenerateSessionVirtualId:988(786ns) PrepareNewSessionHandshake:477(2320ns) {SwitchToState:179(1066ns)}}}}}
2025-11-27T18:17:02.735591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470879037183002:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(69896391ns) {Configure:104(69888822ns) {ProcessConfigured:135(4795458ns) {StartInitialHandshake:157(4793815ns) {DropHandshakes:470(1769ns) {DropIncomingHandshake:442(475ns) DropOutgoingHandshake:457(326ns)} GenerateSessionVirtualId:988(580ns) PrepareNewSessionHandshake:477(2123ns) {SwitchToState:179(113ns)}}}}}
2025-11-27T18:17:02.720852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470881709599898:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(66897266ns) {Configure:104(62739925ns) {ProcessConfigured:135(5296544ns) {StartInitialHandshake:157(5293775ns) {DropHandshakes:470(2214ns) {DropIncomingHandshake:442(535ns) DropOutgoingHandshake:457(355ns)} GenerateSessionVirtualId:988(462ns) PrepareNewSessionHandshake:477(3918ns) {SwitchToState:179(1484ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:02.737909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470882220959706:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(53502501ns) {Configure:104(53494262ns) {ProcessConfigured:135(8833213ns) {StartInitialHandshake:157(8468034ns) {DropHandshakes:470(1627ns) {DropIncomingHandshake:442(428ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(1182ns) PrepareNewSessionHandshake:477(1899ns) {SwitchToState:179(97ns)}}}}}
2025-11-27T18:17:02.745361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470882220959644:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(183146038ns) {Configure:104(183138763ns) {ProcessConfigured:135(6070286ns) {StartInitialHandshake:157(6068497ns) {DropHandshakes:470(722ns) {DropIncomingHandshake:442(263ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:477(4112ns) {SwitchToState:179(1490ns)}}}}}
2025-11-27T18:17:02.761646Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470882220959642:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(79141220ns) {Configure:104(79134665ns) {ProcessConfigured:135(6595430ns) {StartInitialHandshake:157(6593813ns) {DropHandshakes:470(1598ns) {DropIncomingHandshake:442(422ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(433ns) PrepareNewSessionHandshake:477(2789ns) {SwitchToState:179(1078ns)}}}}}
2025-11-27T18:17:02.753479Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470879037182931:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(239663918ns) {Configure:104(239655619ns) {ProcessConfigured:135(3864834ns) {StartInitialHandshake:157(3861992ns) {DropHandshakes:470(2746ns) {DropIncomingHandshake:442(892ns) DropOutgoingHandshake:457(371ns)} GenerateSessionVirtualId:988(424ns) PrepareNewSessionHandshake:477(3204ns) {SwitchToState:179(1291ns)}}}}}
2025-11-27T18:17:02.779122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470882220959643:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(104961379ns) {Configure:104(104957677ns) {ProcessConfigured:135(23780620ns) {StartInitialHandshake:157(23776925ns) {DropHandshakes:470(1311ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(325ns)} GenerateSessionVirtualId:988(481ns) PrepareNewSessionHandshake:477(4370ns) {SwitchToState:179(1963ns)}}}}}
2025-11-27T18:17:02.777809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470879037183003:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(123478716ns) {Configure:104(123020727ns) {ProcessConfigured:135(3859374ns) {StartInitialHandshake:157(3856927ns) {DropHandshakes:470(3655ns) {DropIncomingHandshake:442(995ns) DropOutgoingHandshake:457(363ns)} GenerateSessionVirtualId:988(852ns) PrepareNewSessionHandshake:477(4228ns) {SwitchToState:179(2086ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:02.765853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470882701197084:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(77600128ns) {Configure:104(77577665ns) {ProcessConfigured:135(3041715ns) {StartInitialHandshake:157(3039474ns) {DropHandshakes:470(2966057ns) {DropIncomingHandshake:442(1997ns) DropOutgoingHandshake:457(393ns)} GenerateSessionVirtualId:988(1098ns) PrepareNewSessionHandshake:477(2078ns) {SwitchToState:179(944ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:02.795023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470879037182930:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(298305528ns) {Configure:104(298298422ns) {ProcessConfigured:135(4992862ns) {StartInitialHandshake:157(4972961ns) {DropHandshakes:470(2476ns) {DropIncomingHandshake:442(391ns) DropOutgoingHandshake:457(433ns)} GenerateSessionVirtualId:988(883ns) PrepareNewSessionHandshake:477(4137ns) {SwitchToState:179(2124ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:02.721902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470882935911536:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16982
2025-11-27T18:17:02.727040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470882935911522:2095] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(86518266ns) {Configure:104(86513291ns) {ProcessConfigured:135(1449533ns) {StartInitialHandshake:157(1447522ns) {DropHandshakes:470(1904ns) {DropIncomingHandshake:442(467ns) DropOutgoingHandshake:457(737ns)} GenerateSessionVirtualId:988(1138ns) PrepareNewSessionHandshake:477(1434ns) {SwitchToState:179(549ns)}}}}}
2025-11-27T18:17:02.777781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470882935911536:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.778016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470882935911522:2095] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.826934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470882935911542:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16982
2025-11-27T18:17:02.832270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470882935911542:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.835135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470882935911461:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.854805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470882935911544:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16982
2025-11-27T18:17:02.855932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470882935911541:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16982
2025-11-27T18:17:02.862993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470882935911544:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.868337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470882935911545:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16982
2025-11-27T18:17:02.868983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470882935911462:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.879818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470882935911542:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:02.880956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470882935911536:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:02.886621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470882935911461:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.886699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:02.889141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470882935911522:2095] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.889203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:02.889586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470882935911461:2048] [node 1] ICP22 created new session: [6:7577470882935911547:2048]
2025-11-27T18:17:02.889756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470882935911522:2095] [node 4] ICP22 created new session: [6:7577470882935911549:2095]
2025-11-27T18:17:02.891358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470882935911547:2048] [node 1] ICS09 handshake done sender: [6:7577470882935911542:2117] self: [6:7577470882935911543:0] peer: [1:7577470881112265663:0] socket: 104 qp: -1
2025-11-27T18:17:02.893648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470882935911550:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16982
2025-11-27T18:17:02.894001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470882935911549:2095] [node 4] ICS09 handshake done sender: [6:7577470882935911536:2114] self: [6:7577470882935911540:0] peer: [4:7577470882701197088:0] socket: 65 qp: -1
2025-11-27T18:17:02.895582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470882935911547:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:02.895653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470882935911549:2095] [node 4] ICS10 traffic start
2025-11-27T18:17:02.896421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:02.899207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:02.909175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470882935911550:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.916741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470882935911463:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.924515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470882935911461:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23265621ns) {HandleHandshakeStatus:329(23262375ns) {DropIncomingHandshake:442(1228ns) DropOutgoingHandshake:457(5795ns) SwitchToState:179(2244ns) ProcessPendingSessionEvents:495(6808730ns) {ForwardSessionEventToSession:612(3588944ns) ForwardSessionEventToSession:612(11563ns) ForwardSessionEventToSession:612(996640ns) ForwardSessionEventToSession:612(2294ns) ForwardSessionEventToSession:612(1039525ns) ForwardSessionEventToSession:612(4835ns) ForwardSessionEventToSession:612(2740ns) ForwardSessionEventToSession:612(720384ns) ForwardSessionEventToSession:612(415711ns) ForwardSessionEventToSession:612(6490ns)}}}
2025-11-27T18:17:02.933494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470882935911522:2095] [node 4] ICP35 event processing took too much time PendingConnection:279(22601790ns) {HandleHandshakeStatus:329(22583893ns) {DropIncomingHandshake:442(2201ns) DropOutgoingHandshake:457(14757ns) SwitchToState:179(1107ns) ProcessPendingSessionEvents:495(6983191ns) {ForwardSessionEventToSession:612(2694001ns) ForwardSessionEventToSession:612(402838ns) ForwardSessionEventToSession:612(2503ns) ForwardSessionEventToSession:612(909344ns) ForwardSessionEventToSession:612(2992ns) ForwardSessionEventToSession:612(485183ns) ForwardSessionEventToSession:612(767667ns) ForwardSessionEventToSession:612(1008099ns) ForwardSessionEventToSession:612(6210ns) ForwardSessionEventToSession:612(683578ns)}}}
2025-11-27T18:17:02.935436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470882935911462:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21531) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470882935911544:2118] held: no
2025-11-27T18:17:02.945406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470882935911463:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2167) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470882935911550:2120] held: no
2025-11-27T18:17:02.698075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470881112265653:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58106
2025-11-27T18:17:02.786176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470882220959715:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30544
2025-11-27T18:17:02.798497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470879037183010:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30472
2025-11-27T18:17:02.792134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470882701197089:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.798640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470879037183013:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30472
2025-11-27T18:17:02.792196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470882701197089:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.808483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470882701197089:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:02.824080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470882701197096:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60193
2025-11-27T18:17:02.836875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470882701197096:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.836910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470882701197083:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.836941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:02.836983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470882701197083:2110] [node 6] ICP22 created new session: [4:7577470882701197099:2110]
2025-11-27T18:17:02.837017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470882701197099:2110] [node 6] ICS09 handshake done sender: [4:7577470882701197089:2112] self: [4:7577470882701197088:0] peer: [6:7577470882935911540:0] socket: 63 qp: -1
2025-11-27T18:17:02.837047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470882701197099:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:02.837079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:02.837611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470882701197095:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60193
2025-11-27T18:17:02.837880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470882701197017:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.843355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470882701197095:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.845108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470882701197103:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60193
2025-11-27T18:17:02.849525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470882701197098:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60193
2025-11-27T18:17:02.855385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470882701197103:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.857204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470882701197017:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64226) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470882701197096:2116] held: no
2025-11-27T18:17:02.862107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470882701197098:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.862286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470882701197018:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.862476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470882701197019:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.874013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470882701197104:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60193
2025-11-27T18:17:02.888080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470882701197110:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60193
2025-11-27T18:17:02.890241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470882701197096:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:02.892825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470882701197017:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.892995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470882701197094:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.893028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470882701197094:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.899180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:02.900735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470882701197113:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60193
2025-11-27T18:17:02.900999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470882701197017:2048] [node 1] ICP22 created new session: [4:7577470882701197111:2048]
2025-11-27T18:17:02.901612Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470882701197094:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:02.905163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470882701197111:2048] [node 1] ICS09 handshake done sender: [4:7577470882701197096:2116] self: [4:7577470882701197102:0] peer: [1:7577470881112265666:0] socket: 97 qp: -1
2025-11-27T18:17:02.906573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470882701197111:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:02.907621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:02.913953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470882701197098:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:02.922527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470882701197018:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21531) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470882701197103:2118] held: no
2025-11-27T18:17:02.934043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470882701197084:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.934110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:02.934161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470882701197084:2109] [node 5] ICP22 created new session: [4:7577470882701197115:2109]
2025-11-27T18:17:02.934201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470882701197115:2109] [node 5] ICS09 handshake done sender: [4:7577470882701197094:2114] self: [4:7577470882701197093:0] peer: [5:7577470882220959732:0] socket: 73 qp: -1
2025-11-27T18:17:02.934245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470882701197115:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:02.934280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:02.933510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470882701197017:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(31067624ns) {HandleHandshakeStatus:329(31064364ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(223ns) SwitchToState:179(545ns) ProcessPendingSessionEvents:495(1099ns)}}
2025-11-27T18:17:02.935038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470882701197083:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:02.937540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470882701197019:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.937784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470882701197109:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60193
2025-11-27T18:17:02.938396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:02.938447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470882701197019:2050] [node 3] ICP22 created new session: [4:7577470882701197117:2050]
2025-11-27T18:17:02.938491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470882701197117:2050] [node 3] ICS09 handshake done sender: [4:7577470882701197098:2117] self: [4:7577470882701197107:0] peer: [3:7577470879037183005:0] socket: 99 qp: -1
2025-11-27T18:17:02.938534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470882701197117:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:02.938573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:02.941797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470882701197103:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:02.942130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470882701197018:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.942177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:02.943581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470882701197018:2049] [node 2] ICP22 created new session: [4:7577470882701197119:2049]
2025-11-27T18:17:02.945022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470882701197119:2049] [node 2] ICS09 handshake done sender: [4:7577470882701197103:2118] self: [4:7577470882701197106:0] peer: [2:7577470881709599982:0] socket: 109 qp: -1
2025-11-27T18:17:02.945370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470882701197119:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:02.945706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:02.727961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470881709599970:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6996
2025-11-27T18:17:02.728106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470881709599971:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6996
2025-11-27T18:17:02.728152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470881709599972:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6996
2025-11-27T18:17:02.730831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470881709599970:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.735532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470881709599972:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.737880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470881709599897:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(97626097ns) {Configure:104(97616624ns) {ProcessConfigured:135(2936800ns) {StartInitialHandshake:157(2934412ns) {DropHandshakes:470(1649ns) {DropIncomingHandshake:442(437ns) DropOutgoingHandshake:457(321ns)} GenerateSessionVirtualId:988(629ns) PrepareNewSessionHandshake:477(2696ns) {SwitchToState:179(1146ns)}}}}}
2025-11-27T18:17:02.764821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470881709599897:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.764961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470881709599971:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.789691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470881709599962:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(133041297ns) {Configure:104(133037195ns) {ProcessConfigured:135(9968967ns) {StartInitialHandshake:157(9966833ns) {DropHandshakes:470(2400ns) {DropIncomingHandshake:442(492ns) DropOutgoingHandshake:457(456ns)} GenerateSessionVirtualId:988(588ns) PrepareNewSessionHandshake:477(3928ns) {SwitchToState:179(1261ns)}}}}}
2025-11-27T18:17:02.799736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470881709599966:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.799759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470881709599966:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.806111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470881709599966:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:02.809463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470881709599897:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64226) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577470881709599970:2114] held: no
2025-11-27T18:17:02.814726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470881709599963:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(88073713ns) {Configure:104(88065357ns) {ProcessConfigured:135(2715041ns) {StartInitialHandshake:157(2711968ns) {DropHandshakes:470(2703ns) {DropIncomingHandshake:442(758ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(609ns) PrepareNewSessionHandshake:477(2720ns) {SwitchToState:179(1025ns)}}}}}
2025-11-27T18:17:02.830045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470881709599898:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.830131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:02.830184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470881709599898:2049] [node 3] ICP22 created new session: [2:7577470881709599984:2049]
2025-11-27T18:17:02.830223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470881709599984:2049] [node 3] ICS09 handshake done sender: [2:7577470881709599966:2111] self: [2:7577470881709599965:0] peer: [3:7577470879037183020:0] socket: 70 qp: -1
2025-11-27T18:17:02.830259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470881709599984:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:02.830288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:02.830855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470881709599968:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.830875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470881709599968:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.831907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470881709599968:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:02.837438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470881709599981:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6996
2025-11-27T18:17:02.846123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470881709599961:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.848104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470881709599970:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:02.848251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:02.848998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470881709599961:2107] [node 5] ICP22 created new session: [2:7577470881709599990:2107]
2025-11-27T18:17:02.849643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470881709599990:2107] [node 5] ICS09 handshake done sender: [2:7577470881709599968:2112] self: [2:7577470881709599964:0] peer: [5:7577470882220959723:0] socket: 71 qp: -1
2025-11-27T18:17:02.849878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470881709599897:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.849934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:02.849979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470881709599897:2048] [node 1] ICP22 created new session: [2:7577470881709599991:2048]
2025-11-27T18:17:02.850017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470881709599991:2048] [node 1] ICS09 handshake done sender: [2:7577470881709599970:2114] self: [2:7577470881709599976:0] peer: [1:7577470881112265655:0] socket: 66 qp: -1
2025-11-27T18:17:02.850052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470881709599991:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:02.850526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:02.850981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470881709599990:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:02.851148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:02.868934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470881709599983:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.868978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470881709599983:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.869182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470881709599897:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16074789ns) {HandleHandshakeStatus:329(16071989ns) {DropIncomingHandshake:442(445ns) DropOutgoingHandshake:457(308ns) SwitchToState:179(806ns) ProcessPendingSessionEvents:495(1761ns)}}
2025-11-27T18:17:02.873749Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470881709599961:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(28934168ns) {HandleHandshakeStatus:329(28930816ns) {DropIncomingHandshake:442(1551ns) DropOutgoingHandshake:457(926ns) SwitchToState:179(440ns) ProcessPendingSessionEvents:495(10455751ns) {ForwardSessionEventToSession:612(5806940ns) ForwardSessionEventToSession:612(1126329ns) ForwardSessionEventToSession:612(1308983ns) ForwardSessionEventToSession:612(4902ns) ForwardSessionEventToSession:612(2451ns) ForwardSessionEventToSession:612(2127518ns) ForwardSessionEventToSession:612(7527ns) ForwardSessionEventToSession:612(3905ns) ForwardSessionEventToSession:612(36125ns) ForwardSessionEventToSession:612(3888ns)}}}
2025-11-27T18:17:02.879855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470881709599983:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:02.889538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470881709599978:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.890244Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470881709599978:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.890416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470881709599963:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.890455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:02.893892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470881709599963:2108] [node 4] ICP22 created new session: [2:7577470881709599995:2108]
2025-11-27T18:17:02.896378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470881709599995:2108] [node 4] ICS09 handshake done sender: [2:7577470881709599983:2119] self: [2:7577470881709599982:0] peer: [4:7577470882701197106:0] socket: 107 qp: -1
2025-11-27T18:17:02.900073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470881709599995:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:02.901233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:02.905883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470881709599996:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6996
2025-11-27T18:17:02.922477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470881709599978:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:02.929660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470881709599962:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.929720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:02.929764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470881709599962:2109] [node 6] ICP22 created new session: [2:7577470881709599999:2109]
2025-11-27T18:17:02.929807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470881709599999:2109] [node 6] ICS09 handshake done sender: [2:7577470881709599978:2117] self: [2:7577470881709599977:0] peer: [6:7577470882935911546:0] socket: 110 qp: -1
2025-11-27T18:17:02.929835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470881709599999:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:02.929867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:02.935413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577470881709599961:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:02.944296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470881709599963:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(32048794ns) {HandleHandshakeStatus:329(32046505ns) {DropIncomingHandshake:442(10880ns) DropOutgoingHandshake:457(193ns) SwitchToState:179(115ns) ProcessPendingSessionEvents:495(19137611ns) {ForwardSessionEventToSession:612(5322251ns) ForwardSessionEventToSession:612(734035ns) ForwardSessionEventToSession:612(2967ns) ForwardSessionEventToSession:612(2390ns) ForwardSessionEventToSession:612(414257ns) ForwardSessionEventToSession:612(2567ns) ForwardSessionEventToSession:612(3768180ns) ForwardSessionEventToSession:612(3328779ns) ForwardSessionEventToSession:612(2398ns) ForwardSessionEventToSession:612(5541531ns)}}}
2025-11-27T18:17:02.737386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470881112265652:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(33593227ns) {Configure:104(33569212ns) {ProcessConfigured:135(56816ns) {StartInitialHandshake:157(55040ns) {DropHandshakes:470(2452ns) {DropIncomingHandshake:442(578ns) DropOutgoingHandshake:457(409ns)} GenerateSessionVirtualId:988(1029ns) PrepareNewSessionHandshake:477(1222ns) {SwitchToState:179(686ns)}}}}}
2025-11-27T18:17:02.798821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470879037183016:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30472
2025-11-27T18:17:02.737750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470881112265588:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(71727950ns) {Configure:104(71720377ns) {ProcessConfigured:135(8530541ns) {StartInitialHandshake:157(6315562ns) {DropHandshakes:470(1848ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(145ns)} GenerateSessionVirtualId:988(580ns) PrepareNewSessionHandshake:477(3112ns) {SwitchToState:179(1312ns)}}}}}
2025-11-27T18:17:02.757628Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470881112265651:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(101028435ns) {Configure:104(101024548ns) {ProcessConfigured:135(82988ns) {StartInitialHandshake:157(80454ns) {DropHandshakes:470(3495ns) {DropIncomingHandshake:442(1122ns) DropOutgoingHandshake:457(457ns)} GenerateSessionVirtualId:988(722ns) PrepareNewSessionHandshake:477(2443ns) {SwitchToState:179(988ns)}}}}}
2025-11-27T18:17:02.757812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470881112265653:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.765929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470881112265662:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58106
2025-11-27T18:17:02.785773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470881112265662:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.787138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470881112265657:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.794254Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470881112265657:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.795301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470881112265657:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:02.795796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470881112265587:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.795905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:02.796018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470881112265670:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58106
2025-11-27T18:17:02.801502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470881112265587:2048] [node 2] ICP22 created new session: [1:7577470881112265672:2048]
2025-11-27T18:17:02.804895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470881112265672:2048] [node 2] ICS09 handshake done sender: [1:7577470881112265657:2112] self: [1:7577470881112265655:0] peer: [2:7577470881709599976:0] socket: 59 qp: -1
2025-11-27T18:17:02.808290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470881112265672:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:02.809499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:02.816817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470881112265661:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.816840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470881112265661:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.817857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470881112265661:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:02.818114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470881112265588:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.818159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:02.818202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470881112265588:2049] [node 3] ICP22 created new session: [1:7577470881112265677:2049]
2025-11-27T18:17:02.818240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470881112265677:2049] [node 3] ICS09 handshake done sender: [1:7577470881112265661:2113] self: [1:7577470881112265660:0] peer: [3:7577470879037183022:0] socket: 77 qp: -1
2025-11-27T18:17:02.818271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470881112265677:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:02.818303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:02.821444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470881112265674:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58106
2025-11-27T18:17:02.834953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470881112265656:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.834993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470881112265656:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.835000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470881112265670:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.842944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470881112265664:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.842970Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470881112265664:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.843460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470881112265667:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.843482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470881112265667:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.844909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470881112265656:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:02.848123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470881112265667:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:02.848245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470881112265664:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:02.852039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470881112265674:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.852088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470881112265651:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.852153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:02.852212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470881112265651:2107] [node 4] ICP22 created new session: [1:7577470881112265684:2107]
2025-11-27T18:17:02.852251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470881112265684:2107] [node 4] ICS09 handshake done sender: [1:7577470881112265667:2116] self: [1:7577470881112265666:0] peer: [4:7577470882701197102:0] socket: 90 qp: -1
2025-11-27T18:17:02.852273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470881112265684:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:02.852307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:02.852685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470881112265649:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.852725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:02.852755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470881112265649:2108] [node 5] ICP22 created new session: [1:7577470881112265686:2108]
2025-11-27T18:17:02.852786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470881112265686:2108] [node 5] ICS09 handshake done sender: [1:7577470881112265656:2111] self: [1:7577470881112265654:0] peer: [5:7577470882220959722:0] socket: 58 qp: -1
2025-11-27T18:17:02.852811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470881112265686:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:02.852849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:02.853177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470881112265652:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.853218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:02.853246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470881112265652:2109] [node 6] ICP22 created new session: [1:7577470881112265688:2109]
2025-11-27T18:17:02.853345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470881112265688:2109] [node 6] ICS09 handshake done sender: [1:7577470881112265664:2115] self: [1:7577470881112265663:0] peer: [6:7577470882935911543:0] socket: 103 qp: -1
2025-11-27T18:17:02.853371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470881112265688:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:02.853407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:02.856779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470881112265588:2049] [node 3] ICP18 handshake (actor [1:7577470881112265674:2118]) is too old
2025-11-27T18:17:02.868905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470881112265587:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(50627830ns) {HandleHandshakeStatus:329(50624443ns) {DropIncomingHandshake:442(19615ns) DropOutgoingHandshake:457(488ns) SwitchToState:179(622ns) ProcessPendingSessionEvents:495(14306168ns) {ForwardSessionEventToSession:612(5337079ns) ForwardSessionEventToSession:612(2139459ns) ForwardSessionEventToSession:612(3679ns) ForwardSessionEventToSession:612(1843141ns) ForwardSessionEventToSession:612(2353ns) ForwardSessionEventToSession:612(512716ns) ForwardSessionEventToSession:612(3109ns) ForwardSessionEventToSession:612(10847ns) ForwardSessionEventToSession:612(2891170ns) ForwardSessionEventToSession:612(278336ns)}}}
2025-11-27T18:17:02.791661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470882220959715:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.795575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470882220959716:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30544
2025-11-27T18:17:02.799184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470879037183010:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.799417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470879037182931:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.815563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470879037183013:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.816359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470879037183009:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30472
2025-11-27T18:17:02.816528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470879037182930:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.823637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470879037183009:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.823762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470879037183021:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30472
2025-11-27T18:17:02.835518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470879037183004:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(56389279ns) {Configure:104(56383446ns) {ProcessConfigured:135(56865ns) {StartInitialHandshake:157(49707ns) {DropHandshakes:470(2256ns) {DropIncomingHandshake:442(294ns) DropOutgoingHandshake:457(557ns)} GenerateSessionVirtualId:988(339ns) PrepareNewSessionHandshake:477(618ns) {SwitchToState:179(140ns)}}}}}
2025-11-27T18:17:02.846504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470879037183010:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:02.849251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470879037182931:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.849355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:02.849362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470879037183016:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.850954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470879037182931:2049] [node 2] ICP22 created new session: [3:7577470879037183027:2049]
2025-11-27T18:17:02.851213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470879037183026:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30472
2025-11-27T18:17:02.854931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470879037183027:2049] [node 2] ICS09 handshake done sender: [3:7577470879037183010:2122] self: [3:7577470879037183020:0] peer: [2:7577470881709599965:0] socket: 78 qp: -1
2025-11-27T18:17:02.855430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470879037183027:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:02.857119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:02.873421Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470879037182931:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23817027ns) {HandleHandshakeStatus:329(23815256ns) {DropIncomingHandshake:442(669ns) DropOutgoingHandshake:457(4682ns) SwitchToState:179(202ns) ProcessPendingSessionEvents:495(1806662ns) {ForwardSessionEventToSession:612(1763997ns) ForwardSessionEventToSession:612(4383ns) ForwardSessionEventToSession:612(2491ns) ForwardSessionEventToSession:612(5137ns) ForwardSessionEventToSession:612(2192ns) ForwardSessionEventToSession:612(1982ns) ForwardSessionEventToSession:612(5362ns) ForwardSessionEventToSession:612(1862ns) ForwardSessionEventToSession:612(7445ns) ForwardSessionEventToSession:612(2247ns)}}}
2025-11-27T18:17:02.873735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470879037183013:2124] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:02.874177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470879037183006:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.874198Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470879037183006:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.875000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470879037182930:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64226) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470879037183013:2124] held: no
2025-11-27T18:17:02.881041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470879037182930:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.889545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:02.889765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470879037183012:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.891376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470879037182930:2048] [node 1] ICP22 created new session: [3:7577470879037183031:2048]
2025-11-27T18:17:02.891681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470879037183012:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.893952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470879037183031:2048] [node 1] ICS09 handshake done sender: [3:7577470879037183013:2124] self: [3:7577470879037183022:0] peer: [1:7577470881112265660:0] socket: 80 qp: -1
2025-11-27T18:17:02.894084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470879037183028:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30472
2025-11-27T18:17:02.894343Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470879037183006:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:02.896350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470879037183031:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:02.901011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:02.905298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470879037183002:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.906914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:02.908791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470879037183028:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.909037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470879037183002:2107] [node 4] ICP22 created new session: [3:7577470879037183036:2107]
2025-11-27T18:17:02.909086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470879037183036:2107] [node 4] ICS09 handshake done sender: [3:7577470879037183006:2119] self: [3:7577470879037183005:0] peer: [4:7577470882701197107:0] socket: 92 qp: -1
2025-11-27T18:17:02.909119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470879037183036:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:02.909172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:02.909279Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470879037182930:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(26630913ns) {HandleHandshakeStatus:329(26629316ns) {DropIncomingHandshake:442(982ns) DropOutgoingHandshake:457(626ns) SwitchToState:179(655ns) ProcessPendingSessionEvents:495(935ns)}}
2025-11-27T18:17:02.935467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470879037183024:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.935498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470879037183024:2128] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.938419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470879037183024:2128] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:02.942204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470879037183002:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(29350099ns) {HandleHandshakeStatus:329(29346901ns) {DropIncomingHandshake:442(807ns) DropOutgoingHandshake:457(751ns) SwitchToState:179(2551ns) ProcessPendingSessionEvents:495(24876500ns) {ForwardSessionEventToSession:612(14257848ns) ForwardSessionEventToSession:612(6724ns) ForwardSessionEventToSession:612(1616027ns) ForwardSessionEventToSession:612(3605ns) ForwardSessionEventToSession:612(2622ns) ForwardSessionEventToSession:612(4908593ns) ForwardSessionEventToSession:612(3537ns) ForwardSessionEventToSession:612(2745ns) ForwardSessionEventToSession:612(4050531ns) ForwardSessionEventToSession:612(3302ns)}}}
2025-11-27T18:17:02.942954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577470879037183002:2107] [node 4] ICP18 handshake (actor [3:7577470879037183028:2130]) is too old
2025-11-27T18:17:02.943410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470879037183004:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.943467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:02.943527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470879037183004:2109] [node 6] ICP22 created new session: [3:7577470879037183039:2109]
2025-11-27T18:17:02.944214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470879037183039:2109] [node 6] ICS09 handshake done sender: [3:7577470879037183024:2128] self: [3:7577470879037183023:0] peer: [6:7577470882935911553:0] socket: 122 qp: -1
2025-11-27T18:17:02.944631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470879037183039:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:02.944977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:02.804906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470882220959716:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.808223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470882220959717:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30544
2025-11-27T18:17:02.825852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470882220959642:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.825996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470882220959717:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.826387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470882220959721:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30544
2025-11-27T18:17:02.826472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470882220959643:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.848372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470882220959705:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(62422011ns) {Configure:104(62417114ns) {ProcessConfigured:135(1954340ns) {StartInitialHandshake:157(1951732ns) {DropHandshakes:470(1533ns) {DropIncomingHandshake:442(382ns) DropOutgoingHandshake:457(341ns)} GenerateSessionVirtualId:988(774ns) PrepareNewSessionHandshake:477(3280ns) {SwitchToState:179(1947ns)}}}}}
2025-11-27T18:17:02.855904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470882220959721:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.868921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470882220959642:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64226) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470882220959716:2115] held: no
2025-11-27T18:17:02.869127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470882220959724:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30544
2025-11-27T18:17:02.873133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470882220959644:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.884347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470882220959730:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30544
2025-11-27T18:17:02.886516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470882220959717:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:02.889352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470882220959730:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.889926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470882220959643:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.890129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470882220959706:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.890342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470882220959728:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30544
2025-11-27T18:17:02.890509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:02.890556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470882220959643:2049] [node 2] ICP22 created new session: [5:7577470882220959733:2049]
2025-11-27T18:17:02.890588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470882220959733:2049] [node 2] ICS09 handshake done sender: [5:7577470882220959717:2116] self: [5:7577470882220959723:0] peer: [2:7577470881709599964:0] socket: 81 qp: -1
2025-11-27T18:17:02.890616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470882220959733:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:02.890656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:02.901310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470882220959716:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:02.902600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470882220959736:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30544
2025-11-27T18:17:02.903721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470882220959730:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:02.905171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470882220959642:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.906650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:02.906693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470882220959706:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.906885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470882220959642:2048] [node 1] ICP22 created new session: [5:7577470882220959737:2048]
2025-11-27T18:17:02.909300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470882220959737:2048] [node 1] ICS09 handshake done sender: [5:7577470882220959716:2115] self: [5:7577470882220959722:0] peer: [1:7577470881112265654:0] socket: 79 qp: -1
2025-11-27T18:17:02.909439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:02.913556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470882220959706:2108] [node 4] ICP22 created new session: [5:7577470882220959738:2108]
2025-11-27T18:17:02.921204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470882220959738:2108] [node 4] ICS09 handshake done sender: [5:7577470882220959730:2121] self: [5:7577470882220959732:0] peer: [4:7577470882701197093:0] socket: 82 qp: -1
2025-11-27T18:17:02.922459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470882220959737:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:02.922754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470882220959738:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:02.923997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:02.925671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:02.941338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470882220959642:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32032829ns) {HandleHandshakeStatus:329(32030893ns) {DropIncomingHandshake:442(3739ns) DropOutgoingHandshake:457(184ns) SwitchToState:179(725ns) ProcessPendingSessionEvents:495(978ns)}}
2025-11-27T18:17:02.945490Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470882220959706:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(33917262ns) {HandleHandshakeStatus:329(33914996ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(15153ns) SwitchToState:179(167ns) ProcessPendingSessionEvents:495(5711637ns) {ForwardSessionEventToSession:612(933830ns) ForwardSessionEventToSession:612(1645929ns) ForwardSessionEventToSession:612(298797ns) ForwardSessionEventToSession:612(2686ns) ForwardSessionEventToSession:612(1924ns) ForwardSessionEventToSession:612(2324795ns) ForwardSessionEventToSession:612(2204ns) ForwardSessionEventToSession:612(201509ns) ForwardSessionEventToSession:612(1988ns) ForwardSessionEventToSession:612(278363ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:02.986005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470882935911555:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16982
2025-11-27T18:17:02.986142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470882220959741:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30544
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:02.986344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470882935911544:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:02.986473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470882935911554:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16982
2025-11-27T18:17:02.986559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470882935911556:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16982
2025-11-27T18:17:02.986813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470882935911462:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.986889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470882935911550:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:02.986896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:02.986952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470882935911462:2049] [node 2] ICP22 created new session: [6:7577470882935911557:2049]
2025-11-27T18:17:02.987039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470882935911463:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.987214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:02.987216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470882935911557:2049] [node 2] ICS09 handshake done sender: [6:7577470882935911544:2118] self: [6:7577470882935911546:0] peer: [2:7577470881709599977:0] socket: 111 qp: -1
2025-11-27T18:17:02.987276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470882935911557:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:02.987277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470882935911463:2050] [node 3] ICP22 created new session: [6:7577470882935911558:2050]
2025-11-27T18:17:02.987325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470882935911558:2050] [node 3] ICS09 handshake done sender: [6:7577470882935911550:2120] self: [6:7577470882935911553:0] peer: [3:7577470879037183023:0] socket: 124 qp: -1
2025-11-27T18:17:02.987337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:02.987348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470882935911558:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:02.987376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:02.987869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470882935911554:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.988022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470882935911525:2100] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:02.989548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470879037183012:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:02.989806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470879037183003:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.989883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:02.989930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470879037183003:2108] [node 5] ICP22 created new session: [3:7577470879037183041:2108]
2025-11-27T18:17:02.990007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470879037183041:2108] [node 5] ICS09 handshake done sender: [3:7577470879037183012:2123] self: [3:7577470879037183011:0] peer: [5:7577470882220959729:0] socket: 93 qp: -1
2025-11-27T18:17:02.990034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470879037183041:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:02.990076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:02.993546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470882220959726:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:02.993581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470882220959726:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:02.994017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470882220959721:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:02.994416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470882220959644:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2167) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470882220959721:2117] held: no
2025-11-27T18:17:02.994504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470882220959644:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:02.994563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:02.994606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470882220959644:2050] [node 3] ICP22 created new session: [5:7577470882220959742:2050]
2025-11-27T18:17:02.994641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470882220959742:2050] [node 3] ICS09 handshake done sender: [5:7577470882220959721:2117] self: [5:7577470882220959729:0] peer: [3:7577470879037183011:0] socket: 101 qp: -1
2025-11-27T18:17:02.994669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470882220959742:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:02.994699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:02.997691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470882220959726:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:02.997967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470882220959705:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:02.998096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:02.998152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470882220959705:2109] [node 6] ICP22 created new session: [5:7577470882220959745:2109]
2025-11-27T18:17:02.998198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470882220959745:2109] [node 6] ICS09 handshake done sender: [5:7577470882220959726:2119] self: [5:7577470882220959725:0] peer: [6:7577470882935911561:0] socket: 127 qp: -1
2025-11-27T18:17:02.998229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470882220959745:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:02.998271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:02.998579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470882935911562:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16982
2025-11-27T18:17:02.999089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470882935911554:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:02.999535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470882935911525:2100] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20599) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470882935911554:2121] held: no
2025-11-27T18:17:03.000409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470882935911525:2100] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.000533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:03.000586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470882935911525:2100] [node 5] ICP22 created new session: [6:7577470882935911563:2100]
2025-11-27T18:17:03.000630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470882935911563:2100] [node 5] ICS09 handshake done sender: [6:7577470882935911554:2121] self: [6:7577470882935911561:0] peer: [5:7577470882220959725:0] socket: 56 qp: -1
2025-11-27T18:17:03.000665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470882935911563:2100] [node 5] ICS10 traffic start
2025-11-27T18:17:03.000736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:03.068648Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:03.068894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470881709599991:2048] [node 1] ICS01 socket: 66 reason# EndOfStream
2025-11-27T18:17:03.068929Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:03.068996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470881709599897:2048] [node 1] ICP30 unregister session Session# [2:7577470881709599991:2048] VirtualId# [2:7577470881709599976:0]
2025-11-27T18:17:03.069048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470881709599991:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.069137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:03.069153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470879037183031:2048] [node 1] ICS01 socket: 80 reason# EndOfStream
2025-11-27T18:17:03.069199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470879037182930:2048] [node 1] ICP30 unregister session Session# [3:7577470879037183031:2048] VirtualId# [3:7577470879037183022:0]
2025-11-27T18:17:03.069221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470879037183031:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.069199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:03.069426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:03.069500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:03.073236Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:03.073503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470882935911547:2048] [node 1] ICS07 socket disconnect 104 reason# EndOfStream
2025-11-27T18:17:03.073543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470882935911547:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.073634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:03.073658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:03.073691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470882935911547:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:03.073699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:03.073942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470882701197111:2048] [node 1] ICS01 socket: 97 reason# EndOfStream
2025-11-27T18:17:03.074020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470882701197017:2048] [node 1] ICP30 unregister session Session# [4:7577470882701197111:2048] VirtualId# [4:7577470882701197102:0]
2025-11-27T18:17:03.074049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470882701197111:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.074131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:03.074172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:03.074228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470882220959737:2048] [node 1] ICS01 socket: 79 reason# EndOfStream
2025-11-27T18:17:03.074282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470882220959642:2048] [node 1] ICP30 unregister session Session# [5:7577470882220959737:2048] VirtualId# [5:7577470882220959722:0]
2025-11-27T18:17:03.074329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470882220959737:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.074404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:03.074452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:03.074864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470882935911461:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64226) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:03.074909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470882935911547:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:03.074948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470882935911461:2048] [node 1] ICP30 unregister session Session# [6:7577470882935911547:2048] VirtualId# [6:7577470882935911543:0]
2025-11-27T18:17:03.074987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:03.075012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470882935911461:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64226) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:03.075024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64226) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:03.084966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470882935911461:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:03.109112Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:03.109390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470879037183027:2049] [node 2] ICS07 socket disconnect 78 reason# EndOfStream
2025-11-27T18:17:03.109416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470879037183027:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.109514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:03.109601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470879037183027:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:03.110333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470879037182931:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21531) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:03.110548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470879037183027:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:03.110602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470879037182931:2049] [node 2] ICP30 unregister session Session# [3:7577470879037183027:2049] VirtualId# [3:7577470879037183020:0]
2025-11-27T18:17:03.110649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:03.110675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470879037182931:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21531) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:03.110687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21531) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:03.111776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:03.111980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470882220959733:2049] [node 2] ICS01 socket: 81 reason# EndOfStream
2025-11-27T18:17:03.112020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470882220959643:2049] [node 2] ICP30 unregister session Session# [5:7577470882220959733:2049] VirtualId# [5:7577470882220959723:0]
2025-11-27T18:17:03.112044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470882220959733:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.112110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:03.112150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:03.113141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:03.113441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470882701197119:2049] [node 2] ICS01 socket: 109 reason# EndOfStream
2025-11-27T18:17:03.113490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470882701197018:2049] [node 2] ICP30 unregister session Session# [4:7577470882701197119:2049] VirtualId# [4:7577470882701197106:0]
2025-11-27T18:17:03.113519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470882701197119:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.113615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:03.113669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:03.119248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:03.119495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470882935911557:2049] [node 2] ICS01 socket: 111 reason# EndOfStream
2025-11-27T18:17:03.119543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470882935911462:2049] [node 2] ICP30 unregister session Session# [6:7577470882935911557:2049] VirtualId# [6:7577470882935911546:0]
2025-11-27T18:17:03.119576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470882935911557:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.119666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:03.119713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:03.121400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470879037182931:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:03.159145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:03.159402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470882220959742:2050] [node 3] ICS01 socket: 101 reason# EndOfStream
2025-11-27T18:17:03.159452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470882220959644:2050] [node 3] ICP30 unregister session Session# [5:7577470882220959742:2050] VirtualId# [5:7577470882220959729:0]
2025-11-27T18:17:03.159493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470882220959742:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.159591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:03.159631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:03.160606Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:03.161019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470882701197117:2050] [node 3] ICS01 socket: 99 reason# EndOfStream
2025-11-27T18:17:03.161064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470882701197019:2050] [node 3] ICP30 unregister session Session# [4:7577470882701197117:2050] VirtualId# [4:7577470882701197107:0]
2025-11-27T18:17:03.161099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470882701197117:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.161180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:03.161228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:03.161360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:03.161560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470882935911558:2050] [node 3] ICS01 socket: 124 reason# EndOfStream
2025-11-27T18:17:03.161589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470882935911463:2050] [node 3] ICP30 unregister session Session# [6:7577470882935911558:2050] VirtualId# [6:7577470882935911553:0]
2025-11-27T18:17:03.161608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470882935911558:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.161678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:03.161712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:03.208986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:03.209231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470882220959738:2108] [node 4] ICS07 socket disconnect 82 reason# EndOfStream
2025-11-27T18:17:03.209316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470882220959738:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.209415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:03.209472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470882220959738:2108] [node 4] ICS15 start handshake
2025-11-27T18:17:03.210303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470882220959706:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8683) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:03.210433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470882220959738:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:03.210486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470882220959706:2108] [node 4] ICP30 unregister session Session# [5:7577470882220959738:2108] VirtualId# [5:7577470882220959732:0]
2025-11-27T18:17:03.210545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:03.210581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470882220959706:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8683) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:03.210614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8683) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:03.213429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:03.213669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470882935911549:2095] [node 4] ICS07 socket disconnect 65 reason# EndOfStream
2025-11-27T18:17:03.213692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470882935911549:2095] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.213767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:03.213808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470882935911549:2095] [node 4] ICS15 start handshake
2025-11-27T18:17:03.216115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470882935911522:2095] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8683) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:03.216249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470882935911549:2095] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:03.216290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470882935911522:2095] [node 4] ICP30 unregister session Session# [6:7577470882935911549:2095] VirtualId# [6:7577470882935911540:0]
2025-11-27T18:17:03.216352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:03.216407Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470882935911522:2095] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8683) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:03.216424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8683) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:03.216968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470882220959706:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:17:03.226185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470882935911522:2095] [node 4] ICP33 wake up from error state
2025-11-27T18:17:03.254533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:03.254764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470882935911563:2100] [node 5] ICS01 socket: 56 reason# EndOfStream
2025-11-27T18:17:03.254802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470882935911525:2100] [node 5] ICP30 unregister session Session# [6:7577470882935911563:2100] VirtualId# [6:7577470882935911561:0]
2025-11-27T18:17:03.254836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470882935911563:2100] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:03.254905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:03.254940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:03.334724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470886011804223:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:03.335655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470886011804224:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:03.362352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470884506645476:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:03.362828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470884506645477:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:03.393524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470884538923400:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:03.394738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470884538923401:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:03.422719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470886781282703:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:03.423069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470886781282701:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:03.423084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470886781282702:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:03.456333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470887186853140:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:03.456375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470887186853141:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:03.456399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470887186853142:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:03.491886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470886994509691:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:03.491921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470886994509692:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:03.492438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470886994509693:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:03.598321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470886994509749:2103] [node 5] ICP01 ready to work
2025-11-27T18:17:03.602195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470886011804285:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:03.645141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470886011804290:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:03.666986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470886011804291:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:03.681296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470884506645539:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:03.696385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470884538923464:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:03.709183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470886994509749:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(72969053ns) {Configure:104(63824479ns) {ProcessConfigured:135(60302ns) {StartInitialHandshake:157(58563ns) {DropHandshakes:470(1661ns) {DropIncomingHandshake:442(418ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(1106ns) PrepareNewSessionHandshake:477(1230ns) {SwitchToState:179(393ns)}}}}}
2025-11-27T18:17:03.696592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470887186853205:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:03.696539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470884506645541:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:03.694571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470886781282764:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:03.701918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470884538923465:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:03.725543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470886994509755:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:03.727329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470884538923401:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(87648228ns) {Configure:104(87642779ns) {ProcessConfigured:135(16965940ns) {StartInitialHandshake:157(16962788ns) {DropHandshakes:470(3449ns) {DropIncomingHandshake:442(842ns) DropOutgoingHandshake:457(559ns)} GenerateSessionVirtualId:988(952ns) PrepareNewSessionHandshake:477(4746ns) {SwitchToState:179(1796ns)}}}}}
2025-11-27T18:17:03.705801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470884506645540:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:03.718825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470886781282703:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(77096310ns) {Configure:104(77093200ns) {ProcessConfigured:135(73067ns) {StartInitialHandshake:157(71028ns) {DropHandshakes:470(2320ns) {DropIncomingHandshake:442(651ns) DropOutgoingHandshake:457(314ns)} GenerateSessionVirtualId:988(1256ns) PrepareNewSessionHandshake:477(1839ns) {SwitchToState:179(523ns)}}}}}
2025-11-27T18:17:03.744815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470884538923468:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:03.747025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470887186853206:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:03.749681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470886781282768:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:03.761942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470886781282702:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(106039552ns) {Configure:104(106037446ns) {ProcessConfigured:135(1133600ns) {StartInitialHandshake:157(1130773ns) {DropHandshakes:470(1548ns) {DropIncomingHandshake:442(425ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(506ns) PrepareNewSessionHandshake:477(5683ns) {SwitchToState:179(1437ns)}}}}}
2025-11-27T18:17:03.765220Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470884506645539:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(71755980ns) {Configure:104(71748100ns) {ProcessConfigured:135(46871ns) {StartInitialHandshake:157(42356ns) {DropHandshakes:470(1326ns) {DropIncomingHandshake:442(302ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:477(894ns) {SwitchToState:179(342ns)}}}}}
2025-11-27T18:17:03.765883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470884506645476:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(154783632ns) {Configure:104(135860283ns) {ProcessConfigured:135(57331ns) {StartInitialHandshake:157(56337ns) {DropHandshakes:470(935ns) {DropIncomingHandshake:442(242ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:477(837ns) {SwitchToState:179(352ns)}}}}}
2025-11-27T18:17:03.761850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470886011804223:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(149213074ns) {Configure:104(149206728ns) {ProcessConfigured:135(1777984ns) {StartInitialHandshake:157(1775810ns) {DropHandshakes:470(6034ns) {DropIncomingHandshake:442(587ns) DropOutgoingHandshake:457(3603ns)} GenerateSessionVirtualId:988(1071ns) PrepareNewSessionHandshake:477(2553ns) {SwitchToState:179(1085ns)}}}}}
2025-11-27T18:17:03.763196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470886011804290:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(87864722ns) {Configure:104(87854547ns) {ProcessConfigured:135(2253409ns) {StartInitialHandshake:157(2250443ns) {DropHandshakes:470(2471ns) {DropIncomingHandshake:442(550ns) DropOutgoingHandshake:457(318ns)} GenerateSessionVirtualId:988(1375ns) PrepareNewSessionHandshake:477(2886ns) {SwitchToState:179(946ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.768552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470884506645544:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34578
2025-11-27T18:17:03.771176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470884538923473:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13929
2025-11-27T18:17:03.773469Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470886994509693:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(75151700ns) {Configure:104(75144567ns) {ProcessConfigured:135(59660ns) {StartInitialHandshake:157(56320ns) {DropHandshakes:470(1643ns) {DropIncomingHandshake:442(615ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(619ns) PrepareNewSessionHandshake:477(1571ns) {SwitchToState:179(431ns)}}}}}
2025-11-27T18:17:03.774316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470884506645544:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.775814Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470886011804224:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(123889650ns) {Configure:104(117702976ns) {ProcessConfigured:135(2230210ns) {StartInitialHandshake:157(2228075ns) {DropHandshakes:470(5530ns) {DropIncomingHandshake:442(687ns) DropOutgoingHandshake:457(309ns)} GenerateSessionVirtualId:988(877ns) PrepareNewSessionHandshake:477(2031ns) {SwitchToState:179(1007ns)}}}}}
2025-11-27T18:17:03.777148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470884538923473:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.776016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470886994509692:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(78650232ns) {Configure:104(78645195ns) {ProcessConfigured:135(48199ns) {StartInitialHandshake:157(47007ns) {DropHandshakes:470(1096ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(466ns) PrepareNewSessionHandshake:477(1683ns) {SwitchToState:179(484ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.781519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470884506645477:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(130464127ns) {Configure:104(122343029ns) {ProcessConfigured:135(1822636ns) {StartInitialHandshake:157(1821528ns) {DropHandshakes:470(1305ns) {DropIncomingHandshake:442(527ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(591ns) PrepareNewSessionHandshake:477(1239ns) {SwitchToState:179(527ns)}}}}}
2025-11-27T18:17:03.778303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470886011804291:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(85892055ns) {Configure:104(85881537ns) {ProcessConfigured:135(7011152ns) {StartInitialHandshake:157(7009715ns) {DropHandshakes:470(1466ns) {DropIncomingHandshake:442(589ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(476ns) PrepareNewSessionHandshake:477(2752ns) {SwitchToState:179(1151ns)}}}}}
2025-11-27T18:17:03.781072Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470886781282764:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(70282757ns) {Configure:104(70274935ns) {ProcessConfigured:135(3166912ns) {StartInitialHandshake:157(3164850ns) {DropHandshakes:470(1334ns) {DropIncomingHandshake:442(357ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(462ns) PrepareNewSessionHandshake:477(1280ns) {SwitchToState:179(549ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.778757Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470886994509755:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(35099535ns) {Configure:104(35090009ns) {ProcessConfigured:135(67312ns) {StartInitialHandshake:157(66355ns) {DropHandshakes:470(1176ns) {DropIncomingHandshake:442(388ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(413ns) PrepareNewSessionHandshake:477(905ns) {SwitchToState:179(453ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.783641Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470884506645540:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(50443749ns) {Configure:104(50435930ns) {ProcessConfigured:135(1798823ns) {StartInitialHandshake:157(1797502ns) {DropHandshakes:470(1142ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(398ns) PrepareNewSessionHandshake:477(2470ns) {SwitchToState:179(1182ns)}}}}}
2025-11-27T18:17:03.783082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470886994509691:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(92774186ns) {Configure:104(92767104ns) {ProcessConfigured:135(4635065ns) {StartInitialHandshake:157(3807914ns) {DropHandshakes:470(2257ns) {DropIncomingHandshake:442(419ns) DropOutgoingHandshake:457(657ns)} GenerateSessionVirtualId:988(917ns) PrepareNewSessionHandshake:477(1703ns) {SwitchToState:179(863ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.787022Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470884538923464:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(76359904ns) {Configure:104(76354157ns) {ProcessConfigured:135(4803695ns) {StartInitialHandshake:157(3698264ns) {DropHandshakes:470(977ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(166ns)} GenerateSessionVirtualId:988(697ns) PrepareNewSessionHandshake:477(1248ns) {SwitchToState:179(261ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.786756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470886781282771:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57118
2025-11-27T18:17:03.789301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470886011804302:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47649
2025-11-27T18:17:03.787380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470884506645541:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(65652603ns) {Configure:104(65645300ns) {ProcessConfigured:135(2852280ns) {StartInitialHandshake:157(2851064ns) {DropHandshakes:470(1181ns) {DropIncomingHandshake:442(319ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(360ns) PrepareNewSessionHandshake:477(2530ns) {SwitchToState:179(1216ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.786947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470886781282701:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(137948017ns) {Configure:104(137940416ns) {ProcessConfigured:135(5288578ns) {StartInitialHandshake:157(5286708ns) {DropHandshakes:470(1712ns) {DropIncomingHandshake:442(474ns) DropOutgoingHandshake:457(308ns)} GenerateSessionVirtualId:988(1044ns) PrepareNewSessionHandshake:477(3010ns) {SwitchToState:179(1133ns)}}}}}
2025-11-27T18:17:03.788198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470887186853142:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(74231426ns) {Configure:104(74222488ns) {ProcessConfigured:135(1232915ns) {StartInitialHandshake:157(1231677ns) {DropHandshakes:470(1480ns) {DropIncomingHandshake:442(253ns) DropOutgoingHandshake:457(288ns)} GenerateSessionVirtualId:988(541ns) PrepareNewSessionHandshake:477(1363ns) {SwitchToState:179(152ns)}}}}}
2025-11-27T18:17:03.794391Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470884538923400:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(130370388ns) {Configure:104(130363697ns) {ProcessConfigured:135(5154498ns) {StartInitialHandshake:157(5152171ns) {DropHandshakes:470(1134ns) {DropIncomingHandshake:442(252ns) DropOutgoingHandshake:457(304ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:477(1412ns) {SwitchToState:179(116ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.797312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470886994509769:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37499
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.797795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470886781282778:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57118
2025-11-27T18:17:03.794991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470884538923483:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13929
2025-11-27T18:17:03.788576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470887186853201:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41825
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.805364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470886994509769:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.788737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470887186853207:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41825
2025-11-27T18:17:03.805310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470886781282771:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.805923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470884538923470:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.805946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470884538923470:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.810690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470884538923484:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13929
2025-11-27T18:17:03.811367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470884538923470:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:03.812911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470884538923475:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.812938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470884538923475:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.813424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470886011804302:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.788824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470887186853208:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41825
2025-11-27T18:17:03.810864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470886994509767:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37499
2025-11-27T18:17:03.823338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470884506645552:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34578
2025-11-27T18:17:03.805521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470886781282703:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.827094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470886011804293:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.826683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470884538923483:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.811780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470886994509755:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.825755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470884506645553:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34578
2025-11-27T18:17:03.794318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470887186853208:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.812998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470886781282768:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(41663352ns) {Configure:104(41237021ns) {ProcessConfigured:135(13143840ns) {StartInitialHandshake:157(13141387ns) {DropHandshakes:470(2650ns) {DropIncomingHandshake:442(945ns) DropOutgoingHandshake:457(498ns)} GenerateSessionVirtualId:988(538ns) PrepareNewSessionHandshake:477(5515251ns) {SwitchToState:179(3038ns)}}}}}
2025-11-27T18:17:03.826689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470884538923482:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13929
2025-11-27T18:17:03.815848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470886994509767:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.826751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470884506645552:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.827124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470886011804293:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.795749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470887186853142:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:03.815605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470886781282778:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.826833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470884538923475:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:03.833808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470886011804287:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.825839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470886994509691:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.826833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470884506645476:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.824971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470886781282775:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.833835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470886011804287:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.837824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470886994509773:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37499
2025-11-27T18:17:03.826888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470884538923401:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:03.812209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470887186853140:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(122119975ns) {Configure:104(122112898ns) {ProcessConfigured:135(4660767ns) {StartInitialHandshake:157(4659302ns) {DropHandshakes:470(2005ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:477(1354ns) {SwitchToState:179(415ns)}}}}}
2025-11-27T18:17:03.833715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470884506645553:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.825000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470886781282775:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.835998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470886011804295:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.833389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470884538923468:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.851783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470886994509769:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:03.833913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470884506645549:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.813472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470887186853201:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.836066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470884538923465:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.857611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470886994509755:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.836019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470886011804295:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.813653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470887186853141:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(122885128ns) {Configure:104(122877830ns) {ProcessConfigured:135(8472569ns) {StartInitialHandshake:157(8470571ns) {DropHandshakes:470(1976ns) {DropIncomingHandshake:442(490ns) DropOutgoingHandshake:457(297ns)} GenerateSessionVirtualId:988(472ns) PrepareNewSessionHandshake:477(1619ns) {SwitchToState:179(636ns)}}}}}
2025-11-27T18:17:03.826151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470886781282782:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57118
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.828641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470886781282775:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.833935Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470884506645549:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.857676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:03.838943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:03.837193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470886011804304:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47649
2025-11-27T18:17:03.815252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470887186853207:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.838986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470884538923468:2108] [node 5] ICP22 created new session: [3:7577470884538923489:2108]
2025-11-27T18:17:03.842020Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470886011804293:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.845629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470886781282771:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:03.839268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470884506645543:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.859788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470886994509755:2109] [node 4] ICP22 created new session: [6:7577470886994509776:2109]
2025-11-27T18:17:03.816793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470887186853206:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(46668919ns) {Configure:104(45408142ns) {ProcessConfigured:135(42335ns) {StartInitialHandshake:157(37995ns) {DropHandshakes:470(1327ns) {DropIncomingHandshake:442(245ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(433ns) PrepareNewSessionHandshake:477(605ns) {SwitchToState:179(199ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.846052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470886011804295:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:03.839285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470884506645543:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.826199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470887186853205:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(62536465ns) {Configure:104(62524278ns) {ProcessConfigured:135(13797399ns) {StartInitialHandshake:157(8524830ns) {DropHandshakes:470(1022ns) {DropIncomingHandshake:442(269ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(547ns) PrepareNewSessionHandshake:477(1379ns) {SwitchToState:179(526ns)}}}}}
2025-11-27T18:17:03.839478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:03.849474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470886781282703:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.863524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470886994509776:2109] [node 4] ICS09 handshake done sender: [6:7577470886994509769:2116] self: [6:7577470886994509771:0] peer: [4:7577470886781282774:0] socket: 86 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.852097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470886781282764:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.842809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470884506645549:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:03.828156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470887186853224:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41825
2025-11-27T18:17:03.841714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470884538923482:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.852928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470886011804287:2109] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:03.864132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470886994509776:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:03.828360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470887186853140:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:03.864900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470886994509774:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37499
2025-11-27T18:17:03.828644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470887186853214:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41825
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.841920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470884538923465:2107] [node 4] ICP22 created new session: [3:7577470884538923491:2107]
2025-11-27T18:17:03.858557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470886011804223:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:03.852512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:03.851983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470884506645560:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34578
2025-11-27T18:17:03.867939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:03.858601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:03.852554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470886781282703:2050] [node 3] ICP22 created new session: [4:7577470886781282786:2050]
2025-11-27T18:17:03.830851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470887186853214:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.854593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470884506645477:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:03.877345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470886994509767:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:03.845031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470884538923492:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13929
2025-11-27T18:17:03.884379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470886994509777:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37499
2025-11-27T18:17:03.835452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470887186853141:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:03.858637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470886011804223:2048] [node 2] ICP22 created new session: [1:7577470886011804309:2048]
2025-11-27T18:17:03.854602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470884506645543:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:03.884423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470886994509778:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37499
2025-11-27T18:17:03.836174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470887186853208:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.854632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.845515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470887186853142:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.846007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470884538923489:2108] [node 5] ICS09 handshake done sender: [3:7577470884538923475:2114] self: [3:7577470884538923474:0] peer: [5:7577470887186853212:0] socket: 72 qp: -1
2025-11-27T18:17:03.858663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470886011804309:2048] [node 2] ICS09 handshake done sender: [1:7577470886011804293:2111] self: [1:7577470886011804292:0] peer: [2:7577470884506645559:0] socket: 78 qp: -1
2025-11-27T18:17:03.893720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470886994509775:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37499
2025-11-27T18:17:03.852596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470886781282786:2050] [node 3] ICS09 handshake done sender: [4:7577470886781282771:2112] self: [4:7577470886781282781:0] peer: [3:7577470884538923469:0] socket: 71 qp: -1
2025-11-27T18:17:03.854667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470884506645477:2049] [node 3] ICP22 created new session: [2:7577470884506645565:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.847620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:03.852620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470886781282786:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:03.894027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470886994509691:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.858687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470886011804309:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:03.846696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470884538923489:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.848941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470887186853230:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41825
2025-11-27T18:17:03.858704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:03.847676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:03.854712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470884506645565:2049] [node 3] ICS09 handshake done sender: [2:7577470884506645549:2113] self: [2:7577470884506645548:0] peer: [3:7577470884538923487:0] socket: 89 qp: -1
2025-11-27T18:17:03.896187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:03.852654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:03.849418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470887186853142:2050] [node 3] ICP22 created new session: [5:7577470887186853229:2050]
2025-11-27T18:17:03.852216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470887186853229:2050] [node 3] ICS09 handshake done sender: [5:7577470887186853208:2113] self: [5:7577470887186853212:0] peer: [3:7577470884538923474:0] socket: 73 qp: -1
2025-11-27T18:17:03.854545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470887186853229:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:03.857255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:03.865424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470887186853214:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:03.870592Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470887186853140:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8634) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470887186853201:2108] held: no
2025-11-27T18:17:03.870671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470887186853141:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.870707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:03.870751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470887186853141:2049] [node 2] ICP22 created new session: [5:7577470887186853234:2049]
2025-11-27T18:17:03.870787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470887186853234:2049] [node 2] ICS09 handshake done sender: [5:7577470887186853214:2115] self: [5:7577470887186853227:0] peer: [2:7577470884506645542:0] socket: 96 qp: -1
2025-11-27T18:17:03.870812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470887186853234:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:03.870899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:03.874501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470887186853228:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41825
2025-11-27T18:17:03.877052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470887186853142:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(27052033ns) {HandleHandshakeStatus:329(27050288ns) {DropIncomingHandshake:442(1111ns) DropOutgoingHandshake:457(4138ns) SwitchToState:179(112ns) ProcessPendingSessionEvents:495(1840963ns) {ForwardSessionEventToSession:612(1786874ns) ForwardSessionEventToSession:612(5116ns) ForwardSessionEventToSession:612(8577ns) ForwardSessionEventToSession:612(1907ns) ForwardSessionEventToSession:612(5255ns) ForwardSessionEventToSession:612(1729ns) ForwardSessionEventToSession:612(2393ns) ForwardSessionEventToSession:612(7255ns) ForwardSessionEventToSession:612(1504ns) ForwardSessionEventToSession:612(1509ns)}}}
2025-11-27T18:17:03.894278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470887186853233:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41825
2025-11-27T18:17:03.897798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470887186853233:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.898729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470887186853201:2108] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:03.903870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470887186853219:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.903901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470887186853219:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.914035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470887186853140:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.915014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470887186853206:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:03.915920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:03.915969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470887186853140:2048] [node 1] ICP22 created new session: [5:7577470887186853239:2048]
2025-11-27T18:17:03.916009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470887186853239:2048] [node 1] ICS09 handshake done sender: [5:7577470887186853201:2108] self: [5:7577470887186853222:0] peer: [1:7577470886011804286:0] socket: 61 qp: -1
2025-11-27T18:17:03.916040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470887186853239:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:03.916084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:03.930169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470887186853219:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:03.945280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470887186853205:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:03.945808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:03.945870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470887186853205:2110] [node 6] ICP22 created new session: [5:7577470887186853241:2110]
2025-11-27T18:17:03.953409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470887186853241:2110] [node 6] ICS09 handshake done sender: [5:7577470887186853219:2118] self: [5:7577470887186853216:0] peer: [6:7577470886994509783:0] socket: 91 qp: -1
2025-11-27T18:17:03.859039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470886011804290:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:03.896398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470886994509778:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.953459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470887186853241:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:03.953510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:03.955464Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470887186853205:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(17024792ns) {HandleHandshakeStatus:329(17022640ns) {DropIncomingHandshake:442(17043ns) DropOutgoingHandshake:457(376ns) SwitchToState:179(411ns) ProcessPendingSessionEvents:495(122858ns) {ForwardSessionEventToSession:612(64101ns) ForwardSessionEventToSession:612(3455ns) ForwardSessionEventToSession:612(5744ns) ForwardSessionEventToSession:612(14219ns) ForwardSessionEventToSession:612(5817ns) ForwardSessionEventToSession:612(10455ns) ForwardSessionEventToSession:612(2369ns) ForwardSessionEventToSession:612(4931ns) ForwardSessionEventToSession:612(1916ns) ForwardSessionEventToSession:612(1806ns)}}}
2025-11-27T18:17:03.956471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470887186853242:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41825
2025-11-27T18:17:03.965852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470887186853233:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:03.967069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470887186853206:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.973098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:03.974755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470887186853206:2109] [node 4] ICP22 created new session: [5:7577470887186853244:2109]
2025-11-27T18:17:03.852850Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470886781282703:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:03.848141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470884538923484:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.854730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470884506645565:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:03.857111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:03.896521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470886994509777:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.976761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470887186853244:2109] [node 4] ICS09 handshake done sender: [5:7577470887186853233:2123] self: [5:7577470887186853237:0] peer: [4:7577470886781282779:0] socket: 80 qp: -1
2025-11-27T18:17:03.859815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470886781282764:2109] [node 6] ICP22 created new session: [4:7577470886781282789:2109]
2025-11-27T18:17:03.859062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:03.977402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470887186853244:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:03.849390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470884538923491:2107] [node 4] ICS09 handshake done sender: [3:7577470884538923470:2112] self: [3:7577470884538923469:0] peer: [4:7577470886781282781:0] socket: 67 qp: -1
2025-11-27T18:17:03.861815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470886781282789:2109] [node 6] ICS09 handshake done sender: [4:7577470886781282775:2113] self: [4:7577470886781282774:0] peer: [6:7577470886994509771:0] socket: 85 qp: -1
2025-11-27T18:17:03.859090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470886011804290:2110] [node 6] ICP22 created new session: [1:7577470886011804311:2110]
2025-11-27T18:17:03.863423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470886781282789:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:03.859126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470886011804311:2110] [node 6] ICS09 handshake done sender: [1:7577470886011804295:2112] self: [1:7577470886011804294:0] peer: [6:7577470886994509772:0] socket: 75 qp: -1
2025-11-27T18:17:03.863452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:03.859148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470886011804311:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:03.849603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470884538923483:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:03.854755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:03.987059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:03.859166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:03.851794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470884538923491:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:03.859776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470886011804306:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47649
2025-11-27T18:17:03.860921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470886011804285:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:03.853048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:03.860974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:03.861002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470886011804285:2108] [node 5] ICP22 created new session: [1:7577470886011804313:2108]
2025-11-27T18:17:03.854441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470884538923401:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.861045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470886011804313:2108] [node 5] ICS09 handshake done sender: [1:7577470886011804287:2109] self: [1:7577470886011804286:0] peer: [5:7577470887186853222:0] socket: 60 qp: -1
2025-11-27T18:17:03.856624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:03.861073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470886011804313:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:03.861111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:03.857953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470884538923401:2049] [node 2] ICP22 created new session: [3:7577470884538923496:2049]
2025-11-27T18:17:03.864171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470886011804304:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.857993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470884538923496:2049] [node 2] ICS09 handshake done sender: [3:7577470884538923483:2118] self: [3:7577470884538923487:0] peer: [2:7577470884506645548:0] socket: 93 qp: -1
2025-11-27T18:17:03.864267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470886011804285:2108] [node 5] ICP18 handshake (actor [1:7577470886011804304:2116]) is too old
2025-11-27T18:17:03.858019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470884538923496:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:03.858049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:03.865626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470886011804315:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47649
2025-11-27T18:17:03.874049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470886781282792:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57118
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:03.891598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470884538923465:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(36383428ns) {HandleHandshakeStatus:329(34042414ns) {DropIncomingHandshake:442(15832ns) DropOutgoingHandshake:457(577ns) SwitchToState:179(315ns) ProcessPendingSessionEvents:495(7277895ns) {ForwardSessionEventToSession:612(3783109ns) ForwardSessionEventToSession:612(4708ns) ForwardSessionEventToSession:612(376940ns) ForwardSessionEventToSession:612(1936ns) ForwardSessionEventToSession:612(1377ns) ForwardSessionEventToSession:612(578957ns) ForwardSessionEventToSession:612(2187ns) ForwardSessionEventToSession:612(883775ns) ForwardSessionEventToSession:612(1630246ns) ForwardSessionEventToSession:612(3382ns)}}}
2025-11-27T18:17:03.876941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470886781282792:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.883331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470886781282701:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:03.893453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470884538923468:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(37703491ns) {HandleHandshakeStatus:329(37701243ns) {DropIncomingHandshake:442(1530ns) DropOutgoingHandshake:457(423ns) SwitchToState:179(2224ns) ProcessPendingSessionEvents:495(9692822ns) {ForwardSessionEventToSession:612(4631680ns) ForwardSessionEventToSession:612(3567ns) ForwardSessionEventToSession:612(3148154ns) ForwardSessionEventToSession:612(5577ns) ForwardSessionEventToSession:612(2420ns) ForwardSessionEventToSession:612(968807ns) ForwardSessionEventToSession:612(3899ns) ForwardSessionEventToSession:612(2107ns) ForwardSessionEventToSession:612(897477ns) ForwardSessionEventToSession:612(2808ns)}}}
2025-11-27T18:17:03.897569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470886781282791:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57118
2025-11-27T18:17:03.899486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470884538923498:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13929
2025-11-27T18:17:03.901951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577470884538923468:2108] [node 5] ICP18 handshake (actor [3:7577470884538923484:2119]) is too old
2025-11-27T18:17:03.896681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470886994509692:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:03.861682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470884506645562:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34578
2025-11-27T18:17:03.862177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470884506645560:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.896734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470886994509749:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:03.866323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470886011804306:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.864751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470884506645539:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:03.905872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470886781282764:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(41724954ns) {HandleHandshakeStatus:329(41722186ns) {DropIncomingHandshake:442(12673ns) DropOutgoingHandshake:457(348ns) SwitchToState:179(277ns) ProcessPendingSessionEvents:495(21631764ns) {ForwardSessionEventToSession:612(13290850ns) ForwardSessionEventToSession:612(5911ns) ForwardSessionEventToSession:612(3154145ns) ForwardSessionEventToSession:612(16274ns) ForwardSessionEventToSession:612(695405ns) ForwardSessionEventToSession:612(4470ns) ForwardSessionEventToSession:612(2396ns) ForwardSessionEventToSession:612(4436583ns) ForwardSessionEventToSession:612(4486ns) ForwardSessionEventToSession:612(2682ns)}}}
2025-11-27T18:17:03.913657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470886781282791:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.914323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470886781282701:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8634) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470886781282792:2119] held: no
2025-11-27T18:17:03.934640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470886781282780:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.934673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470886781282780:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.934506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470886781282702:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:03.941531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470886781282794:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57118
2025-11-27T18:17:03.941987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470886781282780:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:03.949700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470886781282795:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57118
2025-11-27T18:17:03.953690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470886781282792:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:03.957499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470886781282768:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:03.957503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470886781282701:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.957544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:03.958908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470886781282701:2048] [node 1] ICP22 created new session: [4:7577470886781282798:2048]
2025-11-27T18:17:03.959012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:03.973465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470886781282795:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.970360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470886781282768:2108] [node 5] ICP22 created new session: [4:7577470886781282799:2108]
2025-11-27T18:17:03.976729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470886781282799:2108] [node 5] ICS09 handshake done sender: [4:7577470886781282780:2116] self: [4:7577470886781282779:0] peer: [5:7577470887186853237:0] socket: 92 qp: -1
2025-11-27T18:17:03.976831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470886781282798:2048] [node 1] ICS09 handshake done sender: [4:7577470886781282792:2119] self: [4:7577470886781282793:0] peer: [1:7577470886011804298:0] socket: 127 qp: -1
2025-11-27T18:17:03.977438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470886781282799:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:03.979127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:03.979530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470886781282798:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:03.986842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:03.988507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470886781282802:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57118
2025-11-27T18:17:03.998370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470886781282791:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:03.998841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470886781282768:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(31093440ns) {HandleHandshakeStatus:329(31089760ns) {DropIncomingHandshake:442(438ns) DropOutgoingHandshake:457(2863ns) SwitchToState:179(738ns) ProcessPendingSessionEvents:495(1022254ns) {ForwardSessionEventToSession:612(978417ns) ForwardSessionEventToSession:612(3327ns) ForwardSessionEventToSession:612(9611ns) ForwardSessionEventToSession:612(1593ns) ForwardSessionEventToSession:612(2132ns) ForwardSessionEventToSession:612(4553ns) ForwardSessionEventToSession:612(4959ns) ForwardSessionEventToSession:612(5047ns) ForwardSessionEventToSession:612(2168ns) ForwardSessionEventToSession:612(1618ns)}}}
2025-11-27T18:17:04.009512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470886781282701:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(40869066ns) {HandleHandshakeStatus:329(40866329ns) {DropIncomingHandshake:442(645ns) DropOutgoingHandshake:457(287ns) SwitchToState:179(966ns) ProcessPendingSessionEvents:495(865ns)}}
2025-11-27T18:17:04.009595Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470886781282702:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4743) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470886781282791:2118] held: no
2025-11-27T18:17:04.009679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470886781282702:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.009723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:04.009783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470886781282702:2049] [node 2] ICP22 created new session: [4:7577470891076250099:2049]
2025-11-27T18:17:04.009824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470891076250099:2049] [node 2] ICS09 handshake done sender: [4:7577470886781282791:2118] self: [4:7577470886781282796:0] peer: [2:7577470884506645550:0] socket: 124 qp: -1
2025-11-27T18:17:04.009856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470891076250099:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:04.009895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:04.013629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577470886781282768:2108] [node 5] ICP18 handshake (actor [4:7577470886781282795:2121]) is too old
2025-11-27T18:17:03.999515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470887186853206:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(32249395ns) {HandleHandshakeStatus:329(32246888ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(19249ns) SwitchToState:179(387ns) ProcessPendingSessionEvents:495(1026404ns) {ForwardSessionEventToSession:612(971663ns) ForwardSessionEventToSession:612(11899ns) ForwardSessionEventToSession:612(4776ns) ForwardSessionEventToSession:612(2209ns) ForwardSessionEventToSession:612(2005ns) ForwardSessionEventToSession:612(4665ns) ForwardSessionEventToSession:612(1967ns) ForwardSessionEventToSession:612(8005ns) ForwardSessionEventToSession:612(1828ns) ForwardSessionEventToSession:612(7890ns)}}}
2025-11-27T18:17:03.898928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470886994509691:2048] [node 1] ICP22 created new session: [6:7577470886994509781:2048]
2025-11-27T18:17:03.892128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470886011804301:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.892162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470886011804301:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.901908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470886994509781:2048] [node 1] ICS09 handshake done sender: [6:7577470886994509767:2115] self: [6:7577470886994509772:0] peer: [1:7577470886011804294:0] socket: 82 qp: -1
2025-11-27T18:17:03.905959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470886994509781:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:03.914743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:03.864784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:03.864812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470884506645539:2108] [node 5] ICP22 created new session: [2:7577470884506645569:2108]
2025-11-27T18:17:03.921654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470886994509755:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(42714373ns) {HandleHandshakeStatus:329(42710873ns) {DropIncomingHandshake:442(1320ns) DropOutgoingHandshake:457(23776ns) SwitchToState:179(502ns) ProcessPendingSessionEvents:495(16346480ns) {ForwardSessionEventToSession:612(8421075ns) ForwardSessionEventToSession:612(9640ns) ForwardSessionEventToSession:612(328270ns) ForwardSessionEventToSession:612(3026ns) ForwardSessionEventToSession:612(1536637ns) ForwardSessionEventToSession:612(4052950ns) ForwardSessionEventToSession:612(5742ns) ForwardSessionEventToSession:612(1965221ns) ForwardSessionEventToSession:612(3655ns) ForwardSessionEventToSession:612(1958ns)}}}
2025-11-27T18:17:03.924778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470886994509786:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37499
2025-11-27T18:17:03.865450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470884506645476:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8634) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577470884506645552:2115] held: no
2025-11-27T18:17:03.933796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470886994509691:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(42202547ns) {HandleHandshakeStatus:329(42199743ns) {DropIncomingHandshake:442(1360ns) DropOutgoingHandshake:457(6276ns) SwitchToState:179(424ns) ProcessPendingSessionEvents:495(130374ns) {ForwardSessionEventToSession:612(59418ns) ForwardSessionEventToSession:612(5975ns) ForwardSessionEventToSession:612(2609ns) ForwardSessionEventToSession:612(1963ns) ForwardSessionEventToSession:612(4651ns) ForwardSessionEventToSession:612(5536ns) ForwardSessionEventToSession:612(2369ns) ForwardSessionEventToSession:612(4502ns) ForwardSessionEventToSession:612(1614ns) ForwardSessionEventToSession:612(31606ns)}}}
2025-11-27T18:17:03.934266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470886994509777:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:03.893820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470886011804315:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.934881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470886994509749:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.893891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470886011804301:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:03.934927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:03.901093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470886011804291:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:03.904123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:03.934979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470886994509749:2103] [node 5] ICP22 created new session: [6:7577470886994509787:2103]
2025-11-27T18:17:03.935035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470886994509787:2103] [node 5] ICS09 handshake done sender: [6:7577470886994509777:2120] self: [6:7577470886994509783:0] peer: [5:7577470887186853216:0] socket: 74 qp: -1
2025-11-27T18:17:03.935063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470886994509787:2103] [node 5] ICS10 traffic start
2025-11-27T18:17:03.935113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:03.936362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470886994509785:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37499
2025-11-27T18:17:03.945739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470886994509778:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:03.908658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470886011804291:2107] [node 4] ICP22 created new session: [1:7577470886011804321:2107]
2025-11-27T18:17:03.966088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470886994509692:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.914679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470886011804321:2107] [node 4] ICS09 handshake done sender: [1:7577470886011804301:2114] self: [1:7577470886011804298:0] peer: [4:7577470886781282793:0] socket: 105 qp: -1
2025-11-27T18:17:03.966148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:03.915186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470886011804321:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:03.966200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470886994509692:2049] [node 2] ICP22 created new session: [6:7577470886994509789:2049]
2025-11-27T18:17:03.925823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:03.966243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470886994509789:2049] [node 2] ICS09 handshake done sender: [6:7577470886994509778:2121] self: [6:7577470886994509782:0] peer: [2:7577470884506645554:0] socket: 66 qp: -1
2025-11-27T18:17:03.983314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470886011804291:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(73665692ns) {HandleHandshakeStatus:329(73661931ns) {DropIncomingHandshake:442(13631ns) DropOutgoingHandshake:457(2620ns) SwitchToState:179(823ns) ProcessPendingSessionEvents:495(32859178ns) {ForwardSessionEventToSession:612(7848421ns) ForwardSessionEventToSession:612(7563602ns) ForwardSessionEventToSession:612(1504136ns) ForwardSessionEventToSession:612(3567ns) ForwardSessionEventToSession:612(2631ns) ForwardSessionEventToSession:612(7725182ns) ForwardSessionEventToSession:612(2443ns) ForwardSessionEventToSession:612(8171767ns) ForwardSessionEventToSession:612(5646ns) ForwardSessionEventToSession:612(2728ns)}}}
2025-11-27T18:17:03.966271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470886994509789:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:03.966324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:03.868266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470884506645569:2108] [node 5] ICS09 handshake done sender: [2:7577470884506645543:2110] self: [2:7577470884506645542:0] peer: [5:7577470887186853227:0] socket: 95 qp: -1
2025-11-27T18:17:03.869408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470884506645569:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:03.869899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:03.870071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470884506645552:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:03.882460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470884506645476:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:03.882536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:03.882582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470884506645476:2048] [node 1] ICP22 created new session: [2:7577470884506645571:2048]
2025-11-27T18:17:03.882627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470884506645571:2048] [node 1] ICS09 handshake done sender: [2:7577470884506645552:2115] self: [2:7577470884506645559:0] peer: [1:7577470886011804292:0] socket: 83 qp: -1
2025-11-27T18:17:03.882661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470884506645571:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:03.882696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:03.896964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470884506645555:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.896985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470884506645555:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.913525Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470884506645539:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(31685622ns) {HandleHandshakeStatus:329(31682729ns) {DropIncomingHandshake:442(860ns) DropOutgoingHandshake:457(219ns) SwitchToState:179(273ns) ProcessPendingSessionEvents:495(10774507ns) {ForwardSessionEventToSession:612(3210533ns) ForwardSessionEventToSession:612(600091ns) ForwardSessionEventToSession:612(4480353ns) ForwardSessionEventToSession:612(9472ns) ForwardSessionEventToSession:612(357709ns) ForwardSessionEventToSession:612(2133ns) ForwardSessionEventToSession:612(1517683ns) ForwardSessionEventToSession:612(3315ns) ForwardSessionEventToSession:612(558914ns) ForwardSessionEventToSession:612(10280ns)}}}
2025-11-27T18:17:03.914123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470884506645539:2108] [node 5] ICP18 handshake (actor [2:7577470884506645560:2118]) is too old
2025-11-27T18:17:03.930251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470884506645555:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:03.941887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470884506645573:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34578
2025-11-27T18:17:03.945272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470884506645541:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:03.958789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470884506645551:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:03.958833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470884506645551:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:03.958911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:03.970406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470884506645541:2109] [node 6] ICP22 created new session: [2:7577470884506645575:2109]
2025-11-27T18:17:03.974807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470884506645575:2109] [node 6] ICS09 handshake done sender: [2:7577470884506645555:2117] self: [2:7577470884506645554:0] peer: [6:7577470886994509782:0] socket: 123 qp: -1
2025-11-27T18:17:03.974849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470884506645575:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:03.976755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:03.987032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470884506645551:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:03.988800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470884506645540:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:03.998402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:04.005849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470884506645540:2107] [node 4] ICP22 created new session: [2:7577470884506645578:2107]
2025-11-27T18:17:04.013719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470884506645578:2107] [node 4] ICS09 handshake done sender: [2:7577470884506645551:2114] self: [2:7577470884506645550:0] peer: [4:7577470886781282796:0] socket: 122 qp: -1
2025-11-27T18:17:04.015709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470884506645578:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:04.015749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:04.016803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470884506645540:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(27722458ns) {HandleHandshakeStatus:329(27719639ns) {DropIncomingHandshake:442(13448ns) DropOutgoingHandshake:457(880ns) SwitchToState:179(564ns) ProcessPendingSessionEvents:495(75147ns) {ForwardSessionEventToSession:612(44255ns) ForwardSessionEventToSession:612(6699ns) ForwardSessionEventToSession:612(1961ns) ForwardSessionEventToSession:612(3033ns) ForwardSessionEventToSession:612(1529ns) ForwardSessionEventToSession:612(1623ns) ForwardSessionEventToSession:612(3188ns) ForwardSessionEventToSession:612(902ns) ForwardSessionEventToSession:612(3497ns) ForwardSessionEventToSession:612(3135ns)}}}
2025-11-27T18:17:04.022448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470884506645541:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(52815281ns) {HandleHandshakeStatus:329(52812253ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(587ns) SwitchToState:179(724ns) ProcessPendingSessionEvents:495(14732648ns) {ForwardSessionEventToSession:612(3372950ns) ForwardSessionEventToSession:612(4622ns) ForwardSessionEventToSession:612(1184213ns) ForwardSessionEventToSession:612(3808ns) ForwardSessionEventToSession:612(2317ns) ForwardSessionEventToSession:612(1148168ns) ForwardSessionEventToSession:612(8741818ns) ForwardSessionEventToSession:612(9857ns) ForwardSessionEventToSession:612(238361ns) ForwardSessionEventToSession:612(3069ns)}}}
2025-11-27T18:17:04.027963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470884538923481:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.028023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470884538923481:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.027287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470886994509775:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.027594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470886994509693:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.029655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470884538923481:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.030106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470891289477088:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37499
2025-11-27T18:17:04.030275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470884538923464:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.030458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:04.030701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470886994509775:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:04.030979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470886994509693:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.030503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470884538923464:2109] [node 6] ICP22 created new session: [3:7577470888833890796:2109]
2025-11-27T18:17:04.031234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:04.030781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470888833890796:2109] [node 6] ICS09 handshake done sender: [3:7577470884538923481:2116] self: [3:7577470884538923478:0] peer: [6:7577470891289477087:0] socket: 116 qp: -1
2025-11-27T18:17:04.031343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470886994509693:2050] [node 3] ICP22 created new session: [6:7577470891289477089:2050]
2025-11-27T18:17:04.031073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470888833890796:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:04.031115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:04.032173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470891289477089:2050] [node 3] ICS09 handshake done sender: [6:7577470886994509775:2119] self: [6:7577470891289477087:0] peer: [3:7577470884538923478:0] socket: 121 qp: -1
2025-11-27T18:17:04.032227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470891289477089:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:04.032273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:04.034913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470884538923498:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.035260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470884538923400:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.036094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470886011804297:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.036128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470886011804297:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.037480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470884506645573:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.037246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470886011804297:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:04.037587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470888833890799:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13929
2025-11-27T18:17:04.037626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470884506645541:2109] [node 6] ICP18 handshake (actor [2:7577470884506645573:2120]) is too old
2025-11-27T18:17:04.038081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470884538923498:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:04.038157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470884538923400:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.038749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:04.037730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470886011804224:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.038800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470884538923400:2048] [node 1] ICP22 created new session: [3:7577470888833890800:2048]
2025-11-27T18:17:04.038879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470888833890800:2048] [node 1] ICS09 handshake done sender: [3:7577470884538923498:2121] self: [3:7577470888833890798:0] peer: [1:7577470886011804296:0] socket: 126 qp: -1
2025-11-27T18:17:04.037938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:04.038287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470886011804224:2049] [node 3] ICP22 created new session: [1:7577470890306771620:2049]
2025-11-27T18:17:04.038910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470888833890800:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:04.038662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470890306771620:2049] [node 3] ICS09 handshake done sender: [1:7577470886011804297:2113] self: [1:7577470886011804296:0] peer: [3:7577470888833890798:0] socket: 125 qp: -1
2025-11-27T18:17:04.038995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:04.038829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470890306771620:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:04.038976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:04.075490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:04.075733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470884506645571:2048] [node 1] ICS01 socket: 83 reason# EndOfStream
2025-11-27T18:17:04.075773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470884506645476:2048] [node 1] ICP30 unregister session Session# [2:7577470884506645571:2048] VirtualId# [2:7577470884506645559:0]
2025-11-27T18:17:04.075819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470884506645571:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.075905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:04.075948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:04.076600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:04.076823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470888833890800:2048] [node 1] ICS01 socket: 126 reason# EndOfStream
2025-11-27T18:17:04.076855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470884538923400:2048] [node 1] ICP30 unregister session Session# [3:7577470888833890800:2048] VirtualId# [3:7577470888833890798:0]
2025-11-27T18:17:04.076878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470888833890800:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.076951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:04.076991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:04.079068Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:04.079262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470886781282798:2048] [node 1] ICS01 socket: 127 reason# EndOfStream
2025-11-27T18:17:04.079294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470886781282701:2048] [node 1] ICP30 unregister session Session# [4:7577470886781282798:2048] VirtualId# [4:7577470886781282793:0]
2025-11-27T18:17:04.079340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470886781282798:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.079427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:04.079460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:04.079987Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:04.080201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470887186853239:2048] [node 1] ICS01 socket: 61 reason# EndOfStream
2025-11-27T18:17:04.080233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470887186853140:2048] [node 1] ICP30 unregister session Session# [5:7577470887186853239:2048] VirtualId# [5:7577470887186853222:0]
2025-11-27T18:17:04.080256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470887186853239:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.080318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:04.080384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:04.080851Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:04.081049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470886994509781:2048] [node 1] ICS07 socket disconnect 82 reason# EndOfStream
2025-11-27T18:17:04.081085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470886994509781:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.081139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:04.081173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470886994509781:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:04.082103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470886994509691:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8634) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:04.082172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470886994509781:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:04.082228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470886994509691:2048] [node 1] ICP30 unregister session Session# [6:7577470886994509781:2048] VirtualId# [6:7577470886994509772:0]
2025-11-27T18:17:04.082286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:04.082316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470886994509691:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8634) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.082340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8634) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.092532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470886994509691:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:04.128209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:04.128448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470884538923496:2049] [node 2] ICS07 socket disconnect 93 reason# EndOfStream
2025-11-27T18:17:04.128474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470884538923496:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.128543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:04.128590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470884538923496:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:04.129665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470884538923401:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4743) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:04.129737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470884538923496:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:04.129786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470884538923401:2049] [node 2] ICP30 unregister session Session# [3:7577470884538923496:2049] VirtualId# [3:7577470884538923487:0]
2025-11-27T18:17:04.129837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:04.129861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470884538923401:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4743) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.129876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4743) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.134258Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:04.134503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470891076250099:2049] [node 2] ICS01 socket: 124 reason# EndOfStream
2025-11-27T18:17:04.134544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470886781282702:2049] [node 2] ICP30 unregister session Session# [4:7577470891076250099:2049] VirtualId# [4:7577470886781282796:0]
2025-11-27T18:17:04.134576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470891076250099:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.134645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:04.134689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:04.135971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:04.136177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470886994509789:2049] [node 2] ICS01 socket: 66 reason# EndOfStream
2025-11-27T18:17:04.136215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470886994509692:2049] [node 2] ICP30 unregister session Session# [6:7577470886994509789:2049] VirtualId# [6:7577470886994509782:0]
2025-11-27T18:17:04.136239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470886994509789:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.136313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:04.136405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:04.137422Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:04.137647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470887186853234:2049] [node 2] ICS01 socket: 96 reason# EndOfStream
2025-11-27T18:17:04.137682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470887186853141:2049] [node 2] ICP30 unregister session Session# [5:7577470887186853234:2049] VirtualId# [5:7577470887186853227:0]
2025-11-27T18:17:04.137705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470887186853234:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.137780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:04.137821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:04.139981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470884538923401:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:04.188868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:04.189126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470887186853229:2050] [node 3] ICS07 socket disconnect 73 reason# EndOfStream
2025-11-27T18:17:04.189168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470887186853229:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.189251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:04.189330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470887186853229:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:04.189390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:04.189610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470886781282786:2050] [node 3] ICS01 socket: 71 reason# EndOfStream
2025-11-27T18:17:04.189646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470886781282703:2050] [node 3] ICP30 unregister session Session# [4:7577470886781282786:2050] VirtualId# [4:7577470886781282781:0]
2025-11-27T18:17:04.189674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470886781282786:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.189752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:04.189803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:04.190132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470887186853142:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26934) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:04.190278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470887186853229:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:04.190682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470887186853142:2050] [node 3] ICP30 unregister session Session# [5:7577470887186853229:2050] VirtualId# [5:7577470887186853212:0]
2025-11-27T18:17:04.190736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:04.190765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470887186853142:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26934) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.190779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26934) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.193478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:04.193797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470891289477089:2050] [node 3] ICS01 socket: 121 reason# EndOfStream
2025-11-27T18:17:04.193841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470886994509693:2050] [node 3] ICP30 unregister session Session# [6:7577470891289477089:2050] VirtualId# [6:7577470891289477087:0]
2025-11-27T18:17:04.197399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470891289477089:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.197542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:04.197612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:04.201010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470887186853142:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:04.234213Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:04.234484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470887186853244:2109] [node 4] ICS01 socket: 80 reason# EndOfStream
2025-11-27T18:17:04.234535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470887186853206:2109] [node 4] ICP30 unregister session Session# [5:7577470887186853244:2109] VirtualId# [5:7577470887186853237:0]
2025-11-27T18:17:04.234568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470887186853244:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.234677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:04.234741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:04.235081Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:04.235296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470886994509776:2109] [node 4] ICS07 socket disconnect 86 reason# EndOfStream
2025-11-27T18:17:04.235342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470886994509776:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.235412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:04.235476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470886994509776:2109] [node 4] ICS15 start handshake
2025-11-27T18:17:04.236440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470886994509755:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7903) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:04.236490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470886994509776:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:04.236528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470886994509755:2109] [node 4] ICP30 unregister session Session# [6:7577470886994509776:2109] VirtualId# [6:7577470886994509771:0]
2025-11-27T18:17:04.236564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:04.236590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470886994509755:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7903) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.236601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7903) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.246119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470886994509755:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:17:04.274796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:04.275035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470886994509787:2103] [node 5] ICS01 socket: 74 reason# EndOfStream
2025-11-27T18:17:04.275070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470886994509749:2103] [node 5] ICP30 unregister session Session# [6:7577470886994509787:2103] VirtualId# [6:7577470886994509783:0]
2025-11-27T18:17:04.275099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470886994509787:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.275176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:04.275216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:04.379368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470891379247517:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:04.379398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470891379247518:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:04.402198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470891440884401:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:04.402233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470891440884402:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:04.424689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470889179978003:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:04.424753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470889179978004:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:04.449832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470888031369897:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:04.449891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470888031369898:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:04.449932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470888031369899:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:04.471810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470890614992736:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:04.471848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470890614992737:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:04.471871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470890614992738:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:04.493421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470891763711040:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:04.493499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470891763711041:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:04.493556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470891763711042:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:04.571918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470891763711100:2105] [node 5] ICP01 ready to work
2025-11-27T18:17:04.641102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470891440884466:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:04.641524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470889179978067:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:04.641702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470891379247517:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(62880662ns) {Configure:104(62872597ns) {ProcessConfigured:135(16059225ns) {StartInitialHandshake:157(16055814ns) {DropHandshakes:470(6372189ns) {DropIncomingHandshake:442(2635ns) DropOutgoingHandshake:457(526ns)} GenerateSessionVirtualId:988(2227ns) PrepareNewSessionHandshake:477(3453ns) {SwitchToState:179(1880ns)}}}}}
2025-11-27T18:17:04.641751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470891440884467:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:04.641711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470891763711107:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:04.642012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470888031369961:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:04.641928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470890614992800:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:04.642283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470891440884468:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:04.642615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470888031369962:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:04.642842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470890614992801:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:04.642260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470889179978068:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:04.646572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470889179978071:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:04.652307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470891379247584:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:04.652968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470891379247585:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:04.653344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470891379247586:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.656629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470891379247587:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31101
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.660829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470891440884479:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26123
2025-11-27T18:17:04.660895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470891440884480:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26123
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.670920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470891379247596:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31101
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.676346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470888031369973:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61770
2025-11-27T18:17:04.676502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470888031369974:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61770
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.678015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470890614992815:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10521
2025-11-27T18:17:04.684637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470890614992815:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.684880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470891440884479:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.685406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470891440884484:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26123
2025-11-27T18:17:04.685634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470891440884485:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26123
2025-11-27T18:17:04.685701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470888031369973:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.687258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470891440884484:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.686169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470888031369898:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.687400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470891440884485:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.687549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470891440884472:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.687567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470891440884472:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.687775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470891379247587:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.688305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470889179978078:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13101
2025-11-27T18:17:04.688451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470889179978079:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13101
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.688538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470889179978080:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13101
2025-11-27T18:17:04.688906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470891440884472:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.689121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470888031369979:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61770
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.689307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470889179978078:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.689443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470889179978079:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.689505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470889179978083:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13101
2025-11-27T18:17:04.689618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470891379247596:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.689722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470891440884466:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.689826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:04.689919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470889179978004:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.689952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470891440884466:2107] [node 4] ICP22 created new session: [2:7577470891440884491:2107]
2025-11-27T18:17:04.690051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470891440884491:2107] [node 4] ICS09 handshake done sender: [2:7577470891440884472:2113] self: [2:7577470891440884471:0] peer: [4:7577470888031369978:0] socket: 75 qp: -1
2025-11-27T18:17:04.690097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470891440884491:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:04.690078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470889179978080:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.690201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470888031369974:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.690163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.690558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470891379247598:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31101
2025-11-27T18:17:04.690450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470888031369973:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.690790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470891379247599:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31101
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.690901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470891379247600:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31101
2025-11-27T18:17:04.690887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470891440884480:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.691035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470888031369980:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61770
2025-11-27T18:17:04.691113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470888031369898:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.691159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:04.691225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470891379247599:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.691198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470888031369898:2049] [node 2] ICP22 created new session: [4:7577470888031369981:2049]
2025-11-27T18:17:04.691237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470888031369981:2049] [node 2] ICS09 handshake done sender: [4:7577470888031369973:2116] self: [4:7577470888031369978:0] peer: [2:7577470891440884471:0] socket: 82 qp: -1
2025-11-27T18:17:04.691263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470888031369981:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:04.691510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.692068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470891440884490:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26123
2025-11-27T18:17:04.692089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470889179978083:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.692350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470891440884474:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.692374Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470891440884474:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.692471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470888031369897:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.692530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470891379247598:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.692590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470890614992817:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10521
2025-11-27T18:17:04.693514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470891440884401:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.693933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470891379247581:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.693960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470891379247581:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.694700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470890614992818:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10521
2025-11-27T18:17:04.695794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470891379247589:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.695820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470891379247589:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.696151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470890614992818:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.695806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470891440884474:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:04.696536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470890614992800:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.696754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470891440884402:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.696807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:04.696842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470891440884402:2049] [node 3] ICP22 created new session: [2:7577470891440884496:2049]
2025-11-27T18:17:04.696870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470890614992817:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.696877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470891440884496:2049] [node 3] ICS09 handshake done sender: [2:7577470891440884474:2114] self: [2:7577470891440884473:0] peer: [3:7577470889179978088:0] socket: 76 qp: -1
2025-11-27T18:17:04.696902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470891379247581:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:04.696925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470891440884496:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:04.696961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:04.697059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470890614992737:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.697118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470888031369970:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.697142Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470888031369970:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.697554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470889179978003:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.697645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470891379247589:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.698425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470888031369980:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.698444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470891379247600:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.698551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470891440884476:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.698486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470891379247517:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.698577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470891440884476:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.698488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470889179978089:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13101
2025-11-27T18:17:04.698531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:04.698567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470891379247517:2048] [node 2] ICP22 created new session: [1:7577470891379247607:2048]
2025-11-27T18:17:04.698624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470891379247607:2048] [node 2] ICS09 handshake done sender: [1:7577470891379247581:2110] self: [1:7577470891379247580:0] peer: [2:7577470891440884494:0] socket: 71 qp: -1
2025-11-27T18:17:04.699140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470888031369985:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61770
2025-11-27T18:17:04.698699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470891379247607:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:04.698746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.699579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470888031369899:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.699688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470890614992821:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10521
2025-11-27T18:17:04.699802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470891379247584:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.699814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470888031369970:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:04.699858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.699906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470891379247584:2107] [node 4] ICP22 created new session: [1:7577470891379247609:2107]
2025-11-27T18:17:04.699918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470889179978073:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.700040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470891379247609:2107] [node 4] ICS09 handshake done sender: [1:7577470891379247589:2113] self: [1:7577470891379247588:0] peer: [4:7577470888031369983:0] socket: 80 qp: -1
2025-11-27T18:17:04.699945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470889179978073:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.700219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470891440884498:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26123
2025-11-27T18:17:04.700267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470891379247609:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.700312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:04.700541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470888031369974:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:04.701003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470888031369961:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.701039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:04.701073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470888031369961:2108] [node 5] ICP22 created new session: [4:7577470888031369988:2108]
2025-11-27T18:17:04.701099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470889179978079:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:04.701067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470891763711119:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44607
2025-11-27T18:17:04.701132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470888031369988:2108] [node 5] ICS09 handshake done sender: [4:7577470888031369970:2114] self: [4:7577470888031369969:0] peer: [5:7577470890614992819:0] socket: 111 qp: -1
2025-11-27T18:17:04.701157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470888031369988:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:04.701215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:04.701280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470891440884490:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.701218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470890614992818:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:04.701462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470891440884480:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:04.701398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470891379247583:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.701423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470891379247583:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.702148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470889179978004:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.702612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:04.702877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470889179978004:2049] [node 2] ICP22 created new session: [3:7577470889179978093:2049]
2025-11-27T18:17:04.703084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470889179978093:2049] [node 2] ICS09 handshake done sender: [3:7577470889179978079:2116] self: [3:7577470889179978088:0] peer: [2:7577470891440884473:0] socket: 86 qp: -1
2025-11-27T18:17:04.703160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470889179978093:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:04.703055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470890614992800:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.703207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:04.703124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:04.703179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470890614992800:2108] [node 4] ICP22 created new session: [5:7577470890614992823:2108]
2025-11-27T18:17:04.703148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470891440884476:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:04.703229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470890614992823:2108] [node 4] ICS09 handshake done sender: [5:7577470890614992818:2118] self: [5:7577470890614992819:0] peer: [4:7577470888031369969:0] socket: 112 qp: -1
2025-11-27T18:17:04.703270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470890614992823:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:04.703212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470888031369897:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.703297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:04.703320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:04.703390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470888031369897:2048] [node 1] ICP22 created new session: [4:7577470888031369990:2048]
2025-11-27T18:17:04.703617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470888031369990:2048] [node 1] ICS09 handshake done sender: [4:7577470888031369974:2117] self: [4:7577470888031369983:0] peer: [1:7577470891379247588:0] socket: 83 qp: -1
2025-11-27T18:17:04.703690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470888031369990:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:04.703731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:04.704131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470891440884401:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.704238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:04.704296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470891440884401:2048] [node 1] ICP22 created new session: [2:7577470891440884501:2048]
2025-11-27T18:17:04.704335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470891440884501:2048] [node 1] ICS09 handshake done sender: [2:7577470891440884480:2118] self: [2:7577470891440884494:0] peer: [1:7577470891379247580:0] socket: 73 qp: -1
2025-11-27T18:17:04.704351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470891440884501:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:04.704369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.704621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470890614992822:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10521
2025-11-27T18:17:04.706498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470891440884402:2049] [node 3] ICP18 handshake (actor [2:7577470891440884490:2121]) is too old
2025-11-27T18:17:04.706967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470890614992817:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:04.707202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470889179978073:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:04.707636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470891440884467:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.707703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:04.707958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470891440884467:2109] [node 5] ICP22 created new session: [2:7577470891440884503:2109]
2025-11-27T18:17:04.708006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470890614992737:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.708102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:04.708080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470891440884503:2109] [node 5] ICS09 handshake done sender: [2:7577470891440884476:2115] self: [2:7577470891440884475:0] peer: [5:7577470890614992820:0] socket: 96 qp: -1
2025-11-27T18:17:04.708128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470891440884503:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:04.708155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470890614992737:2049] [node 2] ICP22 created new session: [5:7577470890614992825:2049]
2025-11-27T18:17:04.708166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:04.708195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470890614992825:2049] [node 2] ICS09 handshake done sender: [5:7577470890614992817:2117] self: [5:7577470890614992820:0] peer: [2:7577470891440884475:0] socket: 110 qp: -1
2025-11-27T18:17:04.708228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470890614992825:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:04.708262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:04.708277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470889179978067:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.708384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470891379247583:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:04.708327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:04.708358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470889179978067:2107] [node 4] ICP22 created new session: [3:7577470889179978096:2107]
2025-11-27T18:17:04.708404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470889179978096:2107] [node 4] ICS09 handshake done sender: [3:7577470889179978073:2112] self: [3:7577470889179978072:0] peer: [4:7577470888031369987:0] socket: 103 qp: -1
2025-11-27T18:17:04.708431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470889179978096:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.708477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:04.709345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470891379247518:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.709357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577470891440884401:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:04.709388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:04.709187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470891763711120:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44607
2025-11-27T18:17:04.709422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470891379247518:2049] [node 3] ICP22 created new session: [1:7577470891379247615:2049]
2025-11-27T18:17:04.709459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470891379247615:2049] [node 3] ICS09 handshake done sender: [1:7577470891379247583:2111] self: [1:7577470891379247582:0] peer: [3:7577470889179978090:0] socket: 77 qp: -1
2025-11-27T18:17:04.709493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470891379247615:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.709515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:04.709576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470888031369992:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61770
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.711362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470889179978095:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13101
2025-11-27T18:17:04.711422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470891763711121:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44607
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.712724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470891763711122:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44607
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.713080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470890614992828:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10521
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.713026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470888031369899:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11571) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470888031369980:2119] held: no
2025-11-27T18:17:04.713176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470888031369980:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:04.715692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470890614992829:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10521
2025-11-27T18:17:04.716977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470889179978080:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.717338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470889179978003:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32121) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470889179978080:2117] held: no
2025-11-27T18:17:04.717447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470889179978003:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.717452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470891763711123:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44607
2025-11-27T18:17:04.717478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:04.717529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470889179978003:2048] [node 1] ICP22 created new session: [3:7577470889179978100:2048]
2025-11-27T18:17:04.717568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470889179978100:2048] [node 1] ICS09 handshake done sender: [3:7577470889179978080:2117] self: [3:7577470889179978090:0] peer: [1:7577470891379247582:0] socket: 88 qp: -1
2025-11-27T18:17:04.717641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470889179978100:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:04.717682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:04.717046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470888031369899:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.717119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:04.717857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470891763711119:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.717195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470888031369899:2050] [node 3] ICP22 created new session: [4:7577470888031369994:2050]
2025-11-27T18:17:04.717245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470888031369994:2050] [node 3] ICS09 handshake done sender: [4:7577470888031369980:2119] self: [4:7577470888031369987:0] peer: [3:7577470889179978072:0] socket: 105 qp: -1
2025-11-27T18:17:04.718421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470891763711041:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.717300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470888031369994:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:04.717343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:04.719801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470891763711121:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.719879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470891763711122:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.719881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470891440884478:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.720101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470891763711107:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.719920Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470891440884478:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.720280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470891763711100:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.720530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470888031369972:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.720566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470891763711120:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.720551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470888031369972:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.721519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470891763711040:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.721788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470891763711128:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44607
2025-11-27T18:17:04.720316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470890614992828:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.721626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470891440884478:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:04.721293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470891379247591:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.720428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470890614992736:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.720911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470890614992811:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.721331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470891379247591:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.724934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470888031369972:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:04.724109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470891379247593:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.720934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470890614992811:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.724133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470891379247593:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.725115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470891763711119:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:04.724937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470890614992811:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:04.725654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470891379247591:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:04.726178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470891440884468:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.726199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470888031369962:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.726305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:04.726402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470888031369962:2110] [node 6] ICP22 created new session: [4:7577470888031369997:2110]
2025-11-27T18:17:04.726520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470888031369997:2110] [node 6] ICS09 handshake done sender: [4:7577470888031369972:2115] self: [4:7577470888031369971:0] peer: [6:7577470891763711125:0] socket: 126 qp: -1
2025-11-27T18:17:04.726551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470888031369997:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:04.726455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470890614992801:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.727206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:04.727255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470890614992801:2109] [node 6] ICP22 created new session: [5:7577470890614992833:2109]
2025-11-27T18:17:04.727289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470890614992833:2109] [node 6] ICS09 handshake done sender: [5:7577470890614992811:2115] self: [5:7577470890614992810:0] peer: [6:7577470891763711126:0] socket: 108 qp: -1
2025-11-27T18:17:04.727313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470890614992833:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:04.727498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:04.726240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:04.726447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470891440884468:2110] [node 6] ICP22 created new session: [2:7577470891440884506:2110]
2025-11-27T18:17:04.727331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470891440884506:2110] [node 6] ICS09 handshake done sender: [2:7577470891440884478:2116] self: [2:7577470891440884477:0] peer: [6:7577470891763711124:0] socket: 121 qp: -1
2025-11-27T18:17:04.727625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470891379247593:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:04.728472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470891379247585:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.727583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470891763711041:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.727783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:04.727831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470891763711041:2049] [node 2] ICP22 created new session: [6:7577470891763711132:2049]
2025-11-27T18:17:04.727889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470891763711132:2049] [node 2] ICS09 handshake done sender: [6:7577470891763711119:2116] self: [6:7577470891763711124:0] peer: [2:7577470891440884477:0] socket: 122 qp: -1
2025-11-27T18:17:04.727917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470891763711132:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:04.727946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:04.726585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:04.728227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470890614992829:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.727441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470891440884506:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:04.727498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:04.728552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:04.728598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470891379247585:2108] [node 5] ICP22 created new session: [1:7577470891379247619:2108]
2025-11-27T18:17:04.728634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470891379247619:2108] [node 5] ICS09 handshake done sender: [1:7577470891379247591:2114] self: [1:7577470891379247590:0] peer: [5:7577470890614992830:0] socket: 107 qp: -1
2025-11-27T18:17:04.728661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470891379247619:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:04.728695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.729675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470891379247586:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.729711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470890614992832:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10521
2025-11-27T18:17:04.729782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:04.729820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470891379247586:2109] [node 6] ICP22 created new session: [1:7577470891379247621:2109]
2025-11-27T18:17:04.729864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470891379247621:2109] [node 6] ICS09 handshake done sender: [1:7577470891379247593:2115] self: [1:7577470891379247592:0] peer: [6:7577470891763711127:0] socket: 124 qp: -1
2025-11-27T18:17:04.729888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470891379247621:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:04.729909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:04.730782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470890614992738:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.730835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470890614992828:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:04.731893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470889179978077:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.731739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470890614992736:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.731805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:04.731847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470890614992736:2048] [node 1] ICP22 created new session: [5:7577470890614992836:2048]
2025-11-27T18:17:04.733555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470890614992836:2048] [node 1] ICS09 handshake done sender: [5:7577470890614992828:2121] self: [5:7577470890614992830:0] peer: [1:7577470891379247590:0] socket: 67 qp: -1
2025-11-27T18:17:04.731923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470889179978077:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.733522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470889179978077:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:04.734009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470890614992836:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.734303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:04.734370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470891763711129:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44607
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.734933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470891763711130:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44607
2025-11-27T18:17:04.735148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470889179978068:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.735207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:04.735391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470889179978068:2109] [node 5] ICP22 created new session: [3:7577470889179978103:2109]
2025-11-27T18:17:04.735448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470889179978103:2109] [node 5] ICS09 handshake done sender: [3:7577470889179978077:2114] self: [3:7577470889179978076:0] peer: [5:7577470890614992835:0] socket: 89 qp: -1
2025-11-27T18:17:04.735478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470889179978103:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:04.735508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:04.735631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470891763711123:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.735733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470891763711100:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6441) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470891763711122:2119] held: no
2025-11-27T18:17:04.737944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470890614992736:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.738261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470891763711131:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44607
2025-11-27T18:17:04.739279Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470891763711122:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:04.739314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470890614992738:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11571) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470890614992829:2122] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.739643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470890614992838:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10521
2025-11-27T18:17:04.739691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470891763711040:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32121) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470891763711120:2117] held: no
2025-11-27T18:17:04.739805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470891763711121:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:04.740043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470891763711100:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.740071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:04.740075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470890614992829:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:04.740096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470891763711100:2105] [node 5] ICP22 created new session: [6:7577470891763711136:2105]
2025-11-27T18:17:04.740119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470891763711136:2105] [node 5] ICS09 handshake done sender: [6:7577470891763711122:2119] self: [6:7577470891763711126:0] peer: [5:7577470890614992810:0] socket: 72 qp: -1
2025-11-27T18:17:04.740144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470891763711136:2105] [node 5] ICS10 traffic start
2025-11-27T18:17:04.740181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:04.740662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470891763711120:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:04.740900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470891763711107:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.740928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:04.740947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470891763711107:2110] [node 4] ICP22 created new session: [6:7577470891763711138:2110]
2025-11-27T18:17:04.740984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470891763711138:2110] [node 4] ICS09 handshake done sender: [6:7577470891763711121:2118] self: [6:7577470891763711125:0] peer: [4:7577470888031369971:0] socket: 98 qp: -1
2025-11-27T18:17:04.741000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470891763711138:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:04.741017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:04.740862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470890614992738:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.740908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:04.740944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470890614992738:2050] [node 3] ICP22 created new session: [5:7577470890614992839:2050]
2025-11-27T18:17:04.740981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470890614992839:2050] [node 3] ICS09 handshake done sender: [5:7577470890614992829:2122] self: [5:7577470890614992835:0] peer: [3:7577470889179978076:0] socket: 106 qp: -1
2025-11-27T18:17:04.741005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470890614992839:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:04.741436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470891763711042:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:04.741027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.741986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470888031369999:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61770
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.742739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470891763711040:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.742777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:04.742811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470891763711040:2048] [node 1] ICP22 created new session: [6:7577470891763711141:2048]
2025-11-27T18:17:04.742873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470891763711141:2048] [node 1] ICS09 handshake done sender: [6:7577470891763711120:2117] self: [6:7577470891763711127:0] peer: [1:7577470891379247592:0] socket: 125 qp: -1
2025-11-27T18:17:04.742497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470889179978105:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13101
2025-11-27T18:17:04.742903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470891763711141:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:04.742933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:04.744696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470889179978082:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.744726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470889179978082:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:04.745674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470889179978105:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:04.745763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470889179978082:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:04.746130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470891763711143:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44607
2025-11-27T18:17:04.746537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470891763711123:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:04.746708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470891763711042:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:04.746750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:04.746797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470891763711042:2050] [node 3] ICP22 created new session: [6:7577470891763711144:2050]
2025-11-27T18:17:04.746834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470891763711144:2050] [node 3] ICS09 handshake done sender: [6:7577470891763711123:2120] self: [6:7577470891763711140:0] peer: [3:7577470889179978081:0] socket: 127 qp: -1
2025-11-27T18:17:04.746871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470891763711144:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:04.746894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:04.746104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470889179978071:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:04.746137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:04.746167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470889179978071:2110] [node 6] ICP22 created new session: [3:7577470889179978107:2110]
2025-11-27T18:17:04.746218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470889179978107:2110] [node 6] ICS09 handshake done sender: [3:7577470889179978082:2118] self: [3:7577470889179978081:0] peer: [6:7577470891763711140:0] socket: 101 qp: -1
2025-11-27T18:17:04.746246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470889179978107:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:04.746283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:04.746798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577470889179978071:2110] [node 6] ICP18 handshake (actor [3:7577470889179978105:2122]) is too old
2025-11-27T18:17:04.757187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:04.757419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470891440884501:2048] [node 1] ICS07 socket disconnect 73 reason# EndOfStream
2025-11-27T18:17:04.757454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470891440884501:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.757524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:04.757567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470891440884501:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:04.758444Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:04.758628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470889179978100:2048] [node 1] ICS01 socket: 88 reason# EndOfStream
2025-11-27T18:17:04.758771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470889179978003:2048] [node 1] ICP30 unregister session Session# [3:7577470889179978100:2048] VirtualId# [3:7577470889179978090:0]
2025-11-27T18:17:04.758820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470889179978100:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.758888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:04.759004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:04.761181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470891440884401:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32121) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:04.761223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470891440884501:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:04.762583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:04.762771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470891440884401:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32121) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:04.762835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470891440884501:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:04.762864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470891440884401:2048] [node 1] ICP30 unregister session Session# [2:7577470891440884501:2048] VirtualId# [2:7577470891440884494:0]
2025-11-27T18:17:04.762904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:04.762951Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470891440884401:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32121) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.762970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32121) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.762919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470888031369990:2048] [node 1] ICS01 socket: 83 reason# EndOfStream
2025-11-27T18:17:04.762965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470888031369897:2048] [node 1] ICP30 unregister session Session# [4:7577470888031369990:2048] VirtualId# [4:7577470888031369983:0]
2025-11-27T18:17:04.763000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470888031369990:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.763071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:04.763122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:04.763165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:04.763345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470890614992836:2048] [node 1] ICS07 socket disconnect 67 reason# EndOfStream
2025-11-27T18:17:04.763366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470890614992836:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.763455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:04.763515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470890614992836:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:04.765032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:04.765187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470891763711141:2048] [node 1] ICS01 socket: 125 reason# EndOfStream
2025-11-27T18:17:04.765221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470891763711040:2048] [node 1] ICP30 unregister session Session# [6:7577470891763711141:2048] VirtualId# [6:7577470891763711127:0]
2025-11-27T18:17:04.765244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470891763711141:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.765448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470890614992736:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32121) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:04.765376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:04.765419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:04.765492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470890614992836:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:04.765522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470890614992736:2048] [node 1] ICP30 unregister session Session# [5:7577470890614992836:2048] VirtualId# [5:7577470890614992830:0]
2025-11-27T18:17:04.765563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:04.765599Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470890614992736:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32121) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.765610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32121) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.772416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577470891440884401:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:04.775496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470890614992736:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:04.793854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:04.794067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470889179978093:2049] [node 2] ICS01 socket: 86 reason# EndOfStream
2025-11-27T18:17:04.794110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470889179978004:2049] [node 2] ICP30 unregister session Session# [3:7577470889179978093:2049] VirtualId# [3:7577470889179978088:0]
2025-11-27T18:17:04.794149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470889179978093:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.794234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:04.794294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:04.798666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:04.798911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470888031369981:2049] [node 2] ICS07 socket disconnect 82 reason# EndOfStream
2025-11-27T18:17:04.798955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470888031369981:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.799025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:04.799067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470888031369981:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:04.799191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:04.799391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470890614992825:2049] [node 2] ICS07 socket disconnect 110 reason# EndOfStream
2025-11-27T18:17:04.799411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470890614992825:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.799879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:04.799936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470890614992825:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:04.800437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:04.800671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470891763711132:2049] [node 2] ICS01 socket: 122 reason# EndOfStream
2025-11-27T18:17:04.800748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470888031369898:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2918) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:04.801886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470888031369981:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:04.801759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470891763711041:2049] [node 2] ICP30 unregister session Session# [6:7577470891763711132:2049] VirtualId# [6:7577470891763711124:0]
2025-11-27T18:17:04.801798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470891763711132:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.801866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:04.802082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470888031369898:2049] [node 2] ICP30 unregister session Session# [4:7577470888031369981:2049] VirtualId# [4:7577470888031369978:0]
2025-11-27T18:17:04.801915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:04.802129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:04.802165Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470888031369898:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2918) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.802123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470890614992737:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2918) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:04.802180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470890614992825:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:04.802179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2918) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.802206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470890614992737:2049] [node 2] ICP30 unregister session Session# [5:7577470890614992825:2049] VirtualId# [5:7577470890614992820:0]
2025-11-27T18:17:04.802256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:04.802304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470890614992737:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2918) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.802329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2918) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:04.812313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470888031369898:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:04.812339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470890614992737:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:04.827301Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:04.827540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470888031369994:2050] [node 3] ICS01 socket: 105 reason# EndOfStream
2025-11-27T18:17:04.827572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470888031369899:2050] [node 3] ICP30 unregister session Session# [4:7577470888031369994:2050] VirtualId# [4:7577470888031369987:0]
2025-11-27T18:17:04.827632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470888031369994:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.827701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:04.827734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:04.828115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:04.828293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470890614992839:2050] [node 3] ICS01 socket: 106 reason# EndOfStream
2025-11-27T18:17:04.828323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470890614992738:2050] [node 3] ICP30 unregister session Session# [5:7577470890614992839:2050] VirtualId# [5:7577470890614992835:0]
2025-11-27T18:17:04.828348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470890614992839:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.828409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:04.828442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:04.828873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:04.829070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470891763711144:2050] [node 3] ICS01 socket: 127 reason# EndOfStream
2025-11-27T18:17:04.829106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470891763711042:2050] [node 3] ICP30 unregister session Session# [6:7577470891763711144:2050] VirtualId# [6:7577470891763711140:0]
2025-11-27T18:17:04.829129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470891763711144:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.829189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:04.829228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:04.844964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:04.845213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470890614992823:2108] [node 4] ICS01 socket: 112 reason# EndOfStream
2025-11-27T18:17:04.845269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470890614992800:2108] [node 4] ICP30 unregister session Session# [5:7577470890614992823:2108] VirtualId# [5:7577470890614992819:0]
2025-11-27T18:17:04.845305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470890614992823:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.845390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:04.845444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:04.849446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:04.849678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470891763711138:2110] [node 4] ICS01 socket: 98 reason# EndOfStream
2025-11-27T18:17:04.849710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470891763711107:2110] [node 4] ICP30 unregister session Session# [6:7577470891763711138:2110] VirtualId# [6:7577470891763711125:0]
2025-11-27T18:17:04.849737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470891763711138:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.849815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:04.849858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:04.884183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:04.884526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470891763711136:2105] [node 5] ICS01 socket: 72 reason# EndOfStream
2025-11-27T18:17:04.884627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470891763711100:2105] [node 5] ICP30 unregister session Session# [6:7577470891763711136:2105] VirtualId# [6:7577470891763711126:0]
2025-11-27T18:17:04.884697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470891763711136:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:04.884815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:04.884889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:04.938379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470891580190320:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:04.938405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470891580190319:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:04.958243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470890150651359:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:04.958276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470890150651360:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:04.980626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470888527887887:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:04.980664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470888527887888:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:05.007017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470894654929442:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:05.007062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470894654929443:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:05.007083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470894654929444:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:05.028651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470892765736464:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:05.028698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470892765736465:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:05.028726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470892765736466:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:05.050471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470895090404721:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:05.050471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470895090404720:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:05.050507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470895090404722:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:05.190329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470894445618719:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:05.204417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470894445618720:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:05.204443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470894654929515:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:05.203293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470895090404778:2102] [node 5] ICP01 ready to work
2025-11-27T18:17:05.205008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470894445618721:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:05.205331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470892765736527:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:05.205159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470892822855272:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:05.205901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470895090404784:2103] [node 4] ICP01 ready to work
2025-11-27T18:17:05.206230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470894654929516:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:05.206642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470892765736528:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:05.206751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470892822855273:2117] [node 5] ICP01 ready to work
2025-11-27T18:17:05.207356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470892822855274:2125] [node 6] ICP01 ready to work
2025-11-27T18:17:05.208968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470891580190319:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(91151100ns) {Configure:104(91139272ns) {ProcessConfigured:135(656687ns) {StartInitialHandshake:157(653963ns) {DropHandshakes:470(2695ns) {DropIncomingHandshake:442(863ns) DropOutgoingHandshake:457(535ns)} GenerateSessionVirtualId:988(1084ns) PrepareNewSessionHandshake:477(2458ns) {SwitchToState:179(898ns)}}}}}
2025-11-27T18:17:05.209335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470895875157682:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:05.209587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470895875157683:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:05.211084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470895090404720:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(48535826ns) {Configure:104(48530514ns) {ProcessConfigured:135(269961ns) {StartInitialHandshake:157(267492ns) {DropHandshakes:470(3458ns) {DropIncomingHandshake:442(312ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(627ns) PrepareNewSessionHandshake:477(1924ns) {SwitchToState:179(805ns)}}}}}
2025-11-27T18:17:05.210669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470891580190320:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(92267347ns) {Configure:104(92264708ns) {ProcessConfigured:135(1302816ns) {StartInitialHandshake:157(1300175ns) {DropHandshakes:470(1361ns) {DropIncomingHandshake:442(339ns) DropOutgoingHandshake:457(333ns)} GenerateSessionVirtualId:988(456ns) PrepareNewSessionHandshake:477(3592ns) {SwitchToState:179(1578ns)}}}}}
2025-11-27T18:17:05.210743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470895875157685:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:05.212512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470894654929443:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(25799146ns) {Configure:104(25787748ns) {ProcessConfigured:135(750931ns) {StartInitialHandshake:157(748087ns) {DropHandshakes:470(2343ns) {DropIncomingHandshake:442(878ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(982ns) PrepareNewSessionHandshake:477(2288ns) {SwitchToState:179(943ns)}}}}}
2025-11-27T18:17:05.212611Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470892765736464:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(61568995ns) {Configure:104(61559631ns) {ProcessConfigured:135(387046ns) {StartInitialHandshake:157(385086ns) {DropHandshakes:470(1253ns) {DropIncomingHandshake:442(412ns) DropOutgoingHandshake:457(212ns)} GenerateSessionVirtualId:988(541ns) PrepareNewSessionHandshake:477(1120ns) {SwitchToState:179(415ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.225846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470895875157690:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59643
2025-11-27T18:17:05.237150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470895090404784:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22301909ns) {Configure:104(22294405ns) {ProcessConfigured:135(1805463ns) {StartInitialHandshake:157(1803615ns) {DropHandshakes:470(842ns) {DropIncomingHandshake:442(284ns) DropOutgoingHandshake:457(117ns)} GenerateSessionVirtualId:988(327ns) PrepareNewSessionHandshake:477(2691ns) {SwitchToState:179(1117ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.237822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470895875157685:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19093404ns) {Configure:104(19086881ns) {ProcessConfigured:135(572045ns) {StartInitialHandshake:157(570501ns) {DropHandshakes:470(1300ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(596ns) PrepareNewSessionHandshake:477(1788ns) {SwitchToState:179(708ns)}}}}}
2025-11-27T18:17:05.238594Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470892765736528:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19351583ns) {Configure:104(19346980ns) {ProcessConfigured:135(960933ns) {StartInitialHandshake:157(958500ns) {DropHandshakes:470(1286ns) {DropIncomingHandshake:442(424ns) DropOutgoingHandshake:457(275ns)} GenerateSessionVirtualId:988(403ns) PrepareNewSessionHandshake:477(1425ns) {SwitchToState:179(548ns)}}}}}
2025-11-27T18:17:05.238962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470892822855273:2117] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19593265ns) {Configure:104(19588324ns) {ProcessConfigured:135(922847ns) {StartInitialHandshake:157(920992ns) {DropHandshakes:470(1618ns) {DropIncomingHandshake:442(450ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(701ns) PrepareNewSessionHandshake:477(2172ns) {SwitchToState:179(1097ns)}}}}}
2025-11-27T18:17:05.244001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470894445618735:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25404
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.245897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470892765736537:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14660
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.247020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470895875157690:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.248765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470895875157696:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59643
2025-11-27T18:17:05.249109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470892765736537:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.249359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470892765736465:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.251605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470894445618735:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.254755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470895875157698:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59643
2025-11-27T18:17:05.254773Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470894654929516:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22771019ns) {Configure:104(22764224ns) {ProcessConfigured:135(4504863ns) {StartInitialHandshake:157(4502057ns) {DropHandshakes:470(1916ns) {DropIncomingHandshake:442(579ns) DropOutgoingHandshake:457(346ns)} GenerateSessionVirtualId:988(1094ns) PrepareNewSessionHandshake:477(2711ns) {SwitchToState:179(1120ns)}}}}}
2025-11-27T18:17:05.254234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470894445618736:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25404
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.256176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470892822855285:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44572
2025-11-27T18:17:05.255431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470895875157680:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.255455Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470895875157680:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.254318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470894445618737:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25404
2025-11-27T18:17:05.257208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470892765736544:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14660
2025-11-27T18:17:05.254383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470890150651359:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.257345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470892822855286:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44572
2025-11-27T18:17:05.257434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470895875157680:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:05.255130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470894445618727:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.261499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470892822855287:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44572
2025-11-27T18:17:05.266397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470892822855285:2141] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.268010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470891580190319:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.267953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470892822855286:2142] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.255157Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470894445618727:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.268750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470892765736544:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.268823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:05.267607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470894445618737:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.268481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470894445618727:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:05.269025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470891580190319:2048] [node 2] ICP22 created new session: [1:7577470895875157700:2048]
2025-11-27T18:17:05.269568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470888527887888:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.269581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470895875157700:2048] [node 2] ICS09 handshake done sender: [1:7577470895875157680:2111] self: [1:7577470895875157679:0] peer: [2:7577470894445618738:0] socket: 77 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:17:05.269787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470895875157700:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:05.269701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470892822855287:2143] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.269935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470888527887887:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.270565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470894654929527:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22116
2025-11-27T18:17:05.270854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470892822855294:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44572
2025-11-27T18:17:05.269798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470895875157696:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.269822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:05.271427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470894445618739:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25404
2025-11-27T18:17:05.270193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470895875157698:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.271453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470892765736545:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14660
2025-11-27T18:17:05.271889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470894445618720:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.271925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:05.272954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470892765736537:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:05.271964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470894445618720:2109] [node 5] ICP22 created new session: [2:7577470894445618744:2109]
2025-11-27T18:17:05.273096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470895875157684:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.273403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470894654929531:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22116
2025-11-27T18:17:05.272212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470894445618744:2109] [node 5] ICS09 handshake done sender: [2:7577470894445618727:2113] self: [2:7577470894445618726:0] peer: [5:7577470892765736543:0] socket: 74 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.273627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470892765736465:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.273120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470895875157684:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.272250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470894445618744:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:05.274595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470894654929527:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.275552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:05.278999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470895875157688:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.272282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:05.278024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470894654929442:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.279017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470895875157688:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.280774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470892765736465:2049] [node 2] ICP22 created new session: [5:7577470892765736547:2049]
2025-11-27T18:17:05.289284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470892822855295:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44572
2025-11-27T18:17:05.283740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470892765736547:2049] [node 2] ICS09 handshake done sender: [5:7577470892765736537:2114] self: [5:7577470892765736543:0] peer: [2:7577470894445618726:0] socket: 82 qp: -1
2025-11-27T18:17:05.279873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470895875157702:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59643
2025-11-27T18:17:05.272993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470894445618725:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.285030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470892765736547:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:05.285076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.295360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470894654929533:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22116
2025-11-27T18:17:05.273015Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470894445618725:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.292332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470892822855286:2142] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:05.279924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470895875157703:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59643
2025-11-27T18:17:05.298440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470888527887887:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.298501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:05.298548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470888527887887:2048] [node 1] ICP22 created new session: [3:7577470892822855296:2048]
2025-11-27T18:17:05.298585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470892822855296:2048] [node 1] ICS09 handshake done sender: [3:7577470892822855286:2142] self: [3:7577470892822855292:0] peer: [1:7577470895875157681:0] socket: 84 qp: -1
2025-11-27T18:17:05.298615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470892822855296:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:05.291267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470894445618736:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.280189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470895875157684:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:05.299036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:05.299878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470894654929527:2125] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.303418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470895090404799:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47954
2025-11-27T18:17:05.292209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470890150651359:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64488) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577470894445618735:2116] held: no
2025-11-27T18:17:05.289003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470891580190320:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.300977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470894654929442:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.301538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470892765736465:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23999586ns) {HandleHandshakeStatus:329(23996892ns) {DropIncomingHandshake:442(622ns) DropOutgoingHandshake:457(4834ns) SwitchToState:179(329ns) ProcessPendingSessionEvents:495(6331969ns) {ForwardSessionEventToSession:612(860422ns) ForwardSessionEventToSession:612(5073ns) ForwardSessionEventToSession:612(3481ns) ForwardSessionEventToSession:612(397361ns) ForwardSessionEventToSession:612(2930ns) ForwardSessionEventToSession:612(2344ns) ForwardSessionEventToSession:612(1092078ns) ForwardSessionEventToSession:612(5303ns) ForwardSessionEventToSession:612(3940458ns) ForwardSessionEventToSession:612(5967ns)}}}
2025-11-27T18:17:05.297563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470894445618735:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:05.289328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:05.314138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470892822855294:2144] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.321284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470892765736549:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14660
2025-11-27T18:17:05.289457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470895875157688:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:05.290699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470891580190320:2049] [node 3] ICP22 created new session: [1:7577470895875157709:2049]
2025-11-27T18:17:05.291413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470895875157709:2049] [node 3] ICS09 handshake done sender: [1:7577470895875157684:2112] self: [1:7577470895875157681:0] peer: [3:7577470892822855292:0] socket: 78 qp: -1
2025-11-27T18:17:05.301036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:05.291714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470895875157709:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:05.292480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:05.309405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470895875157702:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.309495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470895875157682:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.312812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:05.318509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470895875157682:2107] [node 4] ICP22 created new session: [1:7577470895875157711:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.321403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470895875157711:2107] [node 4] ICS09 handshake done sender: [1:7577470895875157688:2113] self: [1:7577470895875157686:0] peer: [4:7577470894654929532:0] socket: 79 qp: -1
2025-11-27T18:17:05.321447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470895875157711:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:05.321494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:05.301738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470894445618743:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25404
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.322355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577470895875157682:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:05.302165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470890150651359:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.330590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470892765736551:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14660
2025-11-27T18:17:05.324797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470895090404801:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47954
2025-11-27T18:17:05.309533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470894654929442:2048] [node 1] ICP22 created new session: [4:7577470894654929534:2048]
2025-11-27T18:17:05.328387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470892822855298:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44572
2025-11-27T18:17:05.330859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470895090404801:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.302197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:05.311561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470894654929531:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.302234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470890150651359:2048] [node 1] ICP22 created new session: [2:7577470894445618748:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.339572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470895090404722:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.333976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470892765736549:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.339678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470891580190320:2049] [node 3] ICP18 handshake (actor [1:7577470895875157702:2119]) is too old
2025-11-27T18:17:05.341443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470892765736464:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.336749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470892822855280:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.302274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470894445618748:2048] [node 1] ICS09 handshake done sender: [2:7577470894445618735:2116] self: [2:7577470894445618738:0] peer: [1:7577470895875157679:0] socket: 81 qp: -1
2025-11-27T18:17:05.312635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470894654929534:2048] [node 1] ICS09 handshake done sender: [4:7577470894654929527:2125] self: [4:7577470894654929532:0] peer: [1:7577470895875157686:0] socket: 85 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.315251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470894654929534:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:05.353878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470895090404800:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47954
2025-11-27T18:17:05.336798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470892822855280:2138] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.366172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470895875157689:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.366227Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470895875157689:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.302301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470894445618748:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:05.302325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:05.302680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470894445618720:2109] [node 5] ICP18 handshake (actor [2:7577470894445618736:2117]) is too old
2025-11-27T18:17:05.303341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470894445618725:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:05.309465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470894445618742:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25404
2025-11-27T18:17:05.322852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470890150651360:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.322891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:05.322935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470890150651360:2049] [node 3] ICP22 created new session: [2:7577470894445618751:2049]
2025-11-27T18:17:05.322992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470894445618751:2049] [node 3] ICS09 handshake done sender: [2:7577470894445618725:2112] self: [2:7577470894445618723:0] peer: [3:7577470892822855291:0] socket: 73 qp: -1
2025-11-27T18:17:05.323015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470894445618751:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:05.323042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:05.324414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470894445618742:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.336885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470894445618743:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.357889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470892822855285:2141] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:05.362675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470895090404799:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.321066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:05.366192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470892765736464:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19217955ns) {IncomingHandshake:261(19130579ns) {DropIncomingHandshake:442(694ns) PrepareNewSessionHandshake:477(1628ns) {SwitchToState:179(578ns)} GenerateSessionVirtualId:988(638ns) IssueIncomingHandshakeReply:193(13810957ns) {SwitchToState:179(5737ns)}}}
2025-11-27T18:17:05.365616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470895090404802:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47954
2025-11-27T18:17:05.378925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470895875157689:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:05.369684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470890150651360:2049] [node 3] ICP18 handshake (actor [2:7577470894445618742:2120]) is too old
2025-11-27T18:17:05.367027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470892822855284:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.380343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470894445618729:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.329883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470894654929444:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.377754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470892765736539:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.365769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470895090404721:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.330204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470894654929536:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22116
2025-11-27T18:17:05.381374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470895875157683:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.381507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470895875157695:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.367051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470892822855284:2140] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.380402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470894445618729:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.377792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470892765736539:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.366284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470895090404800:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.331569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470894654929442:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(30374656ns) {HandleHandshakeStatus:329(30372916ns) {DropIncomingHandshake:442(569ns) DropOutgoingHandshake:457(4621ns) SwitchToState:179(772ns) ProcessPendingSessionEvents:495(6563161ns) {ForwardSessionEventToSession:612(3587940ns) ForwardSessionEventToSession:612(3129ns) ForwardSessionEventToSession:612(3936ns) ForwardSessionEventToSession:612(6661ns) ForwardSessionEventToSession:612(23925ns) ForwardSessionEventToSession:612(3053ns) ForwardSessionEventToSession:612(907004ns) ForwardSessionEventToSession:612(11230ns) ForwardSessionEventToSession:612(1998791ns) ForwardSessionEventToSession:612(4219ns)}}}
2025-11-27T18:17:05.381525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470895875157695:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.381434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470894445618734:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.366802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470895090404778:2102] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.379199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470892765736551:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.367626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470888527887888:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.381455Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470894445618734:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.349593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470894654929536:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.373446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470895090404802:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.367691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:05.367744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470888527887888:2049] [node 2] ICP22 created new session: [3:7577470892822855302:2049]
2025-11-27T18:17:05.367789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470892822855302:2049] [node 2] ICS09 handshake done sender: [3:7577470892822855285:2141] self: [3:7577470892822855291:0] peer: [2:7577470894445618723:0] socket: 83 qp: -1
2025-11-27T18:17:05.367821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470892822855302:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:05.367853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:05.380559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470892822855280:2138] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:05.380779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470892822855284:2140] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:05.391501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470892822855272:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.392196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470892822855274:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.383554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:05.379544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470892765736553:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14660
2025-11-27T18:17:05.392257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:05.392269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:05.392829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470892822855272:2107] [node 4] ICP22 created new session: [3:7577470892822855307:2107]
2025-11-27T18:17:05.393382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470892822855274:2125] [node 6] ICP22 created new session: [3:7577470892822855306:2125]
2025-11-27T18:17:05.394243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470892822855307:2107] [node 4] ICS09 handshake done sender: [3:7577470892822855280:2138] self: [3:7577470892822855279:0] peer: [4:7577470894654929537:0] socket: 106 qp: -1
2025-11-27T18:17:05.394279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470892822855307:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:05.394324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:05.395844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470892822855306:2125] [node 6] ICS09 handshake done sender: [3:7577470892822855284:2140] self: [3:7577470892822855283:0] peer: [6:7577470895090404803:0] socket: 125 qp: -1
2025-11-27T18:17:05.396752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470892822855306:2125] [node 6] ICS10 traffic start
2025-11-27T18:17:05.398337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:05.390988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470894445618729:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:05.377500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470895090404720:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.383607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470895875157683:2108] [node 5] ICP22 created new session: [1:7577470895875157715:2108]
2025-11-27T18:17:05.362873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470894654929443:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.392897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470894445618734:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:05.379598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470892765736554:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14660
2025-11-27T18:17:05.385913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470895090404808:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47954
2025-11-27T18:17:05.395104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470894445618721:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.383700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470895875157715:2108] [node 5] ICS09 handshake done sender: [1:7577470895875157689:2114] self: [1:7577470895875157687:0] peer: [5:7577470892765736552:0] socket: 117 qp: -1
2025-11-27T18:17:05.388719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470892765736527:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.391600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470895090404807:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47954
2025-11-27T18:17:05.381026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470894654929540:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22116
2025-11-27T18:17:05.383738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470895875157715:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:05.395150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:05.389623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470894654929524:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.388905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470892765736549:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:05.383777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:05.399167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470892765736464:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64488) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470892765736549:2118] held: no
2025-11-27T18:17:05.387480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470895875157695:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:05.395183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470894445618721:2110] [node 6] ICP22 created new session: [2:7577470894445618756:2110]
2025-11-27T18:17:05.389642Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470894654929524:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.399247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470892765736464:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.392644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470895875157685:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.399282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:05.395229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470894445618756:2110] [node 6] ICS09 handshake done sender: [2:7577470894445618729:2114] self: [2:7577470894445618728:0] peer: [6:7577470895090404804:0] socket: 122 qp: -1
2025-11-27T18:17:05.399320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470892765736464:2048] [node 1] ICP22 created new session: [5:7577470892765736557:2048]
2025-11-27T18:17:05.390475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470894654929531:2126] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:05.395253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470894445618756:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:05.399355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470892765736557:2048] [node 1] ICS09 handshake done sender: [5:7577470892765736549:2118] self: [5:7577470892765736552:0] peer: [1:7577470895875157687:0] socket: 72 qp: -1
2025-11-27T18:17:05.393470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:05.410532Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470892822855274:2125] [node 6] ICP35 event processing took too much time PendingConnection:279(21658473ns) {HandleHandshakeStatus:329(21656293ns) {DropIncomingHandshake:442(1780ns) DropOutgoingHandshake:457(622ns) SwitchToState:179(388ns) ProcessPendingSessionEvents:495(3114499ns) {ForwardSessionEventToSession:612(1649030ns) ForwardSessionEventToSession:612(123991ns) ForwardSessionEventToSession:612(95344ns) ForwardSessionEventToSession:612(3168ns) ForwardSessionEventToSession:612(1205738ns) ForwardSessionEventToSession:612(5545ns) ForwardSessionEventToSession:612(2405ns) ForwardSessionEventToSession:612(5856ns) ForwardSessionEventToSession:612(4893ns) ForwardSessionEventToSession:612(3402ns)}}}
2025-11-27T18:17:05.395278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:05.393513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470895875157685:2109] [node 6] ICP22 created new session: [1:7577470895875157718:2109]
2025-11-27T18:17:05.392422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470894654929444:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.396676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470895875157718:2109] [node 6] ICS09 handshake done sender: [1:7577470895875157695:2116] self: [1:7577470895875157694:0] peer: [6:7577470895090404806:0] socket: 127 qp: -1
2025-11-27T18:17:05.400551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470894445618719:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.399385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470892765736557:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:05.398348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470895875157718:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:05.411946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470895090404720:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64488) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470895090404802:2119] held: no
2025-11-27T18:17:05.400595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:05.393418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.402995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:05.400643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470894445618719:2107] [node 4] ICP22 created new session: [2:7577470894445618758:2107]
2025-11-27T18:17:05.399408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:05.405554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470892765736539:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:05.409014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470892765736553:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.409205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470892765736528:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.409271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:05.409410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470892765736528:2109] [node 6] ICP22 created new session: [5:7577470892765736560:2109]
2025-11-27T18:17:05.409540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470892765736560:2109] [node 6] ICS09 handshake done sender: [5:7577470892765736539:2115] self: [5:7577470892765736538:0] peer: [6:7577470895090404805:0] socket: 104 qp: -1
2025-11-27T18:17:05.409672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470892765736560:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:05.411386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:05.412727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470892765736466:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.412805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470892765736562:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14660
2025-11-27T18:17:05.400684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470894445618758:2107] [node 4] ICS09 handshake done sender: [2:7577470894445618734:2115] self: [2:7577470894445618733:0] peer: [4:7577470894654929539:0] socket: 112 qp: -1
2025-11-27T18:17:05.400707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470894445618758:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:05.400752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:05.396362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470894654929444:2050] [node 3] ICP22 created new session: [4:7577470894654929543:2050]
2025-11-27T18:17:05.410635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470895875157685:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16422026ns) {HandleHandshakeStatus:329(16419175ns) {DropIncomingHandshake:442(9591ns) DropOutgoingHandshake:457(258ns) SwitchToState:179(133ns) ProcessPendingSessionEvents:495(3514484ns) {ForwardSessionEventToSession:612(1985953ns) ForwardSessionEventToSession:612(3300ns) ForwardSessionEventToSession:612(241997ns) ForwardSessionEventToSession:612(1209593ns) ForwardSessionEventToSession:612(5090ns) ForwardSessionEventToSession:612(33282ns) ForwardSessionEventToSession:612(7329ns) ForwardSessionEventToSession:612(2150ns) ForwardSessionEventToSession:612(4226ns) ForwardSessionEventToSession:612(9682ns)}}}
2025-11-27T18:17:05.414679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470895090404801:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:05.414433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470892822855282:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.397456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470894654929543:2050] [node 3] ICS09 handshake done sender: [4:7577470894654929531:2126] self: [4:7577470894654929537:0] peer: [3:7577470892822855279:0] socket: 108 qp: -1
2025-11-27T18:17:05.397487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470894654929543:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:05.414472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470892822855282:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.401784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:05.415516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470895090404799:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:17:05.411761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470894654929444:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16865522ns) {HandleHandshakeStatus:329(16863482ns) {DropIncomingHandshake:442(997ns) DropOutgoingHandshake:457(4902ns) SwitchToState:179(347ns) ProcessPendingSessionEvents:495(3166394ns) {ForwardSessionEventToSession:612(1608826ns) ForwardSessionEventToSession:612(4473ns) ForwardSessionEventToSession:612(195188ns) ForwardSessionEventToSession:612(2458ns) ForwardSessionEventToSession:612(122580ns) ForwardSessionEventToSession:612(2204ns) ForwardSessionEventToSession:612(2097ns) ForwardSessionEventToSession:612(1208527ns) ForwardSessionEventToSession:612(3831ns) ForwardSessionEventToSession:612(2059ns)}}}
::ffff:127.1.0.0
2025-11-27T18:17:05.416697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470895090404721:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15459) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470895090404799:2116] held: no
2025-11-27T18:17:05.416864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470895090404721:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.417060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:05.417076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470895090404809:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47954
2025-11-27T18:17:05.417109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470895090404721:2049] [node 2] ICP22 created new session: [6:7577470895090404811:2049]
2025-11-27T18:17:05.417185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470895090404811:2049] [node 2] ICS09 handshake done sender: [6:7577470895090404799:2116] self: [6:7577470895090404804:0] peer: [2:7577470894445618728:0] socket: 91 qp: -1
2025-11-27T18:17:05.417224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470895090404811:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:05.417270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.417502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470892822855282:2139] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:05.417360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470894654929443:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15459) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470894654929536:2128] held: no
2025-11-27T18:17:05.417529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470894654929541:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22116
2025-11-27T18:17:05.417561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470894654929524:2123] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.417945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470892822855273:2117] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.418114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:05.418564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470895090404810:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47954
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.419039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470892765736551:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:05.418575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470892822855273:2117] [node 5] ICP22 created new session: [3:7577470892822855311:2117]
2025-11-27T18:17:05.418902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470894654929545:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22116
2025-11-27T18:17:05.418938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470894654929515:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.421053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470892765736564:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14660
2025-11-27T18:17:05.419560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470895090404722:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.419103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:05.418625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470892822855311:2117] [node 5] ICS09 handshake done sender: [3:7577470892822855282:2139] self: [3:7577470892822855281:0] peer: [5:7577470892765736563:0] socket: 132 qp: -1
2025-11-27T18:17:05.419141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470894654929515:2108] [node 5] ICP22 created new session: [4:7577470894654929547:2108]
2025-11-27T18:17:05.419630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:05.418649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470892822855311:2117] [node 5] ICS10 traffic start
2025-11-27T18:17:05.419383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470894654929536:2128] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:05.421106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470894654929547:2108] [node 5] ICS09 handshake done sender: [4:7577470894654929524:2123] self: [4:7577470894654929523:0] peer: [5:7577470892765736555:0] socket: 128 qp: -1
2025-11-27T18:17:05.421430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470894654929547:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:05.421850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:05.419670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470895090404722:2050] [node 3] ICP22 created new session: [6:7577470895090404813:2050]
2025-11-27T18:17:05.418680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:05.419709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470895090404813:2050] [node 3] ICS09 handshake done sender: [6:7577470895090404801:2118] self: [6:7577470895090404803:0] peer: [3:7577470892822855283:0] socket: 126 qp: -1
2025-11-27T18:17:05.419738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470895090404813:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:05.419764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:05.423570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470894654929443:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.420422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470895090404778:2102] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17465) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470895090404800:2117] held: no
2025-11-27T18:17:05.420750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470895090404802:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:05.423775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470892765736527:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.423550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470895090404800:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:05.423612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:05.423943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:05.423648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470894654929443:2049] [node 2] ICP22 created new session: [4:7577470894654929549:2049]
2025-11-27T18:17:05.423980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470892765736527:2108] [node 4] ICP22 created new session: [5:7577470892765736565:2108]
2025-11-27T18:17:05.423986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470895090404720:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.423703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470894654929549:2049] [node 2] ICS09 handshake done sender: [4:7577470894654929536:2128] self: [4:7577470894654929539:0] peer: [2:7577470894445618733:0] socket: 115 qp: -1
2025-11-27T18:17:05.424028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470892765736565:2108] [node 4] ICS09 handshake done sender: [5:7577470892765736551:2119] self: [5:7577470892765736555:0] peer: [4:7577470894654929523:0] socket: 129 qp: -1
2025-11-27T18:17:05.424175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470892765736466:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7342) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470892765736553:2120] held: no
2025-11-27T18:17:05.424193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470892765736565:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:05.424230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:05.424179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:05.423772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470894654929549:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:05.424227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470895090404720:2048] [node 1] ICP22 created new session: [6:7577470895090404817:2048]
2025-11-27T18:17:05.424261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470895090404817:2048] [node 1] ICS09 handshake done sender: [6:7577470895090404802:2119] self: [6:7577470895090404806:0] peer: [1:7577470895875157694:0] socket: 130 qp: -1
2025-11-27T18:17:05.423896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470894654929545:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.424282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470895090404817:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:05.424300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470895090404778:2102] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.423921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:05.424618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577470894654929515:2108] [node 5] ICP18 handshake (actor [4:7577470894654929545:2131]) is too old
2025-11-27T18:17:05.424312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:05.424335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:05.424370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470895090404778:2102] [node 5] ICP22 created new session: [6:7577470895090404820:2102]
2025-11-27T18:17:05.424480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470895090404820:2102] [node 5] ICS09 handshake done sender: [6:7577470895090404800:2117] self: [6:7577470895090404805:0] peer: [5:7577470892765736538:0] socket: 124 qp: -1
2025-11-27T18:17:05.424590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470895090404820:2102] [node 5] ICS10 traffic start
2025-11-27T18:17:05.424622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:05.426026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470892765736553:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.426440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470895090404818:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47954
2025-11-27T18:17:05.426715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470894654929551:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22116
2025-11-27T18:17:05.427065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470892765736466:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.427191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470895090404818:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.427315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470895090404784:2103] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:05.427581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470894654929526:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.427612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470894654929526:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:05.427513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:05.427574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470892765736466:2050] [node 3] ICP22 created new session: [5:7577470892765736567:2050]
2025-11-27T18:17:05.427612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470892765736567:2050] [node 3] ICS09 handshake done sender: [5:7577470892765736553:2120] self: [5:7577470892765736563:0] peer: [3:7577470892822855281:0] socket: 133 qp: -1
2025-11-27T18:17:05.427644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470892765736567:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:05.427786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:05.428715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470894654929551:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:05.428773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470894654929526:2124] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:05.428886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470895090404823:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47954
2025-11-27T18:17:05.429012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470894654929516:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:05.429098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:05.429155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470894654929516:2110] [node 6] ICP22 created new session: [4:7577470894654929554:2110]
2025-11-27T18:17:05.429193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470894654929554:2110] [node 6] ICS09 handshake done sender: [4:7577470894654929526:2124] self: [4:7577470894654929525:0] peer: [6:7577470895090404822:0] socket: 89 qp: -1
2025-11-27T18:17:05.429168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470895090404818:2124] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:05.429219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470894654929554:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:05.429247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:05.429609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470895090404784:2103] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:05.429658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:05.429817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470895090404784:2103] [node 4] ICP22 created new session: [6:7577470895090404824:2103]
2025-11-27T18:17:05.429862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470895090404824:2103] [node 4] ICS09 handshake done sender: [6:7577470895090404818:2124] self: [6:7577470895090404822:0] peer: [4:7577470894654929525:0] socket: 88 qp: -1
2025-11-27T18:17:05.429889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470895090404824:2103] [node 4] ICS10 traffic start
2025-11-27T18:17:05.429914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:05.450428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:05.450601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470894445618748:2048] [node 1] ICS01 socket: 81 reason# EndOfStream
2025-11-27T18:17:05.450634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470890150651359:2048] [node 1] ICP30 unregister session Session# [2:7577470894445618748:2048] VirtualId# [2:7577470894445618738:0]
2025-11-27T18:17:05.450656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470894445618748:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.450724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:05.450760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:05.451386Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:05.451595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470892822855296:2048] [node 1] ICS01 socket: 84 reason# EndOfStream
2025-11-27T18:17:05.451643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470888527887887:2048] [node 1] ICP30 unregister session Session# [3:7577470892822855296:2048] VirtualId# [3:7577470892822855292:0]
2025-11-27T18:17:05.451667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470892822855296:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.451743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:05.451792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:05.453914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:05.454040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470894654929534:2048] [node 1] ICS01 socket: 85 reason# EndOfStream
2025-11-27T18:17:05.454085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470894654929442:2048] [node 1] ICP30 unregister session Session# [4:7577470894654929534:2048] VirtualId# [4:7577470894654929532:0]
2025-11-27T18:17:05.454104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470894654929534:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.454149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:05.454172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:05.454841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:05.454975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470892765736557:2048] [node 1] ICS01 socket: 72 reason# EndOfStream
2025-11-27T18:17:05.454996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470892765736464:2048] [node 1] ICP30 unregister session Session# [5:7577470892765736557:2048] VirtualId# [5:7577470892765736552:0]
2025-11-27T18:17:05.455011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470892765736557:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.455048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:05.455068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:05.455629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:05.455804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470895090404817:2048] [node 1] ICS01 socket: 130 reason# EndOfStream
2025-11-27T18:17:05.455836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470895090404720:2048] [node 1] ICP30 unregister session Session# [6:7577470895090404817:2048] VirtualId# [6:7577470895090404806:0]
2025-11-27T18:17:05.455859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470895090404817:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.455917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:05.455953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:05.472211Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:05.472427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470892822855302:2049] [node 2] ICS01 socket: 83 reason# EndOfStream
2025-11-27T18:17:05.472461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470888527887888:2049] [node 2] ICP30 unregister session Session# [3:7577470892822855302:2049] VirtualId# [3:7577470892822855291:0]
2025-11-27T18:17:05.472487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470892822855302:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.472568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:05.472623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:05.474639Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:05.474837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470894654929549:2049] [node 2] ICS01 socket: 115 reason# EndOfStream
2025-11-27T18:17:05.474880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470894654929443:2049] [node 2] ICP30 unregister session Session# [4:7577470894654929549:2049] VirtualId# [4:7577470894654929539:0]
2025-11-27T18:17:05.474913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470894654929549:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.474980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:05.475019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:05.475530Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:05.475708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470892765736547:2049] [node 2] ICS07 socket disconnect 82 reason# EndOfStream
2025-11-27T18:17:05.475737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470892765736547:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.475798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:05.475836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470892765736547:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:05.476453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:05.476594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470892765736465:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15459) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:05.476626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470895090404811:2049] [node 2] ICS01 socket: 91 reason# EndOfStream
2025-11-27T18:17:05.476694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470892765736547:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:05.476702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470895090404721:2049] [node 2] ICP30 unregister session Session# [6:7577470895090404811:2049] VirtualId# [6:7577470895090404804:0]
2025-11-27T18:17:05.476751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470895090404811:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.476731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470892765736465:2049] [node 2] ICP30 unregister session Session# [5:7577470892765736547:2049] VirtualId# [5:7577470892765736543:0]
2025-11-27T18:17:05.476809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:05.476793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:05.476834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470892765736465:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15459) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:05.476846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15459) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:05.476846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:05.487204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470892765736465:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:05.520864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:05.521096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470894654929543:2050] [node 3] ICS07 socket disconnect 108 reason# EndOfStream
2025-11-27T18:17:05.521128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470894654929543:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.521219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:05.521283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470894654929543:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:05.521914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:05.522143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470892765736567:2050] [node 3] ICS01 socket: 133 reason# EndOfStream
2025-11-27T18:17:05.522186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470892765736466:2050] [node 3] ICP30 unregister session Session# [5:7577470892765736567:2050] VirtualId# [5:7577470892765736563:0]
2025-11-27T18:17:05.522212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470892765736567:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.522216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470894654929444:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7342) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:05.522334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:05.522343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470894654929543:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:05.522400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:05.522400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470894654929444:2050] [node 3] ICP30 unregister session Session# [4:7577470894654929543:2050] VirtualId# [4:7577470894654929537:0]
2025-11-27T18:17:05.522450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:05.522484Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470894654929444:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7342) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:05.522504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7342) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:05.523003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:05.523206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470895090404813:2050] [node 3] ICS01 socket: 126 reason# EndOfStream
2025-11-27T18:17:05.523260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470895090404722:2050] [node 3] ICP30 unregister session Session# [6:7577470895090404813:2050] VirtualId# [6:7577470895090404803:0]
2025-11-27T18:17:05.523284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470895090404813:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.523350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:05.523389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:05.553030Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:05.553284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470892765736565:2108] [node 4] ICS07 socket disconnect 129 reason# EndOfStream
2025-11-27T18:17:05.553337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470892765736565:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.553421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:05.553458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470892765736565:2108] [node 4] ICS15 start handshake
2025-11-27T18:17:05.554239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470892765736527:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25686) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:05.554351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470892765736565:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:05.554400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470892765736527:2108] [node 4] ICP30 unregister session Session# [5:7577470892765736565:2108] VirtualId# [5:7577470892765736555:0]
2025-11-27T18:17:05.554448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:05.554485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470892765736527:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25686) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:05.554497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25686) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:05.554024Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:05.554223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470895090404824:2103] [node 4] ICS07 socket disconnect 88 reason# EndOfStream
2025-11-27T18:17:05.554255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470895090404824:2103] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.554337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:05.554400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470895090404824:2103] [node 4] ICS15 start handshake
2025-11-27T18:17:05.555066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470895090404784:2103] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25686) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:05.555179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470895090404824:2103] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:05.555233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470895090404784:2103] [node 4] ICP30 unregister session Session# [6:7577470895090404824:2103] VirtualId# [6:7577470895090404822:0]
2025-11-27T18:17:05.555271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:05.555296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470895090404784:2103] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25686) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:05.555308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25686) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:05.563995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470892765736527:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:17:05.564991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470895090404784:2103] [node 4] ICP33 wake up from error state
2025-11-27T18:17:05.597128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:05.597401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470895090404820:2102] [node 5] ICS01 socket: 124 reason# EndOfStream
2025-11-27T18:17:05.597458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470895090404778:2102] [node 5] ICP30 unregister session Session# [6:7577470895090404820:2102] VirtualId# [6:7577470895090404805:0]
2025-11-27T18:17:05.597491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470895090404820:2102] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:05.597560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:05.597609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:05.669990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470894305632710:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:05.670139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470894305632711:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:05.689220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470894374010049:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:05.689269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470894374010050:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:05.712779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470895602747266:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:05.712779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470895602747265:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:05.737753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470892931115448:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:05.737801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470892931115449:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:05.743185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470892931115447:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:05.760418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470891780098227:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:05.760532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470891780098228:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:05.760535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470891780098229:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:05.782444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470895943980712:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:05.782474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470895943980713:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:05.782497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470895943980714:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:05.863275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470895943980774:2106] [node 5] ICP01 ready to work
2025-11-27T18:17:05.896457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470894305632777:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:05.896892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470894305632779:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:05.942395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470894305632778:2110] [node 5] ICP01 ready to work
2025-11-27T18:17:05.962440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470895602747329:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:06.001347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470892931115515:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:05.985618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470894374010113:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:06.006569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470892931115449:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(92399899ns) {Configure:104(92393609ns) {ProcessConfigured:135(76131ns) {StartInitialHandshake:157(74979ns) {DropHandshakes:470(3667ns) {DropIncomingHandshake:442(919ns) DropOutgoingHandshake:457(402ns)} GenerateSessionVirtualId:988(1227ns) PrepareNewSessionHandshake:477(2387ns) {SwitchToState:179(841ns)}}}}}
2025-11-27T18:17:06.000822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470895602747332:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:06.008986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470895943980774:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(125554680ns) {Configure:104(125548956ns) {ProcessConfigured:135(37708473ns) {StartInitialHandshake:157(37705192ns) {DropHandshakes:470(3374ns) {DropIncomingHandshake:442(816ns) DropOutgoingHandshake:457(475ns)} GenerateSessionVirtualId:988(1209ns) PrepareNewSessionHandshake:477(4524ns) {SwitchToState:179(2452ns)}}}}}
2025-11-27T18:17:05.998489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470894374010114:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:06.014277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470900238948074:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:06.014311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470896075065586:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:06.009204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470898668977414:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:06.007030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470897226082814:2112] [node 6] ICP01 ready to work
2025-11-27T18:17:06.014054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470895602747266:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(96275669ns) {Configure:104(96262706ns) {ProcessConfigured:135(11922525ns) {StartInitialHandshake:157(11911763ns) {DropHandshakes:470(5370ns) {DropIncomingHandshake:442(827ns) DropOutgoingHandshake:457(429ns)} GenerateSessionVirtualId:988(1434ns) PrepareNewSessionHandshake:477(5304ns) {SwitchToState:179(2660ns)}}}}}
2025-11-27T18:17:06.016890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470896075065587:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:06.014078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470899897714629:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:06.018480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470891780098228:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(91229033ns) {Configure:104(91217581ns) {ProcessConfigured:135(66188ns) {StartInitialHandshake:157(63535ns) {DropHandshakes:470(2287ns) {DropIncomingHandshake:442(497ns) DropOutgoingHandshake:457(310ns)} GenerateSessionVirtualId:988(1110ns) PrepareNewSessionHandshake:477(1680ns) {SwitchToState:179(567ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.022393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470894305632777:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(118800434ns) {Configure:104(118794970ns) {ProcessConfigured:135(1508061ns) {StartInitialHandshake:157(1503822ns) {DropHandshakes:470(2186ns) {DropIncomingHandshake:442(826ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(1210ns) PrepareNewSessionHandshake:477(3120ns) {SwitchToState:179(1234ns)}}}}}
2025-11-27T18:17:06.022955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470894305632711:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(156666816ns) {Configure:104(153038922ns) {ProcessConfigured:135(5323095ns) {StartInitialHandshake:157(5320392ns) {DropHandshakes:470(3495ns) {DropIncomingHandshake:442(1093ns) DropOutgoingHandshake:457(644ns)} GenerateSessionVirtualId:988(767ns) PrepareNewSessionHandshake:477(4087ns) {SwitchToState:179(1743ns)}}}}}
2025-11-27T18:17:06.022794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470894374010109:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21313
2025-11-27T18:17:06.033946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470895602747265:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(141788652ns) {Configure:104(141779184ns) {ProcessConfigured:135(1228399ns) {StartInitialHandshake:157(1212194ns) {DropHandshakes:470(3127ns) {DropIncomingHandshake:442(735ns) DropOutgoingHandshake:457(481ns)} GenerateSessionVirtualId:988(731ns) PrepareNewSessionHandshake:477(2757ns) {SwitchToState:179(980ns)}}}}}
2025-11-27T18:17:06.031168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470891780098227:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(96232890ns) {Configure:104(96225052ns) {ProcessConfigured:135(2379365ns) {StartInitialHandshake:157(2377540ns) {DropHandshakes:470(2512ns) {DropIncomingHandshake:442(555ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(635ns) PrepareNewSessionHandshake:477(3324ns) {SwitchToState:179(1251ns)}}}}}
2025-11-27T18:17:06.034882Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470894374010114:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25240729ns) {Configure:104(25230736ns) {ProcessConfigured:135(684558ns) {StartInitialHandshake:157(675085ns) {DropHandshakes:470(1937ns) {DropIncomingHandshake:442(542ns) DropOutgoingHandshake:457(173ns)} GenerateSessionVirtualId:988(613ns) PrepareNewSessionHandshake:477(1825ns) {SwitchToState:179(937ns)}}}}}
2025-11-27T18:17:06.039701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470895943980713:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24421571ns) {Configure:104(24415468ns) {ProcessConfigured:135(3770149ns) {StartInitialHandshake:157(2456239ns) {DropHandshakes:470(2103ns) {DropIncomingHandshake:442(312ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(1631ns) PrepareNewSessionHandshake:477(4253ns) {SwitchToState:179(2333ns)}}}}}
2025-11-27T18:17:06.040495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470894374010113:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25920342ns) {Configure:104(25912426ns) {ProcessConfigured:135(1600402ns) {StartInitialHandshake:157(1598220ns) {DropHandshakes:470(1418ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(749ns) PrepareNewSessionHandshake:477(6176ns) {SwitchToState:179(4847ns)}}}}}
2025-11-27T18:17:06.044138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470894305632779:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(97517853ns) {Configure:104(97511331ns) {ProcessConfigured:135(1690842ns) {StartInitialHandshake:157(1689164ns) {DropHandshakes:470(1315ns) {DropIncomingHandshake:442(474ns) DropOutgoingHandshake:457(181ns)} GenerateSessionVirtualId:988(349ns) PrepareNewSessionHandshake:477(2563ns) {SwitchToState:179(895ns)}}}}}
2025-11-27T18:17:06.044779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470892931115447:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(111385302ns) {Configure:104(111377297ns) {ProcessConfigured:135(1795805ns) {StartInitialHandshake:157(1794938ns) {DropHandshakes:470(1070ns) {DropIncomingHandshake:442(234ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(371ns) PrepareNewSessionHandshake:477(1966ns) {SwitchToState:179(599ns)}}}}}
2025-11-27T18:17:06.048274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470894374010050:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18257276ns) {Configure:104(18252328ns) {ProcessConfigured:135(1048892ns) {StartInitialHandshake:157(1047709ns) {DropHandshakes:470(1129ns) {DropIncomingHandshake:442(322ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(374ns) PrepareNewSessionHandshake:477(1253ns) {SwitchToState:179(172ns)}}}}}
2025-11-27T18:17:06.048270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470895602747332:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18720097ns) {Configure:104(18711361ns) {ProcessConfigured:135(1226181ns) {StartInitialHandshake:157(1223454ns) {DropHandshakes:470(1082ns) {DropIncomingHandshake:442(284ns) DropOutgoingHandshake:457(140ns)} GenerateSessionVirtualId:988(447ns) PrepareNewSessionHandshake:477(1614ns) {SwitchToState:179(667ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.049557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470895602747329:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(28275742ns) {Configure:104(28271381ns) {ProcessConfigured:135(3070940ns) {StartInitialHandshake:157(3069733ns) {DropHandshakes:470(245922ns) {DropIncomingHandshake:442(1236ns) DropOutgoingHandshake:457(423ns)} GenerateSessionVirtualId:988(1139ns) PrepareNewSessionHandshake:477(2915ns) {SwitchToState:179(1330ns)}}}}}
2025-11-27T18:17:06.057911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470897226082814:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(34153100ns) {Configure:104(34147453ns) {ProcessConfigured:135(12281578ns) {StartInitialHandshake:157(12278753ns) {DropHandshakes:470(1301ns) {DropIncomingHandshake:442(359ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(443ns) PrepareNewSessionHandshake:477(3860ns) {SwitchToState:179(1928ns)}}}}}
2025-11-27T18:17:06.057526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470891780098229:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31228122ns) {Configure:104(31223609ns) {ProcessConfigured:135(7121113ns) {StartInitialHandshake:157(7118836ns) {DropHandshakes:470(1007ns) {DropIncomingHandshake:442(306ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(7609ns) {SwitchToState:179(4637ns)}}}}}
2025-11-27T18:17:06.060856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470894305632778:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(33831723ns) {Configure:104(33813063ns) {ProcessConfigured:135(7343355ns) {StartInitialHandshake:157(7341191ns) {DropHandshakes:470(2761ns) {DropIncomingHandshake:442(903ns) DropOutgoingHandshake:457(300ns)} GenerateSessionVirtualId:988(1160ns) PrepareNewSessionHandshake:477(2335ns) {SwitchToState:179(904ns)}}}}}
2025-11-27T18:17:06.067301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470898668977413:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21313
2025-11-27T18:17:06.056694Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470895943980714:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29495707ns) {Configure:104(29490962ns) {ProcessConfigured:135(71215ns) {StartInitialHandshake:157(69442ns) {DropHandshakes:470(1341ns) {DropIncomingHandshake:442(286ns) DropOutgoingHandshake:457(181ns)} GenerateSessionVirtualId:988(724ns) PrepareNewSessionHandshake:477(905ns) {SwitchToState:179(340ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.069596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470896075065586:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(34458436ns) {Configure:104(34449989ns) {ProcessConfigured:135(4029505ns) {StartInitialHandshake:157(4026981ns) {DropHandshakes:470(3038ns) {DropIncomingHandshake:442(969ns) DropOutgoingHandshake:457(277ns)} GenerateSessionVirtualId:988(601ns) PrepareNewSessionHandshake:477(2987ns) {SwitchToState:179(947ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.067578Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470895943980712:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(70967433ns) {Configure:104(70960477ns) {ProcessConfigured:135(12234916ns) {StartInitialHandshake:157(12233632ns) {DropHandshakes:470(589ns) {DropIncomingHandshake:442(155ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:477(2854ns) {SwitchToState:179(623ns)}}}}}
2025-11-27T18:17:06.085782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470896075065601:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49269
2025-11-27T18:17:06.087189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470898600600083:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25966
2025-11-27T18:17:06.087966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470900238948074:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31266363ns) {Configure:104(31256882ns) {ProcessConfigured:135(48544ns) {StartInitialHandshake:157(46004ns) {DropHandshakes:470(1494ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(362ns) PrepareNewSessionHandshake:477(752ns) {SwitchToState:179(316ns)}}}}}
2025-11-27T18:17:06.086967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470896075065601:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.089121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470899897714629:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48776108ns) {Configure:104(48767934ns) {ProcessConfigured:135(490107ns) {StartInitialHandshake:157(487145ns) {DropHandshakes:470(1739ns) {DropIncomingHandshake:442(391ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(1637ns) PrepareNewSessionHandshake:477(96661ns) {SwitchToState:179(785ns)}}}}}
2025-11-27T18:17:06.087265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470896075065586:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.089233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470898600600083:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.089211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470898668977414:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(49053837ns) {Configure:104(49048145ns) {ProcessConfigured:135(446652ns) {StartInitialHandshake:157(444001ns) {DropHandshakes:470(2116ns) {DropIncomingHandshake:442(513ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(1262ns) PrepareNewSessionHandshake:477(1775ns) {SwitchToState:179(878ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.090590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470897226082816:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.089393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470898668977413:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.090630Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470897226082816:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.093176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470900238948081:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31870
2025-11-27T18:17:06.092547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470897226082816:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:06.092563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470898668977421:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21313
2025-11-27T18:17:06.093480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470898600600088:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25966
2025-11-27T18:17:06.093136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470897226082823:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45814
2025-11-27T18:17:06.094051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470892931115515:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.094561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470899897714642:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41849
2025-11-27T18:17:06.094101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:06.094998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470899897714639:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41849
2025-11-27T18:17:06.095309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470898668977421:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.094859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470892931115515:2109] [node 5] ICP22 created new session: [4:7577470897226082825:2109]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:17:06.095488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470900238948081:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.095017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470897226082823:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.095493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470892931115447:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.096038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470899897714642:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.095619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470894374010109:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:17:06.096384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470895602747265:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.095623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470897226082825:2109] [node 5] ICS09 handshake done sender: [4:7577470897226082816:2114] self: [4:7577470897226082815:0] peer: [5:7577470896075065602:0] socket: 90 qp: -1
2025-11-27T18:17:06.096028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470896075065604:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49269
2025-11-27T18:17:06.101473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470898668977425:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21313
2025-11-27T18:17:06.097475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470899897714643:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41849
2025-11-27T18:17:06.095660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470897226082825:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.102589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470898600600090:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25966
2025-11-27T18:17:06.103707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470896075065603:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49269
2025-11-27T18:17:06.102817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470898668977426:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21313
No RDMA context found for GID: 2025-11-27T18:17:06.102506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470895943980774:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-27T18:17:06.102593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470898600600089:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25966
2025-11-27T18:17:06.095917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:06.102481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470899897714639:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.104428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470894374010049:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.113231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470899897714643:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.107500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470898668977425:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.107481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470897226082829:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45814
2025-11-27T18:17:06.108667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470896075065606:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49269
2025-11-27T18:17:06.102712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470898600600077:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.112689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470896075065601:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:06.113193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470896075065605:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49269
2025-11-27T18:17:06.113229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470896075065593:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.102743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470898600600077:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.119547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470892931115515:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(20149004ns) {HandleHandshakeStatus:329(20145898ns) {DropIncomingHandshake:442(420ns) DropOutgoingHandshake:457(669ns) SwitchToState:179(649ns) ProcessPendingSessionEvents:495(11672367ns) {ForwardSessionEventToSession:612(1766273ns) ForwardSessionEventToSession:612(775169ns) ForwardSessionEventToSession:612(1630373ns) ForwardSessionEventToSession:612(3839387ns) ForwardSessionEventToSession:612(7981ns) ForwardSessionEventToSession:612(2044ns) ForwardSessionEventToSession:612(2057456ns) ForwardSessionEventToSession:612(3431ns) ForwardSessionEventToSession:612(607216ns) ForwardSessionEventToSession:612(956591ns)}}}
2025-11-27T18:17:06.113274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470896075065593:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.105035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470898600600079:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.122218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470898668977426:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.120998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470899897714646:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41849
2025-11-27T18:17:06.118165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470896075065605:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.105066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470898600600079:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.106871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470894305632771:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.125467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470897226082829:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.106911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470894305632771:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.119168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470896075065586:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.109029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470898600600090:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.119221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.110625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470898600600077:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:06.126924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470892931115448:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.119583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470894305632771:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:06.127225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470898668977418:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.119291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470896075065603:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.127831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470899897714642:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:06.127260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470898668977418:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.127973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470900238948088:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31870
2025-11-27T18:17:06.119475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470896075065586:2108] [node 4] ICP22 created new session: [5:7577470896075065608:2108]
2025-11-27T18:17:06.119668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470898600600079:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:06.129168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470895602747265:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.119710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470896075065593:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:06.128776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470898668977433:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21313
2025-11-27T18:17:06.133380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470897226082830:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45814
2025-11-27T18:17:06.126829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470898600600088:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.122132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470896075065608:2108] [node 4] ICS09 handshake done sender: [5:7577470896075065601:2115] self: [5:7577470896075065602:0] peer: [4:7577470897226082815:0] socket: 93 qp: -1
2025-11-27T18:17:06.129597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:06.136368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470898668977418:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:06.126956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470894305632777:2108] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.130321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470895602747265:2048] [node 1] ICP22 created new session: [3:7577470899897714648:2048]
2025-11-27T18:17:06.122177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470896075065608:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:06.122607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470896075065606:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.138443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470900238948081:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:06.138360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470894374010109:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:06.135657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470892931115447:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470897226082823:2117] held: no
2025-11-27T18:17:06.126999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:06.127361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470894305632777:2108] [node 4] ICP22 created new session: [1:7577470898600600095:2108]
2025-11-27T18:17:06.134254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470899897714648:2048] [node 1] ICS09 handshake done sender: [3:7577470899897714642:2117] self: [3:7577470899897714644:0] peer: [1:7577470898600600078:0] socket: 103 qp: -1
2025-11-27T18:17:06.139664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470897226082823:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:06.122681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470891780098229:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.138737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470900238948089:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31870
2025-11-27T18:17:06.125434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:06.140822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470892931115447:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.135802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470899897714648:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:06.140216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470900238948089:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.141833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470898668977416:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.127514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470894305632711:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.127552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:06.135804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470899897714633:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.140397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470895943980774:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.141837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:06.126867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470896075065587:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.141875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470898668977416:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.140786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:06.127699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470898600600095:2108] [node 4] ICS09 handshake done sender: [1:7577470898600600077:2112] self: [1:7577470898600600076:0] peer: [4:7577470897226082827:0] socket: 94 qp: -1
2025-11-27T18:17:06.142642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470892931115447:2048] [node 1] ICP22 created new session: [4:7577470897226082834:2048]
2025-11-27T18:17:06.135836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470899897714633:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.136061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:06.143633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470899897714633:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:06.144171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470895602747332:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.144294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:06.144507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470895602747332:2108] [node 5] ICP22 created new session: [3:7577470899897714653:2108]
2025-11-27T18:17:06.146824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470899897714653:2108] [node 5] ICS09 handshake done sender: [3:7577470899897714633:2113] self: [3:7577470899897714632:0] peer: [5:7577470896075065610:0] socket: 113 qp: -1
2025-11-27T18:17:06.148120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470899897714653:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:06.149993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:06.126907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:06.142095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470894374010049:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.127497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470894305632710:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.142729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470895943980712:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.127653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470896075065587:2109] [node 6] ICP22 created new session: [5:7577470896075065611:2109]
2025-11-27T18:17:06.142651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:06.127781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:06.143845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470897226082834:2048] [node 1] ICS09 handshake done sender: [4:7577470897226082823:2117] self: [4:7577470897226082827:0] peer: [1:7577470898600600076:0] socket: 95 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.143908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470897226082834:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:06.142995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470895943980774:2106] [node 5] ICP22 created new session: [6:7577470900238948091:2106]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.143348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470894374010113:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.127822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470896075065611:2109] [node 6] ICS09 handshake done sender: [5:7577470896075065593:2112] self: [5:7577470896075065592:0] peer: [6:7577470900238948087:0] socket: 85 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.127995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470894305632710:2048] [node 2] ICP22 created new session: [1:7577470898600600097:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.162176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470899897714650:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41849
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.127990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470896075065611:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:06.143906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470900238948091:2106] [node 5] ICS09 handshake done sender: [6:7577470900238948081:2114] self: [6:7577470900238948087:0] peer: [5:7577470896075065592:0] socket: 91 qp: -1
2025-11-27T18:17:06.144408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:06.128020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470894305632711:2049] [node 3] ICP22 created new session: [1:7577470898600600096:2049]
2025-11-27T18:17:06.143400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:06.128032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.173228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470897226082833:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45814
2025-11-27T18:17:06.143445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470894374010113:2108] [node 4] ICP22 created new session: [2:7577470898668977437:2108]
2025-11-27T18:17:06.144168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470900238948091:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:06.128101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470898600600095:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:06.168415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470895602747332:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(22043496ns) {HandleHandshakeStatus:329(22040749ns) {DropIncomingHandshake:442(268ns) DropOutgoingHandshake:457(274ns) SwitchToState:179(650ns) ProcessPendingSessionEvents:495(4250410ns) {ForwardSessionEventToSession:612(2317827ns) ForwardSessionEventToSession:612(8052ns) ForwardSessionEventToSession:612(14670ns) ForwardSessionEventToSession:612(4055ns) ForwardSessionEventToSession:612(2366ns) ForwardSessionEventToSession:612(216221ns) ForwardSessionEventToSession:612(2950ns) ForwardSessionEventToSession:612(277439ns) ForwardSessionEventToSession:612(1385487ns) ForwardSessionEventToSession:612(3073ns)}}}
2025-11-27T18:17:06.135676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470891780098228:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.144572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:06.143486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470898668977437:2108] [node 4] ICS09 handshake done sender: [2:7577470898668977418:2114] self: [2:7577470898668977417:0] peer: [4:7577470897226082831:0] socket: 120 qp: -1
2025-11-27T18:17:06.128132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:06.177577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470892931115448:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16723) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470897226082829:2118] held: no
2025-11-27T18:17:06.139020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577470896075065587:2109] [node 6] ICP18 handshake (actor [5:7577470896075065603:2116]) is too old
2025-11-27T18:17:06.171434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470899897714650:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.129243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470898600600096:2049] [node 3] ICS09 handshake done sender: [1:7577470898600600079:2113] self: [1:7577470898600600078:0] peer: [3:7577470899897714644:0] socket: 96 qp: -1
2025-11-27T18:17:06.143532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470898668977437:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:06.164476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470900238948090:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31870
2025-11-27T18:17:06.173775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470899897714656:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41849
2025-11-27T18:17:06.178111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470897226082836:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45814
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.143570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:06.180328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470897226082829:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:06.172107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470896075065615:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49269
2025-11-27T18:17:06.129322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470898600600096:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:06.178853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577470895602747332:2108] [node 5] ICP18 handshake (actor [3:7577470899897714650:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.166476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470895943980774:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(25889566ns) {HandleHandshakeStatus:329(25887621ns) {DropIncomingHandshake:442(831ns) DropOutgoingHandshake:457(14379ns) SwitchToState:179(523ns) ProcessPendingSessionEvents:495(12277067ns) {ForwardSessionEventToSession:612(9860322ns) ForwardSessionEventToSession:612(10502ns) ForwardSessionEventToSession:612(1772511ns) ForwardSessionEventToSession:612(3292ns) ForwardSessionEventToSession:612(2470ns) ForwardSessionEventToSession:612(5391ns) ForwardSessionEventToSession:612(2534ns) ForwardSessionEventToSession:612(11751ns) ForwardSessionEventToSession:612(1885ns) ForwardSessionEventToSession:612(2356ns)}}}
2025-11-27T18:17:06.182007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470897226082836:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.172492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470896075065617:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49269
2025-11-27T18:17:06.144049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470894374010049:2048] [node 1] ICP22 created new session: [2:7577470898668977436:2048]
2025-11-27T18:17:06.180193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470896075065606:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:06.179490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470900238948090:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.129320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470898600600097:2048] [node 2] ICS09 handshake done sender: [1:7577470894305632771:2107] self: [1:7577470894305632770:0] peer: [2:7577470898668977431:0] socket: 78 qp: -1
2025-11-27T18:17:06.181600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470896075065615:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.182723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470892931115448:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.199988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470899897714635:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.179764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470900238948096:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31870
2025-11-27T18:17:06.131392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470898600600089:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.182249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470896075065616:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49269
2025-11-27T18:17:06.144450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470898668977436:2048] [node 1] ICS09 handshake done sender: [2:7577470894374010109:2107] self: [2:7577470898668977431:0] peer: [1:7577470894305632770:0] socket: 79 qp: -1
2025-11-27T18:17:06.179839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470900238948074:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.205408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470900238948095:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31870
2025-11-27T18:17:06.211831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470900238948095:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.186275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:06.133420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470898600600097:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:06.200025Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470899897714635:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.145271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470898668977436:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:06.186097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470891780098228:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.212124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470895943980714:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.187066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470897226082822:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.133464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:06.211334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470899897714635:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:06.145705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:06.134332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:06.171026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470898668977416:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:06.187109Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470897226082822:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.212189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470900238948099:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31870
2025-11-27T18:17:06.186144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:06.214162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470895602747329:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.142664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470894305632710:2048] [node 2] ICP18 handshake (actor [1:7577470898600600088:2117]) is too old
2025-11-27T18:17:06.212246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470900238948098:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31870
2025-11-27T18:17:06.187113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470891780098228:2049] [node 2] ICP22 created new session: [5:7577470896075065618:2049]
2025-11-27T18:17:06.187424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470892931115448:2049] [node 2] ICP22 created new session: [4:7577470897226082837:2049]
2025-11-27T18:17:06.143075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470894305632711:2049] [node 3] ICP18 handshake (actor [1:7577470898600600089:2118]) is too old
2025-11-27T18:17:06.214217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:06.180319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470894374010114:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.192675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470892931115449:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.218977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470900238948096:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.180379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:06.189368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470896075065618:2049] [node 2] ICS09 handshake done sender: [5:7577470896075065606:2119] self: [5:7577470896075065614:0] peer: [2:7577470898668977415:0] socket: 117 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.164415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470898600600081:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.214290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470899897714656:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.192758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470897226082837:2049] [node 2] ICS09 handshake done sender: [4:7577470897226082829:2118] self: [4:7577470897226082831:0] peer: [2:7577470898668977417:0] socket: 123 qp: -1
2025-11-27T18:17:06.219015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470900238948090:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:06.192807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470897226082837:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:06.180639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470894374010049:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25143309ns) {HandleHandshakeStatus:329(25139409ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(5831ns) SwitchToState:179(473ns) ProcessPendingSessionEvents:495(13192528ns) {ForwardSessionEventToSession:612(7812528ns) ForwardSessionEventToSession:612(4940ns) ForwardSessionEventToSession:612(2198577ns) ForwardSessionEventToSession:612(512966ns) ForwardSessionEventToSession:612(7683ns) ForwardSessionEventToSession:612(137126ns) ForwardSessionEventToSession:612(11877ns) ForwardSessionEventToSession:612(2421ns) ForwardSessionEventToSession:612(2479162ns) ForwardSessionEventToSession:612(2912ns)}}}
2025-11-27T18:17:06.214338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470895602747329:2107] [node 4] ICP22 created new session: [3:7577470899897714658:2107]
2025-11-27T18:17:06.192774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470896075065618:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:06.164463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470898600600081:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.192846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:06.181248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470894374010114:2109] [node 5] ICP22 created new session: [2:7577470898668977443:2109]
2025-11-27T18:17:06.192845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:06.220898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470900238948089:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:06.225883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470895943980713:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.197642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470898600600103:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25966
2025-11-27T18:17:06.214646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470899897714637:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.193330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470891780098229:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31139) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470896075065605:2118] held: no
2025-11-27T18:17:06.200099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470892931115448:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17305406ns) {HandleHandshakeStatus:329(17302436ns) {DropIncomingHandshake:442(992ns) DropOutgoingHandshake:457(366ns) SwitchToState:179(500ns) ProcessPendingSessionEvents:495(1399ns)}}
2025-11-27T18:17:06.181967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470898668977443:2109] [node 5] ICS09 handshake done sender: [2:7577470898668977416:2113] self: [2:7577470898668977415:0] peer: [5:7577470896075065614:0] socket: 116 qp: -1
2025-11-27T18:17:06.225959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470900238948074:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.214682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470899897714637:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.199804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470896075065605:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:06.198983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470898600600081:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:06.205806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470897226082822:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:06.182359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470898668977443:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:06.206763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470891780098227:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.226266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470895943980712:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.206144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470897226082814:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.211320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470898600600103:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.215144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470899897714658:2107] [node 4] ICS09 handshake done sender: [3:7577470899897714635:2114] self: [3:7577470899897714634:0] peer: [4:7577470897226082838:0] socket: 101 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.182398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:06.210874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:06.213277Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470898600600084:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.217465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470899897714658:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:06.226316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.206817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470891780098229:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.218982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:06.199966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470894374010114:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(18752799ns) {HandleHandshakeStatus:329(18749582ns) {DropIncomingHandshake:442(12524ns) DropOutgoingHandshake:457(357ns) SwitchToState:179(274ns) ProcessPendingSessionEvents:495(6156850ns) {ForwardSessionEventToSession:612(4862334ns) ForwardSessionEventToSession:612(25920ns) ForwardSessionEventToSession:612(2713ns) ForwardSessionEventToSession:612(5413ns) ForwardSessionEventToSession:612(2433ns) ForwardSessionEventToSession:612(2011ns) ForwardSessionEventToSession:612(1228424ns) ForwardSessionEventToSession:612(3209ns) ForwardSessionEventToSession:612(5683ns) ForwardSessionEventToSession:612(5191ns)}}}
2025-11-27T18:17:06.220581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470895602747266:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.206858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:06.221133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470898668977420:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.235793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470899897714637:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:06.221155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470898668977420:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.206887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470891780098229:2050] [node 3] ICP22 created new session: [5:7577470896075065621:2050]
2025-11-27T18:17:06.211556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470897226082814:2112] [node 6] ICP22 created new session: [4:7577470897226082841:2112]
2025-11-27T18:17:06.226692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470895943980712:2048] [node 1] ICP22 created new session: [6:7577470900238948102:2048]
2025-11-27T18:17:06.241841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470895602747329:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(23698181ns) {HandleHandshakeStatus:329(23225581ns) {DropIncomingHandshake:442(12834ns) DropOutgoingHandshake:457(325ns) SwitchToState:179(310ns) ProcessPendingSessionEvents:495(3502420ns) {ForwardSessionEventToSession:612(2559609ns) ForwardSessionEventToSession:612(4082ns) ForwardSessionEventToSession:612(113569ns) ForwardSessionEventToSession:612(3413ns) ForwardSessionEventToSession:612(347442ns) ForwardSessionEventToSession:612(3320ns) ForwardSessionEventToSession:612(2506ns) ForwardSessionEventToSession:612(446397ns) ForwardSessionEventToSession:612(6491ns) ForwardSessionEventToSession:612(2642ns)}}}
2025-11-27T18:17:06.213303Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470898600600084:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.226008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470898668977420:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:06.234408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470894374010050:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.234464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:06.235313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470894374010050:2049] [node 3] ICP22 created new session: [2:7577470898668977446:2049]
2025-11-27T18:17:06.236475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470898668977446:2049] [node 3] ICS09 handshake done sender: [2:7577470898668977420:2115] self: [2:7577470898668977419:0] peer: [3:7577470899897714660:0] socket: 143 qp: -1
2025-11-27T18:17:06.236726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470898668977446:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:06.236798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:06.240269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470898668977423:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.240291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470898668977423:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.242277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470898668977423:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:06.260544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470898668977414:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.266445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:06.266508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470898668977414:2110] [node 6] ICP22 created new session: [2:7577470898668977449:2110]
2025-11-27T18:17:06.268153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470898668977449:2110] [node 6] ICS09 handshake done sender: [2:7577470898668977423:2117] self: [2:7577470898668977422:0] peer: [6:7577470900238948101:0] socket: 107 qp: -1
2025-11-27T18:17:06.269802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470898668977449:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:06.269957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:06.235047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470900238948102:2048] [node 1] ICS09 handshake done sender: [6:7577470900238948089:2117] self: [6:7577470900238948092:0] peer: [1:7577470898600600080:0] socket: 135 qp: -1
2025-11-27T18:17:06.242197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470899897714629:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.213477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470894305632779:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.213745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:06.214125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470894305632779:2111] [node 6] ICP22 created new session: [1:7577470898600600105:2111]
2025-11-27T18:17:06.215127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470898600600105:2111] [node 6] ICS09 handshake done sender: [1:7577470898600600081:2114] self: [1:7577470898600600080:0] peer: [6:7577470900238948092:0] socket: 118 qp: -1
2025-11-27T18:17:06.217455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470898600600105:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:06.220980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:06.220981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470898600600084:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:06.234428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470894305632778:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.235269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:06.235713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470894305632778:2110] [node 5] ICP22 created new session: [1:7577470898600600108:2110]
2025-11-27T18:17:06.236831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470898600600108:2110] [node 5] ICS09 handshake done sender: [1:7577470898600600084:2116] self: [1:7577470898600600082:0] peer: [5:7577470896075065620:0] socket: 132 qp: -1
2025-11-27T18:17:06.236868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470898600600108:2110] [node 5] ICS10 traffic start
2025-11-27T18:17:06.236903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:06.240058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470894305632779:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(22653729ns) {HandleHandshakeStatus:329(22284108ns) {DropIncomingHandshake:442(561ns) DropOutgoingHandshake:457(528ns) SwitchToState:179(435ns) ProcessPendingSessionEvents:495(8970091ns) {ForwardSessionEventToSession:612(6798212ns) ForwardSessionEventToSession:612(3789ns) ForwardSessionEventToSession:612(842612ns) ForwardSessionEventToSession:612(2670ns) ForwardSessionEventToSession:612(310542ns) ForwardSessionEventToSession:612(4059ns) ForwardSessionEventToSession:612(2324ns) ForwardSessionEventToSession:612(973302ns) ForwardSessionEventToSession:612(6939ns) ForwardSessionEventToSession:612(2497ns)}}}
2025-11-27T18:17:06.242163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470894305632779:2111] [node 6] ICP18 handshake (actor [1:7577470898600600103:2120]) is too old
2025-11-27T18:17:06.270841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470894305632778:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(26694514ns) {HandleHandshakeStatus:329(26691655ns) {DropIncomingHandshake:442(16408ns) DropOutgoingHandshake:457(377ns) SwitchToState:179(366ns) ProcessPendingSessionEvents:495(18630469ns) {ForwardSessionEventToSession:612(1187355ns) ForwardSessionEventToSession:612(4566ns) ForwardSessionEventToSession:612(1419697ns) ForwardSessionEventToSession:612(3359ns) ForwardSessionEventToSession:612(1237112ns) ForwardSessionEventToSession:612(4960188ns) ForwardSessionEventToSession:612(2260ns) ForwardSessionEventToSession:612(9789074ns) ForwardSessionEventToSession:612(4158ns) ForwardSessionEventToSession:612(2500ns)}}}
2025-11-27T18:17:06.206928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470896075065621:2050] [node 3] ICS09 handshake done sender: [5:7577470896075065605:2118] self: [5:7577470896075065610:0] peer: [3:7577470899897714632:0] socket: 115 qp: -1
2025-11-27T18:17:06.212112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470897226082841:2112] [node 6] ICS09 handshake done sender: [4:7577470897226082822:2116] self: [4:7577470897226082820:0] peer: [6:7577470900238948097:0] socket: 137 qp: -1
2025-11-27T18:17:06.235309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:06.242245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:06.213193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470897226082841:2112] [node 6] ICS10 traffic start
2025-11-27T18:17:06.206958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470896075065621:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:06.244795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470899897714629:2110] [node 6] ICP22 created new session: [3:7577470899897714663:2110]
2025-11-27T18:17:06.213747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:06.207002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:06.235375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470900238948102:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:06.245672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470895602747266:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16723) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470899897714656:2121] held: no
2025-11-27T18:17:06.235735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:06.226652Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470892931115449:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31139) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470897226082836:2121] held: no
2025-11-27T18:17:06.260199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470899897714663:2110] [node 6] ICS09 handshake done sender: [3:7577470899897714637:2115] self: [3:7577470899897714636:0] peer: [6:7577470900238948100:0] socket: 114 qp: -1
2025-11-27T18:17:06.218304Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470891780098228:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(25217845ns) {HandleHandshakeStatus:329(25214056ns) {DropIncomingHandshake:442(681ns) DropOutgoingHandshake:457(4301ns) SwitchToState:179(491ns) ProcessPendingSessionEvents:495(13633565ns) {ForwardSessionEventToSession:612(2085034ns) ForwardSessionEventToSession:612(1145397ns) ForwardSessionEventToSession:612(3565ns) ForwardSessionEventToSession:612(4776186ns) ForwardSessionEventToSession:612(551134ns) ForwardSessionEventToSession:612(2875ns) ForwardSessionEventToSession:612(4614391ns) ForwardSessionEventToSession:612(4374ns) ForwardSessionEventToSession:612(2135ns) ForwardSessionEventToSession:612(426232ns)}}}
2025-11-27T18:17:06.260248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470899897714663:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:06.235958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470897226082842:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45814
2025-11-27T18:17:06.218426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470891780098228:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:06.236391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470900238948074:2109] [node 4] ICP22 created new session: [6:7577470900238948103:2109]
2025-11-27T18:17:06.264784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:06.237125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470897226082836:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:06.243022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470896075065623:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49269
2025-11-27T18:17:06.241771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470892931115449:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.268181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470899897714661:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41849
2025-11-27T18:17:06.241823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:06.272934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470899897714629:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(29367068ns) {HandleHandshakeStatus:329(29364411ns) {DropIncomingHandshake:442(12214ns) DropOutgoingHandshake:457(349ns) SwitchToState:179(376ns) ProcessPendingSessionEvents:495(4359033ns) {ForwardSessionEventToSession:612(2529995ns) ForwardSessionEventToSession:612(207120ns) ForwardSessionEventToSession:612(3131ns) ForwardSessionEventToSession:612(657278ns) ForwardSessionEventToSession:612(9306ns) ForwardSessionEventToSession:612(2714ns) ForwardSessionEventToSession:612(171210ns) ForwardSessionEventToSession:612(155333ns) ForwardSessionEventToSession:612(4998ns) ForwardSessionEventToSession:612(599947ns)}}}
2025-11-27T18:17:06.243078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470891780098227:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470896075065615:2120] held: no
2025-11-27T18:17:06.236818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470900238948103:2109] [node 4] ICS09 handshake done sender: [6:7577470900238948090:2118] self: [6:7577470900238948097:0] peer: [4:7577470897226082820:0] socket: 138 qp: -1
2025-11-27T18:17:06.273123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470899897714656:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:06.241953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470892931115449:2050] [node 3] ICP22 created new session: [4:7577470897226082844:2050]
2025-11-27T18:17:06.237326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470900238948103:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:06.260652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470896075065615:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:06.274321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470895602747266:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.250696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470897226082844:2050] [node 3] ICS09 handshake done sender: [4:7577470897226082836:2121] self: [4:7577470897226082838:0] peer: [3:7577470899897714634:0] socket: 140 qp: -1
2025-11-27T18:17:06.237363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:06.266431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470891780098227:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.274381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:06.260504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470897226082844:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:06.249535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470895943980712:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18109938ns) {HandleHandshakeStatus:329(18106806ns) {DropIncomingHandshake:442(501ns) DropOutgoingHandshake:457(6256ns) SwitchToState:179(243ns) ProcessPendingSessionEvents:495(7123442ns) {ForwardSessionEventToSession:612(2540780ns) ForwardSessionEventToSession:612(312258ns) ForwardSessionEventToSession:612(5165ns) ForwardSessionEventToSession:612(1290838ns) ForwardSessionEventToSession:612(3399ns) ForwardSessionEventToSession:612(2469ns) ForwardSessionEventToSession:612(290645ns) ForwardSessionEventToSession:612(2936ns) ForwardSessionEventToSession:612(220478ns) ForwardSessionEventToSession:612(2436183ns)}}}
2025-11-27T18:17:06.267989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:06.274521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470895602747266:2049] [node 2] ICP22 created new session: [3:7577470899897714665:2049]
2025-11-27T18:17:06.264679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:06.268053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470891780098227:2048] [node 1] ICP22 created new session: [5:7577470896075065624:2048]
2025-11-27T18:17:06.250499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470900238948106:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31870
2025-11-27T18:17:06.274556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470899897714665:2049] [node 2] ICS09 handshake done sender: [3:7577470899897714656:2121] self: [3:7577470899897714660:0] peer: [2:7577470898668977419:0] socket: 144 qp: -1
2025-11-27T18:17:06.268175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470896075065624:2048] [node 1] ICS09 handshake done sender: [5:7577470896075065615:2120] self: [5:7577470896075065620:0] peer: [1:7577470898600600082:0] socket: 133 qp: -1
2025-11-27T18:17:06.268995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470892931115449:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(27151367ns) {HandleHandshakeStatus:329(27148206ns) {DropIncomingHandshake:442(1639ns) DropOutgoingHandshake:457(602ns) SwitchToState:179(473ns) ProcessPendingSessionEvents:495(1049ns)}}
2025-11-27T18:17:06.274586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470899897714665:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:06.257633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470900238948107:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31870
2025-11-27T18:17:06.274620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:06.270556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470896075065624:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:06.267155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470900238948095:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:06.270797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:06.270995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470900238948074:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(41238625ns) {HandleHandshakeStatus:329(41235775ns) {DropIncomingHandshake:442(580ns) DropOutgoingHandshake:457(4740ns) SwitchToState:179(258ns) ProcessPendingSessionEvents:495(21412551ns) {ForwardSessionEventToSession:612(18992787ns) ForwardSessionEventToSession:612(323843ns) ForwardSessionEventToSession:612(2270ns) ForwardSessionEventToSession:612(2047874ns) ForwardSessionEventToSession:612(5794ns) ForwardSessionEventToSession:612(2791ns) ForwardSessionEventToSession:612(6315ns) ForwardSessionEventToSession:612(2218ns) ForwardSessionEventToSession:612(5560ns) ForwardSessionEventToSession:612(7818ns)}}}
2025-11-27T18:17:06.271559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470900238948096:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:06.273037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470895943980714:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.273699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:06.274334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470895943980714:2050] [node 3] ICP22 created new session: [6:7577470900238948108:2050]
2025-11-27T18:17:06.274387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470900238948108:2050] [node 3] ICS09 handshake done sender: [6:7577470900238948095:2119] self: [6:7577470900238948100:0] peer: [3:7577470899897714636:0] socket: 141 qp: -1
2025-11-27T18:17:06.274421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470900238948108:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:06.274858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:06.275253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577470895943980714:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:06.275257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470895943980713:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.275323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:06.275370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470895943980713:2049] [node 2] ICP22 created new session: [6:7577470900238948110:2049]
2025-11-27T18:17:06.275414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470900238948110:2049] [node 2] ICS09 handshake done sender: [6:7577470900238948096:2120] self: [6:7577470900238948101:0] peer: [2:7577470898668977422:0] socket: 146 qp: -1
2025-11-27T18:17:06.275438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470900238948110:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:06.275482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:06.275699Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577470895943980713:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:06.298518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:06.298770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470898668977436:2048] [node 1] ICS07 socket disconnect 79 reason# EndOfStream
2025-11-27T18:17:06.298805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470898668977436:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.298894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:06.298947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470898668977436:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:06.299692Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:06.299925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470899897714648:2048] [node 1] ICS07 socket disconnect 103 reason# EndOfStream
2025-11-27T18:17:06.299966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470899897714648:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.300045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:06.300094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470899897714648:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:06.301107Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470894374010049:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28261) Socket error# Connection reset by peer state# SendExRequestHeader processed# 0 remain# 8 incoming: [0:0:0] held: no
2025-11-27T18:17:06.301163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470898668977436:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:06.302330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470895602747265:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28261) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:06.302827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470899897714648:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:06.303475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470894374010049:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28261) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:06.303572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470898668977436:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:06.303616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470894374010049:2048] [node 1] ICP30 unregister session Session# [2:7577470898668977436:2048] VirtualId# [2:7577470898668977431:0]
2025-11-27T18:17:06.303666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:06.303698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470894374010049:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28261) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.303674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:06.303711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28261) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.303845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470897226082834:2048] [node 1] ICS01 socket: 95 reason# EndOfStream
2025-11-27T18:17:06.303891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470892931115447:2048] [node 1] ICP30 unregister session Session# [4:7577470897226082834:2048] VirtualId# [4:7577470897226082827:0]
2025-11-27T18:17:06.303915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470897226082834:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.303976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:06.304028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:06.304000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470895602747265:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28261) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:06.304065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470899897714648:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:06.304099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470895602747265:2048] [node 1] ICP30 unregister session Session# [3:7577470899897714648:2048] VirtualId# [3:7577470899897714644:0]
2025-11-27T18:17:06.304243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:06.304277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470895602747265:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28261) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.304291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28261) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.304711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:06.304876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470896075065624:2048] [node 1] ICS01 socket: 133 reason# EndOfStream
2025-11-27T18:17:06.304908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470891780098227:2048] [node 1] ICP30 unregister session Session# [5:7577470896075065624:2048] VirtualId# [5:7577470896075065620:0]
2025-11-27T18:17:06.304932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470896075065624:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.304992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:06.305028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:06.305545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:06.305733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470900238948102:2048] [node 1] ICS01 socket: 135 reason# EndOfStream
2025-11-27T18:17:06.305772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470895943980712:2048] [node 1] ICP30 unregister session Session# [6:7577470900238948102:2048] VirtualId# [6:7577470900238948092:0]
2025-11-27T18:17:06.305793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470900238948102:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.305847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:06.305904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:06.314569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470895602747265:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:06.337511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:06.337736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470899897714665:2049] [node 2] ICS01 socket: 144 reason# EndOfStream
2025-11-27T18:17:06.337778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470895602747266:2049] [node 2] ICP30 unregister session Session# [3:7577470899897714665:2049] VirtualId# [3:7577470899897714660:0]
2025-11-27T18:17:06.337828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470899897714665:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.337909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:06.337956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:06.349452Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:06.349487Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:06.349712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470897226082837:2049] [node 2] ICS01 socket: 123 reason# EndOfStream
2025-11-27T18:17:06.349752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470892931115448:2049] [node 2] ICP30 unregister session Session# [4:7577470897226082837:2049] VirtualId# [4:7577470897226082831:0]
2025-11-27T18:17:06.349782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470897226082837:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.349857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:06.349907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:06.349452Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:06.349690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470896075065618:2049] [node 2] ICS07 socket disconnect 117 reason# EndOfStream
2025-11-27T18:17:06.349721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470896075065618:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.349809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:06.349851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470896075065618:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:06.351466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470891780098228:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16723) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:06.351509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470896075065618:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:06.351537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470891780098228:2049] [node 2] ICP30 unregister session Session# [5:7577470896075065618:2049] VirtualId# [5:7577470896075065614:0]
2025-11-27T18:17:06.351574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:06.351601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470891780098228:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16723) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.351617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16723) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.350279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470900238948110:2049] [node 2] ICS07 socket disconnect 146 reason# EndOfStream
2025-11-27T18:17:06.350310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470900238948110:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.350417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:06.350502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470900238948110:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:06.358036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470895943980713:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16723) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:06.358183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470900238948110:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:06.358972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470895943980713:2049] [node 2] ICP30 unregister session Session# [6:7577470900238948110:2049] VirtualId# [6:7577470900238948101:0]
2025-11-27T18:17:06.359077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:06.359188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470895943980713:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16723) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.359230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16723) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.364232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470891780098228:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:06.366050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470895943980713:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:06.421500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:06.421718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470897226082844:2050] [node 3] ICS01 socket: 140 reason# EndOfStream
2025-11-27T18:17:06.421761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470892931115449:2050] [node 3] ICP30 unregister session Session# [4:7577470897226082844:2050] VirtualId# [4:7577470897226082838:0]
2025-11-27T18:17:06.421794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470897226082844:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.422006Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:06.421876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:06.421913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:06.422164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470900238948108:2050] [node 3] ICS07 socket disconnect 141 reason# EndOfStream
2025-11-27T18:17:06.422191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470900238948108:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.422240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:06.422288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470900238948108:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:06.421539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:06.421738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470896075065621:2050] [node 3] ICS01 socket: 115 reason# EndOfStream
2025-11-27T18:17:06.421772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470891780098229:2050] [node 3] ICP30 unregister session Session# [5:7577470896075065621:2050] VirtualId# [5:7577470896075065610:0]
2025-11-27T18:17:06.421794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470896075065621:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.421876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:06.421917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:06.423761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470895943980714:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31139) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:06.423821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470900238948108:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:06.423848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470895943980714:2050] [node 3] ICP30 unregister session Session# [6:7577470900238948108:2050] VirtualId# [6:7577470900238948100:0]
2025-11-27T18:17:06.423884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:06.423910Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470895943980714:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31139) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.423924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31139) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.433506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470895943980714:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:06.465182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:06.465442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470896075065608:2108] [node 4] ICS07 socket disconnect 93 reason# EndOfStream
2025-11-27T18:17:06.465470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470896075065608:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.465548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:06.465590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470896075065608:2108] [node 4] ICS15 start handshake
2025-11-27T18:17:06.466722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:06.477536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470900238948103:2109] [node 4] ICS07 socket disconnect 138 reason# EndOfStream
2025-11-27T18:17:06.477580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470900238948103:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.477681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:06.477723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470900238948103:2109] [node 4] ICS15 start handshake
2025-11-27T18:17:06.478633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470900238948074:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63154) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:06.478685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470900238948103:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:06.478717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470900238948074:2109] [node 4] ICP30 unregister session Session# [6:7577470900238948103:2109] VirtualId# [6:7577470900238948097:0]
2025-11-27T18:17:06.478758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:06.478787Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470900238948074:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63154) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.478802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63154) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.472793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470896075065586:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63154) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:06.472883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470896075065608:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:06.472920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470896075065586:2108] [node 4] ICP30 unregister session Session# [5:7577470896075065608:2108] VirtualId# [5:7577470896075065602:0]
2025-11-27T18:17:06.472969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:06.473008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470896075065586:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63154) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.473025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63154) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:06.483704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470896075065586:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:17:06.489386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470900238948074:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:17:06.573717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:06.574131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470900238948091:2106] [node 5] ICS07 socket disconnect 91 reason# EndOfStream
2025-11-27T18:17:06.574188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470900238948091:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:06.574330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:06.574419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470900238948091:2106] [node 5] ICS15 start handshake
2025-11-27T18:17:06.585473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470895943980774:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30144) Couldn't connect to any resolved address incoming: [0:0:0] held: no
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:06.668393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470899138261393:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:06.668485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470899138261394:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:06.687427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470899947907437:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:06.687645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470899947907438:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:06.708403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470898670940282:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:06.708589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470898670940283:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:06.730688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470898110838076:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:06.730730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470898110838077:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:06.730790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470898110838078:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:06.753356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470899428273388:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:06.753401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470899428273390:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:06.753789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470899428273389:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:06.776399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470898181389598:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:06.776449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470898181389600:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:06.776556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470898181389599:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:06.837744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470898181389653:2102] [node 5] ICP01 ready to work
2025-11-27T18:17:06.896477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470899138261456:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:06.901074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470898670940346:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:06.900898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470899428273454:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:06.900712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470899138261458:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:06.901745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470899947907503:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:06.900741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470899138261457:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:06.902303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470899428273455:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:06.902427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470898670940348:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:06.901878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470899947907501:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:06.906078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470898181389660:2104] [node 4] ICP01 ready to work
2025-11-27T18:17:06.906090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470898110838140:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:06.905664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470898670940347:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:06.901891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470899947907502:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:06.907999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470898110838139:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:06.910065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470898181389653:2102] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(68215721ns) {Configure:104(68198080ns) {ProcessConfigured:135(4146445ns) {StartInitialHandshake:157(4143813ns) {DropHandshakes:470(2304ns) {DropIncomingHandshake:442(684ns) DropOutgoingHandshake:457(329ns)} GenerateSessionVirtualId:988(587ns) PrepareNewSessionHandshake:477(2530ns) {SwitchToState:179(865ns)}}}}}
2025-11-27T18:17:06.915791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470899947907438:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(76282226ns) {Configure:104(76278944ns) {ProcessConfigured:135(480765ns) {StartInitialHandshake:157(478255ns) {DropHandshakes:470(2698ns) {DropIncomingHandshake:442(503ns) DropOutgoingHandshake:457(553ns)} GenerateSessionVirtualId:988(937ns) PrepareNewSessionHandshake:477(2150ns) {SwitchToState:179(684ns)}}}}}
2025-11-27T18:17:06.916058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470898181389600:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21321794ns) {Configure:104(21313544ns) {ProcessConfigured:135(4038499ns) {StartInitialHandshake:157(4036509ns) {DropHandshakes:470(7498ns) {DropIncomingHandshake:442(1505ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(1189ns) PrepareNewSessionHandshake:477(2206ns) {SwitchToState:179(916ns)}}}}}
2025-11-27T18:17:06.917932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470898181389598:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16279074ns) {Configure:104(16266637ns) {ProcessConfigured:135(1134304ns) {StartInitialHandshake:157(1132571ns) {DropHandshakes:470(1416ns) {DropIncomingHandshake:442(344ns) DropOutgoingHandshake:457(292ns)} GenerateSessionVirtualId:988(487ns) PrepareNewSessionHandshake:477(5528ns) {SwitchToState:179(4022ns)}}}}}
2025-11-27T18:17:06.923999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470899947907437:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16616364ns) {Configure:104(16609734ns) {ProcessConfigured:135(89651ns) {StartInitialHandshake:157(87359ns) {DropHandshakes:470(2592ns) {DropIncomingHandshake:442(531ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(807ns) PrepareNewSessionHandshake:477(1916ns) {SwitchToState:179(530ns)}}}}}
2025-11-27T18:17:06.924777Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470899428273389:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28230390ns) {Configure:104(28219726ns) {ProcessConfigured:135(644883ns) {StartInitialHandshake:157(643500ns) {DropHandshakes:470(1115ns) {DropIncomingHandshake:442(100ns) DropOutgoingHandshake:457(185ns)} GenerateSessionVirtualId:988(567ns) PrepareNewSessionHandshake:477(1494ns) {SwitchToState:179(667ns)}}}}}
2025-11-27T18:17:06.924033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470899138261393:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(77279466ns) {Configure:104(77270332ns) {ProcessConfigured:135(3338432ns) {StartInitialHandshake:157(3336729ns) {DropHandshakes:470(1197ns) {DropIncomingHandshake:442(275ns) DropOutgoingHandshake:457(489ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:477(2690ns) {SwitchToState:179(1143ns)}}}}}
2025-11-27T18:17:06.925315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470898181389599:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16526354ns) {Configure:104(16523604ns) {ProcessConfigured:135(3890848ns) {StartInitialHandshake:157(3889709ns) {DropHandshakes:470(1159ns) {DropIncomingHandshake:442(330ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(334ns) PrepareNewSessionHandshake:477(1771ns) {SwitchToState:179(138ns)}}}}}
2025-11-27T18:17:06.925183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470898110838077:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(63152278ns) {Configure:104(63141533ns) {ProcessConfigured:135(504029ns) {StartInitialHandshake:157(502457ns) {DropHandshakes:470(164710ns) {DropIncomingHandshake:442(233ns) DropOutgoingHandshake:457(583ns)} GenerateSessionVirtualId:988(700ns) PrepareNewSessionHandshake:477(2348ns) {SwitchToState:179(1343ns)}}}}}
2025-11-27T18:17:06.924019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470899947907502:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17153367ns) {Configure:104(16961985ns) {ProcessConfigured:135(4418558ns) {StartInitialHandshake:157(4416284ns) {DropHandshakes:470(289784ns) {DropIncomingHandshake:442(1374ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(1233ns) PrepareNewSessionHandshake:477(2717ns) {SwitchToState:179(1315ns)}}}}}
2025-11-27T18:17:06.924949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470899138261456:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20608312ns) {Configure:104(20603111ns) {ProcessConfigured:135(3707616ns) {StartInitialHandshake:157(3573196ns) {DropHandshakes:470(1145ns) {DropIncomingHandshake:442(307ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(426ns) PrepareNewSessionHandshake:477(2176ns) {SwitchToState:179(740ns)}}}}}
2025-11-27T18:17:06.925905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470898110838076:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50950246ns) {Configure:104(50942249ns) {ProcessConfigured:135(1394570ns) {StartInitialHandshake:157(1392177ns) {DropHandshakes:470(2001ns) {DropIncomingHandshake:442(489ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(563ns) PrepareNewSessionHandshake:477(2344ns) {SwitchToState:179(965ns)}}}}}
2025-11-27T18:17:06.925005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470899947907501:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16511947ns) {Configure:104(16503392ns) {ProcessConfigured:135(4039368ns) {StartInitialHandshake:157(4036453ns) {DropHandshakes:470(2002ns) {DropIncomingHandshake:442(343ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(833ns) PrepareNewSessionHandshake:477(2933ns) {SwitchToState:179(905ns)}}}}}
2025-11-27T18:17:06.926942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470898670940348:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17872750ns) {Configure:104(17391400ns) {ProcessConfigured:135(1116461ns) {StartInitialHandshake:157(1115244ns) {DropHandshakes:470(1137ns) {DropIncomingHandshake:442(220ns) DropOutgoingHandshake:457(209ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:477(1400ns) {SwitchToState:179(606ns)}}}}}
2025-11-27T18:17:06.925695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470899138261394:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(91007547ns) {Configure:104(90363592ns) {ProcessConfigured:135(1530342ns) {StartInitialHandshake:157(1527779ns) {DropHandshakes:470(1260ns) {DropIncomingHandshake:442(411ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(715ns) PrepareNewSessionHandshake:477(563617ns) {SwitchToState:179(3941ns)}}}}}
2025-11-27T18:17:06.928093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470899428273390:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24883572ns) {Configure:104(24725979ns) {ProcessConfigured:135(1884493ns) {StartInitialHandshake:157(1882989ns) {DropHandshakes:470(1405ns) {DropIncomingHandshake:442(540ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(1045ns) PrepareNewSessionHandshake:477(1294ns) {SwitchToState:179(103ns)}}}}}
2025-11-27T18:17:06.926976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470898670940283:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(79227246ns) {Configure:104(79223061ns) {ProcessConfigured:135(4582335ns) {StartInitialHandshake:157(4580809ns) {DropHandshakes:470(3290270ns) {DropIncomingHandshake:442(2503ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(1215ns) PrepareNewSessionHandshake:477(1596ns) {SwitchToState:179(591ns)}}}}}
2025-11-27T18:17:06.928805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470899138261458:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25180353ns) {Configure:104(24906793ns) {ProcessConfigured:135(1767459ns) {StartInitialHandshake:157(1765978ns) {DropHandshakes:470(626989ns) {DropIncomingHandshake:442(220ns) DropOutgoingHandshake:457(948ns)} GenerateSessionVirtualId:988(950ns) PrepareNewSessionHandshake:477(1486ns) {SwitchToState:179(751ns)}}}}}
2025-11-27T18:17:06.928471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470899428273454:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19492508ns) {Configure:104(19483591ns) {ProcessConfigured:135(1954349ns) {StartInitialHandshake:157(1952706ns) {DropHandshakes:470(599ns) {DropIncomingHandshake:442(157ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(1121ns) {SwitchToState:179(127ns)}}}}}
2025-11-27T18:17:06.927005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470898670940282:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(79920595ns) {Configure:104(79917135ns) {ProcessConfigured:135(4119642ns) {StartInitialHandshake:157(4117692ns) {DropHandshakes:470(1524ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(396ns) PrepareNewSessionHandshake:477(2984ns) {SwitchToState:179(1130ns)}}}}}
2025-11-27T18:17:06.929846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470899428273388:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27105116ns) {Configure:104(25813539ns) {ProcessConfigured:135(1155837ns) {StartInitialHandshake:157(1152905ns) {DropHandshakes:470(1847ns) {DropIncomingHandshake:442(474ns) DropOutgoingHandshake:457(299ns)} GenerateSessionVirtualId:988(655ns) PrepareNewSessionHandshake:477(2040ns) {SwitchToState:179(818ns)}}}}}
2025-11-27T18:17:06.929208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470898110838078:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18777500ns) {Configure:104(18763766ns) {ProcessConfigured:135(949458ns) {StartInitialHandshake:157(944341ns) {DropHandshakes:470(1207ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(491ns) PrepareNewSessionHandshake:477(2047ns) {SwitchToState:179(968ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.940161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470898670940358:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33302
2025-11-27T18:17:06.941658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470899138261468:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13857
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.943792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470898670940357:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33302
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.943798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470899138261467:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13857
2025-11-27T18:17:06.944587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470899428273464:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32773
2025-11-27T18:17:06.944663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470898110838154:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25679
2025-11-27T18:17:06.945615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470898670940357:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.946186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470899138261469:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13857
2025-11-27T18:17:06.947803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470898110838155:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25679
No RDMA context found for GID: 2025-11-27T18:17:06.948396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470899947907517:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14607
2025-11-27T18:17:06.947756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470899428273467:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32773
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.949160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470899138261467:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.954822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470899138261468:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.955009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470899138261469:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.947889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470899428273464:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.952441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470899947907518:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14607
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.953986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470898110838154:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.955400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470898670940364:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33302
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.955003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470899947907517:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.955035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470898110838155:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.951288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470899428273467:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.956808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470899138261475:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13857
2025-11-27T18:17:06.955383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470899428273471:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32773
2025-11-27T18:17:06.955420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470898670940365:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33302
2025-11-27T18:17:06.956687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470899947907520:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14607
2025-11-27T18:17:06.955056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470898110838077:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.956715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470899947907510:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.955452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470898670940358:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.958077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470899138261476:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13857
2025-11-27T18:17:06.955499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470898110838078:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.956805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470899428273455:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.958614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470899138261475:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.957040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470898670940283:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.958104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470898110838146:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.956745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470899947907510:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.957586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470899428273471:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.957114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470898670940355:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.959366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470899138261476:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.958128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470898110838146:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-27T18:17:06.958588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470899428273388:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.957532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470899947907519:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14607
::ffff:127.1.0.0
2025-11-27T18:17:06.957131Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470898670940355:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.959489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470899138261463:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.958021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470899947907520:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.957918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470898670940365:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.960963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470898110838146:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:06.958596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470899947907505:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.959528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470899138261463:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.958096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470898670940364:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.962493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470898110838159:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25679
2025-11-27T18:17:06.958620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470899947907505:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.963804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470899428273475:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32773
2025-11-27T18:17:06.960898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470899138261464:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.964099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470898110838140:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.958817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470898670940282:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.960927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470899138261464:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.958695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470899947907518:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.964376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.961453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470898670940355:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:06.961435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470899138261462:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.964645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470898670940346:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.958762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470899947907510:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:06.967703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470899428273467:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:06.964611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470898110838140:2108] [node 5] ICP22 created new session: [4:7577470898110838161:2108]
2025-11-27T18:17:06.961462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470899138261462:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.964674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:06.959433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470899947907519:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.969041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470899428273476:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32773
2025-11-27T18:17:06.965215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470898110838161:2108] [node 5] ICS09 handshake done sender: [4:7577470898110838146:2112] self: [4:7577470898110838145:0] peer: [5:7577470899428273473:0] socket: 99 qp: -1
2025-11-27T18:17:06.961783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470899138261463:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.964877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470898670940346:2107] [node 4] ICP22 created new session: [3:7577470898670940373:2107]
2025-11-27T18:17:06.960875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470899947907437:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.964929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470899138261464:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:06.965559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470898110838154:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:06.965569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470898110838161:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:06.965132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470898670940371:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33302
2025-11-27T18:17:06.973127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470899428273455:2108] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.965126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470899138261462:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:06.961271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470899947907505:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:06.973205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:06.965863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.965288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470898670940373:2107] [node 4] ICS09 handshake done sender: [3:7577470898670940355:2113] self: [3:7577470898670940353:0] peer: [4:7577470898110838157:0] socket: 106 qp: -1
2025-11-27T18:17:06.961608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470899947907501:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.973367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470899428273455:2108] [node 4] ICP22 created new session: [5:7577470899428273478:2108]
2025-11-27T18:17:06.965571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470899138261456:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.977227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470898181389676:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:17:06.966463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470898110838077:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3897) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470898110838154:2115] held: no
2025-11-27T18:17:06.980342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470898181389676:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.980462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470898181389599:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.965807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470898670940373:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:06.961766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.965868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:06.973785Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470899428273388:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470899428273471:2120] held: no
2025-11-27T18:17:06.965847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:06.967478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470898110838077:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.962321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470899947907501:2107] [node 4] ICP22 created new session: [2:7577470899947907527:2107]
2025-11-27T18:17:06.967507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:06.974999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470899428273478:2108] [node 4] ICS09 handshake done sender: [5:7577470899428273467:2119] self: [5:7577470899428273473:0] peer: [4:7577470898110838145:0] socket: 102 qp: -1
2025-11-27T18:17:06.965881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470899138261394:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.983558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470898181389677:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:17:06.963009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470899947907527:2107] [node 4] ICS09 handshake done sender: [2:7577470899947907510:2113] self: [2:7577470899947907506:0] peer: [4:7577470898110838156:0] socket: 95 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.972736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470898670940358:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:06.970976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470898110838160:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25679
2025-11-27T18:17:06.965929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:06.976000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470899428273478:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:06.970979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470898110838077:2049] [node 2] ICP22 created new session: [4:7577470898110838163:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.963502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470899947907527:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.976560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:06.973335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470898670940283:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3897) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470898670940358:2116] held: no
2025-11-27T18:17:06.987797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470898181389678:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:17:06.965931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470899138261456:2108] [node 5] ICP22 created new session: [1:7577470899138261485:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.964118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:06.977274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470899428273471:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:06.972012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470898110838163:2049] [node 2] ICS09 handshake done sender: [4:7577470898110838154:2115] self: [4:7577470898110838156:0] peer: [2:7577470899947907506:0] socket: 100 qp: -1
2025-11-27T18:17:06.966484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470899138261485:2108] [node 5] ICS09 handshake done sender: [1:7577470899138261463:2111] self: [1:7577470899138261460:0] peer: [5:7577470899428273474:0] socket: 111 qp: -1
2025-11-27T18:17:06.991109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470898181389679:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:17:06.976489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470898670940283:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.964130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470899947907438:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.976553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470898670940372:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33302
2025-11-27T18:17:06.991522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470898181389678:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.964382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:06.979593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470899428273477:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32773
2025-11-27T18:17:06.977219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.973111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470898110838163:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.980024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470899428273388:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.966488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470899138261394:2049] [node 3] ICP22 created new session: [1:7577470899138261486:2049]
2025-11-27T18:17:06.993324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470898181389681:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:17:06.964422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470899947907438:2049] [node 3] ICP22 created new session: [2:7577470899947907529:2049]
2025-11-27T18:17:06.978667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470898670940283:2049] [node 2] ICP22 created new session: [3:7577470898670940377:2049]
2025-11-27T18:17:06.973525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.967205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470899138261485:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:06.983305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:06.965531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470899947907529:2049] [node 3] ICS09 handshake done sender: [2:7577470899947907505:2111] self: [2:7577470899947907504:0] peer: [3:7577470898670940367:0] socket: 90 qp: -1
2025-11-27T18:17:06.997569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470898181389660:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.976286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470898110838155:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:06.979609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470898670940377:2049] [node 2] ICS09 handshake done sender: [3:7577470898670940358:2116] self: [3:7577470898670940367:0] peer: [2:7577470899947907504:0] socket: 98 qp: -1
2025-11-27T18:17:06.967223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470899138261486:2049] [node 3] ICS09 handshake done sender: [1:7577470899138261464:2112] self: [1:7577470899138261461:0] peer: [3:7577470898670940368:0] socket: 113 qp: -1
2025-11-27T18:17:06.966179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470899947907529:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:06.983582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470899428273477:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.998009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470898181389679:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.978676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470898110838078:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.981743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470898670940377:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:06.967459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470899138261486:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.966501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:06.998841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470898181389681:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.983656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470899428273388:2048] [node 1] ICP22 created new session: [5:7577470899428273481:2048]
2025-11-27T18:17:06.983267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:06.978929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:06.967707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:06.983388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470898670940282:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470898670940365:2119] held: no
2025-11-27T18:17:06.984093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470899428273481:2048] [node 1] ICS09 handshake done sender: [5:7577470899428273471:2120] self: [5:7577470899428273474:0] peer: [1:7577470899138261460:0] socket: 120 qp: -1
2025-11-27T18:17:06.976844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470899947907531:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14607
2025-11-27T18:17:06.998880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470898181389598:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.967721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:06.979637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470898110838078:2050] [node 3] ICP22 created new session: [4:7577470898110838167:2050]
2025-11-27T18:17:06.999152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470898181389653:2102] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.984584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470899428273389:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.983663Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470899947907437:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577470899947907519:2118] held: no
2025-11-27T18:17:06.973052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470899138261393:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.982799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470898110838167:2050] [node 3] ICS09 handshake done sender: [4:7577470898110838155:2116] self: [4:7577470898110838157:0] peer: [3:7577470898670940353:0] socket: 119 qp: -1
2025-11-27T18:17:06.984583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470899428273481:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:07.000828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470898181389682:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:17:06.975974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:06.984570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470898670940365:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:06.983830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470899947907519:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:06.983297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470898110838167:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:06.976034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470899138261393:2048] [node 2] ICP22 created new session: [1:7577470899138261489:2048]
2025-11-27T18:17:07.000885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470898181389677:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.986670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470898670940282:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.984622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:06.983594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.983837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470899947907513:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.987772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:06.998764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470899428273484:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32773
2025-11-27T18:17:07.001357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470898181389600:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.976750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470899138261489:2048] [node 2] ICS09 handshake done sender: [1:7577470899138261462:2110] self: [1:7577470899138261459:0] peer: [2:7577470899947907526:0] socket: 110 qp: -1
2025-11-27T18:17:06.988546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470898670940282:2048] [node 1] ICP22 created new session: [3:7577470898670940380:2048]
2025-11-27T18:17:06.991130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470898110838168:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25679
2025-11-27T18:17:06.983865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470899947907513:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.007006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470898181389676:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:07.000956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470899428273462:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.989509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470898670940380:2048] [node 1] ICS09 handshake done sender: [3:7577470898670940365:2119] self: [3:7577470898670940368:0] peer: [1:7577470899138261461:0] socket: 121 qp: -1
2025-11-27T18:17:06.985021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470899947907437:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.976911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470899138261489:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:07.014247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470898181389599:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.978049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:06.993345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470898110838168:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.999208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470899138261474:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.014755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:07.000982Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470899428273462:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.985057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:06.999536Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470899138261474:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.997595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470898110838076:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:06.991572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470898670940380:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:07.003432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470899428273485:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32773
2025-11-27T18:17:06.985108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470899947907437:2048] [node 1] ICP22 created new session: [2:7577470899947907534:2048]
2025-11-27T18:17:06.998607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470898110838153:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.014820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470898181389599:2049] [node 2] ICP22 created new session: [6:7577470902476356985:2049]
2025-11-27T18:17:07.001347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470899138261466:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.992498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:07.007393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470899428273389:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3897) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470899428273477:2123] held: no
2025-11-27T18:17:06.998706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470898670940376:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33302
2025-11-27T18:17:06.985151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470899947907534:2048] [node 1] ICS09 handshake done sender: [2:7577470899947907519:2118] self: [2:7577470899947907526:0] peer: [1:7577470899138261459:0] socket: 114 qp: -1
2025-11-27T18:17:07.016646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470902476356985:2049] [node 2] ICS09 handshake done sender: [6:7577470898181389676:2115] self: [6:7577470898181389680:0] peer: [2:7577470899947907512:0] socket: 139 qp: -1
2025-11-27T18:17:07.001399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470899138261466:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:06.998632Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470898110838153:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.013661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470899428273484:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:06.985174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470899947907534:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:07.004974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470898110838153:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:07.001550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470899138261474:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:07.000854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577470898670940346:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:07.014070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470899428273477:2123] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:07.014296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470898110838139:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:06.985199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:07.016868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470902476356985:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:07.002832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470899138261466:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:07.016910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:07.003033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470898670940356:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.015242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:07.014796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470899428273462:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:06.986811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470899947907513:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.015409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470899428273389:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.003054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470899138261457:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.003064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470898670940356:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.015288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470898110838139:2109] [node 6] ICP22 created new session: [4:7577470902405805469:2109]
2025-11-27T18:17:07.015586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.003108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:07.029563Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470898181389598:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470898181389679:2118] held: no
2025-11-27T18:17:06.989201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470899947907503:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.013976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470898670940356:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:07.015561Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470898110838076:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470898110838168:2119] held: no
2025-11-27T18:17:07.034499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470898181389600:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5762) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470898181389677:2116] held: no
2025-11-27T18:17:07.015649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470899428273389:2049] [node 2] ICP22 created new session: [5:7577470903723240783:2049]
2025-11-27T18:17:07.003466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470899138261457:2107] [node 4] ICP22 created new session: [1:7577470903433228791:2107]
2025-11-27T18:17:07.035829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470902476356983:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:17:06.989430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470899947907511:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.015806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470903723240783:2049] [node 2] ICS09 handshake done sender: [5:7577470899428273477:2123] self: [5:7577470899428273482:0] peer: [2:7577470899947907507:0] socket: 140 qp: -1
2025-11-27T18:17:07.016346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470902405805469:2109] [node 6] ICS09 handshake done sender: [4:7577470898110838153:2114] self: [4:7577470898110838152:0] peer: [6:7577470898181389683:0] socket: 112 qp: -1
2025-11-27T18:17:07.004164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470903433228791:2107] [node 4] ICS09 handshake done sender: [1:7577470899138261474:2117] self: [1:7577470899138261473:0] peer: [4:7577470898110838170:0] socket: 146 qp: -1
2025-11-27T18:17:07.015629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470898670940348:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:06.989461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470899947907511:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.015828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470899428273390:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.016679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470902405805469:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:07.006954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470903433228791:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:07.016343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:07.036859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470898181389653:2102] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1408) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470898181389681:2119] held: no
2025-11-27T18:17:06.990166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:07.016076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470903723240783:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:07.006999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:07.037702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470902476356984:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:17:07.017449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:07.016677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470898670940348:2109] [node 6] ICP22 created new session: [3:7577470902965907679:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.016111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:06.992496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470899947907503:2110] [node 6] ICP22 created new session: [2:7577470899947907537:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.017443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470902965907679:2109] [node 6] ICS09 handshake done sender: [3:7577470898670940356:2114] self: [3:7577470898670940351:0] peer: [6:7577470902476356982:0] socket: 108 qp: -1
2025-11-27T18:17:07.007562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470899138261458:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.017697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470902405805467:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25679
2025-11-27T18:17:07.020981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470898110838168:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:07.017717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470902965907679:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:06.997717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470899947907511:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:07.007613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:07.017755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:07.016388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470899428273454:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.038702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470902476356987:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:17:07.022105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470898110838076:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:06.998412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470899947907537:2110] [node 6] ICS09 handshake done sender: [2:7577470899947907513:2115] self: [2:7577470899947907512:0] peer: [6:7577470898181389680:0] socket: 109 qp: -1
2025-11-27T18:17:07.016429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:07.013668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470899138261458:2109] [node 6] ICP22 created new session: [1:7577470903433228793:2109]
2025-11-27T18:17:07.038716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470902476356986:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:17:07.023953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:07.016703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470899428273454:2110] [node 6] ICP22 created new session: [5:7577470903723240786:2110]
2025-11-27T18:17:06.998458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470899947907537:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:07.028924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470898110838076:2048] [node 1] ICP22 created new session: [4:7577470902405805471:2048]
2025-11-27T18:17:07.014792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470903433228793:2109] [node 6] ICS09 handshake done sender: [1:7577470899138261466:2113] self: [1:7577470899138261465:0] peer: [6:7577470898181389684:0] socket: 144 qp: -1
2025-11-27T18:17:06.998669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:07.018031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470903723240786:2110] [node 6] ICS09 handshake done sender: [5:7577470899428273462:2115] self: [5:7577470899428273458:0] peer: [6:7577470898181389685:0] socket: 104 qp: -1
2025-11-27T18:17:07.029937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470902405805471:2048] [node 1] ICS09 handshake done sender: [4:7577470898110838168:2119] self: [4:7577470898110838170:0] peer: [1:7577470899138261473:0] socket: 94 qp: -1
2025-11-27T18:17:06.999269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470899947907502:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.015602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470903433228793:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:07.030110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470902405805471:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:07.018392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470903723240786:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:07.018015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470898670940363:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.015648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:07.000206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:07.030156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:07.020287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:07.039057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470898181389679:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:07.018388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470898670940363:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.039110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470898181389678:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:07.021431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470898670940363:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:07.001374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470899947907502:2109] [node 5] ICP22 created new session: [2:7577470899947907540:2109]
2025-11-27T18:17:07.034701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470899428273390:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5762) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470899428273484:2124] held: no
2025-11-27T18:17:07.022184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470898670940347:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.002353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470899947907540:2109] [node 5] ICS09 handshake done sender: [2:7577470899947907511:2114] self: [2:7577470899947907507:0] peer: [5:7577470899428273482:0] socket: 96 qp: -1
2025-11-27T18:17:07.022247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:07.036826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470903723240788:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32773
2025-11-27T18:17:07.002388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470899947907540:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:07.027432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470898670940347:2108] [node 5] ICP22 created new session: [3:7577470902965907682:2108]
2025-11-27T18:17:07.037481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470899428273484:2124] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:07.028961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470902965907682:2108] [node 5] ICS09 handshake done sender: [3:7577470898670940363:2117] self: [3:7577470898670940360:0] peer: [5:7577470903723240784:0] socket: 147 qp: -1
2025-11-27T18:17:07.002650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:07.038173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470899428273390:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.029499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470902965907682:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:07.029559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:07.038561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:07.038696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470899428273390:2050] [node 3] ICP22 created new session: [5:7577470903723240789:2050]
2025-11-27T18:17:07.040048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470898181389660:2104] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.038734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470903723240789:2050] [node 3] ICS09 handshake done sender: [5:7577470899428273484:2124] self: [5:7577470903723240784:0] peer: [3:7577470898670940360:0] socket: 148 qp: -1
2025-11-27T18:17:07.040113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470898181389598:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.038942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470903723240789:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:07.040119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:07.039006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:07.040159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470898181389660:2104] [node 4] ICP22 created new session: [6:7577470902476356990:2104]
No RDMA context found for GID: 2025-11-27T18:17:07.040186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470902476356990:2104] [node 4] ICS09 handshake done sender: [6:7577470898181389678:2117] self: [6:7577470898181389683:0] peer: [4:7577470898110838152:0] socket: 142 qp: -1
::ffff:127.1.0.0
2025-11-27T18:17:07.040189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:07.040195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470898181389677:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:07.040212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470902476356990:2104] [node 4] ICS10 traffic start
2025-11-27T18:17:07.040353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470902405805473:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25679
2025-11-27T18:17:07.040224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470898181389598:2048] [node 1] ICP22 created new session: [6:7577470902476356991:2048]
2025-11-27T18:17:07.040234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:07.040256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470902476356991:2048] [node 1] ICS09 handshake done sender: [6:7577470898181389679:2118] self: [6:7577470898181389684:0] peer: [1:7577470899138261465:0] socket: 145 qp: -1
2025-11-27T18:17:07.040284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470902476356991:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:07.040360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:07.040536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470898181389681:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:07.040859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470898181389600:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.040900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:07.040940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470898181389653:2102] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.040956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470898181389600:2050] [node 3] ICP22 created new session: [6:7577470902476356994:2050]
2025-11-27T18:17:07.040973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:07.041015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470898181389653:2102] [node 5] ICP22 created new session: [6:7577470902476356995:2102]
2025-11-27T18:17:07.041019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470902476356994:2050] [node 3] ICS09 handshake done sender: [6:7577470898181389677:2116] self: [6:7577470902476356982:0] peer: [3:7577470898670940351:0] socket: 141 qp: -1
2025-11-27T18:17:07.041047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470902476356995:2102] [node 5] ICS09 handshake done sender: [6:7577470898181389681:2119] self: [6:7577470898181389685:0] peer: [5:7577470899428273458:0] socket: 117 qp: -1
2025-11-27T18:17:07.041053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470902476356994:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:07.041075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470902476356995:2102] [node 5] ICS10 traffic start
2025-11-27T18:17:07.041094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:07.041094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:07.080670Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:07.080927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470899947907534:2048] [node 1] ICS01 socket: 114 reason# EndOfStream
2025-11-27T18:17:07.080968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470899947907437:2048] [node 1] ICP30 unregister session Session# [2:7577470899947907534:2048] VirtualId# [2:7577470899947907526:0]
2025-11-27T18:17:07.080996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470899947907534:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.081072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:07.081120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:07.081791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:07.082004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470898670940380:2048] [node 1] ICS01 socket: 121 reason# EndOfStream
2025-11-27T18:17:07.082044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470898670940282:2048] [node 1] ICP30 unregister session Session# [3:7577470898670940380:2048] VirtualId# [3:7577470898670940368:0]
2025-11-27T18:17:07.082067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470898670940380:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.082161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:07.082211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:07.084302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:07.084509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470902405805471:2048] [node 1] ICS01 socket: 94 reason# EndOfStream
2025-11-27T18:17:07.084542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470898110838076:2048] [node 1] ICP30 unregister session Session# [4:7577470902405805471:2048] VirtualId# [4:7577470898110838170:0]
2025-11-27T18:17:07.084562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470902405805471:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.084641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:07.084701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:07.084951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:07.085200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470899428273481:2048] [node 1] ICS01 socket: 120 reason# EndOfStream
2025-11-27T18:17:07.085273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470899428273388:2048] [node 1] ICP30 unregister session Session# [5:7577470899428273481:2048] VirtualId# [5:7577470899428273474:0]
2025-11-27T18:17:07.085323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470899428273481:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.085406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:07.085450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:07.085703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:07.085938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470902476356991:2048] [node 1] ICS01 socket: 145 reason# EndOfStream
2025-11-27T18:17:07.085989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470898181389598:2048] [node 1] ICP30 unregister session Session# [6:7577470902476356991:2048] VirtualId# [6:7577470898181389684:0]
2025-11-27T18:17:07.086044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470902476356991:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.086122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:07.086168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:07.102259Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:07.102508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470898670940377:2049] [node 2] ICS01 socket: 98 reason# EndOfStream
2025-11-27T18:17:07.102548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470898670940283:2049] [node 2] ICP30 unregister session Session# [3:7577470898670940377:2049] VirtualId# [3:7577470898670940367:0]
2025-11-27T18:17:07.102579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470898670940377:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.102663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:07.102726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:07.105293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:07.105529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470898110838163:2049] [node 2] ICS01 socket: 100 reason# EndOfStream
2025-11-27T18:17:07.105565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470898110838077:2049] [node 2] ICP30 unregister session Session# [4:7577470898110838163:2049] VirtualId# [4:7577470898110838156:0]
2025-11-27T18:17:07.105601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470898110838163:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.105672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:07.105716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:07.106293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:07.106502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470903723240783:2049] [node 2] ICS01 socket: 140 reason# EndOfStream
2025-11-27T18:17:07.106538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470899428273389:2049] [node 2] ICP30 unregister session Session# [5:7577470903723240783:2049] VirtualId# [5:7577470899428273482:0]
2025-11-27T18:17:07.106564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470903723240783:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.106627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:07.106671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:07.107201Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:07.107403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470902476356985:2049] [node 2] ICS07 socket disconnect 139 reason# EndOfStream
2025-11-27T18:17:07.107437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470902476356985:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.107520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:07.107593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470902476356985:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:07.108320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470898181389599:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3897) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:07.108379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470902476356985:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:07.108460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470898181389599:2049] [node 2] ICP30 unregister session Session# [6:7577470902476356985:2049] VirtualId# [6:7577470898181389680:0]
2025-11-27T18:17:07.108512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:07.108539Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470898181389599:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3897) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:07.108555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3897) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:07.118458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470898181389599:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:07.145581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:07.145870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470898110838167:2050] [node 3] ICS07 socket disconnect 119 reason# EndOfStream
2025-11-27T18:17:07.145911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470898110838167:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.145979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:07.146018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470898110838167:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:07.146566Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:07.146828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470898110838078:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5762) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:07.146758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470903723240789:2050] [node 3] ICS01 socket: 148 reason# EndOfStream
2025-11-27T18:17:07.146881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470898110838167:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:07.146805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470899428273390:2050] [node 3] ICP30 unregister session Session# [5:7577470903723240789:2050] VirtualId# [5:7577470903723240784:0]
2025-11-27T18:17:07.146837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470903723240789:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.146992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470898110838078:2050] [node 3] ICP30 unregister session Session# [4:7577470898110838167:2050] VirtualId# [4:7577470898110838157:0]
2025-11-27T18:17:07.146911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:07.147044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:07.147079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470898110838078:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:07.147022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:07.147099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:07.147853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:07.148074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470902476356994:2050] [node 3] ICS01 socket: 141 reason# EndOfStream
2025-11-27T18:17:07.148111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470898181389600:2050] [node 3] ICP30 unregister session Session# [6:7577470902476356994:2050] VirtualId# [6:7577470902476356982:0]
2025-11-27T18:17:07.148139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470902476356994:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.148210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:07.148261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:07.157360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470898110838078:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:07.203497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:07.203789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470899428273478:2108] [node 4] ICS07 socket disconnect 102 reason# EndOfStream
2025-11-27T18:17:07.203826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470899428273478:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.203906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:07.203962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470899428273478:2108] [node 4] ICS15 start handshake
2025-11-27T18:17:07.207299Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:07.207519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470902476356990:2104] [node 4] ICS07 socket disconnect 142 reason# EndOfStream
2025-11-27T18:17:07.207556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470902476356990:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.207635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:07.207678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470902476356990:2104] [node 4] ICS15 start handshake
2025-11-27T18:17:07.208835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470898181389660:2104] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20324) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:07.209696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470902476356990:2104] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:07.212647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470898181389660:2104] [node 4] ICP30 unregister session Session# [6:7577470902476356990:2104] VirtualId# [6:7577470898181389683:0]
2025-11-27T18:17:07.212968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:07.213020Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470898181389660:2104] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:07.213033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:07.229651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470898181389660:2104] [node 4] ICP33 wake up from error state
2025-11-27T18:17:07.295140Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:07.295406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470902476356995:2102] [node 5] ICS01 socket: 117 reason# EndOfStream
2025-11-27T18:17:07.295448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470898181389653:2102] [node 5] ICP30 unregister session Session# [6:7577470902476356995:2102] VirtualId# [6:7577470898181389685:0]
2025-11-27T18:17:07.295476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470902476356995:2102] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.295576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:07.295613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:07.376833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470903343534889:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:07.376935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470903343534890:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:07.400004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470900950834619:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:07.400616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470900950834618:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:07.423674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470901126675643:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:07.423701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470901126675644:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:07.447702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470901816154316:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:07.447730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470901816154317:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:07.447752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470901816154318:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:07.471403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470903583529811:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:07.471437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470903583529812:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:07.471553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470903583529813:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:07.492116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470901010436477:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:07.492120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470901010436478:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:07.492198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470901010436476:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:07.580797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470901010436538:2106] [node 5] ICP01 ready to work
2025-11-27T18:17:07.636891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470903343534954:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:07.644614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470900950834681:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:07.644372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470903343534953:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:07.657620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470903343534955:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:07.657882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470901816154397:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:07.659020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470901816154398:2116] [node 6] ICP01 ready to work
2025-11-27T18:17:07.659266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470903583529874:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:07.659309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470903583529875:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:07.659310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470901126675707:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:07.659630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470900950834683:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:07.659911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470900950834682:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:07.660778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470901010436538:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(79036157ns) {Configure:104(79033114ns) {ProcessConfigured:135(640359ns) {StartInitialHandshake:157(636223ns) {DropHandshakes:470(2124ns) {DropIncomingHandshake:442(591ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(953ns) PrepareNewSessionHandshake:477(2222ns) {SwitchToState:179(1022ns)}}}}}
2025-11-27T18:17:07.665449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470901010436543:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:07.660243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470901126675708:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:07.660278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470901126675709:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:07.667291Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470903343534890:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(82844960ns) {Configure:104(82837116ns) {ProcessConfigured:135(4947445ns) {StartInitialHandshake:157(4945395ns) {DropHandshakes:470(1670ns) {DropIncomingHandshake:442(575ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(616ns) PrepareNewSessionHandshake:477(3093ns) {SwitchToState:179(897ns)}}}}}
2025-11-27T18:17:07.671598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470901010436477:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(58558186ns) {Configure:104(58548907ns) {ProcessConfigured:135(47266ns) {StartInitialHandshake:157(45489ns) {DropHandshakes:470(2504ns) {DropIncomingHandshake:442(524ns) DropOutgoingHandshake:457(654ns)} GenerateSessionVirtualId:988(532ns) PrepareNewSessionHandshake:477(812ns) {SwitchToState:179(255ns)}}}}}
2025-11-27T18:17:07.673198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470903343534889:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(89497918ns) {Configure:104(89493836ns) {ProcessConfigured:135(4295052ns) {StartInitialHandshake:157(4292436ns) {DropHandshakes:470(3219ns) {DropIncomingHandshake:442(929ns) DropOutgoingHandshake:457(643ns)} GenerateSessionVirtualId:988(655ns) PrepareNewSessionHandshake:477(3174ns) {SwitchToState:179(1101ns)}}}}}
2025-11-27T18:17:07.677473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470901010436478:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(44370062ns) {Configure:104(44362154ns) {ProcessConfigured:135(1242988ns) {StartInitialHandshake:157(1241246ns) {DropHandshakes:470(1405ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(676ns) PrepareNewSessionHandshake:477(1659ns) {SwitchToState:179(787ns)}}}}}
2025-11-27T18:17:07.675877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470901126675643:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38134981ns) {Configure:104(38126433ns) {ProcessConfigured:135(449254ns) {StartInitialHandshake:157(447898ns) {DropHandshakes:470(747ns) {DropIncomingHandshake:442(87ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(976ns) {SwitchToState:179(129ns)}}}}}
2025-11-27T18:17:07.677587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470903343534955:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17027396ns) {Configure:104(17020567ns) {ProcessConfigured:135(690733ns) {StartInitialHandshake:157(689300ns) {DropHandshakes:470(892ns) {DropIncomingHandshake:442(155ns) DropOutgoingHandshake:457(472ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(1425ns) {SwitchToState:179(670ns)}}}}}
2025-11-27T18:17:07.678441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470903583529811:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30618269ns) {Configure:104(30608350ns) {ProcessConfigured:135(1464300ns) {StartInitialHandshake:157(1461970ns) {DropHandshakes:470(1729ns) {DropIncomingHandshake:442(535ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(456ns) PrepareNewSessionHandshake:477(1879ns) {SwitchToState:179(988ns)}}}}}
2025-11-27T18:17:07.678088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470903343534954:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16344178ns) {Configure:104(16338006ns) {ProcessConfigured:135(753950ns) {StartInitialHandshake:157(752878ns) {DropHandshakes:470(1364ns) {DropIncomingHandshake:442(336ns) DropOutgoingHandshake:457(244ns)} GenerateSessionVirtualId:988(379ns) PrepareNewSessionHandshake:477(857ns) {SwitchToState:179(386ns)}}}}}
2025-11-27T18:17:07.678364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470901816154397:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18127033ns) {Configure:104(18122539ns) {ProcessConfigured:135(2127548ns) {StartInitialHandshake:157(2126095ns) {DropHandshakes:470(1028ns) {DropIncomingHandshake:442(261ns) DropOutgoingHandshake:457(341ns)} GenerateSessionVirtualId:988(341ns) PrepareNewSessionHandshake:477(1169ns) {SwitchToState:179(161ns)}}}}}
2025-11-27T18:17:07.681911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470903343534953:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20759924ns) {Configure:104(20756198ns) {ProcessConfigured:135(2497824ns) {StartInitialHandshake:157(2496747ns) {DropHandshakes:470(1013572ns) {DropIncomingHandshake:442(2465ns) DropOutgoingHandshake:457(1213ns)} GenerateSessionVirtualId:988(868ns) PrepareNewSessionHandshake:477(1425ns) {SwitchToState:179(507ns)}}}}}
2025-11-27T18:17:07.678316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470901126675644:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24229031ns) {Configure:104(20794562ns) {ProcessConfigured:135(593532ns) {StartInitialHandshake:157(590665ns) {DropHandshakes:470(1436ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(201ns)} GenerateSessionVirtualId:988(643ns) PrepareNewSessionHandshake:477(1973ns) {SwitchToState:179(898ns)}}}}}
2025-11-27T18:17:07.681173Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470900950834681:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21799592ns) {Configure:104(21791548ns) {ProcessConfigured:135(1644985ns) {StartInitialHandshake:157(1642953ns) {DropHandshakes:470(255416ns) {DropIncomingHandshake:442(215ns) DropOutgoingHandshake:457(1190ns)} GenerateSessionVirtualId:988(545ns) PrepareNewSessionHandshake:477(1127ns) {SwitchToState:179(107ns)}}}}}
2025-11-27T18:17:07.681280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470901010436476:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(71134670ns) {Configure:104(71127488ns) {ProcessConfigured:135(1931420ns) {StartInitialHandshake:157(1929596ns) {DropHandshakes:470(926ns) {DropIncomingHandshake:442(267ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(304ns) PrepareNewSessionHandshake:477(1576ns) {SwitchToState:179(549ns)}}}}}
2025-11-27T18:17:07.678613Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470903583529813:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45176702ns) {Configure:104(31291984ns) {ProcessConfigured:135(786855ns) {StartInitialHandshake:157(785387ns) {DropHandshakes:470(4504ns) {DropIncomingHandshake:442(670ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(588ns) PrepareNewSessionHandshake:477(1988ns) {SwitchToState:179(453ns)}}}}}
2025-11-27T18:17:07.678595Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470901816154316:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(70074190ns) {Configure:104(70067665ns) {ProcessConfigured:135(1458695ns) {StartInitialHandshake:157(1457330ns) {DropHandshakes:470(1138ns) {DropIncomingHandshake:442(270ns) DropOutgoingHandshake:457(200ns)} GenerateSessionVirtualId:988(394ns) PrepareNewSessionHandshake:477(1994ns) {SwitchToState:179(988ns)}}}}}
2025-11-27T18:17:07.689449Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470901126675707:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19006356ns) {Configure:104(19002317ns) {ProcessConfigured:135(1775918ns) {StartInitialHandshake:157(1774755ns) {DropHandshakes:470(921ns) {DropIncomingHandshake:442(218ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(301ns) PrepareNewSessionHandshake:477(1051ns) {SwitchToState:179(76ns)}}}}}
2025-11-27T18:17:07.679304Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470903583529875:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18285467ns) {Configure:104(18280917ns) {ProcessConfigured:135(811765ns) {StartInitialHandshake:157(810838ns) {DropHandshakes:470(708ns) {DropIncomingHandshake:442(208ns) DropOutgoingHandshake:457(158ns)} GenerateSessionVirtualId:988(349ns) PrepareNewSessionHandshake:477(1187ns) {SwitchToState:179(186ns)}}}}}
2025-11-27T18:17:07.696867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470900950834682:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19814424ns) {Configure:104(19810770ns) {ProcessConfigured:135(1109665ns) {StartInitialHandshake:157(1108147ns) {DropHandshakes:470(1031ns) {DropIncomingHandshake:442(319ns) DropOutgoingHandshake:457(184ns)} GenerateSessionVirtualId:988(420ns) PrepareNewSessionHandshake:477(1518ns) {SwitchToState:179(188ns)}}}}}
2025-11-27T18:17:07.679021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470901816154318:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17947050ns) {Configure:104(17935443ns) {ProcessConfigured:135(838375ns) {StartInitialHandshake:157(837222ns) {DropHandshakes:470(4196ns) {DropIncomingHandshake:442(209ns) DropOutgoingHandshake:457(99ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(1085ns) {SwitchToState:179(131ns)}}}}}
2025-11-27T18:17:07.696000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470901126675709:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18003801ns) {Configure:104(17999704ns) {ProcessConfigured:135(1114059ns) {StartInitialHandshake:157(1112341ns) {DropHandshakes:470(1019ns) {DropIncomingHandshake:442(259ns) DropOutgoingHandshake:457(97ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(1096ns) {SwitchToState:179(109ns)}}}}}
2025-11-27T18:17:07.701562Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470901126675708:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21625817ns) {Configure:104(21622168ns) {ProcessConfigured:135(2248634ns) {StartInitialHandshake:157(2247485ns) {DropHandshakes:470(935ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(208ns)} GenerateSessionVirtualId:988(348ns) PrepareNewSessionHandshake:477(1630ns) {SwitchToState:179(823ns)}}}}}
2025-11-27T18:17:07.680042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470903583529874:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18057159ns) {Configure:104(18053281ns) {ProcessConfigured:135(2396633ns) {StartInitialHandshake:157(2395501ns) {DropHandshakes:470(1054ns) {DropIncomingHandshake:442(630ns) DropOutgoingHandshake:457(141ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:477(1483ns) {SwitchToState:179(642ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.705078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470900950834618:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(66112800ns) {Configure:104(48043179ns) {ProcessConfigured:135(946412ns) {StartInitialHandshake:157(945203ns) {DropHandshakes:470(786ns) {DropIncomingHandshake:442(169ns) DropOutgoingHandshake:457(183ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:477(1087ns) {SwitchToState:179(109ns)}}}}}
2025-11-27T18:17:07.679560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470901816154317:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(70145062ns) {Configure:104(70135875ns) {ProcessConfigured:135(762246ns) {StartInitialHandshake:157(760966ns) {DropHandshakes:470(481ns) {DropIncomingHandshake:442(160ns) DropOutgoingHandshake:457(34ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(1206ns) {SwitchToState:179(629ns)}}}}}
2025-11-27T18:17:07.682392Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470903583529812:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40576730ns) {Configure:104(40568408ns) {ProcessConfigured:135(1354744ns) {StartInitialHandshake:157(1352307ns) {DropHandshakes:470(1273ns) {DropIncomingHandshake:442(310ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(502ns) PrepareNewSessionHandshake:477(2079ns) {SwitchToState:179(906ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.708509Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470900950834619:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45345511ns) {Configure:104(45331546ns) {ProcessConfigured:135(2889706ns) {StartInitialHandshake:157(2884516ns) {DropHandshakes:470(982ns) {DropIncomingHandshake:442(292ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(676ns) PrepareNewSessionHandshake:477(2987ns) {SwitchToState:179(1647ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.708874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470900950834683:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(43754969ns) {Configure:104(43750508ns) {ProcessConfigured:135(15437623ns) {StartInitialHandshake:157(15435141ns) {DropHandshakes:470(1917ns) {DropIncomingHandshake:442(467ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(724ns) PrepareNewSessionHandshake:477(4487ns) {SwitchToState:179(2135ns)}}}}}
2025-11-27T18:17:07.722139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470901816154398:2116] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(52298008ns) {Configure:104(52292368ns) {ProcessConfigured:135(3556718ns) {StartInitialHandshake:157(3553644ns) {DropHandshakes:470(3059ns) {DropIncomingHandshake:442(976ns) DropOutgoingHandshake:457(346ns)} GenerateSessionVirtualId:988(393ns) PrepareNewSessionHandshake:477(3095ns) {SwitchToState:179(1282ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.722408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470903583529886:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33650
2025-11-27T18:17:07.711725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470900950834694:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.711779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470900950834695:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-27T18:17:07.731609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470901126675719:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49180
2025-11-27T18:17:07.732516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470903343534969:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54555
2025-11-27T18:17:07.730084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470903583529886:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.736823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470903343534970:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54555
2025-11-27T18:17:07.713471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470900950834695:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.732562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470903583529887:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33650
2025-11-27T18:17:07.734812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470901126675721:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49180
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.732956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470900950834694:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.736539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470903583529887:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.738377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470903343534971:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54555
2025-11-27T18:17:07.736571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470901126675719:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.735199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470900950834618:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.737700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470901126675721:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.738232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470903583529890:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33650
2025-11-27T18:17:07.747605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470901010436555:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18019
2025-11-27T18:17:07.742515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470903343534959:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.737699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470900950834698:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-27T18:17:07.746069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470903583529811:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.737727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470901126675726:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49180
2025-11-27T18:17:07.742549Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470903343534959:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.747604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470900950834701:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-27T18:17:07.737752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470901126675723:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49180
2025-11-27T18:17:07.745979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470903343534972:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54555
2025-11-27T18:17:07.749005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470900950834698:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.748789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470901010436555:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.741513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470901126675643:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.746885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470903343534957:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.749218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470900950834701:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.750536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470901010436477:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.750767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470900950834688:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.748600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470901126675723:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.746883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470903583529893:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33650
2025-11-27T18:17:07.750782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470901816154406:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.749752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470901126675726:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.746922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470903343534957:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.750786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470900950834688:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.750810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470901816154406:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.748487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470903583529894:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33650
2025-11-27T18:17:07.750024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470901126675716:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.757563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470900950834703:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-27T18:17:07.746949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470903343534959:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:07.748588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470903583529893:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.757462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470901816154406:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:07.757985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470900950834690:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.750041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470901126675716:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.748408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470903343534963:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.748695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470903583529890:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.750850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470901126675716:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:07.760969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470901816154397:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.758013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470900950834690:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.748433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470903343534963:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.749090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470903583529875:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.761008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:07.749785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470903343534972:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.761109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470900950834688:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:07.759818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470901126675707:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.749772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470903583529894:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.761201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470901816154397:2108] [node 5] ICP22 created new session: [4:7577470901816154414:2108]
2025-11-27T18:17:07.750553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470903343534970:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.761144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470900950834694:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:07.760028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.749855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470903583529813:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.762126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470901816154414:2108] [node 5] ICS09 handshake done sender: [4:7577470901816154406:2131] self: [4:7577470901816154401:0] peer: [5:7577470903583529897:0] socket: 108 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.762489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470900950834618:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.750710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470903343534957:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:07.760168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470901126675707:2108] [node 5] ICP22 created new session: [3:7577470901126675733:2108]
2025-11-27T18:17:07.763041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.763083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470901816154414:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:07.750635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470903583529812:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.761156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470901126675733:2108] [node 5] ICS09 handshake done sender: [3:7577470901126675716:2113] self: [3:7577470901126675714:0] peer: [5:7577470903583529896:0] socket: 122 qp: -1
2025-11-27T18:17:07.776684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470901010436557:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18019
2025-11-27T18:17:07.750722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470903343534889:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.763924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:07.750761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.761535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470901126675733:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.763084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470900950834690:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:07.773739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470901816154413:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23658
2025-11-27T18:17:07.762177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470903583529813:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7360) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470903583529893:2118] held: no
2025-11-27T18:17:07.751730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470903343534889:2048] [node 2] ICP22 created new session: [1:7577470903343534977:2048]
2025-11-27T18:17:07.761571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470901126675729:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49180
2025-11-27T18:17:07.769491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470900950834618:2048] [node 1] ICP22 created new session: [2:7577470900950834708:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.775500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470903583529899:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33650
2025-11-27T18:17:07.752000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470903343534971:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.779715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470901816154413:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.781324Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470901010436477:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470901010436555:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.762418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:07.780940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470901816154316:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.777934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470903583529900:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33650
2025-11-27T18:17:07.773807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470900950834708:2048] [node 1] ICS09 handshake done sender: [2:7577470900950834694:2115] self: [2:7577470900950834702:0] peer: [1:7577470903343534958:0] socket: 101 qp: -1
2025-11-27T18:17:07.784736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470901010436559:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18019
2025-11-27T18:17:07.763987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470901126675719:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:07.757535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470903343534963:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:07.779410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470903583529901:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33650
2025-11-27T18:17:07.788182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470901816154397:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18684974ns) {HandleHandshakeStatus:329(18681907ns) {DropIncomingHandshake:442(187ns) DropOutgoingHandshake:457(243ns) SwitchToState:179(305ns) ProcessPendingSessionEvents:495(5430944ns) {ForwardSessionEventToSession:612(2714760ns) ForwardSessionEventToSession:612(156920ns) ForwardSessionEventToSession:612(894360ns) ForwardSessionEventToSession:612(7998ns) ForwardSessionEventToSession:612(382762ns) ForwardSessionEventToSession:612(2781ns) ForwardSessionEventToSession:612(460608ns) ForwardSessionEventToSession:612(3395ns) ForwardSessionEventToSession:612(2668ns) ForwardSessionEventToSession:612(783575ns)}}}
2025-11-27T18:17:07.774207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470900950834708:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.784757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470901010436558:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18019
2025-11-27T18:17:07.757553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470903343534977:2048] [node 2] ICS09 handshake done sender: [1:7577470903343534959:2112] self: [1:7577470903343534958:0] peer: [2:7577470900950834702:0] socket: 92 qp: -1
2025-11-27T18:17:07.780229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470903583529811:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7125) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470903583529887:2116] held: no
2025-11-27T18:17:07.775314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470900950834681:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.773845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470901126675643:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7125) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470901126675719:2115] held: no
2025-11-27T18:17:07.792176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470901816154417:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23658
2025-11-27T18:17:07.780985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470903583529893:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:07.788193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470901010436555:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:07.776727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:07.775304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470901126675643:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.794979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470901816154417:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.757610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470903343534977:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:07.784773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470903583529902:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33650
2025-11-27T18:17:07.798879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470901816154318:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.788978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470901010436477:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.784848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470903583529890:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:07.775349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:07.789029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:07.788145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470903583529887:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:07.757768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470903343534969:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:17:07.776330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470901126675643:2048] [node 1] ICP22 created new session: [3:7577470901126675736:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.776745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:07.790759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470901010436559:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.791139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470903583529813:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.758001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:07.777059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470901126675736:2048] [node 1] ICS09 handshake done sender: [3:7577470901126675719:2115] self: [3:7577470901126675727:0] peer: [1:7577470903343534956:0] socket: 97 qp: -1
2025-11-27T18:17:07.777074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470900950834681:2109] [node 6] ICP22 created new session: [2:7577470900950834710:2109]
2025-11-27T18:17:07.791175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:07.758425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470903343534890:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.791139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470901010436477:2049] [node 2] ICP22 created new session: [6:7577470901010436560:2049]
2025-11-27T18:17:07.778815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470901126675736:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:07.799732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470901816154316:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7125) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470901816154413:2133] held: no
2025-11-27T18:17:07.777861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470900950834682:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.791177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470901010436558:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.758478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:07.802345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470901816154421:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23658
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.777929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:07.792199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470903583529811:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.779341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:07.759278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470903343534890:2049] [node 3] ICP22 created new session: [1:7577470903343534981:2049]
2025-11-27T18:17:07.792289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470901010436476:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.792222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470903583529813:2050] [node 3] ICP22 created new session: [5:7577470903583529904:2050]
2025-11-27T18:17:07.802510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470901816154419:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23658
2025-11-27T18:17:07.784442Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470901126675707:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(19860878ns) {HandleHandshakeStatus:329(19858836ns) {DropIncomingHandshake:442(27368ns) DropOutgoingHandshake:457(441ns) SwitchToState:179(202ns) ProcessPendingSessionEvents:495(8040567ns) {ForwardSessionEventToSession:612(4638069ns) ForwardSessionEventToSession:612(806802ns) ForwardSessionEventToSession:612(3777ns) ForwardSessionEventToSession:612(2553ns) ForwardSessionEventToSession:612(915263ns) ForwardSessionEventToSession:612(2943ns) ForwardSessionEventToSession:612(1369054ns) ForwardSessionEventToSession:612(3276ns) ForwardSessionEventToSession:612(2293ns) ForwardSessionEventToSession:612(281469ns)}}}
2025-11-27T18:17:07.759328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470903343534981:2049] [node 3] ICS09 handshake done sender: [1:7577470903343534957:2111] self: [1:7577470903343534956:0] peer: [3:7577470901126675727:0] socket: 91 qp: -1
2025-11-27T18:17:07.792492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470901010436560:2049] [node 2] ICS09 handshake done sender: [6:7577470901010436555:2116] self: [6:7577470901010436556:0] peer: [2:7577470900950834685:0] socket: 129 qp: -1
2025-11-27T18:17:07.792464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470903583529875:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.778334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470900950834682:2108] [node 5] ICP22 created new session: [2:7577470900950834712:2108]
2025-11-27T18:17:07.759354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470903343534981:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:07.803642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470901816154420:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23658
2025-11-27T18:17:07.778765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470900950834710:2109] [node 6] ICS09 handshake done sender: [2:7577470900950834688:2111] self: [2:7577470900950834685:0] peer: [6:7577470901010436556:0] socket: 127 qp: -1
2025-11-27T18:17:07.792927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470901010436560:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.797874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470901126675717:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.792792Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470903583529894:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:07.779635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470900950834712:2108] [node 5] ICS09 handshake done sender: [2:7577470900950834690:2112] self: [2:7577470900950834687:0] peer: [5:7577470903583529898:0] socket: 128 qp: -1
2025-11-27T18:17:07.804619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470901816154421:2137] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.792972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:07.760135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:07.797899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470901126675717:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.793438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.779662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470900950834710:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:07.805526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470901816154420:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.793386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470901010436478:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.800249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470901126675739:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49180
2025-11-27T18:17:07.761219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470903343534955:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.793477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470903583529875:2108] [node 4] ICP22 created new session: [5:7577470903583529907:2108]
2025-11-27T18:17:07.798898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470901010436562:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18019
2025-11-27T18:17:07.779697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:07.805795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470901816154413:2133] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:07.761275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:07.801651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470901126675739:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.793685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.761772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470903343534955:2109] [node 5] ICP22 created new session: [1:7577470903343534984:2109]
2025-11-27T18:17:07.809897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470901816154423:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23658
2025-11-27T18:17:07.793722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470903583529811:2048] [node 1] ICP22 created new session: [5:7577470903583529906:2048]
2025-11-27T18:17:07.779940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470900950834712:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:17:07.805649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470901010436478:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7360) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470901010436558:2118] held: no
::ffff:127.1.0.0
2025-11-27T18:17:07.801768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470901126675717:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:07.763099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470903343534984:2109] [node 5] ICS09 handshake done sender: [1:7577470903343534963:2113] self: [1:7577470903343534960:0] peer: [5:7577470903583529895:0] socket: 94 qp: -1
2025-11-27T18:17:07.809914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470901816154317:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.793736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470903583529904:2050] [node 3] ICS09 handshake done sender: [5:7577470903583529893:2118] self: [5:7577470903583529896:0] peer: [3:7577470901126675714:0] socket: 124 qp: -1
2025-11-27T18:17:07.809854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470901010436567:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18019
2025-11-27T18:17:07.801947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470901126675720:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.779977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.810116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470901816154316:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.788842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470900950834709:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62499
2025-11-27T18:17:07.763907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470903343534984:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:07.763952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:07.809915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470901010436566:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18019
2025-11-27T18:17:07.801960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470901126675644:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.794997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470903583529904:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:07.792856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470900950834709:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.810695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470901010436562:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.801970Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470901126675720:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.795027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:07.810149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:07.813811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470901010436559:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:07.803095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470901126675709:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.774166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470903343534889:2048] [node 2] ICP18 handshake (actor [1:7577470903343534970:2117]) is too old
2025-11-27T18:17:07.810184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470901816154316:2048] [node 1] ICP22 created new session: [4:7577470901816154428:2048]
2025-11-27T18:17:07.792927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470900950834618:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21991694ns) {HandleHandshakeStatus:329(21988697ns) {DropIncomingHandshake:442(435ns) DropOutgoingHandshake:457(4446ns) SwitchToState:179(140ns) ProcessPendingSessionEvents:495(4312695ns) {ForwardSessionEventToSession:612(647416ns) ForwardSessionEventToSession:612(2986ns) ForwardSessionEventToSession:612(5875ns) ForwardSessionEventToSession:612(2396ns) ForwardSessionEventToSession:612(168995ns) ForwardSessionEventToSession:612(4049ns) ForwardSessionEventToSession:612(3112373ns) ForwardSessionEventToSession:612(342478ns) ForwardSessionEventToSession:612(10657ns) ForwardSessionEventToSession:612(2539ns)}}}
2025-11-27T18:17:07.803686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:07.817956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470901010436558:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:07.797283Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470903583529812:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470903583529894:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.797295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470903583529907:2108] [node 4] ICS09 handshake done sender: [5:7577470903583529890:2117] self: [5:7577470903583529897:0] peer: [4:7577470901816154401:0] socket: 110 qp: -1
2025-11-27T18:17:07.799559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470900950834682:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(16876688ns) {HandleHandshakeStatus:329(16873302ns) {DropIncomingHandshake:442(24300ns) DropOutgoingHandshake:457(899ns) SwitchToState:179(530ns) ProcessPendingSessionEvents:495(4392714ns) {ForwardSessionEventToSession:612(767455ns) ForwardSessionEventToSession:612(19417ns) ForwardSessionEventToSession:612(1863793ns) ForwardSessionEventToSession:612(327711ns) ForwardSessionEventToSession:612(585647ns) ForwardSessionEventToSession:612(4027ns) ForwardSessionEventToSession:612(2414ns) ForwardSessionEventToSession:612(801589ns) ForwardSessionEventToSession:612(2915ns) ForwardSessionEventToSession:612(2899ns)}}}
2025-11-27T18:17:07.819048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470901010436538:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.780622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470903343534890:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17783577ns) {HandleHandshakeStatus:329(17781424ns) {DropIncomingHandshake:442(12187ns) DropOutgoingHandshake:457(389ns) SwitchToState:179(247ns) ProcessPendingSessionEvents:495(14404338ns) {ForwardSessionEventToSession:612(2115493ns) ForwardSessionEventToSession:612(4846ns) ForwardSessionEventToSession:612(5538123ns) ForwardSessionEventToSession:612(3495ns) ForwardSessionEventToSession:612(4311825ns) ForwardSessionEventToSession:612(3499ns) ForwardSessionEventToSession:612(378719ns) ForwardSessionEventToSession:612(912968ns) ForwardSessionEventToSession:612(2907ns) ForwardSessionEventToSession:612(2541ns)}}}
2025-11-27T18:17:07.811755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470901816154428:2048] [node 1] ICS09 handshake done sender: [4:7577470901816154413:2133] self: [4:7577470901816154418:0] peer: [1:7577470903343534962:0] socket: 139 qp: -1
2025-11-27T18:17:07.804085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470901126675720:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:07.797444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470903583529906:2048] [node 1] ICS09 handshake done sender: [5:7577470903583529887:2116] self: [5:7577470903583529895:0] peer: [1:7577470903343534960:0] socket: 99 qp: -1
2025-11-27T18:17:07.799998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470900950834681:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(17921535ns) {HandleHandshakeStatus:329(17919387ns) {DropIncomingHandshake:442(16330ns) DropOutgoingHandshake:457(208ns) SwitchToState:179(382ns) ProcessPendingSessionEvents:495(7306358ns) {ForwardSessionEventToSession:612(3186659ns) ForwardSessionEventToSession:612(314772ns) ForwardSessionEventToSession:612(2133ns) ForwardSessionEventToSession:612(2057ns) ForwardSessionEventToSession:612(2677096ns) ForwardSessionEventToSession:612(7136ns) ForwardSessionEventToSession:612(503111ns) ForwardSessionEventToSession:612(2836ns) ForwardSessionEventToSession:612(2111ns) ForwardSessionEventToSession:612(591979ns)}}}
2025-11-27T18:17:07.819760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470901010436476:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.804206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470901126675709:2109] [node 6] ICP22 created new session: [3:7577470901126675743:2109]
2025-11-27T18:17:07.784784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470903343534965:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.797905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470903583529907:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:07.812976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470901816154428:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:07.821727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470901010436478:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.813760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:07.804897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470901126675708:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.798008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470903583529906:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:07.804157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470900950834691:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.821769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:07.798991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:07.805009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470901126675743:2109] [node 6] ICS09 handshake done sender: [3:7577470901126675717:2114] self: [3:7577470901126675715:0] peer: [6:7577470901010436565:0] socket: 148 qp: -1
2025-11-27T18:17:07.784818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470903343534965:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.817867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470901816154318:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7360) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470901816154417:2134] held: no
2025-11-27T18:17:07.804183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470900950834691:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.821807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:07.821907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470901816154425:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23658
2025-11-27T18:17:07.805035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470901126675743:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:07.799576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470903583529812:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.824621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470901010436476:2048] [node 1] ICP22 created new session: [6:7577470901010436571:2048]
2025-11-27T18:17:07.788930Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470903343534955:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(19816144ns) {HandleHandshakeStatus:329(19814073ns) {DropIncomingHandshake:442(10513ns) DropOutgoingHandshake:457(218ns) SwitchToState:179(223ns) ProcessPendingSessionEvents:495(3031403ns) {ForwardSessionEventToSession:612(822510ns) ForwardSessionEventToSession:612(873722ns) ForwardSessionEventToSession:612(2986ns) ForwardSessionEventToSession:612(2053ns) ForwardSessionEventToSession:612(305682ns) ForwardSessionEventToSession:612(3490ns) ForwardSessionEventToSession:612(10233ns) ForwardSessionEventToSession:612(137443ns) ForwardSessionEventToSession:612(473825ns) ForwardSessionEventToSession:612(383855ns)}}}
2025-11-27T18:17:07.809900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470900950834691:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:07.799621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:07.805555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:07.825968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470901816154417:2134] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:07.811481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470900950834619:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.826214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470901816154430:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23658
2025-11-27T18:17:07.812274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:07.828185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470901816154318:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.824968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470901010436571:2048] [node 1] ICS09 handshake done sender: [6:7577470901010436559:2119] self: [6:7577470901010436561:0] peer: [1:7577470903343534966:0] socket: 123 qp: -1
2025-11-27T18:17:07.791125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470903343534965:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:07.793358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470903343534954:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.829894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470901816154317:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470901816154420:2136] held: no
2025-11-27T18:17:07.800045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470903583529812:2049] [node 2] ICP22 created new session: [5:7577470903583529911:2049]
2025-11-27T18:17:07.793416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:07.830220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:07.793816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470903343534954:2107] [node 4] ICP22 created new session: [1:7577470903343534988:2107]
2025-11-27T18:17:07.812385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470900950834693:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.824979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470901010436478:2050] [node 3] ICP22 created new session: [6:7577470901010436572:2050]
2025-11-27T18:17:07.800290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:07.830535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470901816154420:2136] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:07.807136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:07.830549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470901816154318:2050] [node 3] ICP22 created new session: [4:7577470901816154431:2050]
2025-11-27T18:17:07.801059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470903583529911:2049] [node 2] ICS09 handshake done sender: [5:7577470903583529894:2119] self: [5:7577470903583529898:0] peer: [2:7577470900950834687:0] socket: 131 qp: -1
2025-11-27T18:17:07.809599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470901126675708:2107] [node 4] ICP22 created new session: [3:7577470901126675745:2107]
2025-11-27T18:17:07.812412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470900950834693:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.825764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470901010436571:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:07.830727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470901816154431:2050] [node 3] ICS09 handshake done sender: [4:7577470901816154417:2134] self: [4:7577470901816154422:0] peer: [3:7577470901126675718:0] socket: 111 qp: -1
2025-11-27T18:17:07.797583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470903343534967:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.810630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470901126675745:2107] [node 4] ICS09 handshake done sender: [3:7577470901126675720:2116] self: [3:7577470901126675718:0] peer: [4:7577470901816154422:0] socket: 109 qp: -1
2025-11-27T18:17:07.830987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470901816154431:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:07.801088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470903583529911:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:07.797614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470903343534967:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.811446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470901126675745:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:07.813743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470900950834619:2049] [node 3] ICP22 created new session: [2:7577470900950834719:2049]
2025-11-27T18:17:07.825817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:07.802171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:07.831246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:07.798084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470903343534988:2107] [node 4] ICS09 handshake done sender: [1:7577470903343534965:2114] self: [1:7577470903343534962:0] peer: [4:7577470901816154418:0] socket: 138 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:07.811836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:07.825929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470901010436572:2050] [node 3] ICS09 handshake done sender: [6:7577470901010436558:2118] self: [6:7577470901010436565:0] peer: [3:7577470901126675715:0] socket: 149 qp: -1
2025-11-27T18:17:07.826143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470903583529884:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.826138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470901010436572:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:07.831276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470901816154317:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.798939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470903343534988:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:07.826169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470903583529884:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.826171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:07.818688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470900950834693:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:07.818699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470901126675746:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49180
2025-11-27T18:17:07.798987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:07.828290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470903583529884:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:07.827357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470901010436568:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18019
2025-11-27T18:17:07.832018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:07.818583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470900950834719:2049] [node 3] ICS09 handshake done sender: [2:7577470900950834691:2113] self: [2:7577470900950834689:0] peer: [3:7577470901126675741:0] socket: 116 qp: -1
2025-11-27T18:17:07.801759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470903343534967:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:07.818756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470901126675644:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470901126675739:2121] held: no
2025-11-27T18:17:07.818760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470900950834719:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:07.830222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470903583529874:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.831206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470901010436568:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.803108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470903343534953:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.825809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470901126675739:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:07.832190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470901816154317:2049] [node 2] ICP22 created new session: [4:7577470901816154433:2049]
2025-11-27T18:17:07.819033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:07.831020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:07.831315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470901010436575:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18019
2025-11-27T18:17:07.803144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:07.832986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470901816154433:2049] [node 2] ICS09 handshake done sender: [4:7577470901816154420:2136] self: [4:7577470901816154427:0] peer: [2:7577470900950834692:0] socket: 152 qp: -1
2025-11-27T18:17:07.831394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470903583529874:2109] [node 6] ICP22 created new session: [5:7577470903583529914:2109]
2025-11-27T18:17:07.824927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470900950834683:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.831415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470901010436538:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29315) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470901010436562:2120] held: no
2025-11-27T18:17:07.826264Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470901126675708:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(20281325ns) {HandleHandshakeStatus:329(20278798ns) {DropIncomingHandshake:442(9345ns) DropOutgoingHandshake:457(405ns) SwitchToState:179(172ns) ProcessPendingSessionEvents:495(7126685ns) {ForwardSessionEventToSession:612(467121ns) ForwardSessionEventToSession:612(435095ns) ForwardSessionEventToSession:612(735078ns) ForwardSessionEventToSession:612(2105620ns) ForwardSessionEventToSession:612(2803394ns) ForwardSessionEventToSession:612(3901ns) ForwardSessionEventToSession:612(2135ns) ForwardSessionEventToSession:612(300814ns) ForwardSessionEventToSession:612(2316ns) ForwardSessionEventToSession:612(254002ns)}}}
2025-11-27T18:17:07.803989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470903343534953:2110] [node 6] ICP22 created new session: [1:7577470903343534991:2110]
2025-11-27T18:17:07.825798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:07.833186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470901816154433:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:07.831755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470903583529914:2109] [node 6] ICS09 handshake done sender: [5:7577470903583529884:2113] self: [5:7577470903583529879:0] peer: [6:7577470901010436570:0] socket: 93 qp: -1
2025-11-27T18:17:07.804790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470903343534991:2110] [node 6] ICS09 handshake done sender: [1:7577470903343534967:2115] self: [1:7577470903343534966:0] peer: [6:7577470901010436561:0] socket: 150 qp: -1
2025-11-27T18:17:07.826520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470901126675644:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.804827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470903343534991:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:07.831923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470901010436543:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:07.833392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:07.831876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470903583529914:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:07.805803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:07.833510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470901816154411:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:07.826548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:07.826326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470900950834683:2107] [node 4] ICP22 created new session: [2:7577470900950834722:2107]
2025-11-27T18:17:07.832895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470901010436562:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:07.833539Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470901816154411:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:07.829566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470903343534993:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54555
2025-11-27T18:17:07.832025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:07.833598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470901010436538:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.835227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470901816154411:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:07.826779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470901126675644:2049] [node 2] ICP22 created new session: [3:7577470901126675748:2049]
2025-11-27T18:17:07.835567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:07.835818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470901816154398:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:07.828389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470901126675748:2049] [node 2] ICS09 handshake done sender: [3:7577470901126675739:2121] self: [3:7577470901126675741:0] peer: [2:7577470900950834689:0] socket: 121 qp: -1
2025-11-27T18:17:07.826841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470900950834722:2107] [node 4] ICS09 handshake done sender: [2:7577470900950834693:2114] self: [2:7577470900950834692:0] peer: [4:7577470901816154427:0] socket: 151 qp: -1
2025-11-27T18:17:07.835683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470901010436538:2106] [node 5] ICP22 created new session: [6:7577470901010436578:2106]
2025-11-27T18:17:07.835865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:07.827684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470900950834722:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:07.835896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470901816154398:2116] [node 6] ICP22 created new session: [4:7577470901816154436:2116]
2025-11-27T18:17:07.836106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470901010436578:2106] [node 5] ICS09 handshake done sender: [6:7577470901010436562:2120] self: [6:7577470901010436570:0] peer: [5:7577470903583529879:0] socket: 126 qp: -1
2025-11-27T18:17:07.828417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470901126675748:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:07.828173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:07.836251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470901816154436:2116] [node 6] ICS09 handshake done sender: [4:7577470901816154411:2132] self: [4:7577470901816154410:0] peer: [6:7577470901010436576:0] socket: 102 qp: -1
2025-11-27T18:17:07.836367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470901816154436:2116] [node 6] ICS10 traffic start
2025-11-27T18:17:07.829122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:07.836142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470901010436578:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:07.836490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:07.836333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:07.836528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470901010436577:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18019
2025-11-27T18:17:07.836750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470901010436568:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:07.837184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470901010436543:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:07.837231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:07.837396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470901010436543:2109] [node 4] ICP22 created new session: [6:7577470901010436580:2109]
2025-11-27T18:17:07.837459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470901010436580:2109] [node 4] ICS09 handshake done sender: [6:7577470901010436568:2123] self: [6:7577470901010436576:0] peer: [4:7577470901816154410:0] socket: 105 qp: -1
2025-11-27T18:17:07.837545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470901010436580:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:07.837594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:07.838442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577470901010436543:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:07.874259Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:07.874515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470900950834708:2048] [node 1] ICS07 socket disconnect 101 reason# EndOfStream
2025-11-27T18:17:07.874545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470900950834708:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.874616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:07.874667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470900950834708:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:07.875480Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:07.875702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470901126675736:2048] [node 1] ICS01 socket: 97 reason# EndOfStream
2025-11-27T18:17:07.875749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470901126675643:2048] [node 1] ICP30 unregister session Session# [3:7577470901126675736:2048] VirtualId# [3:7577470901126675727:0]
2025-11-27T18:17:07.875779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470901126675736:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.875851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:07.875900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:07.876880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470900950834618:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7125) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:07.876917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470900950834708:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:07.877983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470900950834618:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7125) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:07.878351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470900950834708:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:07.878388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470900950834618:2048] [node 1] ICP30 unregister session Session# [2:7577470900950834708:2048] VirtualId# [2:7577470900950834702:0]
2025-11-27T18:17:07.878425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:07.878485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470900950834618:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7125) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:07.878502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7125) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:07.879444Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:07.879503Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:07.879662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470901816154428:2048] [node 1] ICS01 socket: 139 reason# EndOfStream
2025-11-27T18:17:07.879658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470903583529906:2048] [node 1] ICS01 socket: 99 reason# EndOfStream
2025-11-27T18:17:07.879699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470903583529811:2048] [node 1] ICP30 unregister session Session# [5:7577470903583529906:2048] VirtualId# [5:7577470903583529895:0]
2025-11-27T18:17:07.879727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470903583529906:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.879700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470901816154316:2048] [node 1] ICP30 unregister session Session# [4:7577470901816154428:2048] VirtualId# [4:7577470901816154418:0]
2025-11-27T18:17:07.879733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470901816154428:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.879795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:07.879806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:07.879871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:07.879836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:07.880319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:07.880444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470901010436571:2048] [node 1] ICS07 socket disconnect 123 reason# EndOfStream
2025-11-27T18:17:07.880466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470901010436571:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.880516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:07.880554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470901010436571:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:07.881153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470901010436476:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7125) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:07.881193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470901010436571:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:07.881244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470901010436476:2048] [node 1] ICP30 unregister session Session# [6:7577470901010436571:2048] VirtualId# [6:7577470901010436561:0]
2025-11-27T18:17:07.881386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:07.881412Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470901010436476:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7125) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:07.881429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7125) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:07.891511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470901010436476:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:07.909948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:07.910188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470901126675748:2049] [node 2] ICS01 socket: 121 reason# EndOfStream
2025-11-27T18:17:07.910236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470901126675644:2049] [node 2] ICP30 unregister session Session# [3:7577470901126675748:2049] VirtualId# [3:7577470901126675741:0]
2025-11-27T18:17:07.910264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470901126675748:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.910345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:07.910398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:07.912504Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:07.912732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470901816154433:2049] [node 2] ICS01 socket: 152 reason# EndOfStream
2025-11-27T18:17:07.912773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470901816154317:2049] [node 2] ICP30 unregister session Session# [4:7577470901816154433:2049] VirtualId# [4:7577470901816154427:0]
2025-11-27T18:17:07.912796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470901816154433:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.912852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:07.912884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:07.913424Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:07.913619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470903583529911:2049] [node 2] ICS01 socket: 131 reason# EndOfStream
2025-11-27T18:17:07.913663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470903583529812:2049] [node 2] ICP30 unregister session Session# [5:7577470903583529911:2049] VirtualId# [5:7577470903583529898:0]
2025-11-27T18:17:07.913685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470903583529911:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.913751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:07.913805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:07.914241Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:07.914383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470901010436560:2049] [node 2] ICS01 socket: 129 reason# EndOfStream
2025-11-27T18:17:07.914409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470901010436477:2049] [node 2] ICP30 unregister session Session# [6:7577470901010436560:2049] VirtualId# [6:7577470901010436556:0]
2025-11-27T18:17:07.914426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470901010436560:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.914469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:07.914495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:07.966308Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:07.966525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470901816154431:2050] [node 3] ICS01 socket: 111 reason# EndOfStream
2025-11-27T18:17:07.966593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470901816154318:2050] [node 3] ICP30 unregister session Session# [4:7577470901816154431:2050] VirtualId# [4:7577470901816154422:0]
2025-11-27T18:17:07.966628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470901816154431:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.966685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:07.966718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:07.967276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:07.967419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470903583529904:2050] [node 3] ICS01 socket: 124 reason# EndOfStream
2025-11-27T18:17:07.967449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470903583529813:2050] [node 3] ICP30 unregister session Session# [5:7577470903583529904:2050] VirtualId# [5:7577470903583529896:0]
2025-11-27T18:17:07.967469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470903583529904:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.967526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:07.967560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:07.968061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:07.968260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470901010436572:2050] [node 3] ICS01 socket: 149 reason# EndOfStream
2025-11-27T18:17:07.968295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470901010436478:2050] [node 3] ICP30 unregister session Session# [6:7577470901010436572:2050] VirtualId# [6:7577470901010436565:0]
2025-11-27T18:17:07.968322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470901010436572:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:07.968384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:07.968425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:08.016032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:08.016291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470903583529907:2108] [node 4] ICS07 socket disconnect 110 reason# EndOfStream
2025-11-27T18:17:08.016320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470903583529907:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.016409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:08.016477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470903583529907:2108] [node 4] ICS15 start handshake
2025-11-27T18:17:08.016841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:08.017052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470901010436580:2109] [node 4] ICS07 socket disconnect 105 reason# EndOfStream
2025-11-27T18:17:08.017095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470901010436580:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.017157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:08.017202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470901010436580:2109] [node 4] ICS15 start handshake
2025-11-27T18:17:08.017251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470903583529875:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27228) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:08.017367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470903583529907:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:08.017581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470903583529875:2108] [node 4] ICP30 unregister session Session# [5:7577470903583529907:2108] VirtualId# [5:7577470903583529897:0]
2025-11-27T18:17:08.017718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:08.017769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470903583529875:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27228) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.017800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27228) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.018179Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470901010436543:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27228) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:08.018270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470901010436580:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:08.018318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470901010436543:2109] [node 4] ICP30 unregister session Session# [6:7577470901010436580:2109] VirtualId# [6:7577470901010436576:0]
2025-11-27T18:17:08.018351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:08.018386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470901010436543:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27228) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.018398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27228) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.027713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470903583529875:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:17:08.027709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470901010436543:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:17:08.080144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:08.080306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470901010436578:2106] [node 5] ICS01 socket: 126 reason# EndOfStream
2025-11-27T18:17:08.080336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470901010436538:2106] [node 5] ICP30 unregister session Session# [6:7577470901010436578:2106] VirtualId# [6:7577470901010436570:0]
2025-11-27T18:17:08.080358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470901010436578:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.080420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:08.080454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:08.144157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470906192158455:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:08.144194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470906192158456:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:08.166648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470907280291246:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:08.166722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470907280291247:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:08.192599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470906584898957:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:08.192659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470906584898958:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:08.217282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470907792753585:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:08.217338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470907792753586:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:08.217363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470907792753587:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:08.243441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470908164335293:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:08.243511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470908164335294:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:08.243546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470908164335295:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:08.268418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470905695728003:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:08.268481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470905695728004:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:08.268526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470905695728005:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:08.368741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470905695728064:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.371123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470907280291306:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53562
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.372797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470907280291307:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53562
2025-11-27T18:17:08.374677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470907280291306:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.376288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470907280291246:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:08.376585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470906192158516:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.376626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470906192158516:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.393335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470908164335354:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45137
2025-11-27T18:17:08.390393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470906192158516:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:08.400470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470906192158521:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:08.400668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470906192158522:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.400705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470906192158455:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.400736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:08.400766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470906192158455:2048] [node 2] ICP22 created new session: [1:7577470906192158523:2048]
2025-11-27T18:17:08.400792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470906192158523:2048] [node 2] ICS09 handshake done sender: [1:7577470906192158516:2107] self: [1:7577470906192158515:0] peer: [2:7577470907280291308:0] socket: 96 qp: -1
2025-11-27T18:17:08.400821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470906192158523:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:08.400846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:08.401805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470907280291309:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53562
2025-11-27T18:17:08.404545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470908164335354:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.404864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470907280291306:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:08.417578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470906192158532:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:08.432522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470908164335356:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:08.439872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470907280291246:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.439938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.488036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470908164335356:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:08.488238Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470908164335356:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(55453031ns) {Configure:104(55451075ns) {ProcessConfigured:135(130366ns) {IncomingHandshake:261(104188ns) {DropIncomingHandshake:442(784ns) PrepareNewSessionHandshake:477(1992ns) {SwitchToState:179(953ns)} GenerateSessionVirtualId:988(3977ns) IssueIncomingHandshakeReply:193(71456ns) {SwitchToState:179(208ns)}}}}}
2025-11-27T18:17:08.488704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470908164335358:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45137
2025-11-27T18:17:08.439999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470907280291246:2048] [node 1] ICP22 created new session: [2:7577470907280291313:2048]
2025-11-27T18:17:08.499226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470905695728067:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.440039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470907280291313:2048] [node 1] ICS09 handshake done sender: [2:7577470907280291306:2107] self: [2:7577470907280291308:0] peer: [1:7577470906192158515:0] socket: 97 qp: -1
2025-11-27T18:17:08.499300Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7577470905695728067:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.440076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470907280291313:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:08.440114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:08.510494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470905695728067:2109] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:08.455242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470907280291307:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.550288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7577470905695728064:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.498695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470907280291315:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:08.549609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470906584898957:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(89978421ns) {Configure:104(89968828ns) {ProcessConfigured:135(23837606ns) {StartInitialHandshake:157(15483593ns) {DropHandshakes:470(4320ns) {DropIncomingHandshake:442(785ns) DropOutgoingHandshake:457(480ns)} GenerateSessionVirtualId:988(2611ns) PrepareNewSessionHandshake:477(5801ns) {SwitchToState:179(2146ns)}}}}}
2025-11-27T18:17:08.538503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470907280291316:2111] [node 5] ICP01 ready to work
2025-11-27T18:17:08.550644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470906584899028:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:08.550352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:08.553310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470908164335294:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23249911ns) {Configure:104(23238488ns) {ProcessConfigured:135(63295ns) {StartInitialHandshake:157(57165ns) {DropHandshakes:470(2873ns) {DropIncomingHandshake:442(743ns) DropOutgoingHandshake:457(431ns)} GenerateSessionVirtualId:988(750ns) PrepareNewSessionHandshake:477(1658ns) {SwitchToState:179(446ns)}}}}}
2025-11-27T18:17:08.549852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470907280291317:2112] [node 6] ICP01 ready to work
2025-11-27T18:17:08.550892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470906584899029:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:08.551119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470906584899030:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:08.550413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470905695728064:2106] [node 5] ICP22 created new session: [6:7577470905695728074:2106]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.556604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470908164335365:2111] [node 4] ICP01 ready to work
2025-11-27T18:17:08.552537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470905695728074:2106] [node 5] ICS09 handshake done sender: [6:7577470905695728067:2109] self: [6:7577470905695728066:0] peer: [5:7577470908164335357:0] socket: 99 qp: -1
2025-11-27T18:17:08.559120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470908164335362:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45137
2025-11-27T18:17:08.563977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470907792753585:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(56436305ns) {Configure:104(56430219ns) {ProcessConfigured:135(112949ns) {StartInitialHandshake:157(110681ns) {DropHandshakes:470(8671ns) {DropIncomingHandshake:442(285ns) DropOutgoingHandshake:457(271ns)} GenerateSessionVirtualId:988(515ns) PrepareNewSessionHandshake:477(2612ns) {SwitchToState:179(1318ns)}}}}}
2025-11-27T18:17:08.553238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470905695728074:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:08.565329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470907792753650:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:08.554315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:08.566996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470907792753654:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:08.569422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470905695728076:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.575034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470907792753653:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19479
2025-11-27T18:17:08.577412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470908164335358:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:17:08.577367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470906584899028:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22719450ns) {Configure:104(22713450ns) {ProcessConfigured:135(7453577ns) {StartInitialHandshake:157(7451137ns) {DropHandshakes:470(2125ns) {DropIncomingHandshake:442(419ns) DropOutgoingHandshake:457(708ns)} GenerateSessionVirtualId:988(726ns) PrepareNewSessionHandshake:477(3716ns) {SwitchToState:179(1960ns)}}}}}
2025-11-27T18:17:08.577725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470906192158538:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48154
2025-11-27T18:17:08.577803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470908164335354:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.578053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470906192158539:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48154
2025-11-27T18:17:08.578314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470908164335356:2109] [node 6] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.578408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:08.579824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470906192158531:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.579860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470906192158531:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.578458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470908164335356:2109] [node 6] ICP22 created new session: [5:7577470908164335372:2109]
2025-11-27T18:17:08.578503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470908164335372:2109] [node 6] ICS09 handshake done sender: [5:7577470908164335354:2108] self: [5:7577470908164335357:0] peer: [6:7577470905695728066:0] socket: 103 qp: -1
2025-11-27T18:17:08.578541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470908164335372:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:08.578575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:08.579282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470908164335293:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.582877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470907280291326:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53562
2025-11-27T18:17:08.581974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470907792753653:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.591736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470906192158531:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:08.582925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470907280291327:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53562
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.595829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470908164335376:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45137
2025-11-27T18:17:08.595857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470906192158522:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.596495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470908164335377:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45137
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.601221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470907792753585:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.602393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470907792753664:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19479
2025-11-27T18:17:08.595925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:08.598713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470906192158522:2109] [node 5] ICP22 created new session: [1:7577470906192158541:2109]
2025-11-27T18:17:08.605514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470908164335376:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.605465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470907280291326:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.598846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470906192158541:2109] [node 5] ICS09 handshake done sender: [1:7577470906192158531:2113] self: [1:7577470906192158530:0] peer: [5:7577470908164335375:0] socket: 108 qp: -1
2025-11-27T18:17:08.605878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470908164335358:2110] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:08.599056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470906192158541:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:08.599562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.599975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470906584899031:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60665
2025-11-27T18:17:08.609440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470906192158538:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.600719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470906584899040:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60665
2025-11-27T18:17:08.600836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470906584899041:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60665
2025-11-27T18:17:08.613530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470907280291327:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.613643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470907792753666:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19479
2025-11-27T18:17:08.605732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470906584899040:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.617378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470906584899041:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.620923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470908164335293:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.620888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470907792753664:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.621034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470907792753586:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:08.621067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470906192158529:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.621108Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470906192158529:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.620892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470906584899031:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.623496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:08.625015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470908164335293:2048] [node 1] ICP22 created new session: [5:7577470908164335378:2048]
2025-11-27T18:17:08.625407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470907280291319:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.625451Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470907280291319:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.625438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470908164335378:2048] [node 1] ICS09 handshake done sender: [5:7577470908164335358:2110] self: [5:7577470908164335375:0] peer: [1:7577470906192158530:0] socket: 109 qp: -1
2025-11-27T18:17:08.625490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470908164335378:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.625619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:08.626159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470905695728083:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52272
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.626731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470907280291332:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53562
2025-11-27T18:17:08.627615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470907280291319:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.627918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470907792753668:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19479
2025-11-27T18:17:08.630242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470907280291315:2110] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.628248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470907792753669:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19479
2025-11-27T18:17:08.631759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:08.637547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470908164335294:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:08.632198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470907280291315:2110] [node 4] ICP22 created new session: [2:7577470907280291336:2110]
2025-11-27T18:17:08.637601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470906192158543:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48154
2025-11-27T18:17:08.632397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470907280291336:2110] [node 4] ICS09 handshake done sender: [2:7577470907280291319:2113] self: [2:7577470907280291318:0] peer: [4:7577470907792753667:0] socket: 126 qp: -1
2025-11-27T18:17:08.638202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470907792753664:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:08.638817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470906192158529:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:08.632455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470907280291336:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:08.638889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470905695728083:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.632521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:08.639825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470907280291321:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.639848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470906192158539:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.639866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470907280291321:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.639897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470905695728004:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.639922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470906192158521:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.640056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:08.640079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470905695728086:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52272
2025-11-27T18:17:08.640106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470906192158521:2108] [node 4] ICP22 created new session: [1:7577470906192158548:2108]
2025-11-27T18:17:08.640361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470906192158548:2108] [node 4] ICS09 handshake done sender: [1:7577470906192158529:2112] self: [1:7577470906192158528:0] peer: [4:7577470907792753665:0] socket: 107 qp: -1
2025-11-27T18:17:08.640601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470906192158548:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:08.640651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:08.640650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470906584898957:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:08.640695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470907792753586:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.640990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470907280291332:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.640762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:08.640861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470907792753586:2049] [node 2] ICP22 created new session: [4:7577470907792753670:2049]
2025-11-27T18:17:08.641308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470907280291323:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.641090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470907792753670:2049] [node 2] ICS09 handshake done sender: [4:7577470907792753664:2116] self: [4:7577470907792753667:0] peer: [2:7577470907280291318:0] socket: 131 qp: -1
2025-11-27T18:17:08.641333Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470907280291323:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.641317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470907792753670:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:08.641374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:17:08.641983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470906584899047:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60665
2025-11-27T18:17:08.642045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470907792753586:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:08.641999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470906192158546:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48154
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.642928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470906192158543:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.642973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470908164335382:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45137
2025-11-27T18:17:08.643129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470906192158527:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.643151Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470906192158527:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.643329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470907280291321:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:08.642982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470906584899047:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.643095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470906584899049:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60665
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.643956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470905695728088:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52272
2025-11-27T18:17:08.644268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470907280291316:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.644257Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470908164335376:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:08.644325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:08.644371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470907280291316:2111] [node 5] ICP22 created new session: [2:7577470907280291342:2111]
2025-11-27T18:17:08.644464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470907280291342:2111] [node 5] ICS09 handshake done sender: [2:7577470907280291321:2114] self: [2:7577470907280291320:0] peer: [5:7577470908164335380:0] socket: 127 qp: -1
2025-11-27T18:17:08.644490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470907280291342:2111] [node 5] ICS10 traffic start
2025-11-27T18:17:08.644542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:08.644596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470908164335294:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.645285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:08.645510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470908164335294:2049] [node 2] ICP22 created new session: [5:7577470908164335383:2049]
2025-11-27T18:17:08.645576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470908164335383:2049] [node 2] ICS09 handshake done sender: [5:7577470908164335376:2117] self: [5:7577470908164335380:0] peer: [2:7577470907280291320:0] socket: 132 qp: -1
2025-11-27T18:17:08.645624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470908164335383:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:08.645662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:08.645817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470907792753585:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6844) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470907792753653:2112] held: no
2025-11-27T18:17:08.645937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470906192158546:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.646062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470906192158522:2109] [node 5] ICP18 handshake (actor [1:7577470906192158539:2116]) is too old
2025-11-27T18:17:08.646121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470907280291323:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:08.646250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470907792753666:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.646591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470907792753653:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:08.647235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470905695728086:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.647408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470905695728083:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.647539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470907792753585:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.647731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:08.647774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470907792753585:2048] [node 1] ICP22 created new session: [4:7577470907792753674:2048]
2025-11-27T18:17:08.647816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470906192158527:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:08.647805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470907792753674:2048] [node 1] ICS09 handshake done sender: [4:7577470907792753653:2112] self: [4:7577470907792753665:0] peer: [1:7577470906192158528:0] socket: 113 qp: -1
2025-11-27T18:17:08.647814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470906584899051:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60665
2025-11-27T18:17:08.647916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470905695728004:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.647838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470907792753674:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:08.648100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:08.647889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:08.648447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470906192158456:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.648580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470905695728004:2049] [node 2] ICP22 created new session: [6:7577470905695728089:2049]
2025-11-27T18:17:08.648503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:08.648561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470906192158456:2049] [node 3] ICP22 created new session: [1:7577470906192158553:2049]
2025-11-27T18:17:08.648823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470906192158553:2049] [node 3] ICS09 handshake done sender: [1:7577470906192158527:2111] self: [1:7577470906192158526:0] peer: [3:7577470906584899048:0] socket: 106 qp: -1
2025-11-27T18:17:08.648765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470905695728089:2049] [node 2] ICS09 handshake done sender: [6:7577470905695728083:2115] self: [6:7577470905695728087:0] peer: [2:7577470907280291322:0] socket: 141 qp: -1
2025-11-27T18:17:08.648936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470906192158553:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:08.649382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470906584899031:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:08.648992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:08.648807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470905695728089:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:08.648837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.649975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470907792753672:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19479
2025-11-27T18:17:08.650171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470907280291317:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.650233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:08.650254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470906584898957:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.650281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470907280291317:2112] [node 6] ICP22 created new session: [2:7577470907280291344:2112]
2025-11-27T18:17:08.650300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:08.650439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470907792753587:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.650352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470906584898957:2048] [node 1] ICP22 created new session: [3:7577470906584899052:2048]
2025-11-27T18:17:08.650361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470907280291344:2112] [node 6] ICS09 handshake done sender: [2:7577470907280291323:2115] self: [2:7577470907280291322:0] peer: [6:7577470905695728087:0] socket: 128 qp: -1
2025-11-27T18:17:08.650421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470907280291344:2112] [node 6] ICS10 traffic start
2025-11-27T18:17:08.650620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470907792753672:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.650831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470908164335385:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45137
2025-11-27T18:17:08.650385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470906584899052:2048] [node 1] ICS09 handshake done sender: [3:7577470906584899031:2117] self: [3:7577470906584899048:0] peer: [1:7577470906192158526:0] socket: 111 qp: -1
2025-11-27T18:17:08.650471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:08.650409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470906584899052:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:08.650447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:08.651836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470905695728003:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:08.652964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470906192158534:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.652991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470906192158534:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.653619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470905695728092:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52272
2025-11-27T18:17:08.652833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470906584899035:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.652864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470906584899035:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.653664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470906584899049:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.654170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470906584898958:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:08.654490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470906584899035:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.654662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470906192158534:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:08.655398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470908164335386:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45137
2025-11-27T18:17:08.654812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470907280291325:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.654843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470907280291325:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.655677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470908164335385:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.655820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470906584899028:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.656286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470908164335365:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:08.656258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:08.656304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470906584899028:2107] [node 4] ICP22 created new session: [3:7577470906584899057:2107]
2025-11-27T18:17:08.656348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470906584899057:2107] [node 4] ICS09 handshake done sender: [3:7577470906584899035:2119] self: [3:7577470906584899034:0] peer: [4:7577470907792753677:0] socket: 138 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.656383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470906584899057:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:08.656423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:08.656646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470907280291325:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:08.656646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470905695728094:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52272
2025-11-27T18:17:08.656654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470906192158532:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.656709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:08.656760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470906192158532:2110] [node 6] ICP22 created new session: [1:7577470906192158556:2110]
2025-11-27T18:17:08.656800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470906192158556:2110] [node 6] ICS09 handshake done sender: [1:7577470906192158534:2114] self: [1:7577470906192158533:0] peer: [6:7577470905695728091:0] socket: 150 qp: -1
2025-11-27T18:17:08.656832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470906192158556:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:08.656870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:08.657283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470907280291247:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.657283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470907792753658:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.657396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:08.657324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470907792753658:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.657460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470907280291247:2049] [node 3] ICP22 created new session: [2:7577470907280291347:2049]
2025-11-27T18:17:08.657454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470905695728092:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.657518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470905695728086:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:08.657549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470907280291347:2049] [node 3] ICS09 handshake done sender: [2:7577470907280291325:2116] self: [2:7577470907280291324:0] peer: [3:7577470906584899056:0] socket: 115 qp: -1
2025-11-27T18:17:08.657586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470907280291347:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:08.657621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.658187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470905695728003:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.658244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:08.658319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470905695728003:2048] [node 1] ICP22 created new session: [6:7577470905695728095:2048]
2025-11-27T18:17:08.658294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470907792753679:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19479
2025-11-27T18:17:08.658430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470907792753680:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19479
2025-11-27T18:17:08.658934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470905695728095:2048] [node 1] ICS09 handshake done sender: [6:7577470905695728086:2116] self: [6:7577470905695728091:0] peer: [1:7577470906192158533:0] socket: 120 qp: -1
2025-11-27T18:17:08.658971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470905695728095:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:08.659014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:08.659986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470907792753587:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63980) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470907792753666:2117] held: no
2025-11-27T18:17:08.660217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470907792753666:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:08.661142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470907792753587:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.661220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:08.661273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470907792753587:2050] [node 3] ICP22 created new session: [4:7577470907792753682:2050]
2025-11-27T18:17:08.661329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470907792753682:2050] [node 3] ICS09 handshake done sender: [4:7577470907792753666:2117] self: [4:7577470907792753677:0] peer: [3:7577470906584899034:0] socket: 139 qp: -1
2025-11-27T18:17:08.661352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470908164335388:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45137
2025-11-27T18:17:08.661363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470907792753682:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:08.661425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470905695728076:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:08.661391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:08.662073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470907792753658:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:08.662115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470906584898958:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470906584899049:2125] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.662412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470905695728098:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52272
2025-11-27T18:17:08.662489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470907792753679:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.662540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470908164335386:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.662595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470908164335385:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:08.662624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470907792753660:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.662647Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470907792753660:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.662600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470906584899059:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60665
2025-11-27T18:17:08.663244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470908164335365:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.663339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:08.663378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470908164335365:2111] [node 4] ICP22 created new session: [5:7577470908164335389:2111]
2025-11-27T18:17:08.663421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470908164335389:2111] [node 4] ICS09 handshake done sender: [5:7577470908164335385:2120] self: [5:7577470908164335387:0] peer: [4:7577470907792753657:0] socket: 121 qp: -1
2025-11-27T18:17:08.663362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470906584899049:2125] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:08.663453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470908164335389:2111] [node 4] ICS10 traffic start
2025-11-27T18:17:08.663567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:08.663640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470906584898958:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.663618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470907792753650:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.663683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:08.663688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:08.663727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470906584898958:2049] [node 2] ICP22 created new session: [3:7577470906584899061:2049]
2025-11-27T18:17:08.663728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470907792753650:2108] [node 5] ICP22 created new session: [4:7577470907792753684:2108]
2025-11-27T18:17:08.663765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470907792753684:2108] [node 5] ICS09 handshake done sender: [4:7577470907792753658:2114] self: [4:7577470907792753657:0] peer: [5:7577470908164335387:0] socket: 119 qp: -1
2025-11-27T18:17:08.663788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470906584899061:2049] [node 2] ICS09 handshake done sender: [3:7577470906584899049:2125] self: [3:7577470906584899056:0] peer: [2:7577470907280291324:0] socket: 153 qp: -1
2025-11-27T18:17:08.663789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470907792753684:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:08.663829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470906584899061:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:08.663814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:08.663876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:08.665632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470908164335295:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.666064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470907792753660:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:08.666135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470905695728099:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52272
2025-11-27T18:17:08.666340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470907792753654:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.666373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470906584899037:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.666385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:08.666422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470907792753654:2109] [node 6] ICP22 created new session: [4:7577470907792753688:2109]
2025-11-27T18:17:08.666399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470906584899037:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.666566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470907792753688:2109] [node 6] ICS09 handshake done sender: [4:7577470907792753660:2115] self: [4:7577470907792753659:0] peer: [6:7577470905695728097:0] socket: 125 qp: -1
2025-11-27T18:17:08.666600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470907792753688:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:08.666625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:08.666682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470905695728098:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.666771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470905695728092:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:08.667461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470905695728076:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.667507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:08.667529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470906584899037:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:08.667540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470905695728076:2111] [node 4] ICP22 created new session: [6:7577470905695728100:2111]
2025-11-27T18:17:08.667585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470905695728100:2111] [node 4] ICS09 handshake done sender: [6:7577470905695728092:2118] self: [6:7577470905695728097:0] peer: [4:7577470907792753659:0] socket: 100 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.667614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470905695728100:2111] [node 4] ICS10 traffic start
2025-11-27T18:17:08.667638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:08.667869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470906584899029:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.667857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470908164335392:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45137
2025-11-27T18:17:08.667924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:08.667962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470906584899029:2108] [node 5] ICP22 created new session: [3:7577470906584899064:2108]
2025-11-27T18:17:08.668004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470906584899064:2108] [node 5] ICS09 handshake done sender: [3:7577470906584899037:2120] self: [3:7577470906584899036:0] peer: [5:7577470908164335391:0] socket: 142 qp: -1
2025-11-27T18:17:08.668044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470906584899064:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:08.668075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:08.668295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470908164335386:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:08.668479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470905695728005:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:08.668516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470908164335295:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.668561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:08.668599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470908164335295:2050] [node 3] ICP22 created new session: [5:7577470908164335393:2050]
2025-11-27T18:17:08.668646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470908164335393:2050] [node 3] ICS09 handshake done sender: [5:7577470908164335386:2121] self: [5:7577470908164335391:0] peer: [3:7577470906584899036:0] socket: 159 qp: -1
2025-11-27T18:17:08.668670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470908164335393:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:08.668698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:08.669706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470906584899039:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:08.669733Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470906584899039:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:08.670790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470906584899039:2121] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:08.671240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470906584899030:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:08.671287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:08.671329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470906584899030:2110] [node 6] ICP22 created new session: [3:7577470906584899067:2110]
2025-11-27T18:17:08.671369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470906584899067:2110] [node 6] ICS09 handshake done sender: [3:7577470906584899039:2121] self: [3:7577470906584899038:0] peer: [6:7577470905695728102:0] socket: 137 qp: -1
2025-11-27T18:17:08.671399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470906584899067:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:08.671423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:08.671302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470905695728103:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52272
2025-11-27T18:17:08.671711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470905695728098:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:08.671915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470905695728005:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:08.671979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:08.672024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470905695728005:2050] [node 3] ICP22 created new session: [6:7577470905695728104:2050]
2025-11-27T18:17:08.672063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470905695728104:2050] [node 3] ICS09 handshake done sender: [6:7577470905695728098:2120] self: [6:7577470905695728102:0] peer: [3:7577470906584899038:0] socket: 144 qp: -1
2025-11-27T18:17:08.672120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470905695728104:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:08.672162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:08.687336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:08.687524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470907280291313:2048] [node 1] ICS07 socket disconnect 97 reason# EndOfStream
2025-11-27T18:17:08.687548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470907280291313:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.687621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:08.687662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470907280291313:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:08.688771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:08.689000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470906584899052:2048] [node 1] ICS01 socket: 111 reason# EndOfStream
2025-11-27T18:17:08.689040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470906584898957:2048] [node 1] ICP30 unregister session Session# [3:7577470906584899052:2048] VirtualId# [3:7577470906584899048:0]
2025-11-27T18:17:08.689070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470906584899052:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.689134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:08.689359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:08.691131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470907280291246:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6844) Socket error# Connection reset by peer state# SendExRequestHeader processed# 0 remain# 8 incoming: [0:0:0] held: no
2025-11-27T18:17:08.691171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470907280291313:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:08.691791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:08.692248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470907280291246:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6844) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:08.691979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470907792753674:2048] [node 1] ICS01 socket: 113 reason# EndOfStream
2025-11-27T18:17:08.692013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470907792753585:2048] [node 1] ICP30 unregister session Session# [4:7577470907792753674:2048] VirtualId# [4:7577470907792753665:0]
2025-11-27T18:17:08.692309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470907280291313:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:08.692044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470907792753674:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.692103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:08.692151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:08.692529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470907280291246:2048] [node 1] ICP30 unregister session Session# [2:7577470907280291313:2048] VirtualId# [2:7577470907280291308:0]
2025-11-27T18:17:08.692643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:08.692678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470907280291246:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6844) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.692693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6844) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.693086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:08.693480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470908164335378:2048] [node 1] ICS01 socket: 109 reason# EndOfStream
2025-11-27T18:17:08.693520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470908164335293:2048] [node 1] ICP30 unregister session Session# [5:7577470908164335378:2048] VirtualId# [5:7577470908164335375:0]
2025-11-27T18:17:08.693544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470908164335378:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.693609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:08.693646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:08.695891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:08.696125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470905695728095:2048] [node 1] ICS07 socket disconnect 120 reason# EndOfStream
2025-11-27T18:17:08.696158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470905695728095:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.696250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:08.696359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470905695728095:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:08.697661Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470905695728003:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6844) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:08.697775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470905695728095:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:08.697836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470905695728003:2048] [node 1] ICP30 unregister session Session# [6:7577470905695728095:2048] VirtualId# [6:7577470905695728091:0]
2025-11-27T18:17:08.697915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:08.697967Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470905695728003:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6844) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.697998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6844) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.702544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577470907280291246:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:08.707664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470905695728003:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:08.726953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:08.727179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470906584899061:2049] [node 2] ICS01 socket: 153 reason# EndOfStream
2025-11-27T18:17:08.727227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470906584898958:2049] [node 2] ICP30 unregister session Session# [3:7577470906584899061:2049] VirtualId# [3:7577470906584899056:0]
2025-11-27T18:17:08.727266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470906584899061:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.727338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:08.727378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:08.732235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:08.732491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470908164335383:2049] [node 2] ICS07 socket disconnect 132 reason# EndOfStream
2025-11-27T18:17:08.732521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470908164335383:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.732599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:08.732645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470908164335383:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:08.733412Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:08.733411Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:08.733651Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470908164335294:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15057) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:08.733705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470908164335383:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:08.733738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470908164335294:2049] [node 2] ICP30 unregister session Session# [5:7577470908164335383:2049] VirtualId# [5:7577470908164335380:0]
2025-11-27T18:17:08.733783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:08.733809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470908164335294:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15057) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.733823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15057) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.733690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470905695728089:2049] [node 2] ICS01 socket: 141 reason# EndOfStream
2025-11-27T18:17:08.733724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470905695728004:2049] [node 2] ICP30 unregister session Session# [6:7577470905695728089:2049] VirtualId# [6:7577470905695728087:0]
2025-11-27T18:17:08.733743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470905695728089:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.733793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:08.733839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:08.733637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470907792753670:2049] [node 2] ICS01 socket: 131 reason# EndOfStream
2025-11-27T18:17:08.733701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470907792753586:2049] [node 2] ICP30 unregister session Session# [4:7577470907792753670:2049] VirtualId# [4:7577470907792753667:0]
2025-11-27T18:17:08.733733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470907792753670:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.733797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:08.733833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:08.743589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470908164335294:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:08.754220Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:08.754456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470907792753682:2050] [node 3] ICS01 socket: 139 reason# EndOfStream
2025-11-27T18:17:08.754500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470907792753587:2050] [node 3] ICP30 unregister session Session# [4:7577470907792753682:2050] VirtualId# [4:7577470907792753677:0]
2025-11-27T18:17:08.754535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470907792753682:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.754594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:08.754626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:08.755012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:08.755204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470908164335393:2050] [node 3] ICS07 socket disconnect 159 reason# EndOfStream
2025-11-27T18:17:08.755239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470908164335393:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.755321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:08.755371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470908164335393:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:08.755764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:08.755966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470905695728104:2050] [node 3] ICS07 socket disconnect 144 reason# EndOfStream
2025-11-27T18:17:08.756024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470905695728104:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.756128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:08.756178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470905695728104:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:08.756180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470908164335295:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63980) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:08.756232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470908164335393:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:08.756264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470908164335295:2050] [node 3] ICP30 unregister session Session# [5:7577470908164335393:2050] VirtualId# [5:7577470908164335391:0]
2025-11-27T18:17:08.756320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:08.756353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470908164335295:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63980) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.756366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63980) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.756877Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470905695728005:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63980) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:08.756923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470905695728104:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:08.756953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470905695728005:2050] [node 3] ICP30 unregister session Session# [6:7577470905695728104:2050] VirtualId# [6:7577470905695728102:0]
2025-11-27T18:17:08.756988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:08.757011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470905695728005:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63980) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.757023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63980) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.766040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470908164335295:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:08.767012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470905695728005:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:08.789360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:08.789595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470908164335389:2111] [node 4] ICS01 socket: 121 reason# EndOfStream
2025-11-27T18:17:08.789637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470908164335365:2111] [node 4] ICP30 unregister session Session# [5:7577470908164335389:2111] VirtualId# [5:7577470908164335387:0]
2025-11-27T18:17:08.789699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470908164335389:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.789775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:08.789818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:08.790196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:08.790407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470905695728100:2111] [node 4] ICS07 socket disconnect 100 reason# EndOfStream
2025-11-27T18:17:08.790449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470905695728100:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.790515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:08.790564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470905695728100:2111] [node 4] ICS15 start handshake
2025-11-27T18:17:08.791319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470905695728076:2111] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5964) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:08.791356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470905695728100:2111] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:08.791408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470905695728076:2111] [node 4] ICP30 unregister session Session# [6:7577470905695728100:2111] VirtualId# [6:7577470905695728097:0]
2025-11-27T18:17:08.791453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:08.791482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470905695728076:2111] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5964) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.791496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5964) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:08.800831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470905695728076:2111] [node 4] ICP33 wake up from error state
2025-11-27T18:17:08.804899Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:08.805085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470905695728074:2106] [node 5] ICS01 socket: 99 reason# EndOfStream
2025-11-27T18:17:08.805124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470905695728064:2106] [node 5] ICP30 unregister session Session# [6:7577470905695728074:2106] VirtualId# [6:7577470905695728066:0]
2025-11-27T18:17:08.805161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470905695728074:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:08.805220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:08.805271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:08.870938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470908685243378:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:08.870945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470908685243379:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:08.891095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470907121121856:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:08.891121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470907121121857:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:08.911983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470908410426098:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:08.911982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470908410426097:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:08.936841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470907552789603:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:08.936888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470907552789604:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:08.936916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470907552789605:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:08.956965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470907686522734:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:08.957040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470907686522736:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:08.957094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470907686522735:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:08.978597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470905690247941:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:08.978650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470905690247943:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:08.979061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470905690247942:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:09.071690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470909985215292:2099] [node 4] ICP01 ready to work
2025-11-27T18:17:09.072738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470909985215298:2101] [node 5] ICP01 ready to work
2025-11-27T18:17:09.164362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470912980210740:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:09.173250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470911847756962:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:09.172743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470912980210742:2110] [node 5] ICP01 ready to work
2025-11-27T18:17:09.175612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470911981490104:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:09.175796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470908685243379:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(98201489ns) {Configure:104(98198554ns) {ProcessConfigured:135(2118480ns) {StartInitialHandshake:157(2113929ns) {DropHandshakes:470(13618ns) {DropIncomingHandshake:442(620ns) DropOutgoingHandshake:457(11370ns)} GenerateSessionVirtualId:988(990ns) PrepareNewSessionHandshake:477(2648ns) {SwitchToState:179(980ns)}}}}}
2025-11-27T18:17:09.175924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470912980210745:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:09.175911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470911416089224:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:09.175896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470909985215298:2101] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(100465958ns) {Configure:104(100460958ns) {ProcessConfigured:135(2860440ns) {StartInitialHandshake:157(2707348ns) {DropHandshakes:470(1836ns) {DropIncomingHandshake:442(602ns) DropOutgoingHandshake:457(275ns)} GenerateSessionVirtualId:988(965ns) PrepareNewSessionHandshake:477(2719ns) {SwitchToState:179(1342ns)}}}}}
2025-11-27T18:17:09.175733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470911847756963:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:09.176384Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470909985215292:2099] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(100144136ns) {Configure:104(100137554ns) {ProcessConfigured:135(1588941ns) {StartInitialHandshake:157(1586584ns) {DropHandshakes:470(2422ns) {DropIncomingHandshake:442(589ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(718ns) PrepareNewSessionHandshake:477(2437ns) {SwitchToState:179(1133ns)}}}}}
2025-11-27T18:17:09.176354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470911981490105:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:09.176588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470911416089225:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:09.176988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470911416089226:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:09.179957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470907686522734:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24474888ns) {Configure:104(24467461ns) {ProcessConfigured:135(220774ns) {StartInitialHandshake:157(218134ns) {DropHandshakes:470(3169ns) {DropIncomingHandshake:442(807ns) DropOutgoingHandshake:457(331ns)} GenerateSessionVirtualId:988(814ns) PrepareNewSessionHandshake:477(1466ns) {SwitchToState:179(362ns)}}}}}
2025-11-27T18:17:09.180026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470907121121856:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(85366156ns) {Configure:104(85358466ns) {ProcessConfigured:135(700931ns) {StartInitialHandshake:157(698740ns) {DropHandshakes:470(1943ns) {DropIncomingHandshake:442(516ns) DropOutgoingHandshake:457(218ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:477(1632ns) {SwitchToState:179(704ns)}}}}}
2025-11-27T18:17:09.180979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470908410426098:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(86702138ns) {Configure:104(86698711ns) {ProcessConfigured:135(674777ns) {StartInitialHandshake:157(672373ns) {DropHandshakes:470(1237ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(442ns) PrepareNewSessionHandshake:477(1452ns) {SwitchToState:179(731ns)}}}}}
2025-11-27T18:17:09.181609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470908410426097:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(87885703ns) {Configure:104(87881924ns) {ProcessConfigured:135(1177242ns) {StartInitialHandshake:157(1079726ns) {DropHandshakes:470(1941ns) {DropIncomingHandshake:442(641ns) DropOutgoingHandshake:457(349ns)} GenerateSessionVirtualId:988(1051ns) PrepareNewSessionHandshake:477(1659ns) {SwitchToState:179(703ns)}}}}}
2025-11-27T18:17:09.181993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470912705393460:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:09.186008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470912705393462:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:09.186645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470912705393461:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.196895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470912980210754:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16747
2025-11-27T18:17:09.197111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470911416089231:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18697
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.200796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470911416089231:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.201438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470912980210754:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.202210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470907121121856:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.202718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470911416089242:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18697
2025-11-27T18:17:09.201803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470912980210755:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16747
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.204130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470912980210735:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.204169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470912980210735:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.207041Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470912980210735:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:09.206018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470911981490114:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59517
2025-11-27T18:17:09.207156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470911981490114:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.207525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470912980210757:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16747
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.209341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470908685243378:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.210469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470911847756974:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60457
2025-11-27T18:17:09.209905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:09.210519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470908685243378:2048] [node 2] ICP22 created new session: [1:7577470912980210759:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.211572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470911847756975:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60457
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.211588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470912980210759:2048] [node 2] ICS09 handshake done sender: [1:7577470912980210735:2107] self: [1:7577470912980210734:0] peer: [2:7577470911416089241:0] socket: 102 qp: -1
2025-11-27T18:17:09.213021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470911847756974:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.211692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470912980210759:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:09.213169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470909985215315:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41211
2025-11-27T18:17:09.216454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470911847756977:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60457
2025-11-27T18:17:09.211735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: 2025-11-27T18:17:09.217735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470911847756975:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:17:09.212317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470912980210755:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.221022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470912980210757:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.219089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470911416089243:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18697
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.222905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470907552789603:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.223190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470908685243378:2048] [node 2] ICP18 handshake (actor [1:7577470912980210755:2117]) is too old
2025-11-27T18:17:09.223284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470912705393467:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35851
2025-11-27T18:17:09.223313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470912705393468:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35851
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.223598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470912705393473:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35851
2025-11-27T18:17:09.223651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470912705393474:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35851
2025-11-27T18:17:09.223022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470911416089244:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18697
2025-11-27T18:17:09.223279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470909985215316:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41211
2025-11-27T18:17:09.223286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470911416089231:2121] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.224756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470911416089242:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.223634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470912980210747:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.223667Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470912980210747:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.224478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470909985215315:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.223160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470911847756977:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.224899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470911416089244:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.223547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470907552789604:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.224947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470911416089245:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18697
2025-11-27T18:17:09.225047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470909985215316:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.225734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470912705393473:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.225596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470905690247941:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.227048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470911847756970:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.225472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470907121121856:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.227084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470912980210762:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16747
2025-11-27T18:17:09.225669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470909985215292:2099] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.225697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:09.225989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470912705393468:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.227083Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470911847756970:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.226709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470912705393467:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.225993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470907121121856:2048] [node 1] ICP22 created new session: [2:7577470911416089246:2048]
2025-11-27T18:17:09.227614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470912980210747:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.228597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470912980210750:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.226022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470911416089233:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.228628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470912980210750:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.227094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470912705393474:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.226050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470911416089233:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.230644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470911847756983:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60457
2025-11-27T18:17:09.230679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470912980210740:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.232316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470911847756970:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:09.227979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470908410426098:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.232729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470911847756985:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60457
2025-11-27T18:17:09.231185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:09.226936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470911416089246:2048] [node 1] ICS09 handshake done sender: [2:7577470911416089231:2121] self: [2:7577470911416089241:0] peer: [1:7577470912980210734:0] socket: 103 qp: -1
2025-11-27T18:17:09.228233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470908410426097:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.231736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470912980210740:2108] [node 4] ICP22 created new session: [1:7577470912980210765:2108]
2025-11-27T18:17:09.227122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470911416089246:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:09.232331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470912980210765:2108] [node 4] ICS09 handshake done sender: [1:7577470912980210747:2113] self: [1:7577470912980210746:0] peer: [4:7577470911847756981:0] socket: 108 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.227306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:09.232369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470912980210765:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:09.232860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577470907121121856:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:09.233012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470911416089230:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.233045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470911416089230:2120] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.232735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470912980210744:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.235696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470909985215319:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41211
2025-11-27T18:17:09.235507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470911847756962:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.232771Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470912980210744:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.235648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470911847756975:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:09.232851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:09.235817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:09.235867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470911847756962:2109] [node 6] ICP22 created new session: [4:7577470911847756986:2109]
2025-11-27T18:17:09.236344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470911847756986:2109] [node 6] ICS09 handshake done sender: [4:7577470911847756970:2112] self: [4:7577470911847756968:0] peer: [6:7577470909985215317:0] socket: 133 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.236459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470911847756986:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:09.236709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470911416089245:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.236633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470912980210762:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.236497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:09.237034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470911416089233:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:09.237003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470912980210744:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:09.237227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470912705393480:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35851
2025-11-27T18:17:09.237882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470907552789603:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.237069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470912980210750:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:09.238642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470909985215315:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:09.237946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:09.237853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470911416089224:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.238649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470912980210745:2111] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.238299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470907552789603:2048] [node 1] ICP22 created new session: [4:7577470911847756989:2048]
2025-11-27T18:17:09.239019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470912705393467:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:09.238691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:09.238106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470911416089230:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:09.238967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470911847756989:2048] [node 1] ICS09 handshake done sender: [4:7577470911847756975:2116] self: [4:7577470911847756981:0] peer: [1:7577470912980210746:0] socket: 125 qp: -1
2025-11-27T18:17:09.239507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470911981490121:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59517
2025-11-27T18:17:09.238968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470912980210745:2111] [node 6] ICP22 created new session: [1:7577470912980210770:2111]
2025-11-27T18:17:09.239028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470912980210770:2111] [node 6] ICS09 handshake done sender: [1:7577470912980210750:2114] self: [1:7577470912980210749:0] peer: [6:7577470909985215318:0] socket: 139 qp: -1
2025-11-27T18:17:09.239015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470911847756989:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:09.239977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470912705393481:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35851
2025-11-27T18:17:09.238181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:09.239718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470909985215292:2099] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10732) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470909985215315:2115] held: no
2025-11-27T18:17:09.239088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470908685243379:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.240334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470909985215292:2099] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.239513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:09.238724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470911416089224:2107] [node 4] ICP22 created new session: [2:7577470911416089253:2107]
2025-11-27T18:17:09.240464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:09.239098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470912980210770:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:09.239013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470911416089253:2107] [node 4] ICS09 handshake done sender: [2:7577470911416089233:2122] self: [2:7577470911416089232:0] peer: [4:7577470911847756982:0] socket: 117 qp: -1
2025-11-27T18:17:09.240987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470908410426097:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.240744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470909985215292:2099] [node 4] ICP22 created new session: [6:7577470909985215322:2099]
2025-11-27T18:17:09.240768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470911847756983:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.239293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:09.241090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:09.239045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470911416089253:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:09.241082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470909985215322:2099] [node 4] ICS09 handshake done sender: [6:7577470909985215315:2115] self: [6:7577470909985215317:0] peer: [4:7577470911847756968:0] socket: 134 qp: -1
2025-11-27T18:17:09.239300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:09.241361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470908410426097:2048] [node 1] ICP22 created new session: [3:7577470912705393482:2048]
2025-11-27T18:17:09.239080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:09.239356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470908685243379:2049] [node 3] ICP22 created new session: [1:7577470912980210771:2049]
2025-11-27T18:17:09.241234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470909985215322:2099] [node 4] ICS10 traffic start
2025-11-27T18:17:09.239526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470912980210771:2049] [node 3] ICS09 handshake done sender: [1:7577470912980210744:2112] self: [1:7577470912980210743:0] peer: [3:7577470912705393478:0] socket: 105 qp: -1
2025-11-27T18:17:09.241988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470912705393482:2048] [node 1] ICS09 handshake done sender: [3:7577470912705393467:2114] self: [3:7577470912705393478:0] peer: [1:7577470912980210743:0] socket: 116 qp: -1
2025-11-27T18:17:09.242313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470907121121857:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.242141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470912705393482:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:09.241392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:09.242561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:09.242324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470909985215320:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41211
2025-11-27T18:17:09.239702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470912980210771:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:09.242680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470907121121857:2049] [node 3] ICP22 created new session: [2:7577470911416089255:2049]
2025-11-27T18:17:09.242334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:09.239861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:09.243118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470911416089255:2049] [node 3] ICS09 handshake done sender: [2:7577470911416089230:2120] self: [2:7577470911416089229:0] peer: [3:7577470912705393477:0] socket: 114 qp: -1
2025-11-27T18:17:09.244768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470912705393468:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:09.243835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470908685243379:2049] [node 3] ICP18 handshake (actor [1:7577470912980210762:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.243629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470911416089255:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.246985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470907552789605:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.243660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:09.247696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470911981490122:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59517
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.247077Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470907552789604:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2377) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470911847756977:2117] held: no
2025-11-27T18:17:09.246808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470907121121857:2049] [node 3] ICP18 handshake (actor [2:7577470911416089245:2128]) is too old
2025-11-27T18:17:09.248600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470911847756988:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60457
2025-11-27T18:17:09.248655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470908410426098:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.248688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470909985215316:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:09.249003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:09.249278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470908410426098:2049] [node 2] ICP22 created new session: [3:7577470912705393484:2049]
2025-11-27T18:17:09.248947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470909985215321:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41211
2025-11-27T18:17:09.249334Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470905690247941:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27457) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470909985215316:2116] held: no
2025-11-27T18:17:09.249682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470912705393484:2049] [node 2] ICS09 handshake done sender: [3:7577470912705393468:2115] self: [3:7577470912705393477:0] peer: [2:7577470911416089229:0] socket: 119 qp: -1
2025-11-27T18:17:09.249804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470912705393484:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:09.249839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470905690247941:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.249851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:09.250077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470911981490121:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.249880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:09.250162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470911847756977:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:09.250043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470905690247941:2048] [node 1] ICP22 created new session: [6:7577470909985215325:2048]
2025-11-27T18:17:09.250054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470909985215321:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.250447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470907686522734:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.250263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470909985215325:2048] [node 1] ICS09 handshake done sender: [6:7577470909985215316:2116] self: [6:7577470909985215318:0] peer: [1:7577470912980210749:0] socket: 140 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.250571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470909985215325:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.251019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470907552789604:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.250886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470912980210751:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.251058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:09.250620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:09.251396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470911981490122:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.250710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470905690247942:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.251758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470911416089258:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18697
2025-11-27T18:17:09.251888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470907686522735:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.251433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470907552789604:2049] [node 2] ICP22 created new session: [4:7577470911847756994:2049]
2025-11-27T18:17:09.250920Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470912980210751:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.251958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470911416089237:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.251861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577470908410426098:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:09.251985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470911416089237:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.251900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470911847756994:2049] [node 2] ICS09 handshake done sender: [4:7577470911847756977:2117] self: [4:7577470911847756982:0] peer: [2:7577470911416089232:0] socket: 127 qp: -1
2025-11-27T18:17:09.251987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470911981490124:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59517
2025-11-27T18:17:09.252061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470911847756994:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:09.252096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:09.252723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470911981490124:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.253518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470911981490104:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.253773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470912705393465:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.253766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470912980210751:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:09.253802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470912705393465:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.253564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470911416089235:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.253621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470911416089235:2123] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.258289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470912980210742:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.258423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470911847756973:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.258450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:09.259179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470911416089235:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:09.258460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470911847756973:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.258684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470912980210742:2110] [node 5] ICP22 created new session: [1:7577470912980210775:2110]
2025-11-27T18:17:09.260559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470911416089237:2124] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:09.260583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470911981490127:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59517
2025-11-27T18:17:09.259383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470912980210775:2110] [node 5] ICS09 handshake done sender: [1:7577470912980210751:2115] self: [1:7577470912980210748:0] peer: [5:7577470911981490123:0] socket: 118 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.259421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470912980210775:2110] [node 5] ICS10 traffic start
2025-11-27T18:17:09.260922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470911847756973:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:09.262322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470911847756963:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.260626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:09.262372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:09.262427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470911847756963:2108] [node 5] ICP22 created new session: [4:7577470911847756998:2108]
2025-11-27T18:17:09.262477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470911847756998:2108] [node 5] ICS09 handshake done sender: [4:7577470911847756973:2114] self: [4:7577470911847756972:0] peer: [5:7577470911981490126:0] socket: 132 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.262523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470911847756998:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.262398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470912705393465:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:09.262418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470912705393486:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35851
2025-11-27T18:17:09.262574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:09.262256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470911416089225:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.262595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470909985215329:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41211
2025-11-27T18:17:09.262608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470909985215330:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41211
2025-11-27T18:17:09.264189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470912705393460:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.264411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470909985215321:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:09.262427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:09.264368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:09.262474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470911416089225:2108] [node 5] ICP22 created new session: [2:7577470911416089261:2108]
2025-11-27T18:17:09.264411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470912705393460:2107] [node 4] ICP22 created new session: [3:7577470912705393490:2107]
2025-11-27T18:17:09.264806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470911847756997:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60457
2025-11-27T18:17:09.264828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470912705393490:2107] [node 4] ICS09 handshake done sender: [3:7577470912705393465:2112] self: [3:7577470912705393463:0] peer: [4:7577470911847756992:0] socket: 115 qp: -1
2025-11-27T18:17:09.262538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470911416089261:2108] [node 5] ICS09 handshake done sender: [2:7577470911416089235:2123] self: [2:7577470911416089234:0] peer: [5:7577470911981490125:0] socket: 135 qp: -1
2025-11-27T18:17:09.262570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470911416089261:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:09.264852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470912705393490:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:09.265216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470905690247942:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.262581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470911416089226:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.264926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470912705393486:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.265429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:09.262602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:09.265479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470905690247942:2049] [node 2] ICP22 created new session: [6:7577470909985215331:2049]
2025-11-27T18:17:09.262614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:09.267954Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470907686522734:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27457) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470911981490121:2127] held: no
2025-11-27T18:17:09.265107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:09.269609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470911847756983:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:09.269955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470911981490121:2127] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:09.263901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470911416089226:2109] [node 6] ICP22 created new session: [2:7577470911416089263:2109]
2025-11-27T18:17:09.266129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470909985215331:2049] [node 2] ICS09 handshake done sender: [6:7577470909985215321:2119] self: [6:7577470909985215328:0] peer: [2:7577470911416089236:0] socket: 106 qp: -1
2025-11-27T18:17:09.267126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470909985215331:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:09.264034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470911416089263:2109] [node 6] ICS09 handshake done sender: [2:7577470911416089237:2124] self: [2:7577470911416089236:0] peer: [6:7577470909985215328:0] socket: 159 qp: -1
2025-11-27T18:17:09.267168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:09.264065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470911416089263:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.264227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:09.271417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470907552789605:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2956) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470911847756983:2118] held: no
2025-11-27T18:17:09.269599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470911416089258:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.271968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470907552789605:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.272009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:09.271979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470911981490128:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59517
2025-11-27T18:17:09.272052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470907552789605:2050] [node 3] ICP22 created new session: [4:7577470911847757000:2050]
2025-11-27T18:17:09.272022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470909985215330:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.272096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470911847757000:2050] [node 3] ICS09 handshake done sender: [4:7577470911847756983:2118] self: [4:7577470911847756992:0] peer: [3:7577470912705393463:0] socket: 145 qp: -1
2025-11-27T18:17:09.272131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470911847757000:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:09.272147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470905690247943:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.272143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470911416089226:2109] [node 6] ICP18 handshake (actor [2:7577470911416089258:2129]) is too old
2025-11-27T18:17:09.272201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:09.272495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470912705393470:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.272535Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470912705393470:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.275585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470911981490129:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59517
2025-11-27T18:17:09.275837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470907686522735:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2377) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470911981490122:2128] held: no
2025-11-27T18:17:09.275944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470907686522734:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.276069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.276143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470907686522734:2048] [node 1] ICP22 created new session: [5:7577470911981490131:2048]
2025-11-27T18:17:09.276192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470911981490131:2048] [node 1] ICS09 handshake done sender: [5:7577470911981490121:2127] self: [5:7577470911981490123:0] peer: [1:7577470912980210748:0] socket: 157 qp: -1
2025-11-27T18:17:09.276183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470912705393470:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:09.276225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470911981490131:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.276251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:09.276229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470909985215334:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41211
2025-11-27T18:17:09.276302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470911981490130:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59517
2025-11-27T18:17:09.276403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470909985215330:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:09.276685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470911981490122:2128] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:09.276710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470905690247943:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.276714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470912705393461:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.277099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:09.276926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:09.276998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470905690247943:2050] [node 3] ICP22 created new session: [6:7577470909985215335:2050]
2025-11-27T18:17:09.277217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470909985215335:2050] [node 3] ICS09 handshake done sender: [6:7577470909985215330:2121] self: [6:7577470909985215333:0] peer: [3:7577470912705393469:0] socket: 171 qp: -1
2025-11-27T18:17:09.277250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470909985215335:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:09.277310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:09.277270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470912705393461:2109] [node 6] ICP22 created new session: [3:7577470912705393494:2109]
2025-11-27T18:17:09.277339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470912705393494:2109] [node 6] ICS09 handshake done sender: [3:7577470912705393470:2116] self: [3:7577470912705393469:0] peer: [6:7577470909985215333:0] socket: 170 qp: -1
2025-11-27T18:17:09.277389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470912705393494:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:09.277493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.279533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470911847757002:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60457
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.280111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470909985215337:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41211
2025-11-27T18:17:09.280445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470912705393466:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.280482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470912705393466:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.278201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470911981490124:2129] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:09.278409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470911981490130:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.278586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470907686522735:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.281675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470912705393466:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:09.278956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:09.279009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470907686522735:2049] [node 2] ICP22 created new session: [5:7577470911981490135:2049]
2025-11-27T18:17:09.279057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470911981490135:2049] [node 2] ICS09 handshake done sender: [5:7577470911981490122:2128] self: [5:7577470911981490125:0] peer: [2:7577470911416089234:0] socket: 160 qp: -1
2025-11-27T18:17:09.279090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470911981490104:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.279092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470911981490135:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:09.279142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:09.279146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:09.279199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470911981490104:2109] [node 4] ICP22 created new session: [5:7577470911981490137:2109]
2025-11-27T18:17:09.279237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470911981490137:2109] [node 4] ICS09 handshake done sender: [5:7577470911981490124:2129] self: [5:7577470911981490126:0] peer: [4:7577470911847756972:0] socket: 142 qp: -1
2025-11-27T18:17:09.279267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470911981490137:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:09.282022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470912705393462:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.279310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:09.280152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470907686522736:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.281763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470911981490140:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59517
2025-11-27T18:17:09.282115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:09.282167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470912705393462:2108] [node 5] ICP22 created new session: [3:7577470912705393497:2108]
2025-11-27T18:17:09.282214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470912705393497:2108] [node 5] ICS09 handshake done sender: [3:7577470912705393466:2113] self: [3:7577470912705393464:0] peer: [5:7577470911981490139:0] socket: 166 qp: -1
2025-11-27T18:17:09.282224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470911981490130:2133] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:09.282282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470912705393497:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:09.282262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470909985215337:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.282361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:09.282397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470907686522736:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.282476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470909985215298:2101] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.282650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:09.282764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470907686522736:2050] [node 3] ICP22 created new session: [5:7577470911981490141:2050]
2025-11-27T18:17:09.282832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470911981490141:2050] [node 3] ICS09 handshake done sender: [5:7577470911981490130:2133] self: [5:7577470911981490139:0] peer: [3:7577470912705393464:0] socket: 169 qp: -1
2025-11-27T18:17:09.282883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470911981490141:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:09.282936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:09.283886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470911981490116:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.283914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470911981490116:2126] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.284919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470911981490116:2126] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:09.285228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470911981490105:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.285329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.285391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470911981490105:2111] [node 6] ICP22 created new session: [5:7577470911981490144:2111]
2025-11-27T18:17:09.285438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470911981490144:2111] [node 6] ICS09 handshake done sender: [5:7577470911981490116:2126] self: [5:7577470911981490115:0] peer: [6:7577470909985215338:0] socket: 124 qp: -1
2025-11-27T18:17:09.285449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470909985215339:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41211
2025-11-27T18:17:09.285478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470911981490144:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:09.285535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:09.285983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470909985215298:2101] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470909985215337:2123] held: no
2025-11-27T18:17:09.286105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470909985215337:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:09.286644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470909985215298:2101] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.286680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:09.286719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470909985215298:2101] [node 5] ICP22 created new session: [6:7577470909985215340:2101]
2025-11-27T18:17:09.286753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470909985215340:2101] [node 5] ICS09 handshake done sender: [6:7577470909985215337:2123] self: [6:7577470909985215338:0] peer: [5:7577470911981490115:0] socket: 131 qp: -1
2025-11-27T18:17:09.286829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470909985215340:2101] [node 5] ICS10 traffic start
2025-11-27T18:17:09.286858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:09.310502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:09.310720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470911416089246:2048] [node 1] ICS07 socket disconnect 103 reason# EndOfStream
2025-11-27T18:17:09.310753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470911416089246:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.310824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:09.310905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470911416089246:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:09.311536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:09.311770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470912705393482:2048] [node 1] ICS07 socket disconnect 116 reason# EndOfStream
2025-11-27T18:17:09.311814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470912705393482:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.311906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:09.311989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470912705393482:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:09.312747Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470908410426097:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27457) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:09.313300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470907121121856:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27457) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:09.312793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470912705393482:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:09.313342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470911416089246:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:09.312986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470908410426097:2048] [node 1] ICP30 unregister session Session# [3:7577470912705393482:2048] VirtualId# [3:7577470912705393478:0]
2025-11-27T18:17:09.313198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:09.313234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470908410426097:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27457) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:09.313248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27457) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:09.315134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:09.315535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470911847756989:2048] [node 1] ICS01 socket: 125 reason# EndOfStream
2025-11-27T18:17:09.315597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470907552789603:2048] [node 1] ICP30 unregister session Session# [4:7577470911847756989:2048] VirtualId# [4:7577470911847756981:0]
2025-11-27T18:17:09.315629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470911847756989:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.315595Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470907121121856:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27457) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:09.315653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470911416089246:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:09.315733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:09.315744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470907121121856:2048] [node 1] ICP30 unregister session Session# [2:7577470911416089246:2048] VirtualId# [2:7577470911416089241:0]
2025-11-27T18:17:09.315811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:09.315791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:09.315827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470907121121856:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27457) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:09.315847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27457) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:09.316487Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:09.316687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470911981490131:2048] [node 1] ICS01 socket: 157 reason# EndOfStream
2025-11-27T18:17:09.316724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470907686522734:2048] [node 1] ICP30 unregister session Session# [5:7577470911981490131:2048] VirtualId# [5:7577470911981490123:0]
2025-11-27T18:17:09.316752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470911981490131:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.316814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:09.316854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:09.317552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:09.317781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470909985215325:2048] [node 1] ICS01 socket: 140 reason# EndOfStream
2025-11-27T18:17:09.317824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470905690247941:2048] [node 1] ICP30 unregister session Session# [6:7577470909985215325:2048] VirtualId# [6:7577470909985215318:0]
2025-11-27T18:17:09.317866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470909985215325:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.317955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:09.318005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:09.323099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470908410426097:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:09.326138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577470907121121856:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:09.355108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:09.355394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470912705393484:2049] [node 2] ICS07 socket disconnect 119 reason# EndOfStream
2025-11-27T18:17:09.355427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470912705393484:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.355500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:09.355544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470912705393484:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:09.356400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470908410426098:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2377) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:09.356588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470912705393484:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:09.356630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470908410426098:2049] [node 2] ICP30 unregister session Session# [3:7577470912705393484:2049] VirtualId# [3:7577470912705393477:0]
2025-11-27T18:17:09.356686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:09.356715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470908410426098:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2377) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:09.356728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2377) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:09.358038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:09.358246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470911847756994:2049] [node 2] ICS01 socket: 127 reason# EndOfStream
2025-11-27T18:17:09.358416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470907552789604:2049] [node 2] ICP30 unregister session Session# [4:7577470911847756994:2049] VirtualId# [4:7577470911847756982:0]
2025-11-27T18:17:09.358455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470911847756994:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.358547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:09.358595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:09.359112Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:09.359297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470911981490135:2049] [node 2] ICS01 socket: 160 reason# EndOfStream
2025-11-27T18:17:09.359338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470907686522735:2049] [node 2] ICP30 unregister session Session# [5:7577470911981490135:2049] VirtualId# [5:7577470911981490125:0]
2025-11-27T18:17:09.359374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470911981490135:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.359433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:09.359468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:09.360050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:09.360228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470909985215331:2049] [node 2] ICS01 socket: 106 reason# EndOfStream
2025-11-27T18:17:09.360288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470905690247942:2049] [node 2] ICP30 unregister session Session# [6:7577470909985215331:2049] VirtualId# [6:7577470909985215328:0]
2025-11-27T18:17:09.360319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470909985215331:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.360380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:09.360422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:09.366090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470908410426098:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:09.401809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:09.402044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470911847757000:2050] [node 3] ICS01 socket: 145 reason# EndOfStream
2025-11-27T18:17:09.402094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470907552789605:2050] [node 3] ICP30 unregister session Session# [4:7577470911847757000:2050] VirtualId# [4:7577470911847756992:0]
2025-11-27T18:17:09.402127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470911847757000:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.402202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:09.402255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:09.402786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:09.402977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470911981490141:2050] [node 3] ICS07 socket disconnect 169 reason# EndOfStream
2025-11-27T18:17:09.403008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470911981490141:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.403073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:09.403113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470911981490141:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:09.403654Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:09.403824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470909985215335:2050] [node 3] ICS01 socket: 171 reason# EndOfStream
2025-11-27T18:17:09.403816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470907686522736:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2956) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:09.403852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470905690247943:2050] [node 3] ICP30 unregister session Session# [6:7577470909985215335:2050] VirtualId# [6:7577470909985215333:0]
2025-11-27T18:17:09.403921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470911981490141:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:09.403948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470909985215335:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.403950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470907686522736:2050] [node 3] ICP30 unregister session Session# [5:7577470911981490141:2050] VirtualId# [5:7577470911981490139:0]
2025-11-27T18:17:09.404032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:09.404111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:09.404085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:09.404147Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470907686522736:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2956) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:09.404165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2956) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:09.414228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470907686522736:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:09.431584Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:09.431821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470911981490137:2109] [node 4] ICS07 socket disconnect 142 reason# EndOfStream
2025-11-27T18:17:09.431858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470911981490137:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.431933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:09.431975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470911981490137:2109] [node 4] ICS15 start handshake
2025-11-27T18:17:09.432719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470911981490104:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10732) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:09.432806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470911981490137:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:09.432854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470911981490104:2109] [node 4] ICP30 unregister session Session# [5:7577470911981490137:2109] VirtualId# [5:7577470911981490126:0]
2025-11-27T18:17:09.432904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:09.432886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:09.432955Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470911981490104:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10732) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:09.432967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10732) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:09.433142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470909985215322:2099] [node 4] ICS01 socket: 134 reason# EndOfStream
2025-11-27T18:17:09.433199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470909985215292:2099] [node 4] ICP30 unregister session Session# [6:7577470909985215322:2099] VirtualId# [6:7577470909985215317:0]
2025-11-27T18:17:09.433235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470909985215322:2099] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.433351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:09.433399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:09.442907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470911981490104:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:17:09.474954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:09.475161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470909985215340:2101] [node 5] ICS01 socket: 131 reason# EndOfStream
2025-11-27T18:17:09.475204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470909985215298:2101] [node 5] ICP30 unregister session Session# [6:7577470909985215340:2101] VirtualId# [6:7577470909985215338:0]
2025-11-27T18:17:09.475228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470909985215340:2101] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:09.475293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:09.475322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:09.552660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470909494508908:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:09.552763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470909494508909:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:09.573734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470910796743646:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:09.573734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470910796743647:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:09.595739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470911161189268:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:09.595915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470911161189267:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:09.619285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470912529275051:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:09.619297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470912529275052:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:09.619322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470912529275053:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:09.642452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470912779250315:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:09.642508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470912779250316:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:09.642718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470912779250317:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:09.665808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470912822679589:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:09.665862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470912822679590:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:09.666183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470912822679591:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:09.753733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470912822679646:2102] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.764246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470909494508971:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:09.794136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470909494508972:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:09.798922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470912779250376:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13590
2025-11-27T18:17:09.811929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470909494508973:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:09.837582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470911161189332:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:09.838443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470910796743709:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:09.839084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470911161189333:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:09.838647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470912779250380:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:09.839075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470910796743710:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:09.840077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470912822679657:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:09.840922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470912529275114:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:09.841176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470912779250381:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:09.839933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470910796743711:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:09.841243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470911161189334:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:09.841976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470909494508909:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(88017068ns) {Configure:104(88008191ns) {ProcessConfigured:135(1023135ns) {StartInitialHandshake:157(1020807ns) {DropHandshakes:470(2054ns) {DropIncomingHandshake:442(483ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(863ns) PrepareNewSessionHandshake:477(3181ns) {SwitchToState:179(1533ns)}}}}}
2025-11-27T18:17:09.842068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470912529275115:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:09.842213Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470909494508908:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(90941292ns) {Configure:104(90650304ns) {ProcessConfigured:135(951576ns) {StartInitialHandshake:157(948860ns) {DropHandshakes:470(2240ns) {DropIncomingHandshake:442(657ns) DropOutgoingHandshake:457(368ns)} GenerateSessionVirtualId:988(446ns) PrepareNewSessionHandshake:477(1714ns) {SwitchToState:179(778ns)}}}}}
2025-11-27T18:17:09.843467Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470910796743646:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(74337156ns) {Configure:104(74328378ns) {ProcessConfigured:135(531535ns) {StartInitialHandshake:157(528713ns) {DropHandshakes:470(5337ns) {DropIncomingHandshake:442(491ns) DropOutgoingHandshake:457(372ns)} GenerateSessionVirtualId:988(805ns) PrepareNewSessionHandshake:477(2184ns) {SwitchToState:179(667ns)}}}}}
2025-11-27T18:17:09.843582Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470912822679589:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31146524ns) {Configure:104(17501092ns) {ProcessConfigured:135(548863ns) {StartInitialHandshake:157(543808ns) {DropHandshakes:470(2174ns) {DropIncomingHandshake:442(785ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(559ns) PrepareNewSessionHandshake:477(1648ns) {SwitchToState:179(487ns)}}}}}
2025-11-27T18:17:09.846206Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470911161189268:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(65890154ns) {Configure:104(61153878ns) {ProcessConfigured:135(519834ns) {StartInitialHandshake:157(518138ns) {DropHandshakes:470(2096ns) {DropIncomingHandshake:442(451ns) DropOutgoingHandshake:457(600ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:477(2113ns) {SwitchToState:179(1160ns)}}}}}
2025-11-27T18:17:09.847581Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470911161189267:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(61987656ns) {Configure:104(61977452ns) {ProcessConfigured:135(1134078ns) {StartInitialHandshake:157(1132163ns) {DropHandshakes:470(1197ns) {DropIncomingHandshake:442(293ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(501ns) PrepareNewSessionHandshake:477(2200ns) {SwitchToState:179(582ns)}}}}}
2025-11-27T18:17:09.848110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470912529275051:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(59321101ns) {Configure:104(59314438ns) {ProcessConfigured:135(902608ns) {StartInitialHandshake:157(901586ns) {DropHandshakes:470(1582ns) {DropIncomingHandshake:442(345ns) DropOutgoingHandshake:457(416ns)} GenerateSessionVirtualId:988(488ns) PrepareNewSessionHandshake:477(1578ns) {SwitchToState:179(163ns)}}}}}
2025-11-27T18:17:09.848334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470912779250315:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49511746ns) {Configure:104(49506157ns) {ProcessConfigured:135(899640ns) {StartInitialHandshake:157(897962ns) {DropHandshakes:470(1326ns) {DropIncomingHandshake:442(286ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(494ns) PrepareNewSessionHandshake:477(1996ns) {SwitchToState:179(601ns)}}}}}
2025-11-27T18:17:09.853026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470912779250376:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.854058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470910796743722:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.855055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470910796743724:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-27T18:17:09.856102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470910796743722:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.857450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470909494508986:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9535
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.861306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470910796743724:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.858116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470909494508988:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9535
2025-11-27T18:17:09.860391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470912529275128:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59691
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.861530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470912779250390:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13590
2025-11-27T18:17:09.859075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470911161189345:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23813
2025-11-27T18:17:09.861561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470909494508986:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.860586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470912529275130:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59691
2025-11-27T18:17:09.862008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470910796743728:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-27T18:17:09.862503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470912779250393:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13590
2025-11-27T18:17:09.862062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470911161189345:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.863639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470910796743729:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-27T18:17:09.863532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470912779250390:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.862629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470911161189267:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.864179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470912529275128:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.864158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470910796743646:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.871307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470909494508977:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.866222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470912779250315:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.871353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470909494508977:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.865777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470912529275052:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.872087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470910796743728:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.872743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470912529275130:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.872084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470909494508988:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.872706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470910796743717:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.873093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470909494508979:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.872740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470910796743717:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.875049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470911161189340:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.874264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470912779250393:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.873141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470912529275053:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.874707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470912779250316:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.873126Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470909494508979:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.874616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470910796743730:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-27T18:17:09.875086Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470911161189340:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.874259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470909494508975:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.874284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470909494508975:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.877352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470910796743730:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.876673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470911161189350:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23813
2025-11-27T18:17:09.877359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470910796743729:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.876836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470909494508991:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9535
2025-11-27T18:17:09.877524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470911161189340:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:09.878007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470910796743717:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:09.876941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470909494508992:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9535
2025-11-27T18:17:09.877678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470909494508990:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9535
2025-11-27T18:17:09.878064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470911161189351:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23813
2025-11-27T18:17:09.877850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470909494508977:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:09.878009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470910796743721:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.878170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470909494508979:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:09.878667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470911161189350:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.878040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470910796743721:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.880330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470911161189332:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.880061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470910796743721:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:09.878218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470909494508990:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.880370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:09.881455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470910796743709:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.879457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470909494508992:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.882347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470912779250400:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13590
2025-11-27T18:17:09.880813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470911161189332:2107] [node 4] ICP22 created new session: [3:7577470911161189354:2107]
2025-11-27T18:17:09.881592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:09.884229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470912529275133:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59691
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.884588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470912529275134:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59691
2025-11-27T18:17:09.880876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470911161189354:2107] [node 4] ICS09 handshake done sender: [3:7577470911161189340:2114] self: [3:7577470911161189339:0] peer: [4:7577470912529275132:0] socket: 135 qp: -1
2025-11-27T18:17:09.881658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470910796743709:2107] [node 4] ICP22 created new session: [2:7577470910796743738:2107]
2025-11-27T18:17:09.879752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470909494508908:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.879995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:09.880877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470911161189351:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.894468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470912779250401:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13590
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.881518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470911161189354:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:09.880528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470909494508908:2048] [node 2] ICP22 created new session: [1:7577470909494508997:2048]
2025-11-27T18:17:09.880536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470909494508991:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.880555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470909494508972:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.880709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470909494508975:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:09.880883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470909494508997:2048] [node 2] ICS09 handshake done sender: [1:7577470909494508977:2111] self: [1:7577470909494508976:0] peer: [2:7577470910796743731:0] socket: 115 qp: -1
2025-11-27T18:17:09.881493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:09.881543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470909494508997:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:09.881587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:09.881656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470909494508972:2108] [node 5] ICP22 created new session: [1:7577470909494508998:2108]
2025-11-27T18:17:09.882150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470909494508998:2108] [node 5] ICS09 handshake done sender: [1:7577470909494508979:2112] self: [1:7577470909494508978:0] peer: [5:7577470912779250398:0] socket: 116 qp: -1
2025-11-27T18:17:09.882330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470909494508998:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:09.882369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:09.895229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470909494508909:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.881948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470910796743738:2107] [node 4] ICS09 handshake done sender: [2:7577470910796743717:2112] self: [2:7577470910796743716:0] peer: [4:7577470912529275131:0] socket: 124 qp: -1
2025-11-27T18:17:09.881846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:09.896765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470912529275130:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:09.882063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470910796743738:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:09.882532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:09.899179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:09.899535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470909494508909:2049] [node 3] ICP22 created new session: [1:7577470909494509003:2049]
2025-11-27T18:17:09.898982Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470912779250316:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470912779250393:2118] held: no
2025-11-27T18:17:09.894567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470911161189355:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23813
2025-11-27T18:17:09.884820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470910796743710:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.897026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470912529275052:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470912529275128:2115] held: no
2025-11-27T18:17:09.899130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470912779250390:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:09.900060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470909494508972:2108] [node 5] ICP18 handshake (actor [1:7577470909494508990:2117]) is too old
2025-11-27T18:17:09.895162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577470911161189332:2107] [node 4] ICP18 handshake (actor [3:7577470911161189351:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.884875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:09.899676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470912779250393:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:09.899090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470911161189345:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:09.885904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470910796743710:2108] [node 5] ICP22 created new session: [2:7577470910796743740:2108]
2025-11-27T18:17:09.900330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470911161189267:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.903452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470912779250402:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13590
2025-11-27T18:17:09.892223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470910796743740:2108] [node 5] ICS09 handshake done sender: [2:7577470910796743721:2114] self: [2:7577470910796743720:0] peer: [5:7577470912779250399:0] socket: 125 qp: -1
2025-11-27T18:17:09.900650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:09.894529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470910796743740:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:09.900601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470909494509003:2049] [node 3] ICS09 handshake done sender: [1:7577470909494508975:2110] self: [1:7577470909494508974:0] peer: [3:7577470911161189349:0] socket: 114 qp: -1
2025-11-27T18:17:09.902130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470911161189267:2048] [node 1] ICP22 created new session: [3:7577470911161189358:2048]
2025-11-27T18:17:09.899188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470912529275128:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:09.894683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:09.900082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470912529275053:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.903487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470911161189358:2048] [node 1] ICS09 handshake done sender: [3:7577470911161189345:2117] self: [3:7577470911161189349:0] peer: [1:7577470909494508974:0] socket: 122 qp: -1
2025-11-27T18:17:09.900785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470909494509003:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:09.903587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470912779250315:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.903525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470911161189358:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:09.903617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470912779250316:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.902065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:09.900803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470912529275052:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.905159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:09.898730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470910796743734:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58202
2025-11-27T18:17:09.903602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:09.900859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:09.905602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470912779250316:2049] [node 2] ICP22 created new session: [5:7577470912779250403:2049]
2025-11-27T18:17:09.910199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:09.902410Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470910796743709:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16707462ns) {HandleHandshakeStatus:329(16705348ns) {DropIncomingHandshake:442(12555ns) DropOutgoingHandshake:457(407ns) SwitchToState:179(341ns) ProcessPendingSessionEvents:495(3322516ns) {ForwardSessionEventToSession:612(380960ns) ForwardSessionEventToSession:612(5122ns) ForwardSessionEventToSession:612(448701ns) ForwardSessionEventToSession:612(3130ns) ForwardSessionEventToSession:612(1056137ns) ForwardSessionEventToSession:612(4648ns) ForwardSessionEventToSession:612(209656ns) ForwardSessionEventToSession:612(1193350ns) ForwardSessionEventToSession:612(3944ns) ForwardSessionEventToSession:612(2504ns)}}}
2025-11-27T18:17:09.902084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.902121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470912529275053:2050] [node 3] ICP22 created new session: [4:7577470912529275136:2050]
2025-11-27T18:17:09.910844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470910796743724:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:09.910240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470912779250403:2049] [node 2] ICS09 handshake done sender: [5:7577470912779250393:2118] self: [5:7577470912779250399:0] peer: [2:7577470910796743720:0] socket: 126 qp: -1
2025-11-27T18:17:09.922915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470909494508909:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17600780ns) {HandleHandshakeStatus:329(17598062ns) {DropIncomingHandshake:442(835ns) DropOutgoingHandshake:457(358ns) SwitchToState:179(478ns) ProcessPendingSessionEvents:495(9287150ns) {ForwardSessionEventToSession:612(1987705ns) ForwardSessionEventToSession:612(8696ns) ForwardSessionEventToSession:612(3471ns) ForwardSessionEventToSession:612(3968399ns) ForwardSessionEventToSession:612(3550ns) ForwardSessionEventToSession:612(734183ns) ForwardSessionEventToSession:612(365491ns) ForwardSessionEventToSession:612(6980ns) ForwardSessionEventToSession:612(1953170ns) ForwardSessionEventToSession:612(4066ns)}}}
2025-11-27T18:17:09.902156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470912529275052:2049] [node 2] ICP22 created new session: [4:7577470912529275137:2049]
2025-11-27T18:17:09.910299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470912779250403:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:09.910546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470912779250315:2048] [node 1] ICP22 created new session: [5:7577470912779250404:2048]
2025-11-27T18:17:09.923218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470912822679668:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63270
2025-11-27T18:17:09.913094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470910796743646:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16165) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577470910796743724:2116] held: no
2025-11-27T18:17:09.911055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:09.915395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470912779250404:2048] [node 1] ICS09 handshake done sender: [5:7577470912779250390:2116] self: [5:7577470912779250398:0] peer: [1:7577470909494508978:0] socket: 123 qp: -1
2025-11-27T18:17:09.923035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470909494508909:2049] [node 3] ICP18 handshake (actor [1:7577470909494508991:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.902487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470912529275136:2050] [node 3] ICS09 handshake done sender: [4:7577470912529275130:2116] self: [4:7577470912529275132:0] peer: [3:7577470911161189339:0] socket: 141 qp: -1
2025-11-27T18:17:09.917803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470910796743646:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.917850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:09.922640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470910796743646:2048] [node 1] ICP22 created new session: [2:7577470910796743744:2048]
2025-11-27T18:17:09.924889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470912822679668:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.903041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470912529275137:2049] [node 2] ICS09 handshake done sender: [4:7577470912529275128:2115] self: [4:7577470912529275131:0] peer: [2:7577470910796743716:0] socket: 129 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.903041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470912529275136:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:09.915440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470912779250404:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:09.903715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.922946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470910796743744:2048] [node 1] ICS09 handshake done sender: [2:7577470910796743724:2116] self: [2:7577470910796743731:0] peer: [1:7577470909494508976:0] socket: 118 qp: -1
2025-11-27T18:17:09.922450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.905667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470912529275137:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:09.925276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470911161189361:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23813
2025-11-27T18:17:09.922987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470910796743744:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:09.922498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470912779250402:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.910004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.932563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470912822679671:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63270
2025-11-27T18:17:09.925333Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470912529275053:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(23220879ns) {HandleHandshakeStatus:329(23218129ns) {DropIncomingHandshake:442(647ns) DropOutgoingHandshake:457(3329ns) SwitchToState:179(498ns) ProcessPendingSessionEvents:495(12743833ns) {ForwardSessionEventToSession:612(11523690ns) ForwardSessionEventToSession:612(6337ns) ForwardSessionEventToSession:612(2605ns) ForwardSessionEventToSession:612(326410ns) ForwardSessionEventToSession:612(15552ns) ForwardSessionEventToSession:612(2469ns) ForwardSessionEventToSession:612(327875ns) ForwardSessionEventToSession:612(2642ns) ForwardSessionEventToSession:612(219769ns) ForwardSessionEventToSession:612(2674ns)}}}
2025-11-27T18:17:09.934598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470912822679669:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63270
2025-11-27T18:17:09.935277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470912822679670:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63270
2025-11-27T18:17:09.923231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:09.925317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470912779250317:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.925475Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470911161189267:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23194331ns) {HandleHandshakeStatus:329(23192128ns) {DropIncomingHandshake:442(840ns) DropOutgoingHandshake:457(4802ns) SwitchToState:179(289ns) ProcessPendingSessionEvents:495(10434072ns) {ForwardSessionEventToSession:612(9549300ns) ForwardSessionEventToSession:612(165861ns) ForwardSessionEventToSession:612(3127ns) ForwardSessionEventToSession:612(2642ns) ForwardSessionEventToSession:612(91833ns) ForwardSessionEventToSession:612(2569ns) ForwardSessionEventToSession:612(158352ns) ForwardSessionEventToSession:612(4014ns) ForwardSessionEventToSession:612(204778ns) ForwardSessionEventToSession:612(239891ns)}}}
2025-11-27T18:17:09.935593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470910796743715:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.935688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470912822679590:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.932371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470911161189361:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.931080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470912529275140:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59691
2025-11-27T18:17:09.932170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470912529275053:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:09.934442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470912529275142:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59691
2025-11-27T18:17:09.935391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470912529275140:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.936712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470912529275051:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.935613Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470910796743715:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.932492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470911161189344:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.936713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470912779250315:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(28629897ns) {HandleHandshakeStatus:329(28626255ns) {DropIncomingHandshake:442(1012ns) DropOutgoingHandshake:457(2482ns) SwitchToState:179(2549ns) ProcessPendingSessionEvents:495(7885494ns) {ForwardSessionEventToSession:612(1262079ns) ForwardSessionEventToSession:612(4412ns) ForwardSessionEventToSession:612(2766418ns) ForwardSessionEventToSession:612(6840ns) ForwardSessionEventToSession:612(2357ns) ForwardSessionEventToSession:612(2706266ns) ForwardSessionEventToSession:612(3576ns) ForwardSessionEventToSession:612(2423ns) ForwardSessionEventToSession:612(1107090ns) ForwardSessionEventToSession:612(2272ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.935756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470912822679669:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.932523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470911161189344:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.936324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470912822679671:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.938237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470910796743719:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.937839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470912779250410:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13590
2025-11-27T18:17:09.932595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470911161189268:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.938660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470909494508983:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.941504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470912529275125:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.936497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470912822679670:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.938276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470910796743719:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.939559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470912779250411:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13590
2025-11-27T18:17:09.938698Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470909494508983:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.940675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470911161189343:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.943137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470910796743715:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:09.941540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470912529275125:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.946167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470912779250410:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.937188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470912822679589:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.944679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470909494508983:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:09.946700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470912779250389:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.937581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470912822679591:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.946189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470910796743719:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:09.945105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470909494508981:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.940701Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470911161189343:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.948600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470912529275125:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:09.945138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470909494508981:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.947614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470909494508971:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.948867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.946723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470912779250389:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.946494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470910796743647:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.937660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470912822679672:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63270
2025-11-27T18:17:09.943622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470911161189344:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:09.937904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470912822679657:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.946752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:09.947015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470911161189333:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.949712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470912529275144:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59691
2025-11-27T18:17:09.950190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470912779250402:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:09.949915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470909494508971:2107] [node 4] ICP22 created new session: [1:7577470909494509008:2107]
2025-11-27T18:17:09.946793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470910796743647:2049] [node 3] ICP22 created new session: [2:7577470910796743748:2049]
2025-11-27T18:17:09.939586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470912822679672:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.947658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:09.954926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470912779250381:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.950330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470912529275142:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.953934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470909494509008:2107] [node 4] ICS09 handshake done sender: [1:7577470909494508983:2114] self: [1:7577470909494508982:0] peer: [4:7577470912529275143:0] socket: 121 qp: -1
2025-11-27T18:17:09.949245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470910796743748:2049] [node 3] ICS09 handshake done sender: [2:7577470910796743715:2111] self: [2:7577470910796743714:0] peer: [3:7577470911161189362:0] socket: 120 qp: -1
2025-11-27T18:17:09.941545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470912822679646:2102] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:09.948939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470911161189333:2109] [node 5] ICP22 created new session: [3:7577470911161189366:2109]
2025-11-27T18:17:09.956861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470912779250317:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.949514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470910796743711:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.951546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470912529275140:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:09.950172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470912822679678:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63270
2025-11-27T18:17:09.949562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470911161189366:2109] [node 5] ICS09 handshake done sender: [3:7577470911161189344:2116] self: [3:7577470911161189342:0] peer: [5:7577470912779250408:0] socket: 162 qp: -1
2025-11-27T18:17:09.953977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470909494509008:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:09.954869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470912529275114:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.949602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:09.949603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470911161189366:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:09.956917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.954883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470909494508981:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:09.954911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:09.958165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470912779250317:2050] [node 3] ICP22 created new session: [5:7577470912779250414:2050]
2025-11-27T18:17:09.949609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470910796743748:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:09.949645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:09.956118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:09.956112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470912529275114:2109] [node 6] ICP22 created new session: [4:7577470912529275147:2109]
2025-11-27T18:17:09.959957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470912779250414:2050] [node 3] ICS09 handshake done sender: [5:7577470912779250402:2121] self: [5:7577470912779250408:0] peer: [3:7577470911161189342:0] socket: 144 qp: -1
2025-11-27T18:17:09.961520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470909494508973:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.949634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470910796743711:2109] [node 6] ICP22 created new session: [2:7577470910796743749:2109]
2025-11-27T18:17:09.963169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:09.964028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470909494508973:2109] [node 6] ICP22 created new session: [1:7577470909494509010:2109]
2025-11-27T18:17:09.964342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470909494509010:2109] [node 6] ICS09 handshake done sender: [1:7577470909494508981:2113] self: [1:7577470909494508980:0] peer: [6:7577470912822679675:0] socket: 134 qp: -1
2025-11-27T18:17:09.969606Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470912822679590:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470912822679668:2116] held: no
2025-11-27T18:17:09.966141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470909494509010:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:09.971090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470912822679668:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:09.956590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470912529275051:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16165) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470912529275140:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.949666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:09.960768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470911161189268:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470911161189361:2121] held: no
2025-11-27T18:17:09.958288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470912529275147:2109] [node 6] ICS09 handshake done sender: [4:7577470912529275125:2114] self: [4:7577470912529275123:0] peer: [6:7577470912822679676:0] socket: 140 qp: -1
2025-11-27T18:17:09.960826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470912779250414:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:09.967699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:09.963781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470911161189364:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23813
2025-11-27T18:17:09.949693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470910796743749:2109] [node 6] ICS09 handshake done sender: [2:7577470910796743719:2113] self: [2:7577470910796743718:0] peer: [6:7577470912822679673:0] socket: 131 qp: -1
2025-11-27T18:17:09.960892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:09.958330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470912529275147:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:09.973305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470909494508971:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(19454991ns) {HandleHandshakeStatus:329(19451540ns) {DropIncomingHandshake:442(11840ns) DropOutgoingHandshake:457(296ns) SwitchToState:179(514ns) ProcessPendingSessionEvents:495(6914234ns) {ForwardSessionEventToSession:612(2128748ns) ForwardSessionEventToSession:612(4419ns) ForwardSessionEventToSession:612(1088583ns) ForwardSessionEventToSession:612(2747ns) ForwardSessionEventToSession:612(609131ns) ForwardSessionEventToSession:612(3506ns) ForwardSessionEventToSession:612(11258ns) ForwardSessionEventToSession:612(1751039ns) ForwardSessionEventToSession:612(1192769ns) ForwardSessionEventToSession:612(4924ns)}}}
2025-11-27T18:17:09.975935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470912822679679:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63270
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.963048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470912779250389:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:09.958373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:09.949727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470910796743749:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:09.965933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470911161189343:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:09.968720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470912779250380:2111] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.949756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:09.959933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470912529275051:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.969438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:09.980201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470912822679680:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63270
2025-11-27T18:17:09.959984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:09.960172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470912529275051:2048] [node 1] ICP22 created new session: [4:7577470912529275149:2048]
2025-11-27T18:17:09.970300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470912779250380:2111] [node 6] ICP22 created new session: [5:7577470912779250416:2111]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.971350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470912779250317:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:09.974679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470912779250416:2111] [node 6] ICS09 handshake done sender: [5:7577470912779250389:2115] self: [5:7577470912779250388:0] peer: [6:7577470912822679677:0] socket: 132 qp: -1
2025-11-27T18:17:09.969243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470911161189361:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:09.974717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470912779250416:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:09.976357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:09.980954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470912822679681:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63270
2025-11-27T18:17:09.960516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470912529275149:2048] [node 1] ICS09 handshake done sender: [4:7577470912529275140:2119] self: [4:7577470912529275143:0] peer: [1:7577470909494508982:0] socket: 139 qp: -1
2025-11-27T18:17:09.969340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470911161189334:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.981027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470912779250417:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13590
2025-11-27T18:17:09.969373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:09.960542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470912529275149:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:09.971196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470911161189334:2110] [node 6] ICP22 created new session: [3:7577470911161189368:2110]
2025-11-27T18:17:09.962248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470912529275124:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:09.971244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470911161189368:2110] [node 6] ICS09 handshake done sender: [3:7577470911161189343:2115] self: [3:7577470911161189341:0] peer: [6:7577470912822679674:0] socket: 163 qp: -1
2025-11-27T18:17:09.962287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470912529275124:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:09.982949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470912779250381:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470912779250410:2122] held: no
2025-11-27T18:17:09.971275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470911161189368:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:09.962320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:09.971309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:09.969570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470912529275124:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:09.975681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470911161189268:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.971723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470912529275115:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:09.975733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:09.975849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:09.976101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470912529275115:2108] [node 5] ICP22 created new session: [4:7577470912529275152:2108]
2025-11-27T18:17:09.975780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470911161189268:2049] [node 2] ICP22 created new session: [3:7577470911161189370:2049]
2025-11-27T18:17:09.976914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470912529275152:2108] [node 5] ICS09 handshake done sender: [4:7577470912529275124:2113] self: [4:7577470912529275122:0] peer: [5:7577470912779250412:0] socket: 146 qp: -1
2025-11-27T18:17:09.984194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470912822679589:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16165) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470912822679669:2117] held: no
2025-11-27T18:17:09.976476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470911161189370:2049] [node 2] ICS09 handshake done sender: [3:7577470911161189361:2121] self: [3:7577470911161189362:0] peer: [2:7577470910796743714:0] socket: 148 qp: -1
2025-11-27T18:17:09.984428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470912779250410:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:09.976968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470912529275152:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:09.977014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:09.976512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470911161189370:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:09.984263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470912822679590:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.977818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:09.986409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470912779250381:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.985757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:09.986677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:09.985889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470912822679646:2102] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5685) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470912822679672:2120] held: no
2025-11-27T18:17:09.986776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470912779250381:2109] [node 4] ICP22 created new session: [5:7577470912779250419:2109]
2025-11-27T18:17:09.985975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470912822679669:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:09.986832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470912779250419:2109] [node 4] ICS09 handshake done sender: [5:7577470912779250410:2122] self: [5:7577470912779250412:0] peer: [4:7577470912529275122:0] socket: 167 qp: -1
2025-11-27T18:17:09.985984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470912822679590:2049] [node 2] ICP22 created new session: [6:7577470912822679684:2049]
2025-11-27T18:17:09.986865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470912779250419:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:09.986248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470912822679684:2049] [node 2] ICS09 handshake done sender: [6:7577470912822679668:2116] self: [6:7577470912822679673:0] peer: [2:7577470910796743718:0] socket: 133 qp: -1
2025-11-27T18:17:09.986963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:09.986284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470912822679670:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.986390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470912822679684:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:09.986447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:09.986576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470912822679671:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:09.987355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470912822679682:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63270
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:09.987744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470911161189372:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23813
2025-11-27T18:17:09.987378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470912822679589:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.987454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:09.987488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470912822679589:2048] [node 1] ICP22 created new session: [6:7577470912822679686:2048]
2025-11-27T18:17:09.987566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470912822679686:2048] [node 1] ICS09 handshake done sender: [6:7577470912822679669:2117] self: [6:7577470912822679675:0] peer: [1:7577470909494508980:0] socket: 142 qp: -1
2025-11-27T18:17:09.987579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470912822679657:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.987603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470912822679686:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:09.987631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:09.987647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:09.987679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470912822679657:2110] [node 4] ICP22 created new session: [6:7577470912822679688:2110]
2025-11-27T18:17:09.987721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470912822679688:2110] [node 4] ICS09 handshake done sender: [6:7577470912822679670:2118] self: [6:7577470912822679676:0] peer: [4:7577470912529275123:0] socket: 164 qp: -1
2025-11-27T18:17:09.987747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470912822679688:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:09.987776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:09.988469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470912822679591:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.988557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:09.988594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470912822679591:2050] [node 3] ICP22 created new session: [6:7577470912822679691:2050]
2025-11-27T18:17:09.988626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470912822679691:2050] [node 3] ICS09 handshake done sender: [6:7577470912822679671:2119] self: [6:7577470912822679674:0] peer: [3:7577470911161189341:0] socket: 165 qp: -1
2025-11-27T18:17:09.988651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470912822679691:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:09.988681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:09.989493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470912822679672:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:09.989639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470912822679646:2102] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:09.989716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:09.989771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470912822679646:2102] [node 5] ICP22 created new session: [6:7577470912822679693:2102]
2025-11-27T18:17:09.989810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470912822679693:2102] [node 5] ICS09 handshake done sender: [6:7577470912822679672:2120] self: [6:7577470912822679677:0] peer: [5:7577470912779250388:0] socket: 166 qp: -1
2025-11-27T18:17:09.989834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470912822679693:2102] [node 5] ICS10 traffic start
2025-11-27T18:17:09.989884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:10.014793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:10.015021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470910796743744:2048] [node 1] ICS01 socket: 118 reason# EndOfStream
2025-11-27T18:17:10.015113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470910796743646:2048] [node 1] ICP30 unregister session Session# [2:7577470910796743744:2048] VirtualId# [2:7577470910796743731:0]
2025-11-27T18:17:10.015140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470910796743744:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.015230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:10.015290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:10.015866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:10.016031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470911161189358:2048] [node 1] ICS07 socket disconnect 122 reason# EndOfStream
2025-11-27T18:17:10.016083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470911161189358:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.016151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:10.016212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470911161189358:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:10.018353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470911161189267:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16165) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:10.018409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470911161189358:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:10.019507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:10.019773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470912529275149:2048] [node 1] ICS01 socket: 139 reason# EndOfStream
2025-11-27T18:17:10.019826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470912529275051:2048] [node 1] ICP30 unregister session Session# [4:7577470912529275149:2048] VirtualId# [4:7577470912529275143:0]
2025-11-27T18:17:10.019861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470912529275149:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.019935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:10.019995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:10.020081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470911161189267:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16165) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.020203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470911161189358:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.020242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470911161189267:2048] [node 1] ICP30 unregister session Session# [3:7577470911161189358:2048] VirtualId# [3:7577470911161189349:0]
2025-11-27T18:17:10.020306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:10.020340Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470911161189267:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16165) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.020360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16165) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.020444Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:10.020669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470912779250404:2048] [node 1] ICS07 socket disconnect 123 reason# EndOfStream
2025-11-27T18:17:10.020701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470912779250404:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.020765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:10.020810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470912779250404:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:10.021500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470912779250315:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16165) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.021557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470912779250404:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.021652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470912779250315:2048] [node 1] ICP30 unregister session Session# [5:7577470912779250404:2048] VirtualId# [5:7577470912779250398:0]
2025-11-27T18:17:10.021703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:10.021730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470912779250315:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16165) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.021748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16165) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.021707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:10.021941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470912822679686:2048] [node 1] ICS01 socket: 142 reason# EndOfStream
2025-11-27T18:17:10.021980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470912822679589:2048] [node 1] ICP30 unregister session Session# [6:7577470912822679686:2048] VirtualId# [6:7577470912822679675:0]
2025-11-27T18:17:10.022022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470912822679686:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.022086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:10.022126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:10.029641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470911161189267:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:10.031635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470912779250315:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:10.070160Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:10.070359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470911161189370:2049] [node 2] ICS01 socket: 148 reason# EndOfStream
2025-11-27T18:17:10.070392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470911161189268:2049] [node 2] ICP30 unregister session Session# [3:7577470911161189370:2049] VirtualId# [3:7577470911161189362:0]
2025-11-27T18:17:10.070416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470911161189370:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.070481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:10.070532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:10.072548Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:10.072786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470912529275137:2049] [node 2] ICS01 socket: 129 reason# EndOfStream
2025-11-27T18:17:10.072839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470912529275052:2049] [node 2] ICP30 unregister session Session# [4:7577470912529275137:2049] VirtualId# [4:7577470912529275131:0]
2025-11-27T18:17:10.072876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470912529275137:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.072937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:10.072985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:10.073516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:10.073730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470912779250403:2049] [node 2] ICS01 socket: 126 reason# EndOfStream
2025-11-27T18:17:10.073763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470912779250316:2049] [node 2] ICP30 unregister session Session# [5:7577470912779250403:2049] VirtualId# [5:7577470912779250399:0]
2025-11-27T18:17:10.073788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470912779250403:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.073844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:10.073882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:10.074429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:10.074618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470912822679684:2049] [node 2] ICS01 socket: 133 reason# EndOfStream
2025-11-27T18:17:10.074658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470912822679590:2049] [node 2] ICP30 unregister session Session# [6:7577470912822679684:2049] VirtualId# [6:7577470912822679673:0]
2025-11-27T18:17:10.074678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470912822679684:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.074728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:10.074765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:10.107674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:10.107880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470912529275136:2050] [node 3] ICS07 socket disconnect 141 reason# EndOfStream
2025-11-27T18:17:10.107910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470912529275136:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.107975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:10.108017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470912529275136:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:10.108666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:10.108847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470912779250414:2050] [node 3] ICS07 socket disconnect 144 reason# EndOfStream
2025-11-27T18:17:10.108875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470912779250414:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.108863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470912529275053:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1373) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.108966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:10.108935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470912529275136:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.109028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470912779250414:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:10.109021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470912529275053:2050] [node 3] ICP30 unregister session Session# [4:7577470912529275136:2050] VirtualId# [4:7577470912529275132:0]
2025-11-27T18:17:10.109102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:10.109130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470912529275053:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1373) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.109144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1373) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.109770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:10.109832Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470912779250317:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1373) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.109935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470912779250414:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.110007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470912779250317:2050] [node 3] ICP30 unregister session Session# [5:7577470912779250414:2050] VirtualId# [5:7577470912779250408:0]
2025-11-27T18:17:10.110000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470912822679691:2050] [node 3] ICS07 socket disconnect 165 reason# EndOfStream
2025-11-27T18:17:10.110034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470912822679691:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.110053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:10.110102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:10.110079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470912779250317:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1373) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.110091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1373) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.110140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470912822679691:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:10.110842Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470912822679591:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1373) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.110890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470912822679691:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.110967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470912822679591:2050] [node 3] ICP30 unregister session Session# [6:7577470912822679691:2050] VirtualId# [6:7577470912822679674:0]
2025-11-27T18:17:10.111003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:10.111020Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470912822679591:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1373) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.111027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1373) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.118721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470912529275053:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:10.119720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470912779250317:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:10.120746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470912822679591:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:10.163411Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:10.163657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470912779250419:2109] [node 4] ICS01 socket: 167 reason# EndOfStream
2025-11-27T18:17:10.163704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470912779250381:2109] [node 4] ICP30 unregister session Session# [5:7577470912779250419:2109] VirtualId# [5:7577470912779250412:0]
2025-11-27T18:17:10.163743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470912779250419:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.163825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:10.163909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:10.164343Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:10.164567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470912822679688:2110] [node 4] ICS07 socket disconnect 164 reason# EndOfStream
2025-11-27T18:17:10.164598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470912822679688:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.164678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:10.164726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470912822679688:2110] [node 4] ICS15 start handshake
2025-11-27T18:17:10.165515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470912822679657:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11241) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.165630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470912822679688:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.165684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470912822679657:2110] [node 4] ICP30 unregister session Session# [6:7577470912822679688:2110] VirtualId# [6:7577470912822679676:0]
2025-11-27T18:17:10.165724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:10.165749Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470912822679657:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11241) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.165763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11241) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.176077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470912822679657:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:17:10.193341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:10.193589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470912822679693:2102] [node 5] ICS01 socket: 166 reason# EndOfStream
2025-11-27T18:17:10.193629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470912822679646:2102] [node 5] ICP30 unregister session Session# [6:7577470912822679693:2102] VirtualId# [6:7577470912822679677:0]
2025-11-27T18:17:10.193659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470912822679693:2102] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.193737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:10.193791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:10.284475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470915508439841:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:10.284549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470915508439840:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:10.304070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470915066731760:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:10.304098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470915066731761:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:10.326764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470916393163333:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:10.326775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470916393163334:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:10.349665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470916757239997:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:10.349777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470916757239998:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:10.349900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470916757239999:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:10.370646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470914159808032:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:10.370686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470914159808033:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:10.370709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470914159808034:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:10.392473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470916258572991:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:10.392512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470916258572992:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:10.392533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470916258572993:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:10.546036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470916258573054:2105] [node 5] ICP01 ready to work
2025-11-27T18:17:10.536211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470915508439905:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:10.556005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470915508439907:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:10.562438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470915508439906:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:10.570581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470916757240062:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:10.572565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470916258573055:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:10.565329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470914159808096:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:10.570772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470914159808097:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:10.572875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470916757240063:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:10.573104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470915066731823:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:10.573856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470915066731824:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:10.574666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470915066731825:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:10.572475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470916393163397:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:10.572573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470916393163396:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:10.573777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470916393163398:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:10.576386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470915508439840:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(83910765ns) {Configure:104(83901096ns) {ProcessConfigured:135(1157541ns) {StartInitialHandshake:157(1155564ns) {DropHandshakes:470(8052ns) {DropIncomingHandshake:442(1311ns) DropOutgoingHandshake:457(497ns)} GenerateSessionVirtualId:988(1517ns) PrepareNewSessionHandshake:477(2764ns) {SwitchToState:179(824ns)}}}}}
2025-11-27T18:17:10.586876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470916757239999:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(72016876ns) {Configure:104(72013402ns) {ProcessConfigured:135(663886ns) {StartInitialHandshake:157(662238ns) {DropHandshakes:470(1497ns) {DropIncomingHandshake:442(472ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(271ns) PrepareNewSessionHandshake:477(1492ns) {SwitchToState:179(167ns)}}}}}
2025-11-27T18:17:10.588910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470916757239997:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(82784385ns) {Configure:104(82778496ns) {ProcessConfigured:135(2097948ns) {StartInitialHandshake:157(2094932ns) {DropHandshakes:470(3453ns) {DropIncomingHandshake:442(904ns) DropOutgoingHandshake:457(796ns)} GenerateSessionVirtualId:988(1111ns) PrepareNewSessionHandshake:477(3356ns) {SwitchToState:179(1877ns)}}}}}
2025-11-27T18:17:10.590824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470915066731761:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(87274544ns) {Configure:104(80683046ns) {ProcessConfigured:135(1174790ns) {StartInitialHandshake:157(1173212ns) {DropHandshakes:470(1765ns) {DropIncomingHandshake:442(763ns) DropOutgoingHandshake:457(200ns)} GenerateSessionVirtualId:988(316ns) PrepareNewSessionHandshake:477(2288ns) {SwitchToState:179(212ns)}}}}}
2025-11-27T18:17:10.598010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470914159808096:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17130226ns) {Configure:104(17124029ns) {ProcessConfigured:135(960604ns) {StartInitialHandshake:157(959428ns) {DropHandshakes:470(928ns) {DropIncomingHandshake:442(307ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(400ns) PrepareNewSessionHandshake:477(2512ns) {SwitchToState:179(1545ns)}}}}}
2025-11-27T18:17:10.598120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470915066731760:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(83602350ns) {Configure:104(83592671ns) {ProcessConfigured:135(2885487ns) {StartInitialHandshake:157(2882746ns) {DropHandshakes:470(1006ns) {DropIncomingHandshake:442(276ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(335ns) PrepareNewSessionHandshake:477(2667ns) {SwitchToState:179(187ns)}}}}}
2025-11-27T18:17:10.598161Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470916757239998:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(76459538ns) {Configure:104(76453978ns) {ProcessConfigured:135(1434576ns) {StartInitialHandshake:157(1433624ns) {DropHandshakes:470(706698ns) {DropIncomingHandshake:442(1100ns) DropOutgoingHandshake:457(209ns)} GenerateSessionVirtualId:988(869ns) PrepareNewSessionHandshake:477(1052ns) {SwitchToState:179(242ns)}}}}}
2025-11-27T18:17:10.597417Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470915508439906:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19919270ns) {Configure:104(19906670ns) {ProcessConfigured:135(3256189ns) {StartInitialHandshake:157(3253830ns) {DropHandshakes:470(1516ns) {DropIncomingHandshake:442(518ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:477(1524ns) {SwitchToState:179(129ns)}}}}}
2025-11-27T18:17:10.601415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470916258572991:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24289246ns) {Configure:104(24282891ns) {ProcessConfigured:135(3644074ns) {StartInitialHandshake:157(3642480ns) {DropHandshakes:470(5091ns) {DropIncomingHandshake:442(4140ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(1840ns) {SwitchToState:179(314ns)}}}}}
2025-11-27T18:17:10.605523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470916393163334:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17069339ns) {Configure:104(17065027ns) {ProcessConfigured:135(795048ns) {StartInitialHandshake:157(793100ns) {DropHandshakes:470(848ns) {DropIncomingHandshake:442(275ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(326ns) PrepareNewSessionHandshake:477(964ns) {SwitchToState:179(172ns)}}}}}
2025-11-27T18:17:10.606741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470914159808034:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27496232ns) {Configure:104(27492322ns) {ProcessConfigured:135(6888529ns) {StartInitialHandshake:157(6886825ns) {DropHandshakes:470(1091ns) {DropIncomingHandshake:442(403ns) DropOutgoingHandshake:457(272ns)} GenerateSessionVirtualId:988(287ns) PrepareNewSessionHandshake:477(1967ns) {SwitchToState:179(876ns)}}}}}
2025-11-27T18:17:10.606843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470916757240062:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17677897ns) {Configure:104(17672446ns) {ProcessConfigured:135(508776ns) {StartInitialHandshake:157(506830ns) {DropHandshakes:470(1289ns) {DropIncomingHandshake:442(414ns) DropOutgoingHandshake:457(291ns)} GenerateSessionVirtualId:988(795ns) PrepareNewSessionHandshake:477(968ns) {SwitchToState:179(173ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.612035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470914159808032:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53566361ns) {Configure:104(53561397ns) {ProcessConfigured:135(1029019ns) {StartInitialHandshake:157(629983ns) {DropHandshakes:470(1991ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(599ns)} GenerateSessionVirtualId:988(893ns) PrepareNewSessionHandshake:477(1642ns) {SwitchToState:179(93ns)}}}}}
2025-11-27T18:17:10.608744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470916258572992:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(25074777ns) {Configure:104(25066346ns) {ProcessConfigured:135(1255177ns) {StartInitialHandshake:157(1253000ns) {DropHandshakes:470(1030ns) {DropIncomingHandshake:442(299ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(509ns) PrepareNewSessionHandshake:477(1740ns) {SwitchToState:179(184ns)}}}}}
2025-11-27T18:17:10.622762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470915508439905:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32272871ns) {Configure:104(32260083ns) {ProcessConfigured:135(6663052ns) {StartInitialHandshake:157(6660229ns) {DropHandshakes:470(196525ns) {DropIncomingHandshake:442(296ns) DropOutgoingHandshake:457(658ns)} GenerateSessionVirtualId:988(600ns) PrepareNewSessionHandshake:477(3510ns) {SwitchToState:179(1431ns)}}}}}
2025-11-27T18:17:10.611951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470916393163333:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34289236ns) {Configure:104(34279775ns) {ProcessConfigured:135(1352207ns) {StartInitialHandshake:157(1350913ns) {DropHandshakes:470(612ns) {DropIncomingHandshake:442(119ns) DropOutgoingHandshake:457(124ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:477(1540ns) {SwitchToState:179(560ns)}}}}}
2025-11-27T18:17:10.609105Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470916258573054:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27930461ns) {Configure:104(27913302ns) {ProcessConfigured:135(949672ns) {StartInitialHandshake:157(948444ns) {DropHandshakes:470(249231ns) {DropIncomingHandshake:442(662ns) DropOutgoingHandshake:457(156ns)} GenerateSessionVirtualId:988(401ns) PrepareNewSessionHandshake:477(1149ns) {SwitchToState:179(427ns)}}}}}
2025-11-27T18:17:10.622552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470914159808033:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41129647ns) {Configure:104(41124268ns) {ProcessConfigured:135(6476654ns) {StartInitialHandshake:157(6473972ns) {DropHandshakes:470(967ns) {DropIncomingHandshake:442(216ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(3137ns) {SwitchToState:179(351ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.611985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470916393163397:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32515186ns) {Configure:104(32509084ns) {ProcessConfigured:135(14576119ns) {StartInitialHandshake:157(14574558ns) {DropHandshakes:470(959ns) {DropIncomingHandshake:442(286ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(368ns) PrepareNewSessionHandshake:477(4365ns) {SwitchToState:179(573ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.623200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470915508439920:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30034
2025-11-27T18:17:10.622825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470916258572993:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32258116ns) {Configure:104(32253141ns) {ProcessConfigured:135(14338721ns) {StartInitialHandshake:157(14336601ns) {DropHandshakes:470(112534ns) {DropIncomingHandshake:442(250ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(323ns) PrepareNewSessionHandshake:477(3247ns) {SwitchToState:179(2050ns)}}}}}
2025-11-27T18:17:10.628944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470915066731823:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(47688711ns) {Configure:104(47684113ns) {ProcessConfigured:135(20896945ns) {StartInitialHandshake:157(20895298ns) {DropHandshakes:470(2700ns) {DropIncomingHandshake:442(726ns) DropOutgoingHandshake:457(491ns)} GenerateSessionVirtualId:988(698ns) PrepareNewSessionHandshake:477(4262ns) {SwitchToState:179(1594ns)}}}}}
2025-11-27T18:17:10.629636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470915066731824:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(51867473ns) {Configure:104(51862114ns) {ProcessConfigured:135(2618856ns) {StartInitialHandshake:157(2614769ns) {DropHandshakes:470(3437ns) {DropIncomingHandshake:442(897ns) DropOutgoingHandshake:457(553ns)} GenerateSessionVirtualId:988(748ns) PrepareNewSessionHandshake:477(2606ns) {SwitchToState:179(1267ns)}}}}}
2025-11-27T18:17:10.633565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470916258573067:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22603
2025-11-27T18:17:10.629157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470915508439920:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:17:10.633388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470915508439919:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30034
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.635910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470914159808106:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.634463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470915508439919:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.635816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470916393163398:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(44565213ns) {Configure:104(44015866ns) {ProcessConfigured:135(446212ns) {StartInitialHandshake:157(443206ns) {DropHandshakes:470(2589ns) {DropIncomingHandshake:442(551ns) DropOutgoingHandshake:457(537ns)} GenerateSessionVirtualId:988(766ns) PrepareNewSessionHandshake:477(1500ns) {SwitchToState:179(455ns)}}}}}
2025-11-27T18:17:10.636694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470916757240077:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51957
2025-11-27T18:17:10.636003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470916258573067:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.637059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470916393163407:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1110
2025-11-27T18:17:10.637595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470915066731830:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13581
2025-11-27T18:17:10.636500Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470916258573055:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25037818ns) {Configure:104(25029519ns) {ProcessConfigured:135(168991ns) {StartInitialHandshake:157(167010ns) {DropHandshakes:470(1305ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(305ns)} GenerateSessionVirtualId:988(476ns) PrepareNewSessionHandshake:477(1241ns) {SwitchToState:179(616ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.637092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470914159808106:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.638432Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470914159808097:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30072296ns) {Configure:104(30061378ns) {ProcessConfigured:135(818669ns) {StartInitialHandshake:157(816115ns) {DropHandshakes:470(2440ns) {DropIncomingHandshake:442(560ns) DropOutgoingHandshake:457(388ns)} GenerateSessionVirtualId:988(706ns) PrepareNewSessionHandshake:477(2207ns) {SwitchToState:179(912ns)}}}}}
2025-11-27T18:17:10.638210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470916757240069:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.638739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470916393163408:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1110
2025-11-27T18:17:10.636956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470916258573068:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22603
2025-11-27T18:17:10.638248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470916757240069:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.645620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470915066731840:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13581
2025-11-27T18:17:10.646798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470915508439922:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30034
2025-11-27T18:17:10.648147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470915508439902:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.637032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470916258573055:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.642115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470916393163409:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1110
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.638784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470916258573068:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.646757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470915066731830:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.648188Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470915508439902:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.638422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470916757240078:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51957
2025-11-27T18:17:10.642062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470916757240077:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.645607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470916393163407:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.648117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470916258572991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.650953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470915508439912:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.645772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470916393163333:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.646778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470915066731840:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:17:10.648090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470916757239998:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.650958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470915508439909:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.648127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470915066731760:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.645901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470916393163408:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.651222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470916757240069:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.649712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470915066731827:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.649734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470915066731827:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.654643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470916757240079:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51957
2025-11-27T18:17:10.650982Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470915508439912:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.646036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470916393163409:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.649767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470915066731841:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13581
2025-11-27T18:17:10.650991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470915508439909:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.659331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470916757240063:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.648136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470916393163334:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.661595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470916258573073:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22603
2025-11-27T18:17:10.651821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470915508439922:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.659614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470916757240078:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.651260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470915066731834:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.662654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470916393163401:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.657760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470915508439902:2108] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:10.660384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:10.657868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470915508439925:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30034
2025-11-27T18:17:10.662688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470916393163401:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.651288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470915066731834:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.667837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470916393163418:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1110
2025-11-27T18:17:10.660935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470916757240063:2110] [node 6] ICP22 created new session: [4:7577470916757240084:2110]
2025-11-27T18:17:10.667865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470916258573067:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:10.657975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470915508439924:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30034
2025-11-27T18:17:10.653276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470915066731827:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:10.661547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470916757240079:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.659190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470915508439909:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:10.668443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470916258573055:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.662353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470916757239999:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.668485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:10.668174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470916393163401:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.665824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470916757240084:2110] [node 6] ICS09 handshake done sender: [4:7577470916757240069:2114] self: [4:7577470916757240068:0] peer: [6:7577470916258573071:0] socket: 118 qp: -1
2025-11-27T18:17:10.660153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470915066731841:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.669108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470916258573055:2109] [node 4] ICP22 created new session: [6:7577470916258573075:2109]
2025-11-27T18:17:10.668727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470916393163419:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1110
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.661192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470915508439924:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.660195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470915066731834:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:10.666225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470916757240082:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51957
2025-11-27T18:17:10.669168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470916258573075:2109] [node 4] ICS09 handshake done sender: [6:7577470916258573067:2114] self: [6:7577470916258573071:0] peer: [4:7577470916757240068:0] socket: 124 qp: -1
2025-11-27T18:17:10.661985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470915508439841:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.670142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470916393163407:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:10.660270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470915066731843:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13581
2025-11-27T18:17:10.665882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470916757240084:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:10.669213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470916258573075:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:10.662027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:10.671720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470916393163396:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.660731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470915066731761:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.669246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.671788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:10.667202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:10.662709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:10.662266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470915508439925:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.671649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470916258573074:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22603
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.668071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470916757240077:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.671837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470916393163396:2107] [node 4] ICP22 created new session: [3:7577470916393163421:2107]
2025-11-27T18:17:10.667008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470915066731761:2049] [node 3] ICP22 created new session: [2:7577470915066731848:2049]
2025-11-27T18:17:10.681884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470916258573068:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:10.669038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470916757239998:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.672039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470916393163333:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.662687Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470915508439912:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.670229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:10.672220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470916393163409:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:10.667193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470915066731848:2049] [node 3] ICS09 handshake done sender: [2:7577470915066731827:2110] self: [2:7577470915066731826:0] peer: [3:7577470916393163417:0] socket: 136 qp: -1
2025-11-27T18:17:10.662729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470915508439840:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.683034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470916258572991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470916258573068:2115] held: no
2025-11-27T18:17:10.672364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:10.667285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470915066731824:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.671343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470916757239998:2049] [node 2] ICP22 created new session: [4:7577470916757240089:2049]
2025-11-27T18:17:10.683681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470916258572991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.672606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470916393163333:2048] [node 1] ICP22 created new session: [3:7577470916393163422:2048]
2025-11-27T18:17:10.672513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470916757240089:2049] [node 2] ICS09 handshake done sender: [4:7577470916757240077:2117] self: [4:7577470916757240080:0] peer: [2:7577470915066731833:0] socket: 139 qp: -1
2025-11-27T18:17:10.683741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:10.662780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470915508439841:2049] [node 3] ICP22 created new session: [1:7577470915508439931:2049]
2025-11-27T18:17:10.672569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470916757240089:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:10.685456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470916258572991:2048] [node 1] ICP22 created new session: [6:7577470916258573078:2048]
2025-11-27T18:17:10.677545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:10.662807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:10.685617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470916757240088:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51957
2025-11-27T18:17:10.677674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470916393163422:2048] [node 1] ICS09 handshake done sender: [3:7577470916393163407:2114] self: [3:7577470916393163415:0] peer: [1:7577470915508439901:0] socket: 123 qp: -1
2025-11-27T18:17:10.685968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470916258573077:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22603
2025-11-27T18:17:10.679387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470916393163421:2107] [node 4] ICS09 handshake done sender: [3:7577470916393163401:2110] self: [3:7577470916393163399:0] peer: [4:7577470916757240086:0] socket: 152 qp: -1
2025-11-27T18:17:10.686116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470916258573078:2048] [node 1] ICS09 handshake done sender: [6:7577470916258573068:2115] self: [6:7577470916258573072:0] peer: [1:7577470915508439911:0] socket: 132 qp: -1
2025-11-27T18:17:10.693494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470916757239998:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16248393ns) {HandleHandshakeStatus:329(16245173ns) {DropIncomingHandshake:442(1730ns) DropOutgoingHandshake:457(6254ns) SwitchToState:179(413ns) ProcessPendingSessionEvents:495(3292644ns) {ForwardSessionEventToSession:612(1039347ns) ForwardSessionEventToSession:612(4067ns) ForwardSessionEventToSession:612(530465ns) ForwardSessionEventToSession:612(846388ns) ForwardSessionEventToSession:612(128012ns) ForwardSessionEventToSession:612(2248ns) ForwardSessionEventToSession:612(2523ns) ForwardSessionEventToSession:612(717099ns) ForwardSessionEventToSession:612(2962ns) ForwardSessionEventToSession:612(2997ns)}}}
2025-11-27T18:17:10.665926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470915508439931:2049] [node 3] ICS09 handshake done sender: [1:7577470915508439902:2108] self: [1:7577470915508439901:0] peer: [3:7577470916393163415:0] socket: 114 qp: -1
2025-11-27T18:17:10.679450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470916393163422:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:10.686975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470916258573078:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:10.667008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470915508439931:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:10.688266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:10.680918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470916393163421:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:10.667220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470915508439840:2048] [node 2] ICP22 created new session: [1:7577470915508439932:2048]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:17:10.685298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470914159808114:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:17:10.667579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470915508439932:2048] [node 2] ICS09 handshake done sender: [1:7577470915508439909:2111] self: [1:7577470915508439908:0] peer: [2:7577470915066731842:0] socket: 115 qp: -1
2025-11-27T18:17:10.667920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470915508439932:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:10.668017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:10.668062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:10.668936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470915508439907:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.669638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:10.669860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470915508439907:2110] [node 6] ICP22 created new session: [1:7577470915508439935:2110]
2025-11-27T18:17:10.670208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470915508439935:2110] [node 6] ICS09 handshake done sender: [1:7577470915508439912:2112] self: [1:7577470915508439911:0] peer: [6:7577470916258573072:0] socket: 119 qp: -1
2025-11-27T18:17:10.670962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470915508439841:2049] [node 3] ICP18 handshake (actor [1:7577470915508439925:2119]) is too old
2025-11-27T18:17:10.671652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470915508439935:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:10.671697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:10.689188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470914159808113:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.702289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470916258573077:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.693600Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470916757239999:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22020) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470916757240079:2119] held: no
2025-11-27T18:17:10.680970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:10.702692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470914159808113:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.702795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470916258572991:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18308267ns) {HandleHandshakeStatus:329(18307030ns) {DropIncomingHandshake:442(735ns) DropOutgoingHandshake:457(138ns) SwitchToState:179(347ns) ProcessPendingSessionEvents:495(1136ns)}}
2025-11-27T18:17:10.703136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470916757240079:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:10.667824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:10.667889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470915066731824:2107] [node 4] ICP22 created new session: [2:7577470915066731851:2107]
2025-11-27T18:17:10.667888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470915066731848:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:10.667927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:10.668282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470915066731851:2107] [node 4] ICS09 handshake done sender: [2:7577470915066731834:2114] self: [2:7577470915066731833:0] peer: [4:7577470916757240080:0] socket: 138 qp: -1
2025-11-27T18:17:10.668323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470915066731851:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:10.668355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:10.669872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470915066731843:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.672632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470915066731847:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13581
2025-11-27T18:17:10.681856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470915066731760:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577470915066731830:2112] held: no
2025-11-27T18:17:10.684164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470915066731850:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13581
2025-11-27T18:17:10.685350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470915066731761:2049] [node 3] ICP18 handshake (actor [2:7577470915066731843:2118]) is too old
2025-11-27T18:17:10.688327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470915066731830:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:10.699491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470915066731760:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.699542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:10.699597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470915066731760:2048] [node 1] ICP22 created new session: [2:7577470915066731856:2048]
2025-11-27T18:17:10.699639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470915066731856:2048] [node 1] ICS09 handshake done sender: [2:7577470915066731830:2112] self: [2:7577470915066731842:0] peer: [1:7577470915508439908:0] socket: 121 qp: -1
2025-11-27T18:17:10.699674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470915066731856:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:10.699707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:10.702662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577470915066731824:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:10.707003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470915066731836:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.707052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470915066731836:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.680984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:10.703106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470914159808032:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.702820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470916258572992:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.704020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470916757240091:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51957
2025-11-27T18:17:10.711333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470915066731836:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:10.708313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470914159808115:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:17:10.708354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470915508439918:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.708915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470916757239999:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.704801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470916258573082:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22603
2025-11-27T18:17:10.682320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470916393163334:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.711591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470915066731825:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.708402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470915508439918:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.715721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470914159808114:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.682785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:10.713592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470916258573082:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.713587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470916757240091:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.715726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470916258572992:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3381) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470916258573077:2119] held: no
2025-11-27T18:17:10.716174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470914159808033:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.715383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:10.715911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470916757239999:2050] [node 3] ICP22 created new session: [4:7577470916757240093:2050]
2025-11-27T18:17:10.711670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.683557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470916393163334:2049] [node 2] ICP22 created new session: [3:7577470916393163425:2049]
2025-11-27T18:17:10.717100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470916258572993:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.717742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470915508439915:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.716164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470916757239997:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.711728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470915066731825:2109] [node 6] ICP22 created new session: [2:7577470915066731859:2109]
2025-11-27T18:17:10.684303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470916393163425:2049] [node 2] ICS09 handshake done sender: [3:7577470916393163409:2116] self: [3:7577470916393163417:0] peer: [2:7577470915066731826:0] socket: 141 qp: -1
2025-11-27T18:17:10.720262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470914159808115:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.716594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470916757240093:2050] [node 3] ICS09 handshake done sender: [4:7577470916757240079:2119] self: [4:7577470916757240086:0] peer: [3:7577470916393163399:0] socket: 153 qp: -1
2025-11-27T18:17:10.684347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470916393163425:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:10.716636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470916757240093:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:10.717790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470915508439915:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.720374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470914159808117:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:17:10.684523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:10.717872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:10.721425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470914159808096:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.711779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470915066731859:2109] [node 6] ICS09 handshake done sender: [2:7577470915066731836:2115] self: [2:7577470915066731835:0] peer: [6:7577470916258573081:0] socket: 137 qp: -1
2025-11-27T18:17:10.721004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470916258573083:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22603
2025-11-27T18:17:10.722263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470916757240076:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.719782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470915508439918:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:10.711819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470915066731859:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:10.707067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470916393163334:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20371428ns) {HandleHandshakeStatus:329(20368811ns) {DropIncomingHandshake:442(775ns) DropOutgoingHandshake:457(3939ns) SwitchToState:179(460ns) ProcessPendingSessionEvents:495(15351185ns) {ForwardSessionEventToSession:612(1819615ns) ForwardSessionEventToSession:612(810839ns) ForwardSessionEventToSession:612(2307ns) ForwardSessionEventToSession:612(10208046ns) ForwardSessionEventToSession:612(3957ns) ForwardSessionEventToSession:612(622063ns) ForwardSessionEventToSession:612(1104653ns) ForwardSessionEventToSession:612(3764ns) ForwardSessionEventToSession:612(2300ns) ForwardSessionEventToSession:612(754229ns)}}}
2025-11-27T18:17:10.722386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470914159808117:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.722293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470916258573077:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:10.711860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:10.724472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470916258572992:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.717834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470915066731832:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.717367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470916393163428:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1110
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.723826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470914159808034:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.721846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470915508439905:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.722285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470916757240076:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.717862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470915066731832:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.721919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:10.717862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470916393163411:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.724757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:10.722936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470915066731832:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.726313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470914159808032:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470914159808113:2117] held: no
2025-11-27T18:17:10.721976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470915508439905:2109] [node 5] ICP22 created new session: [1:7577470915508439940:2109]
2025-11-27T18:17:10.717882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470916393163411:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.725121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470916258572992:2049] [node 2] ICP22 created new session: [6:7577470916258573086:2049]
2025-11-27T18:17:10.726528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470914159808119:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:17:10.728650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470914159808121:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:17:10.725839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470915066731823:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.722028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470915508439940:2109] [node 5] ICS09 handshake done sender: [1:7577470915508439918:2114] self: [1:7577470915508439917:0] peer: [5:7577470914159808116:0] socket: 174 qp: -1
2025-11-27T18:17:10.726563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470916258573086:2049] [node 2] ICS09 handshake done sender: [6:7577470916258573077:2119] self: [6:7577470916258573081:0] peer: [2:7577470915066731835:0] socket: 172 qp: -1
2025-11-27T18:17:10.719677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470916393163428:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.726800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470916258573086:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:10.722068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470915508439940:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:10.722406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470916393163411:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:10.737538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470916757239997:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470916757240091:2122] held: no
2025-11-27T18:17:10.725887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.727234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:10.740302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470914159808033:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3381) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470914159808114:2118] held: no
2025-11-27T18:17:10.739750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470916757240076:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:10.725162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470916393163398:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.723972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:10.726580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470915066731823:2108] [node 5] ICP22 created new session: [2:7577470915066731862:2108]
2025-11-27T18:17:10.725213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:10.726307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470915508439915:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:10.725916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470916393163398:2109] [node 6] ICP22 created new session: [3:7577470916393163432:2109]
2025-11-27T18:17:10.726894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470915508439906:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.726571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470916393163406:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.726605Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470916393163406:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.728799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:10.726658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470916393163432:2109] [node 6] ICS09 handshake done sender: [3:7577470916393163411:2117] self: [3:7577470916393163410:0] peer: [6:7577470916258573084:0] socket: 151 qp: -1
2025-11-27T18:17:10.729444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470915508439906:2107] [node 4] ICP22 created new session: [1:7577470915508439943:2107]
2025-11-27T18:17:10.741432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470914159808114:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:10.726833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470916393163432:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:10.737689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470915508439943:2107] [node 4] ICS09 handshake done sender: [1:7577470915508439915:2113] self: [1:7577470915508439914:0] peer: [4:7577470916757240094:0] socket: 173 qp: -1
2025-11-27T18:17:10.743051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470914159808033:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.728732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:10.738004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470915508439943:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:10.732672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470916393163406:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:10.741377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470916757240096:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51957
2025-11-27T18:17:10.731641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470916258573085:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22603
2025-11-27T18:17:10.728693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470915066731862:2108] [node 5] ICS09 handshake done sender: [2:7577470915066731832:2113] self: [2:7577470915066731831:0] peer: [5:7577470914159808118:0] socket: 130 qp: -1
2025-11-27T18:17:10.731795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470916258572993:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22020) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470916258573082:2120] held: no
2025-11-27T18:17:10.743059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470914159808113:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:10.739593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:10.739818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470916258573082:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:10.740913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470916393163397:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.742581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470916757240062:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.729397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470915066731862:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:10.740010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470916258572993:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.740079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:10.743138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:10.743029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:10.742683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:10.731672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:10.743652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470914159808033:2049] [node 2] ICP22 created new session: [5:7577470914159808126:2049]
2025-11-27T18:17:10.740130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470916258572993:2050] [node 3] ICP22 created new session: [6:7577470916258573088:2050]
2025-11-27T18:17:10.743011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470916393163397:2108] [node 5] ICP22 created new session: [3:7577470916393163435:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.743096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470916757240062:2108] [node 5] ICP22 created new session: [4:7577470916757240098:2108]
2025-11-27T18:17:10.740172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470916258573088:2050] [node 3] ICS09 handshake done sender: [6:7577470916258573082:2120] self: [6:7577470916258573084:0] peer: [3:7577470916393163410:0] socket: 154 qp: -1
No RDMA context found for GID: 2025-11-27T18:17:10.744276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470914159808126:2049] [node 2] ICS09 handshake done sender: [5:7577470914159808114:2118] self: [5:7577470914159808118:0] peer: [2:7577470915066731831:0] socket: 176 qp: -1
::ffff:127.1.0.0
2025-11-27T18:17:10.743057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470916393163435:2108] [node 5] ICS09 handshake done sender: [3:7577470916393163406:2113] self: [3:7577470916393163405:0] peer: [5:7577470914159808122:0] socket: 149 qp: -1
2025-11-27T18:17:10.744462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470914159808126:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:10.740324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470916258573088:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:10.745005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470914159808032:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.743121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470916393163435:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:10.745594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:10.740936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:10.746728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470915066731823:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(16834449ns) {HandleHandshakeStatus:329(16832178ns) {DropIncomingHandshake:442(13622ns) DropOutgoingHandshake:457(148ns) SwitchToState:179(130ns) ProcessPendingSessionEvents:495(5169371ns) {ForwardSessionEventToSession:612(850334ns) ForwardSessionEventToSession:612(5487ns) ForwardSessionEventToSession:612(1200186ns) ForwardSessionEventToSession:612(5304ns) ForwardSessionEventToSession:612(1194250ns) ForwardSessionEventToSession:612(210226ns) ForwardSessionEventToSession:612(6135ns) ForwardSessionEventToSession:612(372874ns) ForwardSessionEventToSession:612(1288493ns) ForwardSessionEventToSession:612(3873ns)}}}
2025-11-27T18:17:10.746746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:10.743204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:10.747031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470914159808032:2048] [node 1] ICP22 created new session: [5:7577470914159808129:2048]
2025-11-27T18:17:10.748014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470916393163437:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1110
2025-11-27T18:17:10.747838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470914159808129:2048] [node 1] ICS09 handshake done sender: [5:7577470914159808113:2117] self: [5:7577470914159808116:0] peer: [1:7577470915508439917:0] socket: 175 qp: -1
2025-11-27T18:17:10.748090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470914159808129:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:10.743153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470916757240098:2108] [node 5] ICS09 handshake done sender: [4:7577470916757240076:2116] self: [4:7577470916757240075:0] peer: [5:7577470914159808120:0] socket: 117 qp: -1
2025-11-27T18:17:10.748434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:10.743195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470916757240098:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:10.743582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:10.745377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470916757240091:2122] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.746949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470916757239997:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.748111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:10.748152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470916757239997:2048] [node 1] ICP22 created new session: [4:7577470916757240100:2048]
2025-11-27T18:17:10.748545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470914159808123:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:17:10.748669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470916258573090:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22603
2025-11-27T18:17:10.748756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470916757240100:2048] [node 1] ICS09 handshake done sender: [4:7577470916757240091:2122] self: [4:7577470916757240094:0] peer: [1:7577470915508439914:0] socket: 166 qp: -1
2025-11-27T18:17:10.748801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470916757240100:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:10.748864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.749359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470914159808096:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62922) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470914159808115:2119] held: no
2025-11-27T18:17:10.749245Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470915508439906:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17740204ns) {HandleHandshakeStatus:329(17736618ns) {DropIncomingHandshake:442(23313ns) DropOutgoingHandshake:457(505ns) SwitchToState:179(500ns) ProcessPendingSessionEvents:495(2995976ns) {ForwardSessionEventToSession:612(1049236ns) ForwardSessionEventToSession:612(737458ns) ForwardSessionEventToSession:612(3384ns) ForwardSessionEventToSession:612(289326ns) ForwardSessionEventToSession:612(158942ns) ForwardSessionEventToSession:612(2526ns) ForwardSessionEventToSession:612(429260ns) ForwardSessionEventToSession:612(3084ns) ForwardSessionEventToSession:612(2494ns) ForwardSessionEventToSession:612(305394ns)}}}
2025-11-27T18:17:10.749873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470914159808124:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:17:10.750013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470914159808115:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:10.750355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470914159808096:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.750445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:10.750487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470914159808117:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:10.750500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470914159808096:2108] [node 4] ICP22 created new session: [5:7577470914159808131:2108]
2025-11-27T18:17:10.750543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470914159808131:2108] [node 4] ICS09 handshake done sender: [5:7577470914159808115:2119] self: [5:7577470914159808120:0] peer: [4:7577470916757240075:0] socket: 135 qp: -1
2025-11-27T18:17:10.750574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470914159808131:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:10.750604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:10.750831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470914159808034:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.750904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:10.750950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470914159808034:2050] [node 3] ICP22 created new session: [5:7577470914159808133:2050]
2025-11-27T18:17:10.751353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470914159808133:2050] [node 3] ICS09 handshake done sender: [5:7577470914159808117:2120] self: [5:7577470914159808122:0] peer: [3:7577470916393163405:0] socket: 150 qp: -1
2025-11-27T18:17:10.751660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470914159808133:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:10.751709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:10.752241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470916258573090:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.752432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470916258573054:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:10.752678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470914159808111:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:10.752709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470914159808111:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:10.752726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577470916393163397:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:10.753715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470914159808111:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:10.753904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470914159808097:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:10.753982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470916258573092:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22603
2025-11-27T18:17:10.754047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:10.754117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470914159808097:2109] [node 6] ICP22 created new session: [5:7577470914159808136:2109]
2025-11-27T18:17:10.754162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470914159808136:2109] [node 6] ICS09 handshake done sender: [5:7577470914159808111:2116] self: [5:7577470914159808110:0] peer: [6:7577470916258573091:0] socket: 146 qp: -1
2025-11-27T18:17:10.754204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470914159808136:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:10.754190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470916258573090:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:10.754310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:10.754616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470916258573054:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:10.754689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:10.754728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470916258573054:2105] [node 5] ICP22 created new session: [6:7577470916258573093:2105]
2025-11-27T18:17:10.754799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470916258573093:2105] [node 5] ICS09 handshake done sender: [6:7577470916258573090:2123] self: [6:7577470916258573091:0] peer: [5:7577470914159808110:0] socket: 147 qp: -1
2025-11-27T18:17:10.754826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470916258573093:2105] [node 5] ICS10 traffic start
2025-11-27T18:17:10.754861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:10.755303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577470916258573054:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:10.779230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:10.779413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470915066731856:2048] [node 1] ICS01 socket: 121 reason# EndOfStream
2025-11-27T18:17:10.779462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470915066731760:2048] [node 1] ICP30 unregister session Session# [2:7577470915066731856:2048] VirtualId# [2:7577470915066731842:0]
2025-11-27T18:17:10.779492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470915066731856:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.779557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:10.779614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:10.792309Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:10.792561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470916393163422:2048] [node 1] ICS07 socket disconnect 123 reason# EndOfStream
2025-11-27T18:17:10.792591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470916393163422:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.792680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:10.792743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470916393163422:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:10.795184Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:10.795321Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470916393163333:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21109) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:10.795380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470916393163422:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:10.795458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470916757240100:2048] [node 1] ICS01 socket: 166 reason# EndOfStream
2025-11-27T18:17:10.795497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470916757239997:2048] [node 1] ICP30 unregister session Session# [4:7577470916757240100:2048] VirtualId# [4:7577470916757240094:0]
2025-11-27T18:17:10.795788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470916757240100:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.795874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:10.795922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:10.796459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470916393163333:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21109) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.796510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470916393163422:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.796595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470916393163333:2048] [node 1] ICP30 unregister session Session# [3:7577470916393163422:2048] VirtualId# [3:7577470916393163415:0]
2025-11-27T18:17:10.796641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:10.796690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470916393163333:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21109) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.796705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21109) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.797063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:10.797296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470914159808129:2048] [node 1] ICS01 socket: 175 reason# EndOfStream
2025-11-27T18:17:10.797362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470914159808032:2048] [node 1] ICP30 unregister session Session# [5:7577470914159808129:2048] VirtualId# [5:7577470914159808116:0]
2025-11-27T18:17:10.797404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470914159808129:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.797464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:10.797502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:10.797714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:10.797935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470916258573078:2048] [node 1] ICS01 socket: 132 reason# EndOfStream
2025-11-27T18:17:10.797974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470916258572991:2048] [node 1] ICP30 unregister session Session# [6:7577470916258573078:2048] VirtualId# [6:7577470916258573072:0]
2025-11-27T18:17:10.797997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470916258573078:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.798053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:10.798097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:10.806856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470916393163333:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:10.845608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:10.845859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470916393163425:2049] [node 2] ICS07 socket disconnect 141 reason# EndOfStream
2025-11-27T18:17:10.845902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470916393163425:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.846004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:10.846064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470916393163425:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:10.846983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470916393163334:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3381) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.847049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470916393163425:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.847204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470916393163334:2049] [node 2] ICP30 unregister session Session# [3:7577470916393163425:2049] VirtualId# [3:7577470916393163417:0]
2025-11-27T18:17:10.847277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:10.847324Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470916393163334:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.847341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.848123Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:10.848327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470916757240089:2049] [node 2] ICS07 socket disconnect 139 reason# EndOfStream
2025-11-27T18:17:10.848362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470916757240089:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.848426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:10.848466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470916757240089:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:10.849108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470916757239998:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3381) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.849217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:10.849379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470916757240089:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.849416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470916757239998:2049] [node 2] ICP30 unregister session Session# [4:7577470916757240089:2049] VirtualId# [4:7577470916757240080:0]
2025-11-27T18:17:10.849456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:10.849664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470914159808126:2049] [node 2] ICS01 socket: 176 reason# EndOfStream
2025-11-27T18:17:10.849502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470916757239998:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.849712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470914159808033:2049] [node 2] ICP30 unregister session Session# [5:7577470914159808126:2049] VirtualId# [5:7577470914159808118:0]
2025-11-27T18:17:10.849517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.849742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470914159808126:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.849843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:10.849901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:10.850418Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:10.850673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470916258573086:2049] [node 2] ICS01 socket: 172 reason# EndOfStream
2025-11-27T18:17:10.850708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470916258572992:2049] [node 2] ICP30 unregister session Session# [6:7577470916258573086:2049] VirtualId# [6:7577470916258573081:0]
2025-11-27T18:17:10.850740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470916258573086:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.850808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:10.850848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:10.858536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470916393163334:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:10.859036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470916757239998:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:10.897853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:10.898108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470916757240093:2050] [node 3] ICS01 socket: 153 reason# EndOfStream
2025-11-27T18:17:10.898155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470916757239999:2050] [node 3] ICP30 unregister session Session# [4:7577470916757240093:2050] VirtualId# [4:7577470916757240086:0]
2025-11-27T18:17:10.898189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470916757240093:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.898259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:10.898351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:10.898856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:10.899038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470914159808133:2050] [node 3] ICS07 socket disconnect 150 reason# EndOfStream
2025-11-27T18:17:10.899071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470914159808133:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.899142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:10.899187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470914159808133:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:10.899746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:10.899885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470914159808034:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22020) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.899960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470916258573088:2050] [node 3] ICS01 socket: 154 reason# EndOfStream
2025-11-27T18:17:10.899938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470914159808133:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.899971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470914159808034:2050] [node 3] ICP30 unregister session Session# [5:7577470914159808133:2050] VirtualId# [5:7577470914159808122:0]
2025-11-27T18:17:10.900050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470916258572993:2050] [node 3] ICP30 unregister session Session# [6:7577470916258573088:2050] VirtualId# [6:7577470916258573084:0]
2025-11-27T18:17:10.900051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:10.900150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470914159808034:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22020) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.900197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470916258573088:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.900165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22020) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.900289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:10.900362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:10.910292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470914159808034:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:10.938916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:10.939211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470914159808131:2108] [node 4] ICS01 socket: 135 reason# EndOfStream
2025-11-27T18:17:10.939278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470914159808096:2108] [node 4] ICP30 unregister session Session# [5:7577470914159808131:2108] VirtualId# [5:7577470914159808120:0]
2025-11-27T18:17:10.939322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470914159808131:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.939401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:10.939466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:10.939916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:10.940139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470916258573075:2109] [node 4] ICS07 socket disconnect 124 reason# EndOfStream
2025-11-27T18:17:10.940176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470916258573075:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.940246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:10.940321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470916258573075:2109] [node 4] ICS15 start handshake
2025-11-27T18:17:10.941070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470916258573055:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62922) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.941133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470916258573075:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.941215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470916258573055:2109] [node 4] ICP30 unregister session Session# [6:7577470916258573075:2109] VirtualId# [6:7577470916258573071:0]
2025-11-27T18:17:10.941290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:10.941329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470916258573055:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62922) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.941344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62922) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.952700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470916258573055:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:17:10.998232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:10.998508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470916258573093:2105] [node 5] ICS07 socket disconnect 147 reason# EndOfStream
2025-11-27T18:17:10.998543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470916258573093:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:10.998626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:10.998689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470916258573093:2105] [node 5] ICS15 start handshake
2025-11-27T18:17:10.999442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470916258573054:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8307) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:10.999487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470916258573093:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:10.999527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470916258573054:2105] [node 5] ICP30 unregister session Session# [6:7577470916258573093:2105] VirtualId# [6:7577470916258573091:0]
2025-11-27T18:17:10.999569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:17:10.999602Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470916258573054:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8307) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:10.999612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8307) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:11.009627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470916258573054:2105] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:11.095489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470921784047842:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:11.095537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470921784047843:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:11.118223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470920131796630:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:11.118278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470920131796631:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:11.144311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470921149799140:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:11.144382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470921149799141:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:11.167975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470921075542628:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:11.168030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470921075542629:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:11.168902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470921075542627:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:11.201825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470921101395682:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:11.202962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470921101395684:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:11.203132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470921101395683:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:11.245866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470918501441533:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:11.245906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470918501441534:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:11.245930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470918501441535:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:11.662287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470918501441593:2097] [node 4] ICP01 ready to work
2025-11-27T18:17:11.694835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470921075542690:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:11.699202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470921101395746:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:11.694993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470921784047905:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:11.769297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470921075542691:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:11.714946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470921784047906:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:11.765761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470920131796705:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:11.754993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470918501441596:2102] [node 5] ICP01 ready to work
2025-11-27T18:17:11.777771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470921149799208:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:11.787230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470918501441593:2097] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(107529777ns) {Configure:104(107517418ns) {ProcessConfigured:135(67693ns) {StartInitialHandshake:157(65606ns) {DropHandshakes:470(2367ns) {DropIncomingHandshake:442(578ns) DropOutgoingHandshake:457(309ns)} GenerateSessionVirtualId:988(1182ns) PrepareNewSessionHandshake:477(1883ns) {SwitchToState:179(473ns)}}}}}
2025-11-27T18:17:11.798646Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470918501441534:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(30061464ns) {Configure:104(30052759ns) {ProcessConfigured:135(66183ns) {StartInitialHandshake:157(62898ns) {DropHandshakes:470(2245ns) {DropIncomingHandshake:442(515ns) DropOutgoingHandshake:457(385ns)} GenerateSessionVirtualId:988(1446ns) PrepareNewSessionHandshake:477(1401ns) {SwitchToState:179(484ns)}}}}}
2025-11-27T18:17:11.804992Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470918501441533:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(36253689ns) {Configure:104(36246062ns) {ProcessConfigured:135(88803ns) {StartInitialHandshake:157(85799ns) {DropHandshakes:470(1958ns) {DropIncomingHandshake:442(453ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(879ns) PrepareNewSessionHandshake:477(1569ns) {SwitchToState:179(551ns)}}}}}
2025-11-27T18:17:11.777551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470920131796704:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:11.801504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470921149799207:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:11.814738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470921101395683:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(127032861ns) {Configure:104(127028264ns) {ProcessConfigured:135(108391ns) {StartInitialHandshake:157(105312ns) {DropHandshakes:470(3158ns) {DropIncomingHandshake:442(761ns) DropOutgoingHandshake:457(455ns)} GenerateSessionVirtualId:988(1382ns) PrepareNewSessionHandshake:477(1805ns) {SwitchToState:179(700ns)}}}}}
2025-11-27T18:17:11.808873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470921784047907:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:11.811569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470921149799209:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:11.818338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470921101395682:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(148798998ns) {Configure:104(148794565ns) {ProcessConfigured:135(42806ns) {StartInitialHandshake:157(41165ns) {DropHandshakes:470(1976ns) {DropIncomingHandshake:442(467ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(682ns) PrepareNewSessionHandshake:477(1349ns) {SwitchToState:179(548ns)}}}}}
2025-11-27T18:17:11.824804Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470920131796630:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(148338725ns) {Configure:104(148325022ns) {ProcessConfigured:135(73381ns) {StartInitialHandshake:157(70535ns) {DropHandshakes:470(3067ns) {DropIncomingHandshake:442(582ns) DropOutgoingHandshake:457(744ns)} GenerateSessionVirtualId:988(1397ns) PrepareNewSessionHandshake:477(2062ns) {SwitchToState:179(1017ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.828369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470921149799207:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18081251ns) {Configure:104(18072560ns) {ProcessConfigured:135(67042ns) {StartInitialHandshake:157(63894ns) {DropHandshakes:470(3009ns) {DropIncomingHandshake:442(759ns) DropOutgoingHandshake:457(589ns)} GenerateSessionVirtualId:988(885ns) PrepareNewSessionHandshake:477(1739ns) {SwitchToState:179(657ns)}}}}}
2025-11-27T18:17:11.823880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470921101395747:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.834269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470920131796711:2120] [node 6] ICP01 ready to work
2025-11-27T18:17:11.839374Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470918501441596:2102] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(53476959ns) {Configure:104(53470484ns) {ProcessConfigured:135(91754ns) {StartInitialHandshake:157(89305ns) {DropHandshakes:470(3207ns) {DropIncomingHandshake:442(1051ns) DropOutgoingHandshake:457(361ns)} GenerateSessionVirtualId:988(801ns) PrepareNewSessionHandshake:477(2042ns) {SwitchToState:179(739ns)}}}}}
2025-11-27T18:17:11.841381Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470921149799141:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(165964469ns) {Configure:104(165957798ns) {ProcessConfigured:135(12186274ns) {StartInitialHandshake:157(12183228ns) {DropHandshakes:470(2547ns) {DropIncomingHandshake:442(489ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(909ns) PrepareNewSessionHandshake:477(3665ns) {SwitchToState:179(1212ns)}}}}}
2025-11-27T18:17:11.857101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470918501441609:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60249
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.846494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470921149799209:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22105483ns) {Configure:104(22096982ns) {ProcessConfigured:135(72849ns) {StartInitialHandshake:157(70532ns) {DropHandshakes:470(2736ns) {DropIncomingHandshake:442(807ns) DropOutgoingHandshake:457(512ns)} GenerateSessionVirtualId:988(1165ns) PrepareNewSessionHandshake:477(2121ns) {SwitchToState:179(847ns)}}}}}
2025-11-27T18:17:11.849991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470920131796712:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42525
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.874842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470921075542628:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(115700970ns) {Configure:104(115691280ns) {ProcessConfigured:135(98605ns) {StartInitialHandshake:157(94861ns) {DropHandshakes:470(3742ns) {DropIncomingHandshake:442(766ns) DropOutgoingHandshake:457(646ns)} GenerateSessionVirtualId:988(778ns) PrepareNewSessionHandshake:477(2965ns) {SwitchToState:179(957ns)}}}}}
2025-11-27T18:17:11.883075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470920131796713:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42525
2025-11-27T18:17:11.883191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470921149799140:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(200865673ns) {Configure:104(200860292ns) {ProcessConfigured:135(5552682ns) {StartInitialHandshake:157(5550001ns) {DropHandshakes:470(2475ns) {DropIncomingHandshake:442(614ns) DropOutgoingHandshake:457(300ns)} GenerateSessionVirtualId:988(480ns) PrepareNewSessionHandshake:477(3438ns) {SwitchToState:179(1539ns)}}}}}
2025-11-27T18:17:11.881421Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470921784047906:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(121339810ns) {Configure:104(121331854ns) {ProcessConfigured:135(75091ns) {StartInitialHandshake:157(72570ns) {DropHandshakes:470(3406ns) {DropIncomingHandshake:442(815ns) DropOutgoingHandshake:457(405ns)} GenerateSessionVirtualId:988(839ns) PrepareNewSessionHandshake:477(2206ns) {SwitchToState:179(781ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.899384Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470921101395684:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(206529577ns) {Configure:104(206524566ns) {ProcessConfigured:135(13098974ns) {StartInitialHandshake:157(13095991ns) {DropHandshakes:470(983ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(580ns) PrepareNewSessionHandshake:477(4111ns) {SwitchToState:179(1978ns)}}}}}
2025-11-27T18:17:11.892687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470921075542629:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(230950057ns) {Configure:104(230944480ns) {ProcessConfigured:135(7103352ns) {StartInitialHandshake:157(7099930ns) {DropHandshakes:470(2985ns) {DropIncomingHandshake:442(650ns) DropOutgoingHandshake:457(423ns)} GenerateSessionVirtualId:988(580ns) PrepareNewSessionHandshake:477(3311ns) {SwitchToState:179(1812ns)}}}}}
2025-11-27T18:17:11.882137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470921784047908:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54330
2025-11-27T18:17:11.891813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470920131796712:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.909610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470921075542699:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32065
2025-11-27T18:17:11.882291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470921784047909:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54330
2025-11-27T18:17:11.908018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470920131796631:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(243646154ns) {Configure:104(243640348ns) {ProcessConfigured:135(35645388ns) {StartInitialHandshake:157(29635689ns) {DropHandshakes:470(4019ns) {DropIncomingHandshake:442(627ns) DropOutgoingHandshake:457(379ns)} GenerateSessionVirtualId:988(2387ns) PrepareNewSessionHandshake:477(6707ns) {SwitchToState:179(2395ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.882460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470921784047913:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54330
2025-11-27T18:17:11.918796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470921101395746:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(162068498ns) {Configure:104(152991518ns) {ProcessConfigured:135(1048724ns) {StartInitialHandshake:157(1045957ns) {DropHandshakes:470(993195ns) {DropIncomingHandshake:442(633ns) DropOutgoingHandshake:457(876ns)} GenerateSessionVirtualId:988(1169ns) PrepareNewSessionHandshake:477(873ns) {SwitchToState:179(580ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.913309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470920131796713:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.889904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470921784047843:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(230421221ns) {Configure:104(230410903ns) {ProcessConfigured:135(6309515ns) {StartInitialHandshake:157(6307180ns) {DropHandshakes:470(2740ns) {DropIncomingHandshake:442(526ns) DropOutgoingHandshake:457(431ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(3184ns) {SwitchToState:179(1592ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.917388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470920131796704:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(109944544ns) {Configure:104(102650198ns) {ProcessConfigured:135(10954271ns) {StartInitialHandshake:157(10952400ns) {DropHandshakes:470(2719ns) {DropIncomingHandshake:442(686ns) DropOutgoingHandshake:457(377ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:477(3039ns) {SwitchToState:179(1273ns)}}}}}
2025-11-27T18:17:11.925359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470918501441609:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.920690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470921075542627:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(147356091ns) {Configure:104(147347565ns) {ProcessConfigured:135(8571481ns) {StartInitialHandshake:157(8567218ns) {DropHandshakes:470(2632ns) {DropIncomingHandshake:442(647ns) DropOutgoingHandshake:457(528ns)} GenerateSessionVirtualId:988(1284ns) PrepareNewSessionHandshake:477(3013ns) {SwitchToState:179(732ns)}}}}}
2025-11-27T18:17:11.921392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470921149799221:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28730
2025-11-27T18:17:11.920582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470920131796716:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42525
2025-11-27T18:17:11.921170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470921075542690:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(166724178ns) {Configure:104(166713679ns) {ProcessConfigured:135(3142592ns) {StartInitialHandshake:157(3140546ns) {DropHandshakes:470(643ns) {DropIncomingHandshake:442(238ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:477(2751ns) {SwitchToState:179(934ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.929828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470921149799222:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28730
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.928678Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470918501441535:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(140634408ns) {Configure:104(140628636ns) {ProcessConfigured:135(7842517ns) {StartInitialHandshake:157(7839131ns) {DropHandshakes:470(1628ns) {DropIncomingHandshake:442(430ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(1172ns) PrepareNewSessionHandshake:477(3268ns) {SwitchToState:179(2002ns)}}}}}
2025-11-27T18:17:11.921225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470921075542691:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(120164323ns) {Configure:104(120157487ns) {ProcessConfigured:135(1488375ns) {StartInitialHandshake:157(1486756ns) {DropHandshakes:470(1229ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(378ns) PrepareNewSessionHandshake:477(1419ns) {SwitchToState:179(189ns)}}}}}
2025-11-27T18:17:11.897365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470921784047909:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.921696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470921101395757:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51569
2025-11-27T18:17:11.929666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470920131796711:2120] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(66987319ns) {Configure:104(66976261ns) {ProcessConfigured:135(2793904ns) {StartInitialHandshake:157(2788092ns) {DropHandshakes:470(1235ns) {DropIncomingHandshake:442(263ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(444ns) PrepareNewSessionHandshake:477(2150ns) {SwitchToState:179(802ns)}}}}}
2025-11-27T18:17:11.932298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470921149799208:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(111671682ns) {Configure:104(109539352ns) {ProcessConfigured:135(1553728ns) {StartInitialHandshake:157(1550661ns) {DropHandshakes:470(1326ns) {DropIncomingHandshake:442(399ns) DropOutgoingHandshake:457(276ns)} GenerateSessionVirtualId:988(551ns) PrepareNewSessionHandshake:477(1846ns) {SwitchToState:179(725ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.898985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470921784047908:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.925919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470921101395747:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(95209738ns) {Configure:104(95202533ns) {ProcessConfigured:135(3560753ns) {StartInitialHandshake:157(3558003ns) {DropHandshakes:470(1786ns) {DropIncomingHandshake:442(503ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:477(2318ns) {SwitchToState:179(1307ns)}}}}}
2025-11-27T18:17:11.933642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470920131796716:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.932573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470918501441535:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:11.935455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470920131796720:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42525
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.945583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470918501441613:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60249
2025-11-27T18:17:11.945890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470918501441609:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:11.946048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470918501441535:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:11.925617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470921075542699:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.936484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470921101395757:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.933684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470921149799221:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.943654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470920131796721:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42525
2025-11-27T18:17:11.935623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470921075542702:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32065
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.948059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:11.948129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470918501441535:2050] [node 3] ICP22 created new session: [6:7577470918501441614:2050]
2025-11-27T18:17:11.948174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470918501441614:2050] [node 3] ICS09 handshake done sender: [6:7577470918501441609:2114] self: [6:7577470918501441612:0] peer: [3:7577470921149799216:0] socket: 135 qp: -1
2025-11-27T18:17:11.948210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470918501441614:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:11.948253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:11.913115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470921784047913:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.934365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470921149799217:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.943820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470920131796707:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.915149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470921784047921:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54330
2025-11-27T18:17:11.936644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470921075542628:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:11.941954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470921101395682:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:11.943854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470920131796707:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:11.934387Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470921149799217:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:11.936374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470921149799222:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.936793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470921149799223:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28730
2025-11-27T18:17:11.938189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470921149799217:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:11.943586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470921149799226:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28730
2025-11-27T18:17:11.947939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470921149799209:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:11.949347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470921149799223:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.949947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470921149799226:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.955120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470921149799140:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:11.962941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470921149799141:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:11.917730Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470921784047842:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(260089402ns) {Configure:104(254930974ns) {ProcessConfigured:135(38082923ns) {StartInitialHandshake:157(38080961ns) {DropHandshakes:470(3766ns) {DropIncomingHandshake:442(639ns) DropOutgoingHandshake:457(606ns)} GenerateSessionVirtualId:988(1005ns) PrepareNewSessionHandshake:477(2846ns) {SwitchToState:179(976ns)}}}}}
2025-11-27T18:17:11.947159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470921075542702:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.947874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470920131796720:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.920560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470921784047905:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(165275217ns) {Configure:104(165270840ns) {ProcessConfigured:135(3529851ns) {StartInitialHandshake:157(2261274ns) {DropHandshakes:470(2734ns) {DropIncomingHandshake:442(483ns) DropOutgoingHandshake:457(348ns)} GenerateSessionVirtualId:988(1088ns) PrepareNewSessionHandshake:477(2852ns) {SwitchToState:179(1495ns)}}}}}
2025-11-27T18:17:11.966684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:11.951271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470920131796707:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.970054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470921149799209:2108] [node 6] ICP22 created new session: [3:7577470921149799233:2108]
2025-11-27T18:17:11.971153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470921075542628:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470921075542699:2112] held: no
2025-11-27T18:17:11.955798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470920131796705:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:11.933224Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470921784047907:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(92656862ns) {Configure:104(92648358ns) {ProcessConfigured:135(7275697ns) {StartInitialHandshake:157(7272956ns) {DropHandshakes:470(2993119ns) {DropIncomingHandshake:442(1492ns) DropOutgoingHandshake:457(682ns)} GenerateSessionVirtualId:988(1178ns) PrepareNewSessionHandshake:477(2298ns) {SwitchToState:179(304ns)}}}}}
2025-11-27T18:17:11.955924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:11.971120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470921149799233:2108] [node 6] ICS09 handshake done sender: [3:7577470921149799217:2117] self: [3:7577470921149799216:0] peer: [6:7577470918501441612:0] socket: 134 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.949423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470921784047912:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.955987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470920131796705:2108] [node 4] ICP22 created new session: [2:7577470920131796730:2108]
2025-11-27T18:17:11.972517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470921149799233:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:11.974749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470921075542709:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32065
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.983067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470918501441617:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60249
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.949458Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470921784047912:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:11.976079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470921075542699:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:11.962515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470920131796721:2129] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.973740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:11.984767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470921101395763:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51569
2025-11-27T18:17:11.951624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470921784047928:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54330
2025-11-27T18:17:11.985017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470921075542628:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:11.962542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470920131796730:2108] [node 4] ICS09 handshake done sender: [2:7577470920131796707:2121] self: [2:7577470920131796706:0] peer: [4:7577470921075542706:0] socket: 120 qp: -1
2025-11-27T18:17:11.977423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470921149799140:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15060) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470921149799223:2120] held: no
2025-11-27T18:17:11.989155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.993883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470918501441617:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.984793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470921101395764:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51569
2025-11-27T18:17:11.962586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470920131796730:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:11.991545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470921149799141:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470921149799226:2122] held: no
2025-11-27T18:17:11.951742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470921784047921:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.967676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:11.986688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470921101395682:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15060) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470921101395757:2114] held: no
2025-11-27T18:17:11.994316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470918501441533:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.961551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470921784047916:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.986729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470921101395766:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51569
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.991584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470921075542628:2049] [node 2] ICP22 created new session: [4:7577470921075542711:2049]
2025-11-27T18:17:11.993702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470921149799209:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(31091225ns) {HandleHandshakeStatus:329(31088865ns) {DropIncomingHandshake:442(168ns) DropOutgoingHandshake:457(283ns) SwitchToState:179(315ns) ProcessPendingSessionEvents:495(4572821ns) {ForwardSessionEventToSession:612(591857ns) ForwardSessionEventToSession:612(5594ns) ForwardSessionEventToSession:612(201375ns) ForwardSessionEventToSession:612(9496ns) ForwardSessionEventToSession:612(3289ns) ForwardSessionEventToSession:612(506084ns) ForwardSessionEventToSession:612(409159ns) ForwardSessionEventToSession:612(2386ns) ForwardSessionEventToSession:612(2826068ns) ForwardSessionEventToSession:612(2381ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.970293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470920131796630:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:11.961595Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470921784047916:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:11.993661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470921075542711:2049] [node 2] ICS09 handshake done sender: [4:7577470921075542699:2112] self: [4:7577470921075542706:0] peer: [2:7577470920131796706:0] socket: 138 qp: -1
2025-11-27T18:17:11.987719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470921101395764:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.996132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470921149799235:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28730
2025-11-27T18:17:11.972472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470920131796715:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.965468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470921784047916:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:12.009473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470918501441618:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60249
2025-11-27T18:17:11.994193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470921075542711:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:11.974105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470921784047928:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.974201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470921784047912:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:11.996903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470921101395765:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51569
2025-11-27T18:17:11.972506Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470920131796715:2125] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.999021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470921149799236:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28730
2025-11-27T18:17:11.994246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:11.999247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470921101395757:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:11.974535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470921784047918:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.014491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470918501441618:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.000174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470921149799226:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:11.974417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470920131796715:2125] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:11.974555Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470921784047918:2115] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:12.001383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470921075542712:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32065
2025-11-27T18:17:12.000353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470921149799141:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:11.977095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470920131796631:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:12.021458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470918501441534:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:12.005050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470921101395763:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.017677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470921075542712:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.027890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470918501441619:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60249
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.974780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470921784047843:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:12.009607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470921101395682:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:11.977159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:12.000394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:11.978149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470921784047906:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:12.023052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470921075542627:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:11.977208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470920131796631:2049] [node 3] ICP22 created new session: [2:7577470920131796735:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:12.000432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470921149799141:2049] [node 2] ICP22 created new session: [3:7577470921149799238:2049]
2025-11-27T18:17:12.009662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:12.030544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470918501441533:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15060) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470918501441617:2117] held: no
2025-11-27T18:17:12.036873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470921075542714:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32065
2025-11-27T18:17:11.977269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470920131796735:2049] [node 3] ICS09 handshake done sender: [2:7577470920131796715:2125] self: [2:7577470920131796714:0] peer: [3:7577470921149799232:0] socket: 158 qp: -1
2025-11-27T18:17:11.978193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:12.013715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470921101395765:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.034943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470922796408919:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60249
2025-11-27T18:17:12.000484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470921149799238:2049] [node 2] ICS09 handshake done sender: [3:7577470921149799226:2122] self: [3:7577470921149799232:0] peer: [2:7577470920131796714:0] socket: 159 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.978236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470921784047906:2108] [node 5] ICP22 created new session: [1:7577470921784047934:2108]
2025-11-27T18:17:12.057406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470921075542714:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.000515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470921149799238:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:11.977320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470920131796735:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:12.058153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470922796408920:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60249
2025-11-27T18:17:12.057629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470921075542629:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:11.978281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470921784047934:2108] [node 5] ICS09 handshake done sender: [1:7577470921784047912:2112] self: [1:7577470921784047911:0] peer: [5:7577470921101395762:0] socket: 140 qp: -1
2025-11-27T18:17:12.014643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470921101395682:2048] [node 1] ICP22 created new session: [5:7577470925396363064:2048]
No RDMA context found for GID: 2025-11-27T18:17:11.977362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:12.000541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:12.058158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470918501441619:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:17:12.021437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470921101395683:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:12.085540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470921075542704:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.978306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470921784047934:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:12.065223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470918501441593:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:12.021534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470921101395684:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:12.004536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470921149799223:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:12.085576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470921075542704:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:11.985027Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470920131796705:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(18822420ns) {HandleHandshakeStatus:329(18819617ns) {DropIncomingHandshake:442(35834ns) DropOutgoingHandshake:457(767ns) SwitchToState:179(833ns) ProcessPendingSessionEvents:495(1774957ns) {ForwardSessionEventToSession:612(1451388ns) ForwardSessionEventToSession:612(4742ns) ForwardSessionEventToSession:612(277397ns) ForwardSessionEventToSession:612(3553ns) ForwardSessionEventToSession:612(2436ns) ForwardSessionEventToSession:612(7575ns) ForwardSessionEventToSession:612(3071ns) ForwardSessionEventToSession:612(3302ns) ForwardSessionEventToSession:612(8722ns) ForwardSessionEventToSession:612(2198ns)}}}
2025-11-27T18:17:12.069591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470922796408918:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60249
2025-11-27T18:17:11.978329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:12.017419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470921149799140:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:11.986669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470920131796737:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42525
2025-11-27T18:17:12.022786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470925396363064:2048] [node 1] ICS09 handshake done sender: [5:7577470921101395757:2114] self: [5:7577470921101395762:0] peer: [1:7577470921784047911:0] socket: 142 qp: -1
2025-11-27T18:17:12.085675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470921075542627:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15060) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470921075542712:2118] held: no
2025-11-27T18:17:12.081176Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470918501441534:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470918501441618:2118] held: no
2025-11-27T18:17:11.983693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:12.028350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:12.027988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470925396363064:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:12.081416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470918501441618:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:11.991814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470920131796721:2129] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:12.117962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470925370510012:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32065
2025-11-27T18:17:12.028575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:12.030058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470921149799213:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.038914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470921101395682:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(29348661ns) {HandleHandshakeStatus:329(29346036ns) {DropIncomingHandshake:442(3009ns) DropOutgoingHandshake:457(423ns) SwitchToState:179(859ns) ProcessPendingSessionEvents:495(924ns)}}
2025-11-27T18:17:12.081689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470918501441617:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:11.984846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470921784047843:2049] [node 3] ICP22 created new session: [1:7577470921784047933:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:12.067995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470921101395684:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14960) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470921101395763:2117] held: no
2025-11-27T18:17:12.125370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470921075542704:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:12.093150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470925396363070:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51569
No RDMA context found for GID: 2025-11-27T18:17:12.117726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470925396363069:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51569
2025-11-27T18:17:12.142615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470921075542690:2108] [node 6] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:17:12.122031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470921101395765:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:12.147053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470921075542712:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:11.996115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470920131796630:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:12.138421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470921101395683:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470921101395765:2119] held: no
2025-11-27T18:17:12.147085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:11.998151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:12.138513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470921101395683:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:12.147498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470921075542629:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14960) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470921075542714:2119] held: no
2025-11-27T18:17:11.998194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470920131796630:2048] [node 1] ICP22 created new session: [2:7577470920131796739:2048]
2025-11-27T18:17:12.148187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470925370510015:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32065
2025-11-27T18:17:12.139028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470921101395763:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:12.030106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470921149799140:2048] [node 1] ICP22 created new session: [3:7577470925444766536:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:11.984972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470921784047918:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:12.082186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470918501441534:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:12.030118Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470921149799213:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:12.082236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:12.034640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470925444766536:2048] [node 1] ICS09 handshake done sender: [3:7577470921149799223:2120] self: [3:7577470921149799230:0] peer: [1:7577470921784047915:0] socket: 156 qp: -1
2025-11-27T18:17:11.999694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470920131796739:2048] [node 1] ICS09 handshake done sender: [2:7577470920131796721:2129] self: [2:7577470920131796733:0] peer: [1:7577470921784047910:0] socket: 157 qp: -1
2025-11-27T18:17:12.082270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470918501441534:2049] [node 2] ICP22 created new session: [6:7577470922796408922:2049]
2025-11-27T18:17:12.148478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470921075542690:2108] [node 6] ICP22 created new session: [4:7577470925370510018:2108]
2025-11-27T18:17:11.986950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470921784047933:2049] [node 3] ICS09 handshake done sender: [1:7577470921784047916:2114] self: [1:7577470921784047915:0] peer: [3:7577470921149799230:0] socket: 150 qp: -1
2025-11-27T18:17:12.010772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470920131796739:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:12.142221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470921101395684:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:12.037151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470921149799213:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:12.082300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470922796408922:2049] [node 2] ICS09 handshake done sender: [6:7577470918501441618:2118] self: [6:7577470922796408917:0] peer: [2:7577470920131796723:0] socket: 179 qp: -1
2025-11-27T18:17:12.150546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470925370510018:2108] [node 6] ICS09 handshake done sender: [4:7577470921075542704:2115] self: [4:7577470921075542701:0] peer: [6:7577470922796408921:0] socket: 178 qp: -1
2025-11-27T18:17:12.142272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:11.986996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470921784047933:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:11.987629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:11.989188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470921784047842:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:11.991537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:11.991812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470921784047842:2048] [node 2] ICP22 created new session: [1:7577470921784047940:2048]
2025-11-27T18:17:11.993668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470921784047940:2048] [node 2] ICS09 handshake done sender: [1:7577470921784047918:2115] self: [1:7577470921784047910:0] peer: [2:7577470920131796733:0] socket: 155 qp: -1
2025-11-27T18:17:11.994205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470921784047940:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:11.997130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470921784047926:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:11.997154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470921784047926:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:11.997250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:12.013963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470921784047926:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:12.021688Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470921784047843:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(26719319ns) {HandleHandshakeStatus:329(26716954ns) {DropIncomingHandshake:442(108968ns) DropOutgoingHandshake:457(763ns) SwitchToState:179(424ns) ProcessPendingSessionEvents:495(7306392ns) {ForwardSessionEventToSession:612(2832979ns) ForwardSessionEventToSession:612(4342ns) ForwardSessionEventToSession:612(231011ns) ForwardSessionEventToSession:612(1828350ns) ForwardSessionEventToSession:612(3603ns) ForwardSessionEventToSession:612(3704ns) ForwardSessionEventToSession:612(537000ns) ForwardSessionEventToSession:612(2557ns) ForwardSessionEventToSession:612(1845350ns) ForwardSessionEventToSession:612(2730ns)}}}
2025-11-27T18:17:12.021829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470921784047907:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:12.021896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:12.021945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470921784047907:2109] [node 6] ICP22 created new session: [1:7577470926079015239:2109]
2025-11-27T18:17:12.022003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470926079015239:2109] [node 6] ICS09 handshake done sender: [1:7577470921784047926:2118] self: [1:7577470921784047925:0] peer: [6:7577470918501441620:0] socket: 141 qp: -1
2025-11-27T18:17:12.022047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470926079015239:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:12.022080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:12.037132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470921784047923:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.037159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470921784047923:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:12.039710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470921784047923:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:12.049341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470921784047842:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(42752877ns) {HandleHandshakeStatus:329(42750149ns) {DropIncomingHandshake:442(18036ns) DropOutgoingHandshake:457(4894ns) SwitchToState:179(338ns) ProcessPendingSessionEvents:495(20892878ns) {ForwardSessionEventToSession:612(4595119ns) ForwardSessionEventToSession:612(5716ns) ForwardSessionEventToSession:612(401734ns) ForwardSessionEventToSession:612(3531ns) ForwardSessionEventToSession:612(2170ns) ForwardSessionEventToSession:612(6757168ns) ForwardSessionEventToSession:612(7202368ns) ForwardSessionEventToSession:612(1892940ns) ForwardSessionEventToSession:612(4409ns) ForwardSessionEventToSession:612(3458ns)}}}
2025-11-27T18:17:12.061922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470921784047905:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:12.068026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:12.070913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470921784047905:2107] [node 4] ICP22 created new session: [1:7577470926079015242:2107]
2025-11-27T18:17:12.085579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470926079015242:2107] [node 4] ICS09 handshake done sender: [1:7577470921784047923:2117] self: [1:7577470921784047922:0] peer: [4:7577470925370510011:0] socket: 180 qp: -1
2025-11-27T18:17:12.090165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470926079015242:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:12.092966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:12.148218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470921784047905:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(81426623ns) {HandleHandshakeStatus:329(81423544ns) {DropIncomingHandshake:442(24247ns) DropOutgoingHandshake:457(412ns) SwitchToState:179(750ns) ProcessPendingSessionEvents:495(31788267ns) {ForwardSessionEventToSession:612(13532459ns) ForwardSessionEventToSession:612(19102ns) ForwardSessionEventToSession:612(1693483ns) ForwardSessionEventToSession:612(5258494ns) ForwardSessionEventToSession:612(6121ns) ForwardSessionEventToSession:612(9025263ns) ForwardSessionEventToSession:612(893744ns) ForwardSessionEventToSession:612(613305ns) ForwardSessionEventToSession:612(706453ns) ForwardSessionEventToSession:612(8899ns)}}}
2025-11-27T18:17:12.013746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:12.038613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470925444766536:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:12.082326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470922796408922:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:12.033492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470920131796724:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.033528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470920131796724:2130] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:12.034178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470920131796719:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.034210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470920131796719:2127] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:12.036596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470920131796724:2130] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:12.048094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470920131796711:2120] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:12.058124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:12.062073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470920131796711:2120] [node 6] ICP22 created new session: [2:7577470924426764038:2120]
2025-11-27T18:17:12.065377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470924426764038:2120] [node 6] ICS09 handshake done sender: [2:7577470920131796724:2130] self: [2:7577470920131796723:0] peer: [6:7577470922796408917:0] socket: 129 qp: -1
2025-11-27T18:17:12.068151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470920131796719:2127] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:12.069947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470924426764038:2120] [node 6] ICS10 traffic start
2025-11-27T18:17:12.071186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:12.071781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470920131796630:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(45043865ns) {HandleHandshakeStatus:329(42885179ns) {DropIncomingHandshake:442(663ns) DropOutgoingHandshake:457(16154ns) SwitchToState:179(557ns) ProcessPendingSessionEvents:495(10488619ns) {ForwardSessionEventToSession:612(2714401ns) ForwardSessionEventToSession:612(4153ns) ForwardSessionEventToSession:612(1818803ns) ForwardSessionEventToSession:612(2377331ns) ForwardSessionEventToSession:612(1486366ns) ForwardSessionEventToSession:612(5470ns) ForwardSessionEventToSession:612(2882ns) ForwardSessionEventToSession:612(406104ns) ForwardSessionEventToSession:612(2923ns) ForwardSessionEventToSession:612(2501ns)}}}
2025-11-27T18:17:12.090049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470920131796704:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:12.090103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:12.093500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470920131796704:2107] [node 5] ICP22 created new session: [2:7577470924426764041:2107]
2025-11-27T18:17:12.093552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470924426764041:2107] [node 5] ICS09 handshake done sender: [2:7577470920131796719:2127] self: [2:7577470920131796718:0] peer: [5:7577470925396363067:0] socket: 170 qp: -1
2025-11-27T18:17:12.093588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470924426764041:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:12.094321Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470920131796711:2120] [node 6] ICP35 event processing took too much time PendingConnection:279(23368551ns) {HandleHandshakeStatus:329(23364793ns) {DropIncomingHandshake:442(15920ns) DropOutgoingHandshake:457(3831ns) SwitchToState:179(1035ns) ProcessPendingSessionEvents:495(124200ns) {ForwardSessionEventToSession:612(50716ns) ForwardSessionEventToSession:612(2502ns) ForwardSessionEventToSession:612(5071ns) ForwardSessionEventToSession:612(2168ns) ForwardSessionEventToSession:612(2110ns) ForwardSessionEventToSession:612(20582ns) ForwardSessionEventToSession:612(2923ns) ForwardSessionEventToSession:612(22397ns) ForwardSessionEventToSession:612(2289ns) ForwardSessionEventToSession:612(2394ns)}}}
2025-11-27T18:17:12.097379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:12.150394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470920131796704:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(37945595ns) {HandleHandshakeStatus:329(37942576ns) {DropIncomingHandshake:442(13767ns) DropOutgoingHandshake:457(370ns) SwitchToState:179(861ns) ProcessPendingSessionEvents:495(26785468ns) {ForwardSessionEventToSession:612(7307220ns) ForwardSessionEventToSession:612(3165514ns) ForwardSessionEventToSession:612(1878ns) ForwardSessionEventToSession:612(6257730ns) ForwardSessionEventToSession:612(4660ns) ForwardSessionEventToSession:612(2580ns) ForwardSessionEventToSession:612(4126907ns) ForwardSessionEventToSession:612(4763347ns) ForwardSessionEventToSession:612(15780ns) ForwardSessionEventToSession:612(1113720ns)}}}
2025-11-27T18:17:12.151556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470925370510014:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32065
2025-11-27T18:17:12.142625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:12.038664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:12.049785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470921149799207:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:12.058208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:12.058251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470921149799207:2109] [node 5] ICP22 created new session: [3:7577470925444766539:2109]
2025-11-27T18:17:12.070461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470921149799225:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.070502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470921149799225:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:12.071052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470925444766539:2109] [node 5] ICS09 handshake done sender: [3:7577470921149799213:2115] self: [3:7577470921149799212:0] peer: [5:7577470925396363066:0] socket: 165 qp: -1
2025-11-27T18:17:12.071868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470925444766539:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:12.085899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:12.089792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470921149799140:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(54310922ns) {HandleHandshakeStatus:329(54307117ns) {DropIncomingHandshake:442(607ns) DropOutgoingHandshake:457(1293ns) SwitchToState:179(1957ns) ProcessPendingSessionEvents:495(812ns)}}
2025-11-27T18:17:12.093807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470921149799225:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:12.105546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470921149799208:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:12.105621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:12.117871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470921149799208:2107] [node 4] ICP22 created new session: [3:7577470925444766542:2107]
2025-11-27T18:17:12.125466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470925444766542:2107] [node 4] ICS09 handshake done sender: [3:7577470921149799225:2121] self: [3:7577470921149799224:0] peer: [4:7577470925370510013:0] socket: 137 qp: -1
2025-11-27T18:17:12.125516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470925444766542:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:12.125569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:12.125998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470921149799208:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(24208862ns) {HandleHandshakeStatus:329(20303043ns) {DropIncomingHandshake:442(27266ns) DropOutgoingHandshake:457(461ns) SwitchToState:179(597ns) ProcessPendingSessionEvents:495(98263ns) {ForwardSessionEventToSession:612(54127ns) ForwardSessionEventToSession:612(3690ns) ForwardSessionEventToSession:612(2480ns) ForwardSessionEventToSession:612(8345ns) ForwardSessionEventToSession:612(1677ns) ForwardSessionEventToSession:612(1676ns) ForwardSessionEventToSession:612(4740ns) ForwardSessionEventToSession:612(4771ns) ForwardSessionEventToSession:612(4788ns) ForwardSessionEventToSession:612(1744ns)}}}
2025-11-27T18:17:12.147738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470921149799207:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(78822408ns) {HandleHandshakeStatus:329(78818305ns) {DropIncomingHandshake:442(22418ns) DropOutgoingHandshake:457(617ns) SwitchToState:179(594ns) ProcessPendingSessionEvents:495(32671079ns) {ForwardSessionEventToSession:612(7526180ns) ForwardSessionEventToSession:612(4988ns) ForwardSessionEventToSession:612(3819687ns) ForwardSessionEventToSession:612(5859196ns) ForwardSessionEventToSession:612(10072ns) ForwardSessionEventToSession:612(6544082ns) ForwardSessionEventToSession:612(3722ns) ForwardSessionEventToSession:612(4210301ns) ForwardSessionEventToSession:612(4659698ns) ForwardSessionEventToSession:612(4761ns)}}}
2025-11-27T18:17:12.143498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470921101395683:2049] [node 2] ICP22 created new session: [5:7577470925396363074:2049]
2025-11-27T18:17:12.147701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470921101395684:2050] [node 3] ICP22 created new session: [5:7577470925396363073:2050]
2025-11-27T18:17:12.148589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470925396363073:2050] [node 3] ICS09 handshake done sender: [5:7577470921101395763:2117] self: [5:7577470925396363066:0] peer: [3:7577470921149799212:0] socket: 166 qp: -1
2025-11-27T18:17:12.150613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470925396363073:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:12.150654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:12.151703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470925396363072:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51569
2025-11-27T18:17:12.153505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470925396363074:2049] [node 2] ICS09 handshake done sender: [5:7577470921101395765:2119] self: [5:7577470925396363067:0] peer: [2:7577470920131796718:0] socket: 173 qp: -1
2025-11-27T18:17:12.153725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470925396363074:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:12.155785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:12.152782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470921075542627:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:12.183488Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470921101395683:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(37281299ns) {HandleHandshakeStatus:329(37279862ns) {DropIncomingHandshake:442(786ns) DropOutgoingHandshake:457(99ns) SwitchToState:179(375ns) ProcessPendingSessionEvents:495(1001ns)}}
2025-11-27T18:17:12.082359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:12.152822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:12.082670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470918501441533:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:12.153133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470925370510018:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:12.082733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:12.153431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470921075542627:2048] [node 1] ICP22 created new session: [4:7577470925370510019:2048]
2025-11-27T18:17:12.082761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470918501441533:2048] [node 1] ICP22 created new session: [6:7577470922796408924:2048]
2025-11-27T18:17:12.153549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:12.082781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470922796408924:2048] [node 1] ICS09 handshake done sender: [6:7577470918501441617:2117] self: [6:7577470918501441620:0] peer: [1:7577470921784047925:0] socket: 144 qp: -1
2025-11-27T18:17:12.155802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470925370510019:2048] [node 1] ICS09 handshake done sender: [4:7577470921075542712:2118] self: [4:7577470925370510011:0] peer: [1:7577470921784047922:0] socket: 133 qp: -1
2025-11-27T18:17:12.082802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470922796408924:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:12.155839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470925370510019:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:12.082819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:12.153398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470922796408926:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60249
2025-11-27T18:17:12.156191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470925370510015:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.156100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470918501441593:2097] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16765) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470918501441619:2119] held: no
2025-11-27T18:17:12.156634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:12.171168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470918501441619:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:12.161681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470921075542714:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:12.183903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470918501441593:2097] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:12.176125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470921075542629:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:12.183948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:12.185758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470921075542627:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23087905ns) {HandleHandshakeStatus:329(23084811ns) {DropIncomingHandshake:442(1571ns) DropOutgoingHandshake:457(216ns) SwitchToState:179(322ns) ProcessPendingSessionEvents:495(725ns)}}
2025-11-27T18:17:12.185874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470918501441593:2097] [node 4] ICP22 created new session: [6:7577470922796408927:2097]
2025-11-27T18:17:12.185920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470922796408927:2097] [node 4] ICS09 handshake done sender: [6:7577470918501441619:2119] self: [6:7577470922796408921:0] peer: [4:7577470921075542701:0] socket: 181 qp: -1
2025-11-27T18:17:12.193661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:12.185951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470922796408927:2097] [node 4] ICS10 traffic start
2025-11-27T18:17:12.185999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:12.197010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470922796408919:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.197196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470918501441596:2102] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:12.195908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470921075542690:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(51157957ns) {HandleHandshakeStatus:329(51155359ns) {DropIncomingHandshake:442(15127ns) DropOutgoingHandshake:457(318ns) SwitchToState:179(595ns) ProcessPendingSessionEvents:495(37539850ns) {ForwardSessionEventToSession:612(13145551ns) ForwardSessionEventToSession:612(5609694ns) ForwardSessionEventToSession:612(6354ns) ForwardSessionEventToSession:612(4415792ns) ForwardSessionEventToSession:612(4272631ns) ForwardSessionEventToSession:612(4450ns) ForwardSessionEventToSession:612(3031ns) ForwardSessionEventToSession:612(2040688ns) ForwardSessionEventToSession:612(4260778ns) ForwardSessionEventToSession:612(3756972ns)}}}
2025-11-27T18:17:12.197174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470921075542629:2050] [node 3] ICP22 created new session: [4:7577470925370510023:2050]
2025-11-27T18:17:12.197220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470925370510023:2050] [node 3] ICS09 handshake done sender: [4:7577470921075542714:2119] self: [4:7577470925370510013:0] peer: [3:7577470921149799224:0] socket: 153 qp: -1
2025-11-27T18:17:12.197247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470925370510023:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:12.197333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:12.197510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470921075542629:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(22208842ns) {HandleHandshakeStatus:329(21333323ns) {DropIncomingHandshake:442(998ns) DropOutgoingHandshake:457(535ns) SwitchToState:179(363ns) ProcessPendingSessionEvents:495(918ns)}}
2025-11-27T18:17:12.197699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470921101395759:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.197743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470921101395759:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:12.202486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470922796408930:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60249
2025-11-27T18:17:12.208009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470921101395759:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:12.208215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470925396363072:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.208310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470921101395746:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:12.208947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:12.209115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470921101395747:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:12.213473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470921101395746:2109] [node 6] ICP22 created new session: [5:7577470925396363078:2109]
2025-11-27T18:17:12.214735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470925396363078:2109] [node 6] ICS09 handshake done sender: [5:7577470921101395759:2115] self: [5:7577470921101395758:0] peer: [6:7577470922796408929:0] socket: 182 qp: -1
2025-11-27T18:17:12.215550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470922796408919:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:12.215758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470925396363078:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:12.218452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:12.218960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470921075542705:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:12.215646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470918501441596:2102] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29129) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470922796408919:2121] held: no
2025-11-27T18:17:12.219009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470921075542705:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:12.223421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470921075542705:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:12.220247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470918501441596:2102] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:12.224266Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470921101395747:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16765) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470925396363072:2123] held: no
2025-11-27T18:17:12.223656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470921075542691:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:12.223747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:12.223794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470921075542691:2109] [node 5] ICP22 created new session: [4:7577470925370510026:2109]
2025-11-27T18:17:12.223833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470925370510026:2109] [node 5] ICS09 handshake done sender: [4:7577470921075542705:2116] self: [4:7577470921075542703:0] peer: [5:7577470925396363079:0] socket: 161 qp: -1
2025-11-27T18:17:12.223872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470925370510026:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:12.223909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:12.221795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:12.224602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470925396363081:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51569
2025-11-27T18:17:12.228675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470925396363072:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:12.229287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470921101395747:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:12.230174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:12.230250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470921101395747:2108] [node 4] ICP22 created new session: [5:7577470925396363082:2108]
2025-11-27T18:17:12.230297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470925396363082:2108] [node 4] ICS09 handshake done sender: [5:7577470925396363072:2123] self: [5:7577470925396363079:0] peer: [4:7577470921075542703:0] socket: 131 qp: -1
2025-11-27T18:17:12.230329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470925396363082:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:12.230376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:12.229010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470918501441596:2102] [node 5] ICP22 created new session: [6:7577470922796408931:2102]
2025-11-27T18:17:12.230932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470922796408931:2102] [node 5] ICS09 handshake done sender: [6:7577470922796408919:2121] self: [6:7577470922796408929:0] peer: [5:7577470921101395758:0] socket: 183 qp: -1
2025-11-27T18:17:12.232185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470922796408931:2102] [node 5] ICS10 traffic start
2025-11-27T18:17:12.232229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:12.305479Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:12.301209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:12.301463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470920131796739:2048] [node 1] ICS07 socket disconnect 157 reason# EndOfStream
2025-11-27T18:17:12.301492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470920131796739:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.301574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:12.301610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470920131796739:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:12.312774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:12.313042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470925370510019:2048] [node 1] ICS01 socket: 133 reason# EndOfStream
2025-11-27T18:17:12.313081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470921075542627:2048] [node 1] ICP30 unregister session Session# [4:7577470925370510019:2048] VirtualId# [4:7577470925370510011:0]
2025-11-27T18:17:12.313112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470925370510019:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.313205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:12.313247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:12.309571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470925444766536:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-27T18:17:12.309614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470921149799140:2048] [node 1] ICP30 unregister session Session# [3:7577470925444766536:2048] VirtualId# [3:7577470921149799230:0]
2025-11-27T18:17:12.309648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470925444766536:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.309736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:12.309796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:12.314092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470920131796630:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15060) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:12.314135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470920131796739:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:12.316002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470920131796630:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15060) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:12.316069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470920131796739:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:12.316107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470920131796630:2048] [node 1] ICP30 unregister session Session# [2:7577470920131796739:2048] VirtualId# [2:7577470920131796733:0]
2025-11-27T18:17:12.316160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:12.316190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470920131796630:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15060) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:12.316207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15060) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:12.317032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:12.317494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470922796408924:2048] [node 1] ICS01 socket: 144 reason# EndOfStream
2025-11-27T18:17:12.317537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470918501441533:2048] [node 1] ICP30 unregister session Session# [6:7577470922796408924:2048] VirtualId# [6:7577470918501441620:0]
2025-11-27T18:17:12.317563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470922796408924:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.317627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:12.317665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:12.318525Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:12.318724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470925396363064:2048] [node 1] ICS01 socket: 142 reason# EndOfStream
2025-11-27T18:17:12.318760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470921101395682:2048] [node 1] ICP30 unregister session Session# [5:7577470925396363064:2048] VirtualId# [5:7577470921101395762:0]
2025-11-27T18:17:12.318790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470925396363064:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.318865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:12.318906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:12.367257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:12.367500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470921149799238:2049] [node 2] ICS01 socket: 159 reason# EndOfStream
2025-11-27T18:17:12.367552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470921149799141:2049] [node 2] ICP30 unregister session Session# [3:7577470921149799238:2049] VirtualId# [3:7577470921149799232:0]
2025-11-27T18:17:12.367582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470921149799238:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.367674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:12.367733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:12.370753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:12.370990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470922796408922:2049] [node 2] ICS01 socket: 179 reason# EndOfStream
2025-11-27T18:17:12.371051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470918501441534:2049] [node 2] ICP30 unregister session Session# [6:7577470922796408922:2049] VirtualId# [6:7577470922796408917:0]
2025-11-27T18:17:12.371097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470922796408922:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.371172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:12.371230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:12.373501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:12.373734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470925396363074:2049] [node 2] ICS01 socket: 173 reason# EndOfStream
2025-11-27T18:17:12.373774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470921101395683:2049] [node 2] ICP30 unregister session Session# [5:7577470925396363074:2049] VirtualId# [5:7577470925396363067:0]
2025-11-27T18:17:12.373808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470925396363074:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.373379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:12.373886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:12.373937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:12.374159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470921075542711:2049] [node 2] ICS01 socket: 138 reason# EndOfStream
2025-11-27T18:17:12.374211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470921075542628:2049] [node 2] ICP30 unregister session Session# [4:7577470921075542711:2049] VirtualId# [4:7577470921075542706:0]
2025-11-27T18:17:12.374235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470921075542711:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.374293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:12.374341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:12.434111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:12.434361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470925370510023:2050] [node 3] ICS01 socket: 153 reason# EndOfStream
2025-11-27T18:17:12.434411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470921075542629:2050] [node 3] ICP30 unregister session Session# [4:7577470925370510023:2050] VirtualId# [4:7577470925370510013:0]
2025-11-27T18:17:12.434443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470925370510023:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.434582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:12.434640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:12.435206Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:12.435408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470918501441614:2050] [node 3] ICS01 socket: 135 reason# EndOfStream
2025-11-27T18:17:12.435452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470918501441535:2050] [node 3] ICP30 unregister session Session# [6:7577470918501441614:2050] VirtualId# [6:7577470918501441612:0]
2025-11-27T18:17:12.435475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470918501441614:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.435538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:12.435579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:12.437364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:12.437499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470925396363073:2050] [node 3] ICS01 socket: 166 reason# EndOfStream
2025-11-27T18:17:12.437541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470921101395684:2050] [node 3] ICP30 unregister session Session# [5:7577470925396363073:2050] VirtualId# [5:7577470925396363066:0]
2025-11-27T18:17:12.437571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470925396363073:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.437660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:12.437700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:12.525003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:12.525230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470925396363082:2108] [node 4] ICS01 socket: 131 reason# EndOfStream
2025-11-27T18:17:12.525295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470921101395747:2108] [node 4] ICP30 unregister session Session# [5:7577470925396363082:2108] VirtualId# [5:7577470925396363079:0]
2025-11-27T18:17:12.525345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470925396363082:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.525440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:12.525516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:12.529478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:12.529698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470922796408927:2097] [node 4] ICS01 socket: 181 reason# EndOfStream
2025-11-27T18:17:12.529739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470918501441593:2097] [node 4] ICP30 unregister session Session# [6:7577470922796408927:2097] VirtualId# [6:7577470922796408921:0]
2025-11-27T18:17:12.529770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470922796408927:2097] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.529849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:12.529888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:12.593440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:12.593666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470922796408931:2102] [node 5] ICS01 socket: 183 reason# EndOfStream
2025-11-27T18:17:12.593706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470918501441596:2102] [node 5] ICP30 unregister session Session# [6:7577470922796408931:2102] VirtualId# [6:7577470922796408929:0]
2025-11-27T18:17:12.593734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470922796408931:2102] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:12.593830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:12.593884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:12.681160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470923586136570:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:12.681215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470923586136571:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:12.704397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470924742216523:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:12.704439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470924742216524:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:12.731264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470921969429306:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:12.731307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470921969429307:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:12.751533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470923668355590:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:12.751577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470923668355591:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:12.751600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470923668355592:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:12.775743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470923684553199:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:12.775796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470923684553200:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:12.775937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470923684553201:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:12.798203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470924884255735:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:12.798262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470924884255736:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:12.798306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470924884255737:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:12.987571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470924884255797:2106] [node 5] ICP01 ready to work
2025-11-27T18:17:13.101941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470927979520559:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:13.109795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470929037183882:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:13.110424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470927979520560:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:13.112985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470929037183883:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:13.113630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470927963322949:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:13.117207Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470921969429306:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(118700886ns) {Configure:104(118692732ns) {ProcessConfigured:135(3300687ns) {StartInitialHandshake:157(3296543ns) {DropHandshakes:470(12416ns) {DropIncomingHandshake:442(741ns) DropOutgoingHandshake:457(536ns)} GenerateSessionVirtualId:988(795ns) PrepareNewSessionHandshake:477(3589ns) {SwitchToState:179(1782ns)}}}}}
2025-11-27T18:17:13.125070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470927963322950:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:13.117725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470926264396668:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:13.118041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470926264396669:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:13.118279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470926264396670:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:13.132651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470923684553199:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19449695ns) {Configure:104(19443453ns) {ProcessConfigured:135(59524ns) {StartInitialHandshake:157(57478ns) {DropHandshakes:470(2163ns) {DropIncomingHandshake:442(623ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(854ns) PrepareNewSessionHandshake:477(1533ns) {SwitchToState:179(519ns)}}}}}
2025-11-27T18:17:13.128964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470923586136570:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(162352242ns) {Configure:104(162345202ns) {ProcessConfigured:135(7644873ns) {StartInitialHandshake:157(7642477ns) {DropHandshakes:470(558569ns) {DropIncomingHandshake:442(597ns) DropOutgoingHandshake:457(984ns)} GenerateSessionVirtualId:988(1007ns) PrepareNewSessionHandshake:477(3325ns) {SwitchToState:179(1190ns)}}}}}
2025-11-27T18:17:13.134836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470927881103936:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:13.135108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470924884255797:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(147231359ns) {Configure:104(147226730ns) {ProcessConfigured:135(38564ns) {StartInitialHandshake:157(35457ns) {DropHandshakes:470(604ns) {DropIncomingHandshake:442(111ns) DropOutgoingHandshake:457(167ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:477(702ns) {SwitchToState:179(219ns)}}}}}
2025-11-27T18:17:13.137358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470927881103937:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:13.137585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470929179223097:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:13.143176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470924742216523:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18423352ns) {Configure:104(18414092ns) {ProcessConfigured:135(53318ns) {StartInitialHandshake:157(51315ns) {DropHandshakes:470(2126ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(380ns)} GenerateSessionVirtualId:988(318ns) PrepareNewSessionHandshake:477(3169ns) {SwitchToState:179(135ns)}}}}}
2025-11-27T18:17:13.143264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470927881103938:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:13.143792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470929037183886:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.163680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470927881103939:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54891
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.170248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470926264396682:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
2025-11-27T18:17:13.170313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470926264396683:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.171611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470927963322959:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17947
2025-11-27T18:17:13.171820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470927963322962:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17947
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.172853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470926264396684:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.175978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470929037183893:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29002
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.176137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470929037183894:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29002
2025-11-27T18:17:13.176785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470927963322959:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.181461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470929037183895:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29002
2025-11-27T18:17:13.188171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470927881103939:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.182856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470929037183896:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29002
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.197956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470929037183893:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.198519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470923668355592:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.202572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470926264396683:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.202482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470929037183895:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.208581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470927963322967:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17947
2025-11-27T18:17:13.203196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470929037183902:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29002
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.215654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470926264396682:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.220466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470929037183896:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.220748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470927881103951:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54891
2025-11-27T18:17:13.221009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470927881103952:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54891
2025-11-27T18:17:13.227633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470926264396674:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.230233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470927979520574:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54834
2025-11-27T18:17:13.221357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470929037183894:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.227706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470926264396674:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.228685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470926264396684:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.230883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470927963322962:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.228901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470921969429306:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.232039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470927979520574:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.229154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470926264396685:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
2025-11-27T18:17:13.232131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470927963322967:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.231952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470927881103941:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.231189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470926264396674:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:13.233859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470926264396685:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.233903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470926264396668:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.233947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:13.233978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470926264396668:2107] [node 4] ICP22 created new session: [3:7577470926264396690:2107]
2025-11-27T18:17:13.234016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470926264396690:2107] [node 4] ICS09 handshake done sender: [3:7577470926264396674:2113] self: [3:7577470926264396673:0] peer: [4:7577470927963322966:0] socket: 128 qp: -1
2025-11-27T18:17:13.231997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470927881103941:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.232244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470924742216523:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.234043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470926264396690:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:13.234072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:13.232502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470929037183902:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.234863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470923668355590:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.235075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470923668355591:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.235158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470927963322968:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17947
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.238324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470929037183908:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29002
2025-11-27T18:17:13.235824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470927881103952:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.236148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470927881103953:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54891
2025-11-27T18:17:13.236292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470927881103951:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.236409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470927881103935:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.236423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470927881103935:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.237248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470927881103954:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54891
2025-11-27T18:17:13.238184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470927881103941:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.239447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470927979520576:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54834
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.238680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470921969429307:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.245355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470927881103935:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:13.239520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470929037183890:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.245232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470926264396694:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
2025-11-27T18:17:13.239540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470929037183890:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.240242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470929037183888:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.240257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470929037183888:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.246305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470927881103953:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.247759Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470923668355592:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24328) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470927963322959:2114] held: no
2025-11-27T18:17:13.243749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470929037183890:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:13.246558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470929037183888:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:13.248396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470929037183882:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.246796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470927881103954:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.247991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470927963322959:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:13.248447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.249372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470923668355592:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.249500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:13.252197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470926264396695:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
2025-11-27T18:17:13.248490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470929037183882:2107] [node 4] ICP22 created new session: [2:7577470929037183911:2107]
2025-11-27T18:17:13.248537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470929037183911:2107] [node 4] ICS09 handshake done sender: [2:7577470929037183890:2112] self: [2:7577470929037183889:0] peer: [4:7577470927963322970:0] socket: 161 qp: -1
2025-11-27T18:17:13.248570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470929037183911:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:13.248610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:13.249031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470924742216524:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.249070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:13.249390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470924742216524:2049] [node 3] ICP22 created new session: [2:7577470929037183913:2049]
2025-11-27T18:17:13.250090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470929037183913:2049] [node 3] ICS09 handshake done sender: [2:7577470929037183888:2111] self: [2:7577470929037183887:0] peer: [3:7577470926264396693:0] socket: 159 qp: -1
2025-11-27T18:17:13.250129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470929037183913:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:13.250162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:13.246892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470927881103943:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.246907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470927881103943:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.247482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470923586136571:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.247554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.249380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470923586136571:2049] [node 3] ICP22 created new session: [1:7577470927881103960:2049]
2025-11-27T18:17:13.252148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470923668355592:2050] [node 3] ICP22 created new session: [4:7577470927963322972:2050]
2025-11-27T18:17:13.252878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470927963322972:2050] [node 3] ICS09 handshake done sender: [4:7577470927963322959:2114] self: [4:7577470927963322966:0] peer: [3:7577470926264396673:0] socket: 131 qp: -1
2025-11-27T18:17:13.253103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470927963322972:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:13.253133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:13.254908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470927963322971:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17947
2025-11-27T18:17:13.255374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470929037183894:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:13.249452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470927881103960:2049] [node 3] ICS09 handshake done sender: [1:7577470927881103941:2117] self: [1:7577470927881103940:0] peer: [3:7577470926264396688:0] socket: 133 qp: -1
2025-11-27T18:17:13.250975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470927881103960:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:13.251024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:13.251862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470923586136570:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.251890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:13.251916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470923586136570:2048] [node 2] ICP22 created new session: [1:7577470927881103962:2048]
2025-11-27T18:17:13.251940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470927881103962:2048] [node 2] ICS09 handshake done sender: [1:7577470927881103935:2115] self: [1:7577470927881103934:0] peer: [2:7577470929037183906:0] socket: 132 qp: -1
2025-11-27T18:17:13.251958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470927881103962:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:13.252008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.258856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470923668355591:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470927963322967:2117] held: no
2025-11-27T18:17:13.256104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470923586136570:2048] [node 2] ICP18 handshake (actor [1:7577470927881103954:2124]) is too old
2025-11-27T18:17:13.256885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470927881103943:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:13.258066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470929179223109:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-27T18:17:13.260190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470927963322967:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:13.259086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470927881103936:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.259004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470927979520576:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.261703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470921969429307:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470926264396685:2119] held: no
2025-11-27T18:17:13.261982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470926264396682:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:13.259774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:13.259994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470927881103936:2107] [node 4] ICP22 created new session: [1:7577470927881103968:2107]
2025-11-27T18:17:13.261938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470923668355591:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.260045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470927881103968:2107] [node 4] ICS09 handshake done sender: [1:7577470927881103943:2118] self: [1:7577470927881103942:0] peer: [4:7577470927963322969:0] socket: 134 qp: -1
2025-11-27T18:17:13.261988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:13.262030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470923668355591:2049] [node 2] ICP22 created new session: [4:7577470927963322976:2049]
2025-11-27T18:17:13.260094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470927881103968:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:13.262074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470927963322976:2049] [node 2] ICS09 handshake done sender: [4:7577470927963322967:2117] self: [4:7577470927963322970:0] peer: [2:7577470929037183889:0] socket: 166 qp: -1
2025-11-27T18:17:13.260142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:13.262101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470927963322976:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:13.262132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.263202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470923684553201:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.263276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470927979520577:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54834
2025-11-27T18:17:13.263547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470927979520578:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54834
2025-11-27T18:17:13.264237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470921969429306:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470926264396682:2116] held: no
2025-11-27T18:17:13.264347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470921969429306:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.264389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:13.266597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470924742216523:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.266648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:13.266689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470924742216523:2048] [node 1] ICP22 created new session: [2:7577470929037183917:2048]
2025-11-27T18:17:13.264478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470921969429306:2048] [node 1] ICP22 created new session: [3:7577470926264396696:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.278147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470927979520580:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54834
2025-11-27T18:17:13.267206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470929037183917:2048] [node 1] ICS09 handshake done sender: [2:7577470929037183894:2115] self: [2:7577470929037183906:0] peer: [1:7577470927881103934:0] socket: 148 qp: -1
2025-11-27T18:17:13.264529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470926264396696:2048] [node 1] ICS09 handshake done sender: [3:7577470926264396682:2116] self: [3:7577470926264396688:0] peer: [1:7577470927881103940:0] socket: 140 qp: -1
2025-11-27T18:17:13.264563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470926264396696:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:13.264596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:13.265040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470926264396685:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.267248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470929037183917:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:13.267278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:13.267538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577470924742216523:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:13.279262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470926264396676:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.279311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470926264396676:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.280576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470929179223110:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-27T18:17:13.281909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470921969429307:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.287981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470929179223111:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-27T18:17:13.283562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:13.287703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470927963322975:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17947
2025-11-27T18:17:13.290805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470923668355590:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470927963322962:2116] held: no
2025-11-27T18:17:13.283816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470921969429307:2049] [node 2] ICP22 created new session: [3:7577470926264396698:2049]
2025-11-27T18:17:13.291182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470927979520578:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.291757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470927979520559:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.288705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470926264396698:2049] [node 2] ICS09 handshake done sender: [3:7577470926264396685:2119] self: [3:7577470926264396693:0] peer: [2:7577470929037183887:0] socket: 163 qp: -1
2025-11-27T18:17:13.288749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470926264396698:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:13.288790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:13.291533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470927963322962:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:13.293614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470927963322958:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.293636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470927963322958:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.294319Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470926264396676:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:13.295750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470926264396669:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.295887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470927979520580:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.296277Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470929037183898:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.296057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470927979520577:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.296207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470927979520582:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54834
2025-11-27T18:17:13.295978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:13.296322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470929037183898:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.296279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470923668355590:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.293696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470929179223110:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.294400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470929179223109:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.294506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470929179223112:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-27T18:17:13.295922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470924884255736:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.296158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470924884255735:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.296039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470926264396669:2109] [node 5] ICP22 created new session: [3:7577470926264396702:2109]
2025-11-27T18:17:13.313865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470929037183898:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:13.314351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470929037183886:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.296329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:13.309326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470927881103947:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.309391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470927881103947:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.316496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470927881103947:2120] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:13.296103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470926264396702:2109] [node 5] ICS09 handshake done sender: [3:7577470926264396676:2114] self: [3:7577470926264396675:0] peer: [5:7577470927979520579:0] socket: 172 qp: -1
2025-11-27T18:17:13.296385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470923668355590:2048] [node 1] ICP22 created new session: [4:7577470927963322979:2048]
2025-11-27T18:17:13.314425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:13.296164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470926264396702:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:13.314353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470929179223112:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.296209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:13.320012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470923684553200:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.320124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470923684553199:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.320216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470927979520576:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:13.296426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470927963322979:2048] [node 1] ICS09 handshake done sender: [4:7577470927963322962:2116] self: [4:7577470927963322969:0] peer: [1:7577470927881103942:0] socket: 142 qp: -1
2025-11-27T18:17:13.314762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470929037183886:2109] [node 6] ICP22 created new session: [2:7577470929037183920:2109]
2025-11-27T18:17:13.314826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470929037183920:2109] [node 6] ICS09 handshake done sender: [2:7577470929037183898:2118] self: [2:7577470929037183897:0] peer: [6:7577470929179223113:0] socket: 153 qp: -1
2025-11-27T18:17:13.314869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470929037183920:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:13.314902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:13.297307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470927963322979:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:13.320138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470927881103938:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.297345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:13.320214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:13.320260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470927881103938:2109] [node 6] ICP22 created new session: [1:7577470927881103971:2109]
2025-11-27T18:17:13.320305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470927881103971:2109] [node 6] ICS09 handshake done sender: [1:7577470927881103947:2120] self: [1:7577470927881103946:0] peer: [6:7577470929179223114:0] socket: 147 qp: -1
2025-11-27T18:17:13.320337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470927881103971:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:13.320367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:13.324084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470927963322958:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.328006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470929037183892:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.329057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470924884255737:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.331412Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470924884255736:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470929179223110:2116] held: no
2025-11-27T18:17:13.323849Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470923684553201:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24328) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470927979520576:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.333645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470929179223115:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-27T18:17:13.324381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470923684553201:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.328043Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470929037183892:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.333301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470927963322949:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.333351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.324424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:13.329725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470929037183892:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:13.334011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470927963322949:2108] [node 5] ICP22 created new session: [4:7577470927963322982:2108]
2025-11-27T18:17:13.334909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470927963322982:2108] [node 5] ICS09 handshake done sender: [4:7577470927963322958:2113] self: [4:7577470927963322957:0] peer: [5:7577470927979520581:0] socket: 126 qp: -1
2025-11-27T18:17:13.336575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470927881103945:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.336605Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470927881103945:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.338095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470927881103945:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:13.338326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470927881103937:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.338370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:13.338408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470927881103937:2108] [node 5] ICP22 created new session: [1:7577470927881103974:2108]
2025-11-27T18:17:13.338446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470927881103974:2108] [node 5] ICS09 handshake done sender: [1:7577470927881103945:2119] self: [1:7577470927881103944:0] peer: [5:7577470927979520584:0] socket: 144 qp: -1
2025-11-27T18:17:13.338473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470927881103974:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:13.338495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:13.336139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470929179223116:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-27T18:17:13.329953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470929037183883:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.330040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:13.330089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470929037183883:2108] [node 5] ICP22 created new session: [2:7577470929037183923:2108]
2025-11-27T18:17:13.330143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470929037183923:2108] [node 5] ICS09 handshake done sender: [2:7577470929037183892:2113] self: [2:7577470929037183891:0] peer: [5:7577470927979520583:0] socket: 127 qp: -1
2025-11-27T18:17:13.330178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470929037183923:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:13.330219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:13.324470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470923684553201:2050] [node 3] ICP22 created new session: [5:7577470927979520585:2050]
2025-11-27T18:17:13.324507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470927979520585:2050] [node 3] ICS09 handshake done sender: [5:7577470927979520576:2117] self: [5:7577470927979520579:0] peer: [3:7577470926264396675:0] socket: 173 qp: -1
2025-11-27T18:17:13.324547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470927979520585:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:13.324579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:13.327776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470927979520587:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54834
2025-11-27T18:17:13.328902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470927979520578:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:13.334944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470927963322982:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:13.336044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:13.334408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470927979520559:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.334443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:13.334488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470927979520559:2108] [node 4] ICP22 created new session: [5:7577470927979520590:2108]
2025-11-27T18:17:13.334524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470927979520590:2108] [node 4] ICS09 handshake done sender: [5:7577470927979520578:2119] self: [5:7577470927979520581:0] peer: [4:7577470927963322957:0] socket: 178 qp: -1
2025-11-27T18:17:13.334574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470927979520590:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:13.334602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:13.348701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470926264396678:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.348742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470926264396678:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.349679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470926264396678:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:13.351551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470926264396670:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.351589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:13.351640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470926264396670:2110] [node 6] ICP22 created new session: [3:7577470926264396705:2110]
2025-11-27T18:17:13.351740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470926264396705:2110] [node 6] ICS09 handshake done sender: [3:7577470926264396678:2115] self: [3:7577470926264396677:0] peer: [6:7577470929179223117:0] socket: 183 qp: -1
2025-11-27T18:17:13.351772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470926264396705:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:13.351807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:13.350376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470924884255735:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470929179223109:2115] held: no
2025-11-27T18:17:13.351897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470929179223111:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.352176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470929179223109:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:13.352918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470923684553200:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470927979520580:2120] held: no
2025-11-27T18:17:13.355035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470929179223110:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.355559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470924884255735:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.355634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:13.355673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470924884255735:2048] [node 1] ICP22 created new session: [6:7577470929179223121:2048]
2025-11-27T18:17:13.355712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470929179223121:2048] [node 1] ICS09 handshake done sender: [6:7577470929179223109:2115] self: [6:7577470929179223114:0] peer: [1:7577470927881103946:0] socket: 150 qp: -1
2025-11-27T18:17:13.355749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470929179223121:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:13.355780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:13.355526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470927979520589:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54834
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.356753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470929179223118:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-27T18:17:13.356727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470926264396707:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
2025-11-27T18:17:13.356847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470924884255736:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.356882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:13.356915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470924884255736:2049] [node 2] ICP22 created new session: [6:7577470929179223124:2049]
2025-11-27T18:17:13.356950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470929179223124:2049] [node 2] ICS09 handshake done sender: [6:7577470929179223110:2116] self: [6:7577470929179223113:0] peer: [2:7577470929037183897:0] socket: 180 qp: -1
2025-11-27T18:17:13.356978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470929179223124:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:13.357004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:13.356800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470923684553199:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470927979520577:2118] held: no
2025-11-27T18:17:13.357015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470927979520580:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:13.357591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470929179223097:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.358022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470929179223120:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-27T18:17:13.358642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470929179223112:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:13.362213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470924884255737:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.362278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:13.362324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470924884255737:2050] [node 3] ICP22 created new session: [6:7577470929179223127:2050]
2025-11-27T18:17:13.362362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470929179223127:2050] [node 3] ICS09 handshake done sender: [6:7577470929179223112:2118] self: [6:7577470929179223117:0] peer: [3:7577470926264396677:0] socket: 184 qp: -1
2025-11-27T18:17:13.362397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470929179223127:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.363891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470927979520592:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54834
2025-11-27T18:17:13.364438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470923684553200:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.364483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:13.362128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470927963322961:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.362439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:13.364523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470923684553200:2049] [node 2] ICP22 created new session: [5:7577470927979520593:2049]
2025-11-27T18:17:13.364719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470927979520593:2049] [node 2] ICS09 handshake done sender: [5:7577470927979520580:2120] self: [5:7577470927979520583:0] peer: [2:7577470929037183891:0] socket: 158 qp: -1
2025-11-27T18:17:13.362166Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470927963322961:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.365311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470927979520593:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:13.369903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:13.364698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470927963322984:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17947
2025-11-27T18:17:13.368833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470927963322961:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:13.369114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470927963322950:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.369154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:13.365022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470926264396707:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.369197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470927963322950:2109] [node 6] ICP22 created new session: [4:7577470927963322986:2109]
2025-11-27T18:17:13.378524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470927979520577:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:13.374110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577470926264396670:2110] [node 6] ICP18 handshake (actor [3:7577470926264396707:2122]) is too old
2025-11-27T18:17:13.379551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470923684553199:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.379602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:13.379644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470923684553199:2048] [node 1] ICP22 created new session: [5:7577470927979520595:2048]
2025-11-27T18:17:13.379693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470927979520595:2048] [node 1] ICS09 handshake done sender: [5:7577470927979520577:2118] self: [5:7577470927979520584:0] peer: [1:7577470927881103944:0] socket: 167 qp: -1
2025-11-27T18:17:13.380208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470927979520595:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:13.380253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:13.381341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470929179223118:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.369242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470927963322986:2109] [node 6] ICS09 handshake done sender: [4:7577470927963322961:2115] self: [4:7577470927963322960:0] peer: [6:7577470929179223126:0] socket: 143 qp: -1
2025-11-27T18:17:13.369302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470927963322986:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:13.369337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:13.382076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470927963322984:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.382175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577470927963322950:2109] [node 6] ICP18 handshake (actor [4:7577470927963322984:2121]) is too old
2025-11-27T18:17:13.384567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470929179223097:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6982) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470929179223111:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.384742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470929179223129:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-27T18:17:13.385100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470924884255797:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:13.385689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470929179223111:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:13.385837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470927979520568:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:13.385865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470927979520568:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:13.387007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470927979520568:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:13.387211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470927979520560:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:13.387251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:13.387281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470927979520560:2109] [node 6] ICP22 created new session: [5:7577470927979520598:2109]
2025-11-27T18:17:13.387311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470927979520598:2109] [node 6] ICS09 handshake done sender: [5:7577470927979520568:2114] self: [5:7577470927979520567:0] peer: [6:7577470929179223130:0] socket: 157 qp: -1
2025-11-27T18:17:13.387335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470927979520598:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:13.387357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:13.388095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470929179223097:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.388161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:13.388205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470929179223097:2109] [node 4] ICP22 created new session: [6:7577470929179223132:2109]
2025-11-27T18:17:13.388255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470929179223132:2109] [node 4] ICS09 handshake done sender: [6:7577470929179223111:2117] self: [6:7577470929179223126:0] peer: [4:7577470927963322960:0] socket: 181 qp: -1
2025-11-27T18:17:13.388282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470929179223132:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:13.388307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.389720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470929179223131:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-27T18:17:13.390268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470924884255797:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13014) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470929179223118:2121] held: no
2025-11-27T18:17:13.390423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470929179223118:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:13.390565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470924884255797:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:13.390605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:13.390638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470924884255797:2106] [node 5] ICP22 created new session: [6:7577470929179223134:2106]
2025-11-27T18:17:13.390678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470929179223134:2106] [node 5] ICS09 handshake done sender: [6:7577470929179223118:2121] self: [6:7577470929179223130:0] peer: [5:7577470927979520567:0] socket: 149 qp: -1
2025-11-27T18:17:13.390708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470929179223134:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:13.390733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:13.412222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:13.412487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470929037183917:2048] [node 1] ICS07 socket disconnect 148 reason# EndOfStream
2025-11-27T18:17:13.412515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470929037183917:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.412594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:13.412634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470929037183917:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:13.413422Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:13.413554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470926264396696:2048] [node 1] ICS01 socket: 140 reason# EndOfStream
2025-11-27T18:17:13.413595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470921969429306:2048] [node 1] ICP30 unregister session Session# [3:7577470926264396696:2048] VirtualId# [3:7577470926264396688:0]
2025-11-27T18:17:13.413623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470926264396696:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.413683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:13.413721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:13.417338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:13.417187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:13.417425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470927979520595:2048] [node 1] ICS01 socket: 167 reason# EndOfStream
2025-11-27T18:17:13.418720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:13.417460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470923684553199:2048] [node 1] ICP30 unregister session Session# [5:7577470927979520595:2048] VirtualId# [5:7577470927979520584:0]
2025-11-27T18:17:13.417487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470927979520595:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.418881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470929179223121:2048] [node 1] ICS01 socket: 150 reason# EndOfStream
2025-11-27T18:17:13.417566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:13.418918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470924884255735:2048] [node 1] ICP30 unregister session Session# [6:7577470929179223121:2048] VirtualId# [6:7577470929179223114:0]
2025-11-27T18:17:13.417609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:13.424280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470927963322979:2048] [node 1] ICS01 socket: 142 reason# EndOfStream
2025-11-27T18:17:13.424085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470929179223121:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.424178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:13.424232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:13.424689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470923668355590:2048] [node 1] ICP30 unregister session Session# [4:7577470927963322979:2048] VirtualId# [4:7577470927963322969:0]
2025-11-27T18:17:13.425996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470927963322979:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.426072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:13.426117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:13.442430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:13.442657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470926264396698:2049] [node 2] ICS01 socket: 163 reason# EndOfStream
2025-11-27T18:17:13.442706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470921969429307:2049] [node 2] ICP30 unregister session Session# [3:7577470926264396698:2049] VirtualId# [3:7577470926264396693:0]
2025-11-27T18:17:13.442742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470926264396698:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.442806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:13.442843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:13.446804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:13.447028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470927979520593:2049] [node 2] ICS01 socket: 158 reason# EndOfStream
2025-11-27T18:17:13.447080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470923684553200:2049] [node 2] ICP30 unregister session Session# [5:7577470927979520593:2049] VirtualId# [5:7577470927979520583:0]
2025-11-27T18:17:13.447121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470927979520593:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.447212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:13.447263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:13.447502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:13.447710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470929179223124:2049] [node 2] ICS01 socket: 180 reason# EndOfStream
2025-11-27T18:17:13.447745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470924884255736:2049] [node 2] ICP30 unregister session Session# [6:7577470929179223124:2049] VirtualId# [6:7577470929179223113:0]
2025-11-27T18:17:13.447769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470929179223124:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.447837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:13.447899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:13.449408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:13.449613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470927963322976:2049] [node 2] ICS01 socket: 166 reason# EndOfStream
2025-11-27T18:17:13.449647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470923668355591:2049] [node 2] ICP30 unregister session Session# [4:7577470927963322976:2049] VirtualId# [4:7577470927963322970:0]
2025-11-27T18:17:13.449674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470927963322976:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.449736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:13.449769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:13.497549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:13.497808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470927979520585:2050] [node 3] ICS01 socket: 173 reason# EndOfStream
2025-11-27T18:17:13.497849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470923684553201:2050] [node 3] ICP30 unregister session Session# [5:7577470927979520585:2050] VirtualId# [5:7577470927979520579:0]
2025-11-27T18:17:13.497891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470927979520585:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.497970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:13.498017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:13.497545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:13.497794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470927963322972:2050] [node 3] ICS01 socket: 131 reason# EndOfStream
2025-11-27T18:17:13.497838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470923668355592:2050] [node 3] ICP30 unregister session Session# [4:7577470927963322972:2050] VirtualId# [4:7577470927963322966:0]
2025-11-27T18:17:13.497872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470927963322972:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.497948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:13.497990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:13.497549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:13.497805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470929179223127:2050] [node 3] ICS07 socket disconnect 184 reason# EndOfStream
2025-11-27T18:17:13.497835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470929179223127:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.497927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:13.497971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470929179223127:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:13.499098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470924884255737:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24328) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:13.499150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470929179223127:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:13.499179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470924884255737:2050] [node 3] ICP30 unregister session Session# [6:7577470929179223127:2050] VirtualId# [6:7577470929179223117:0]
2025-11-27T18:17:13.499229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:13.499268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470924884255737:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24328) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:13.499281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24328) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:13.509419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470924884255737:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:13.529384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:13.529626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470927979520590:2108] [node 4] ICS01 socket: 178 reason# EndOfStream
2025-11-27T18:17:13.529668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470927979520559:2108] [node 4] ICP30 unregister session Session# [5:7577470927979520590:2108] VirtualId# [5:7577470927979520581:0]
2025-11-27T18:17:13.529699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470927979520590:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.529794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:13.529854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:13.530561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:13.530823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470929179223132:2109] [node 4] ICS01 socket: 181 reason# EndOfStream
2025-11-27T18:17:13.530875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470929179223097:2109] [node 4] ICP30 unregister session Session# [6:7577470929179223132:2109] VirtualId# [6:7577470929179223126:0]
2025-11-27T18:17:13.530907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470929179223132:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.530992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:13.531040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:13.554577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:13.554901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470929179223134:2106] [node 5] ICS01 socket: 149 reason# EndOfStream
2025-11-27T18:17:13.554952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470924884255797:2106] [node 5] ICP30 unregister session Session# [6:7577470929179223134:2106] VirtualId# [6:7577470929179223130:0]
2025-11-27T18:17:13.554991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470929179223134:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:13.555090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:13.555143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:13.633853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470929410265019:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:13.633923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470929410265020:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:13.656436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470929562050291:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:13.656508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470929562050292:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:13.677748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470926763740449:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:13.677790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470926763740450:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:13.700617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470929599643102:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:13.700683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470929599643103:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:13.700712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470929599643104:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:13.722274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470926651415587:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:13.722324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470926651415588:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:13.722369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470926651415589:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:13.745873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470929244413121:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:13.745932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470929244413122:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:13.746149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470929244413123:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:13.856851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470929244413183:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:13.867783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470929410265082:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:13.876498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470929410265083:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:13.881447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470929410265084:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:13.909695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470926651415648:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5462
2025-11-27T18:17:13.940503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470926763740512:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:13.970081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470929599643165:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:13.967430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470926763740513:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:13.979433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470929244413122:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19980538ns) {Configure:104(19973580ns) {ProcessConfigured:135(65263ns) {StartInitialHandshake:157(62401ns) {DropHandshakes:470(1528ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(649ns) PrepareNewSessionHandshake:477(4226ns) {SwitchToState:179(556ns)}}}}}
2025-11-27T18:17:13.982748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470929244413195:2111] [node 4] ICP01 ready to work
2025-11-27T18:17:13.981167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470929599643166:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:13.982371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470926651415651:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:13.983936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470929562050292:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(131250005ns) {Configure:104(131246236ns) {ProcessConfigured:135(8422456ns) {StartInitialHandshake:157(8419410ns) {DropHandshakes:470(12622ns) {DropIncomingHandshake:442(859ns) DropOutgoingHandshake:457(778ns)} GenerateSessionVirtualId:988(1330ns) PrepareNewSessionHandshake:477(1339747ns) {SwitchToState:179(3766ns)}}}}}
2025-11-27T18:17:13.989823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470929562050291:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(144849566ns) {Configure:104(144844631ns) {ProcessConfigured:135(2227641ns) {StartInitialHandshake:157(2224084ns) {DropHandshakes:470(959256ns) {DropIncomingHandshake:442(757ns) DropOutgoingHandshake:457(2223ns)} GenerateSessionVirtualId:988(2147ns) PrepareNewSessionHandshake:477(2961ns) {SwitchToState:179(1000ns)}}}}}
2025-11-27T18:17:13.986853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470929410265083:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(93379316ns) {Configure:104(93373276ns) {ProcessConfigured:135(39224ns) {StartInitialHandshake:157(37291ns) {DropHandshakes:470(1628ns) {DropIncomingHandshake:442(514ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(486ns) PrepareNewSessionHandshake:477(1358ns) {SwitchToState:179(741ns)}}}}}
2025-11-27T18:17:13.988822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470926763740449:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(112204591ns) {Configure:104(112190702ns) {ProcessConfigured:135(4697088ns) {StartInitialHandshake:157(4694605ns) {DropHandshakes:470(1897ns) {DropIncomingHandshake:442(431ns) DropOutgoingHandshake:457(288ns)} GenerateSessionVirtualId:988(510ns) PrepareNewSessionHandshake:477(4006ns) {SwitchToState:179(2543ns)}}}}}
2025-11-27T18:17:13.988785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470929410265019:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(153903461ns) {Configure:104(153891362ns) {ProcessConfigured:135(5001819ns) {StartInitialHandshake:157(4999669ns) {DropHandshakes:470(2695ns) {DropIncomingHandshake:442(762ns) DropOutgoingHandshake:457(354ns)} GenerateSessionVirtualId:988(1634ns) PrepareNewSessionHandshake:477(3337ns) {SwitchToState:179(1282ns)}}}}}
2025-11-27T18:17:13.992731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470929562050358:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:13.995821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470926763740516:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:13.995765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470929562050359:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:13.999117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470926651415587:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16454900ns) {Configure:104(16451162ns) {ProcessConfigured:135(61925ns) {StartInitialHandshake:157(60401ns) {DropHandshakes:470(3025ns) {DropIncomingHandshake:442(536ns) DropOutgoingHandshake:457(694ns)} GenerateSessionVirtualId:988(419ns) PrepareNewSessionHandshake:477(1234ns) {SwitchToState:179(504ns)}}}}}
2025-11-27T18:17:14.000616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470926651415648:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.000969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470930946382952:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.004514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470929599643102:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25352001ns) {Configure:104(25345537ns) {ProcessConfigured:135(606710ns) {StartInitialHandshake:157(604576ns) {DropHandshakes:470(2233ns) {DropIncomingHandshake:442(441ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(488ns) PrepareNewSessionHandshake:477(1725ns) {SwitchToState:179(676ns)}}}}}
2025-11-27T18:17:14.008737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470933894610471:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36184
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.013926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470929410265095:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55117
2025-11-27T18:17:14.014206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470933705232395:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55117
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.019551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470930946382958:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5462
2025-11-27T18:17:14.011936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470933857017656:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:14.020498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470926763740450:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45964762ns) {Configure:104(42146698ns) {ProcessConfigured:135(41489ns) {StartInitialHandshake:157(39943ns) {DropHandshakes:470(879ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(152ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:477(591ns) {SwitchToState:179(250ns)}}}}}
2025-11-27T18:17:14.021153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470929410265095:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.027958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470926651415589:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(77296460ns) {Configure:104(77287524ns) {ProcessConfigured:135(1626339ns) {StartInitialHandshake:157(1624070ns) {DropHandshakes:470(1501ns) {DropIncomingHandshake:442(575ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(597ns) PrepareNewSessionHandshake:477(2526ns) {SwitchToState:179(1287ns)}}}}}
2025-11-27T18:17:14.029427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470929599643104:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(41260830ns) {Configure:104(41256368ns) {ProcessConfigured:135(8066687ns) {StartInitialHandshake:157(8064050ns) {DropHandshakes:470(5532535ns) {DropIncomingHandshake:442(1909ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(2817ns) PrepareNewSessionHandshake:477(1616ns) {SwitchToState:179(800ns)}}}}}
2025-11-27T18:17:14.029206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470933705232396:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55117
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.037340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470933894610471:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.039232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470933705232395:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.044056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470933857017657:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55321
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.038990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470929599643102:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.049702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470930946382958:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.049855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470926651415651:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.044226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470933857017658:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55321
2025-11-27T18:17:14.046102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470933539380495:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14619
2025-11-27T18:17:14.045789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470929410265088:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.045939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470933894610477:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36184
2025-11-27T18:17:14.044351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470933857017659:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55321
2025-11-27T18:17:14.050971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470926763740512:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(46893942ns) {Configure:104(46884963ns) {ProcessConfigured:135(4680700ns) {StartInitialHandshake:157(4678800ns) {DropHandshakes:470(1659536ns) {DropIncomingHandshake:442(1680ns) DropOutgoingHandshake:457(366ns)} GenerateSessionVirtualId:988(1158ns) PrepareNewSessionHandshake:477(12598ns) {SwitchToState:179(11635ns)}}}}}
2025-11-27T18:17:14.045832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470929410265088:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.050252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470933894610466:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.044451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470933857017660:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55321
2025-11-27T18:17:14.050285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470933894610466:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.044836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470933857017657:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.052944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470933894610477:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.044915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470933857017659:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.054407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470933705232396:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.056367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470933857017658:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.056232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470926763740513:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32819252ns) {Configure:104(32814319ns) {ProcessConfigured:135(789247ns) {StartInitialHandshake:157(787228ns) {DropHandshakes:470(1921ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(533ns)} GenerateSessionVirtualId:988(354ns) PrepareNewSessionHandshake:477(2439ns) {SwitchToState:179(1071ns)}}}}}
2025-11-27T18:17:14.059104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470933539380495:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.059229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470929244413122:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.060429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470933539380497:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14619
2025-11-27T18:17:14.057187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470933894610466:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:14.056709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470933705232399:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55117
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.057149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470929562050358:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27730052ns) {Configure:104(27724102ns) {ProcessConfigured:135(409647ns) {StartInitialHandshake:157(407765ns) {DropHandshakes:470(1869ns) {DropIncomingHandshake:442(248ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(507ns) PrepareNewSessionHandshake:477(1243ns) {SwitchToState:179(490ns)}}}}}
2025-11-27T18:17:14.061572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470929599643165:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.056939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470929410265088:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:14.061618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:14.064791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470929599643165:2108] [node 5] ICP22 created new session: [4:7577470933894610482:2108]
2025-11-27T18:17:14.065528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470933539380495:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:14.057861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470933857017660:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.065526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470929410265090:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.058153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470929562050291:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.065767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470933894610482:2108] [node 5] ICS09 handshake done sender: [4:7577470933894610466:2111] self: [4:7577470933894610465:0] peer: [5:7577470930946382964:0] socket: 144 qp: -1
2025-11-27T18:17:14.065832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470933894610482:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:14.065700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470929410265082:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.067414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:14.067478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470929410265082:2107] [node 4] ICP22 created new session: [1:7577470933705232402:2107]
2025-11-27T18:17:14.065650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470930946382965:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5462
2025-11-27T18:17:14.059437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470933857017662:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.067477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:14.067530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470929410265090:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.068856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470930946382958:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:14.059465Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470933857017662:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.068258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470933705232402:2107] [node 4] ICS09 handshake done sender: [1:7577470929410265088:2111] self: [1:7577470929410265087:0] peer: [4:7577470933894610478:0] socket: 139 qp: -1
2025-11-27T18:17:14.072983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470926651415651:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.073057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:14.073107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470926651415651:2109] [node 4] ICP22 created new session: [5:7577470930946382966:2109]
2025-11-27T18:17:14.073152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470930946382966:2109] [node 4] ICS09 handshake done sender: [5:7577470930946382958:2115] self: [5:7577470930946382964:0] peer: [4:7577470933894610465:0] socket: 148 qp: -1
2025-11-27T18:17:14.073183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470930946382966:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:14.073212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:14.070565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470929244413122:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.060653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470933857017662:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.076665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470926651415651:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:14.077139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470930946382968:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5462
2025-11-27T18:17:14.068311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470933705232402:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:14.061051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470933857017656:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.070982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:14.068354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:14.061126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:14.081114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470930946382968:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.072610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470929244413122:2049] [node 2] ICP22 created new session: [6:7577470933539380498:2049]
2025-11-27T18:17:14.081502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470933539380498:2049] [node 2] ICS09 handshake done sender: [6:7577470933539380495:2115] self: [6:7577470933539380496:0] peer: [2:7577470933857017661:0] socket: 162 qp: -1
2025-11-27T18:17:14.085613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470933539380498:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:14.089413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:14.061192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470933857017656:2109] [node 6] ICP22 created new session: [2:7577470933857017674:2109]
2025-11-27T18:17:14.072430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470933705232399:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.091037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470933894610481:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36184
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.061236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470933857017674:2109] [node 6] ICS09 handshake done sender: [2:7577470933857017662:2116] self: [2:7577470933857017661:0] peer: [6:7577470933539380496:0] socket: 161 qp: -1
2025-11-27T18:17:14.061292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470933857017674:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.072834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470929410265090:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:14.061700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.093433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470926651415588:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.093569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470930946382969:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5462
2025-11-27T18:17:14.087051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470929410265019:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.092552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470931058707818:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54846
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.095663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470931058707817:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54846
2025-11-27T18:17:14.087553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:14.097721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470929244413122:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(24992428ns) {HandleHandshakeStatus:329(24989232ns) {DropIncomingHandshake:442(878ns) DropOutgoingHandshake:457(4758ns) SwitchToState:179(490ns) ProcessPendingSessionEvents:495(4881609ns) {ForwardSessionEventToSession:612(1077502ns) ForwardSessionEventToSession:612(789413ns) ForwardSessionEventToSession:612(3425ns) ForwardSessionEventToSession:612(5464ns) ForwardSessionEventToSession:612(195488ns) ForwardSessionEventToSession:612(2330ns) ForwardSessionEventToSession:612(2781198ns) ForwardSessionEventToSession:612(3582ns) ForwardSessionEventToSession:612(3305ns) ForwardSessionEventToSession:612(6994ns)}}}
2025-11-27T18:17:14.095575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470929599643102:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19927) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470933894610471:2114] held: no
2025-11-27T18:17:14.098583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470930946382969:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.098176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470931058707818:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.099198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470931058707817:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.090046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470929410265019:2048] [node 2] ICP22 created new session: [1:7577470933705232408:2048]
2025-11-27T18:17:14.097441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470933894610471:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.099773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470926651415587:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.100688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470933857017664:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.099916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470926763740450:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.091581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470933705232408:2048] [node 2] ICS09 handshake done sender: [1:7577470929410265090:2112] self: [1:7577470929410265089:0] peer: [2:7577470933857017672:0] socket: 140 qp: -1
2025-11-27T18:17:14.100738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470933857017664:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.101250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577470929244413122:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:14.101895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470929599643102:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.103704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:14.103762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470929599643102:2048] [node 1] ICP22 created new session: [4:7577470933894610486:2048]
2025-11-27T18:17:14.103815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470933894610486:2048] [node 1] ICS09 handshake done sender: [4:7577470933894610471:2114] self: [4:7577470933894610478:0] peer: [1:7577470929410265087:0] socket: 142 qp: -1
2025-11-27T18:17:14.103852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470933894610486:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:14.103891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:14.104892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470933894610485:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36184
2025-11-27T18:17:14.107974Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470929562050291:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19927) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577470933857017658:2113] held: no
2025-11-27T18:17:14.108553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470933857017678:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55321
2025-11-27T18:17:14.114416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470933857017658:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:14.108795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470933539380500:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14619
2025-11-27T18:17:14.109209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470933539380503:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14619
2025-11-27T18:17:14.112712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470933539380503:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.112848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470929244413195:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.100277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470926763740449:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.106845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470931058707826:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54846
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.091625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470933705232408:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.115937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470929562050355:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.120660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470930946382974:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5462
2025-11-27T18:17:14.116500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470933539380500:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.092030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:14.118137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470933894610470:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.115979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470929562050355:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.118626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470931058707826:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.118602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470933539380502:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14619
2025-11-27T18:17:14.098424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470933705232410:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55117
2025-11-27T18:17:14.118183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470933894610470:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.116983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470933857017679:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55321
2025-11-27T18:17:14.120992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470929244413121:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.121817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470930946382969:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:14.121061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470933894610470:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.122076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470933539380502:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.103546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470929410265092:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.117100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470933857017664:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:14.123065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470929244413183:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.103584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470929410265092:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.121623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470929562050291:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.108579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470929410265086:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.108612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470929410265086:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.121679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:14.118804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470929410265092:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:14.120897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470929410265086:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:14.123683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470929410265094:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.123705Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470929410265094:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.124229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470929410265020:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.124286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:14.124328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470929410265020:2049] [node 3] ICP22 created new session: [1:7577470933705232413:2049]
2025-11-27T18:17:14.124368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470933705232413:2049] [node 3] ICS09 handshake done sender: [1:7577470929410265086:2110] self: [1:7577470929410265085:0] peer: [3:7577470931058707828:0] socket: 138 qp: -1
2025-11-27T18:17:14.124401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470933705232413:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:14.124990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470930946382973:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5462
2025-11-27T18:17:14.122147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470929599643166:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.121783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470929562050291:2048] [node 1] ICP22 created new session: [2:7577470933857017682:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.124438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:14.124719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.125509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470929410265083:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.129089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470931058707816:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.128255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470926651415587:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.124931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470929562050359:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.128325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.125545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:14.125096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470933857017682:2048] [node 1] ICS09 handshake done sender: [2:7577470933857017658:2113] self: [2:7577470933857017672:0] peer: [1:7577470929410265089:0] socket: 156 qp: -1
2025-11-27T18:17:14.129123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470931058707816:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.128370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470926651415587:2048] [node 1] ICP22 created new session: [5:7577470930946382975:2048]
2025-11-27T18:17:14.125622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470933894610485:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.130866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470933539380506:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14619
2025-11-27T18:17:14.130170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470931058707830:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54846
2025-11-27T18:17:14.125605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470929410265083:2108] [node 5] ICP22 created new session: [1:7577470933705232415:2108]
2025-11-27T18:17:14.125813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470933894610491:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36184
2025-11-27T18:17:14.126220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470933857017682:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:14.128410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470930946382975:2048] [node 1] ICS09 handshake done sender: [5:7577470930946382969:2119] self: [5:7577470930946382972:0] peer: [1:7577470929410265091:0] socket: 174 qp: -1
2025-11-27T18:17:14.132463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470933539380508:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14619
2025-11-27T18:17:14.132523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470933539380503:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:14.125637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470933705232415:2108] [node 5] ICS09 handshake done sender: [1:7577470929410265092:2113] self: [1:7577470929410265091:0] peer: [5:7577470930946382972:0] socket: 136 qp: -1
2025-11-27T18:17:14.128440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470930946382975:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:14.131166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470931058707829:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54846
2025-11-27T18:17:14.127480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:14.125866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470929599643104:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.139513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470929244413195:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.127182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470933705232415:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:14.137883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470931058707816:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:14.128490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:14.127225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:14.128344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:14.126163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470929599643166:2109] [node 6] ICP22 created new session: [4:7577470933894610490:2109]
2025-11-27T18:17:14.141225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.128389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470929562050359:2108] [node 5] ICP22 created new session: [2:7577470933857017683:2108]
2025-11-27T18:17:14.139284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470931058707829:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.127516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470933894610490:2109] [node 6] ICS09 handshake done sender: [4:7577470933894610470:2113] self: [4:7577470933894610469:0] peer: [6:7577470933539380504:0] socket: 177 qp: -1
2025-11-27T18:17:14.130556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470930946382956:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.141455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470929244413195:2111] [node 4] ICP22 created new session: [6:7577470933539380509:2111]
2025-11-27T18:17:14.127804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470933705232410:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.130573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470930946382956:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.131561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470929410265020:2049] [node 3] ICP18 handshake (actor [1:7577470933705232410:2119]) is too old
2025-11-27T18:17:14.141393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470926763740449:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19927) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470931058707817:2113] held: no
2025-11-27T18:17:14.141768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470933539380509:2111] [node 4] ICS09 handshake done sender: [6:7577470933539380503:2119] self: [6:7577470933539380504:0] peer: [4:7577470933894610469:0] socket: 179 qp: -1
2025-11-27T18:17:14.128631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470933894610490:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:14.128439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470933857017683:2108] [node 5] ICS09 handshake done sender: [2:7577470933857017664:2117] self: [2:7577470933857017663:0] peer: [5:7577470930946382970:0] socket: 154 qp: -1
2025-11-27T18:17:14.139060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470929410265094:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:14.142384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470933539380509:2111] [node 4] ICS10 traffic start
2025-11-27T18:17:14.128479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470933857017683:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:14.130728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:14.134537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470930946382968:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:14.141998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470926763740512:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.142852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:14.141075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470926651415588:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.142356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470929410265084:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.128537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:14.145454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470929599643166:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18800085ns) {HandleHandshakeStatus:329(18796282ns) {DropIncomingHandshake:442(260ns) DropOutgoingHandshake:457(615ns) SwitchToState:179(637ns) ProcessPendingSessionEvents:495(9556696ns) {ForwardSessionEventToSession:612(6053915ns) ForwardSessionEventToSession:612(589110ns) ForwardSessionEventToSession:612(3459ns) ForwardSessionEventToSession:612(2086ns) ForwardSessionEventToSession:612(918013ns) ForwardSessionEventToSession:612(3032ns) ForwardSessionEventToSession:612(965978ns) ForwardSessionEventToSession:612(7749ns) ForwardSessionEventToSession:612(2685ns) ForwardSessionEventToSession:612(993532ns)}}}
2025-11-27T18:17:14.142458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:14.142693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:14.142684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:14.145932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470933894610494:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36184
2025-11-27T18:17:14.156396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470933894610485:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:14.141335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470933857017679:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.143137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470929562050355:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:14.142747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470931058707817:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:14.149902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470929562050292:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.149942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:14.150011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470929562050292:2049] [node 3] ICP22 created new session: [2:7577470933857017687:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.145456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470929410265084:2109] [node 6] ICP22 created new session: [1:7577470933705232418:2109]
2025-11-27T18:17:14.145630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470926651415588:2049] [node 2] ICP22 created new session: [5:7577470930946382978:2049]
2025-11-27T18:17:14.143391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470926763740512:2107] [node 4] ICP22 created new session: [3:7577470931058707833:2107]
2025-11-27T18:17:14.147137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470933705232418:2109] [node 6] ICS09 handshake done sender: [1:7577470929410265094:2114] self: [1:7577470929410265093:0] peer: [6:7577470933539380505:0] socket: 175 qp: -1
2025-11-27T18:17:14.147535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470933705232418:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:14.147565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:14.147194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470930946382978:2049] [node 2] ICS09 handshake done sender: [5:7577470930946382968:2118] self: [5:7577470930946382970:0] peer: [2:7577470933857017663:0] socket: 173 qp: -1
2025-11-27T18:17:14.153746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470933857017687:2049] [node 3] ICS09 handshake done sender: [2:7577470929562050355:2110] self: [2:7577470929562050354:0] peer: [3:7577470931058707827:0] socket: 159 qp: -1
2025-11-27T18:17:14.153781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470933857017687:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:14.145897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470931058707833:2107] [node 4] ICS09 handshake done sender: [3:7577470931058707816:2112] self: [3:7577470931058707815:0] peer: [4:7577470933894610492:0] socket: 181 qp: -1
2025-11-27T18:17:14.170899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470933539380500:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:14.147223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470930946382978:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:14.163604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470929599643104:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16086) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470933894610485:2118] held: no
2025-11-27T18:17:14.156370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:14.165548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470929599643104:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.165587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:14.170826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470929599643104:2050] [node 3] ICP22 created new session: [4:7577470933894610495:2050]
2025-11-27T18:17:14.172291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470933894610495:2050] [node 3] ICS09 handshake done sender: [4:7577470933894610485:2118] self: [4:7577470933894610492:0] peer: [3:7577470931058707815:0] socket: 184 qp: -1
2025-11-27T18:17:14.172812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470933894610495:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:14.147259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:14.146056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470931058707833:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:14.169050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470929562050292:2049] [node 3] ICP18 handshake (actor [2:7577470933857017679:2120]) is too old
2025-11-27T18:17:14.173864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:14.147202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:14.149071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470926651415588:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:14.150206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470926763740449:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.174406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470933894610491:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.172794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470929244413121:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19927) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470933539380500:2117] held: no
2025-11-27T18:17:14.175205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470929599643103:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.174205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470929244413121:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.150261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:14.156535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470930946382956:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.150289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470926763740449:2048] [node 1] ICP22 created new session: [3:7577470931058707835:2048]
2025-11-27T18:17:14.172115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470930946382952:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.174803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:14.174867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470930946382952:2110] [node 6] ICP22 created new session: [5:7577470930946382982:2110]
2025-11-27T18:17:14.175137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470930946382982:2110] [node 6] ICS09 handshake done sender: [5:7577470930946382956:2114] self: [5:7577470930946382955:0] peer: [6:7577470933539380507:0] socket: 151 qp: -1
2025-11-27T18:17:14.175464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470930946382982:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:14.175884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:14.177997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470930946382981:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5462
2025-11-27T18:17:14.174243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:14.176111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470933857017670:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.178795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470930946382981:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.178931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470926651415589:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.150333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470931058707835:2048] [node 1] ICS09 handshake done sender: [3:7577470931058707817:2113] self: [3:7577470931058707828:0] peer: [1:7577470929410265085:0] socket: 164 qp: -1
2025-11-27T18:17:14.174294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470929244413121:2048] [node 1] ICP22 created new session: [6:7577470933539380513:2048]
2025-11-27T18:17:14.176140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470933857017670:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.174328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470933539380513:2048] [node 1] ICS09 handshake done sender: [6:7577470933539380500:2117] self: [6:7577470933539380505:0] peer: [1:7577470929410265093:0] socket: 176 qp: -1
2025-11-27T18:17:14.150353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470931058707835:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:14.185107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470933857017670:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:14.174760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470933539380513:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:14.150372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:14.186189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470933894610498:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36184
2025-11-27T18:17:14.150792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577470926763740512:2107] [node 4] ICP18 handshake (actor [3:7577470931058707829:2118]) is too old
2025-11-27T18:17:14.186189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470929562050358:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.175062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:14.177469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470933539380512:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14619
2025-11-27T18:17:14.177963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470933539380511:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14619
2025-11-27T18:17:14.157445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470931058707839:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54846
2025-11-27T18:17:14.181073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470933539380512:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.188861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:14.189241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470929562050358:2107] [node 4] ICP22 created new session: [2:7577470933857017690:2107]
2025-11-27T18:17:14.160996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470931058707818:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:14.189720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470933894610491:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:14.181226Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470929244413183:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22037) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470933539380502:2118] held: no
2025-11-27T18:17:14.181820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470929244413123:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:14.189514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470933857017690:2107] [node 4] ICS09 handshake done sender: [2:7577470933857017670:2118] self: [2:7577470933857017669:0] peer: [4:7577470933894610497:0] socket: 195 qp: -1
2025-11-27T18:17:14.183458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470933539380516:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14619
2025-11-27T18:17:14.193121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470933857017690:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:14.184051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470933539380512:2123] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:14.161240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470931058707840:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54846
2025-11-27T18:17:14.161347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470926763740450:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.162292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:14.162336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470926763740450:2049] [node 2] ICP22 created new session: [3:7577470931058707841:2049]
2025-11-27T18:17:14.194887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470929599643103:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:17:14.162376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470931058707841:2049] [node 2] ICS09 handshake done sender: [3:7577470931058707818:2114] self: [3:7577470931058707827:0] peer: [2:7577470929562050354:0] socket: 165 qp: -1
::ffff:127.1.0.0
2025-11-27T18:17:14.194940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:14.162409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470931058707841:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:14.194923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:14.162458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:14.184675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470933539380502:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:14.195838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470930946382985:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5462
2025-11-27T18:17:14.180076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470931058707840:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.184855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470929244413123:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.181083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470931058707820:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.185463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:14.181117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470931058707820:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.185899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470929244413123:2050] [node 3] ICP22 created new session: [6:7577470933539380517:2050]
2025-11-27T18:17:14.186431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470933539380517:2050] [node 3] ICS09 handshake done sender: [6:7577470933539380512:2123] self: [6:7577470933539380515:0] peer: [3:7577470931058707821:0] socket: 202 qp: -1
2025-11-27T18:17:14.182040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470931058707822:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:14.187523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470929244413183:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.182055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470931058707822:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:14.187568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:14.183633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470931058707822:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:14.187593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470929244413183:2107] [node 5] ICP22 created new session: [6:7577470933539380518:2107]
2025-11-27T18:17:14.184227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470926763740516:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.187623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470933539380518:2107] [node 5] ICS09 handshake done sender: [6:7577470933539380502:2118] self: [6:7577470933539380507:0] peer: [5:7577470930946382955:0] socket: 166 qp: -1
2025-11-27T18:17:14.195170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470929599643103:2049] [node 2] ICP22 created new session: [4:7577470933894610499:2049]
2025-11-27T18:17:14.196598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470930946382981:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:14.196083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470933894610499:2049] [node 2] ICS09 handshake done sender: [4:7577470933894610491:2119] self: [4:7577470933894610497:0] peer: [2:7577470933857017669:0] socket: 197 qp: -1
2025-11-27T18:17:14.196121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470933894610499:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:14.186034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:14.197960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470926651415589:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:14.196155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:14.187639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470933539380518:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:14.186088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470926763740516:2109] [node 6] ICP22 created new session: [3:7577470931058707845:2109]
2025-11-27T18:17:14.186740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470931058707845:2109] [node 6] ICS09 handshake done sender: [3:7577470931058707822:2116] self: [3:7577470931058707821:0] peer: [6:7577470933539380515:0] socket: 149 qp: -1
2025-11-27T18:17:14.186779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470931058707845:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:14.186823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:14.193115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470931058707820:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:14.196942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470926763740513:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:14.198547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:14.198605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470926763740513:2108] [node 5] ICP22 created new session: [3:7577470931058707848:2108]
2025-11-27T18:17:14.198662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470931058707848:2108] [node 5] ICS09 handshake done sender: [3:7577470931058707820:2115] self: [3:7577470931058707819:0] peer: [5:7577470930946382984:0] socket: 143 qp: -1
2025-11-27T18:17:14.198698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470931058707848:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:14.198732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:14.197320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470929599643103:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:14.187657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:14.188822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470933539380517:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:14.188874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:14.200336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:14.200393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470926651415589:2050] [node 3] ICP22 created new session: [5:7577470930946382986:2050]
2025-11-27T18:17:14.200450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470930946382986:2050] [node 3] ICS09 handshake done sender: [5:7577470930946382981:2122] self: [5:7577470930946382984:0] peer: [3:7577470931058707819:0] socket: 150 qp: -1
2025-11-27T18:17:14.200479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470930946382986:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:14.200534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:14.201530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470926651415589:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:14.228614Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:14.228897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470933857017682:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-27T18:17:14.228938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470929562050291:2048] [node 1] ICP30 unregister session Session# [2:7577470933857017682:2048] VirtualId# [2:7577470933857017672:0]
2025-11-27T18:17:14.228979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470933857017682:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.229081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:14.229133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:14.229382Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:14.232488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470931058707835:2048] [node 1] ICS01 socket: 164 reason# EndOfStream
2025-11-27T18:17:14.232541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470926763740449:2048] [node 1] ICP30 unregister session Session# [3:7577470931058707835:2048] VirtualId# [3:7577470931058707828:0]
2025-11-27T18:17:14.232570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470931058707835:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.232652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:14.232697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:14.236951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:14.237178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470933539380513:2048] [node 1] ICS01 socket: 176 reason# EndOfStream
2025-11-27T18:17:14.237238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470929244413121:2048] [node 1] ICP30 unregister session Session# [6:7577470933539380513:2048] VirtualId# [6:7577470933539380505:0]
2025-11-27T18:17:14.237309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470933539380513:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.237394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:14.237432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:14.237284Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:14.237501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470933894610486:2048] [node 1] ICS01 socket: 142 reason# EndOfStream
2025-11-27T18:17:14.237530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470929599643102:2048] [node 1] ICP30 unregister session Session# [4:7577470933894610486:2048] VirtualId# [4:7577470933894610478:0]
2025-11-27T18:17:14.237553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470933894610486:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.237621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:14.237665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:14.236954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:14.237199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470930946382975:2048] [node 1] ICS01 socket: 174 reason# EndOfStream
2025-11-27T18:17:14.237246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470926651415587:2048] [node 1] ICP30 unregister session Session# [5:7577470930946382975:2048] VirtualId# [5:7577470930946382972:0]
2025-11-27T18:17:14.237306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470930946382975:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.237389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:14.237431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:14.297343Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:14.297596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470931058707841:2049] [node 2] ICS01 socket: 165 reason# EndOfStream
2025-11-27T18:17:14.297634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470926763740450:2049] [node 2] ICP30 unregister session Session# [3:7577470931058707841:2049] VirtualId# [3:7577470931058707827:0]
2025-11-27T18:17:14.297674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470931058707841:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.297742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:14.297783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:14.303786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:14.304008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470930946382978:2049] [node 2] ICS07 socket disconnect 173 reason# EndOfStream
2025-11-27T18:17:14.304041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470930946382978:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.304115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:14.304152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470930946382978:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:14.304564Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:14.304789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470933894610499:2049] [node 2] ICS01 socket: 197 reason# EndOfStream
2025-11-27T18:17:14.304825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470929599643103:2049] [node 2] ICP30 unregister session Session# [4:7577470933894610499:2049] VirtualId# [4:7577470933894610497:0]
2025-11-27T18:17:14.304852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470933894610499:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.304921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:14.304954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:14.305707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:14.307387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470933539380498:2049] [node 2] ICS07 socket disconnect 162 reason# EndOfStream
2025-11-27T18:17:14.307440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470933539380498:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.307528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:14.305697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470926651415588:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6616) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:14.307606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470930946382978:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:14.307593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470933539380498:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:14.308614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470929244413122:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6616) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:14.309589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470933539380498:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:14.309623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470929244413122:2049] [node 2] ICP30 unregister session Session# [6:7577470933539380498:2049] VirtualId# [6:7577470933539380496:0]
2025-11-27T18:17:14.309661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:14.309688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470929244413122:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6616) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:14.309699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6616) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:14.308917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470926651415588:2049] [node 2] ICP30 unregister session Session# [5:7577470930946382978:2049] VirtualId# [5:7577470930946382970:0]
2025-11-27T18:17:14.308970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:14.309428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470926651415588:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6616) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:14.309446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6616) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:14.319158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470926651415588:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:14.319171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470929244413122:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:14.345390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:14.345597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470933539380517:2050] [node 3] ICS01 socket: 202 reason# EndOfStream
2025-11-27T18:17:14.345637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470929244413123:2050] [node 3] ICP30 unregister session Session# [6:7577470933539380517:2050] VirtualId# [6:7577470933539380515:0]
2025-11-27T18:17:14.345676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470933539380517:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.345785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:14.345832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:14.347729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:14.347565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:14.347941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470933894610495:2050] [node 3] ICS01 socket: 184 reason# EndOfStream
2025-11-27T18:17:14.347885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470930946382986:2050] [node 3] ICS07 socket disconnect 150 reason# EndOfStream
2025-11-27T18:17:14.347913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470930946382986:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.347981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470929599643104:2050] [node 3] ICP30 unregister session Session# [4:7577470933894610495:2050] VirtualId# [4:7577470933894610492:0]
2025-11-27T18:17:14.348007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470933894610495:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.348079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:14.348118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:14.347987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:14.348027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470930946382986:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:14.351711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470926651415589:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16086) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:14.352269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470930946382986:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:14.352307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470926651415589:2050] [node 3] ICP30 unregister session Session# [5:7577470930946382986:2050] VirtualId# [5:7577470930946382984:0]
2025-11-27T18:17:14.352358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:14.352393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470926651415589:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16086) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:14.352407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16086) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:14.365414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470926651415589:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:14.405829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:14.406070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470933539380509:2111] [node 4] ICS01 socket: 179 reason# EndOfStream
2025-11-27T18:17:14.406112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470929244413195:2111] [node 4] ICP30 unregister session Session# [6:7577470933539380509:2111] VirtualId# [6:7577470933539380504:0]
2025-11-27T18:17:14.406145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470933539380509:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.406236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:14.406281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:14.409595Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:14.409826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470930946382966:2109] [node 4] ICS01 socket: 148 reason# EndOfStream
2025-11-27T18:17:14.409865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470926651415651:2109] [node 4] ICP30 unregister session Session# [5:7577470930946382966:2109] VirtualId# [5:7577470930946382964:0]
2025-11-27T18:17:14.409895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470930946382966:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.409974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:14.410016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:14.449532Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:14.449766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470933539380518:2107] [node 5] ICS01 socket: 166 reason# EndOfStream
2025-11-27T18:17:14.449809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470929244413183:2107] [node 5] ICP30 unregister session Session# [6:7577470933539380518:2107] VirtualId# [6:7577470933539380507:0]
2025-11-27T18:17:14.449839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470933539380518:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:14.449922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:14.449976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:14.535908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470932923790418:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:14.536380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470932923790417:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:14.559800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470934432584386:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:14.559838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470934432584387:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:14.583996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470931513266704:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:14.584022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470931513266705:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:14.610933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470931435855180:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:14.610998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470931435855181:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:14.611027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470931435855182:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:14.634674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470931830270968:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:14.634726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470931830270969:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:14.634920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470931830270970:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:14.661227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470934622404787:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:14.661287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470934622404788:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:14.661314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470934622404789:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:14.784849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470934622404845:2104] [node 5] ICP01 ready to work
2025-11-27T18:17:14.977903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470931513266769:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:14.989650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470934432584449:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:14.985943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470931513266768:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:14.998993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470932923790417:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(215676996ns) {Configure:104(215670441ns) {ProcessConfigured:135(74940ns) {StartInitialHandshake:157(72388ns) {DropHandshakes:470(2580ns) {DropIncomingHandshake:442(478ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(1080ns) PrepareNewSessionHandshake:477(1612ns) {SwitchToState:179(522ns)}}}}}
2025-11-27T18:17:15.000035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470934432584450:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:15.008912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470936125238328:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:14.996985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470932923790480:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:14.997524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470931513266779:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:15.012322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470935730822539:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:15.016744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470934622404845:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(223557393ns) {Configure:104(223553020ns) {ProcessConfigured:135(3100423ns) {StartInitialHandshake:157(3097224ns) {DropHandshakes:470(2633ns) {DropIncomingHandshake:442(823ns) DropOutgoingHandshake:457(396ns)} GenerateSessionVirtualId:988(927ns) PrepareNewSessionHandshake:477(3072ns) {SwitchToState:179(1598ns)}}}}}
2025-11-27T18:17:15.015426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470938727551747:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:15.018675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470937218757779:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:15.017070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470935730822540:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:15.033382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470938917372149:2106] [node 4] ICP01 ready to work
2025-11-27T18:17:15.019241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470934432584387:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(140531850ns) {Configure:104(140498677ns) {ProcessConfigured:135(65092ns) {StartInitialHandshake:157(62471ns) {DropHandshakes:470(2488ns) {DropIncomingHandshake:442(513ns) DropOutgoingHandshake:457(379ns)} GenerateSessionVirtualId:988(536ns) PrepareNewSessionHandshake:477(1550ns) {SwitchToState:179(706ns)}}}}}
2025-11-27T18:17:15.018777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470937218757780:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:15.037041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470936125238329:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:15.056367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470932923790418:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(272184372ns) {Configure:104(272173240ns) {ProcessConfigured:135(3599954ns) {StartInitialHandshake:157(3597378ns) {DropHandshakes:470(2257ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(1266ns) PrepareNewSessionHandshake:477(3657ns) {SwitchToState:179(2138ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.060140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470931435855182:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(117933314ns) {Configure:104(117917466ns) {ProcessConfigured:135(1108177ns) {StartInitialHandshake:157(1106514ns) {DropHandshakes:470(1723ns) {DropIncomingHandshake:442(745ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(529ns) PrepareNewSessionHandshake:477(1888ns) {SwitchToState:179(856ns)}}}}}
2025-11-27T18:17:15.061638Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470934622404787:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(220036386ns) {Configure:104(220030839ns) {ProcessConfigured:135(4236910ns) {StartInitialHandshake:157(4235155ns) {DropHandshakes:470(1150ns) {DropIncomingHandshake:442(244ns) DropOutgoingHandshake:457(459ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:477(2010ns) {SwitchToState:179(908ns)}}}}}
2025-11-27T18:17:15.064290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470935808234076:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6203
2025-11-27T18:17:15.068111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470931830270968:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(246512450ns) {Configure:104(246508540ns) {ProcessConfigured:135(3302948ns) {StartInitialHandshake:157(3079713ns) {DropHandshakes:470(832ns) {DropIncomingHandshake:442(137ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(615ns) PrepareNewSessionHandshake:477(2875ns) {SwitchToState:179(1716ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.071693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470937218757781:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2372
2025-11-27T18:17:15.061583Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470934432584386:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(51224266ns) {Configure:104(51217683ns) {ProcessConfigured:135(2521982ns) {StartInitialHandshake:157(2520326ns) {DropHandshakes:470(745ns) {DropIncomingHandshake:442(291ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(263ns) PrepareNewSessionHandshake:477(2431ns) {SwitchToState:179(1417ns)}}}}}
2025-11-27T18:17:15.071086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470934622404788:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(214452367ns) {Configure:104(214447044ns) {ProcessConfigured:135(7497904ns) {StartInitialHandshake:157(7495125ns) {DropHandshakes:470(2196ns) {DropIncomingHandshake:442(503ns) DropOutgoingHandshake:457(318ns)} GenerateSessionVirtualId:988(1094ns) PrepareNewSessionHandshake:477(1845087ns) {SwitchToState:179(3136ns)}}}}}
2025-11-27T18:17:15.072566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470937218757781:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.061625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470938727551747:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(38996622ns) {Configure:104(38993673ns) {ProcessConfigured:135(2789134ns) {StartInitialHandshake:157(2786805ns) {DropHandshakes:470(1631ns) {DropIncomingHandshake:442(270ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:477(2043ns) {SwitchToState:179(1010ns)}}}}}
2025-11-27T18:17:15.078148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470937218757783:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.073921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470938917372151:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-11-27T18:17:15.078180Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470937218757783:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.072593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470935808234076:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.074456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470938917372159:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-11-27T18:17:15.079692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470937218757783:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:15.072990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470931513266779:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(65572635ns) {Configure:104(65566177ns) {ProcessConfigured:135(39174ns) {StartInitialHandshake:157(38122ns) {DropHandshakes:470(684ns) {DropIncomingHandshake:442(164ns) DropOutgoingHandshake:457(46ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:477(681ns) {SwitchToState:179(153ns)}}}}}
2025-11-27T18:17:15.085863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470935730822539:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(41446761ns) {Configure:104(41442746ns) {ProcessConfigured:135(5326333ns) {StartInitialHandshake:157(5324461ns) {DropHandshakes:470(2152ns) {DropIncomingHandshake:442(593ns) DropOutgoingHandshake:457(341ns)} GenerateSessionVirtualId:988(1136ns) PrepareNewSessionHandshake:477(2326ns) {SwitchToState:179(1030ns)}}}}}
2025-11-27T18:17:15.089215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470931435855181:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(268167950ns) {Configure:104(268163123ns) {ProcessConfigured:135(11693172ns) {StartInitialHandshake:157(11691424ns) {DropHandshakes:470(1323ns) {DropIncomingHandshake:442(362ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(580ns) PrepareNewSessionHandshake:477(2053ns) {SwitchToState:179(815ns)}}}}}
2025-11-27T18:17:15.090132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470935730822551:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59981
2025-11-27T18:17:15.092113Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470935730822540:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18681654ns) {Configure:104(18678210ns) {ProcessConfigured:135(5292818ns) {StartInitialHandshake:157(5290844ns) {DropHandshakes:470(796ns) {DropIncomingHandshake:442(144ns) DropOutgoingHandshake:457(175ns)} GenerateSessionVirtualId:988(589ns) PrepareNewSessionHandshake:477(2330ns) {SwitchToState:179(821ns)}}}}}
2025-11-27T18:17:15.092274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470931830270970:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(66314086ns) {Configure:104(66308955ns) {ProcessConfigured:135(1835248ns) {StartInitialHandshake:157(1833358ns) {DropHandshakes:470(1003809ns) {DropIncomingHandshake:442(427ns) DropOutgoingHandshake:457(1498ns)} GenerateSessionVirtualId:988(1292ns) PrepareNewSessionHandshake:477(2059ns) {SwitchToState:179(1085ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.079784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470938917372151:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.073053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470931513266704:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.094670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470936125238329:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17628519ns) {Configure:104(17616540ns) {ProcessConfigured:135(5461042ns) {StartInitialHandshake:157(5458697ns) {DropHandshakes:470(1642ns) {DropIncomingHandshake:442(551ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(2040ns) {SwitchToState:179(985ns)}}}}}
2025-11-27T18:17:15.080471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470931513266768:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(67664389ns) {Configure:104(67658841ns) {ProcessConfigured:135(8633657ns) {StartInitialHandshake:157(8631527ns) {DropHandshakes:470(1156ns) {DropIncomingHandshake:442(290ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(340ns) PrepareNewSessionHandshake:477(1252ns) {SwitchToState:179(87ns)}}}}}
2025-11-27T18:17:15.080120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470938917372159:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.100220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470935730822551:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.094916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470931513266775:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.102152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470938727551754:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3581
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.081218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470934622404787:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.094950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470931513266775:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.101451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470932923790418:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.101534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470937218757785:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.104953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470938727551759:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3581
2025-11-27T18:17:15.082513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470934622404789:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.105778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470936125238328:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(47717339ns) {Configure:104(47711810ns) {ProcessConfigured:135(2005689ns) {StartInitialHandshake:157(2003844ns) {DropHandshakes:470(2246ns) {DropIncomingHandshake:442(587ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(871ns) PrepareNewSessionHandshake:477(996ns) {SwitchToState:179(717ns)}}}}}
2025-11-27T18:17:15.107286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470935730822556:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59981
2025-11-27T18:17:15.096133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470935808234083:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6203
2025-11-27T18:17:15.100992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470935808234083:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.104235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470931513266775:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:15.107063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470935808234082:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6203
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.101562Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470937218757785:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.105177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470938727551754:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.098436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470938917372165:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-11-27T18:17:15.108432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470935730822549:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.102261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470938917372165:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.104370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470932923790480:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(50691183ns) {Configure:104(50686614ns) {ProcessConfigured:135(2385985ns) {StartInitialHandshake:157(2384208ns) {DropHandshakes:470(1698ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(446ns) PrepareNewSessionHandshake:477(2291ns) {SwitchToState:179(833ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.105252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470938727551760:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3581
2025-11-27T18:17:15.103931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:15.108461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470935730822549:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.106248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470938917372149:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.118209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470931830270969:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(25780094ns) {Configure:104(25774135ns) {ProcessConfigured:135(6106001ns) {StartInitialHandshake:157(6103156ns) {DropHandshakes:470(2145ns) {DropIncomingHandshake:442(281ns) DropOutgoingHandshake:457(406ns)} GenerateSessionVirtualId:988(395ns) PrepareNewSessionHandshake:477(2293ns) {SwitchToState:179(1350ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.104626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470932923790418:2049] [node 3] ICP22 created new session: [1:7577470937218757796:2049]
2025-11-27T18:17:15.110869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470938727551759:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.111289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470934432584386:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.112324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470935730822549:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.118632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470936125238338:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27262
2025-11-27T18:17:15.116984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470938917372169:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-11-27T18:17:15.121449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470931513266704:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17417) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470935808234076:2114] held: no
2025-11-27T18:17:15.118744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470936125238340:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27262
2025-11-27T18:17:15.130517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470936125238338:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.138055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470936125238346:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27262
2025-11-27T18:17:15.107130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470937218757796:2049] [node 3] ICS09 handshake done sender: [1:7577470937218757783:2112] self: [1:7577470937218757782:0] peer: [3:7577470935808234080:0] socket: 146 qp: -1
2025-11-27T18:17:15.122241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470938917372170:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-11-27T18:17:15.134423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470935730822540:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.120459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470938727551760:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.121522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470931513266769:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.144254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470936125238340:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.107525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470937218757796:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:15.122754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470938917372159:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:15.124003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470938727551766:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3581
2025-11-27T18:17:15.141930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:15.121559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.147671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470931830270968:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.145119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470935730822540:2109] [node 6] ICP22 created new session: [4:7577470935730822558:2109]
2025-11-27T18:17:15.107570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:15.138247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470938727551767:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3581
2025-11-27T18:17:15.147966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470934622404787:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.121583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470931513266769:2109] [node 6] ICP22 created new session: [3:7577470935808234086:2109]
2025-11-27T18:17:15.108878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470937218757792:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2372
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.148692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470938727551767:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.148029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470938917372167:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-11-27T18:17:15.121610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470935808234086:2109] [node 6] ICS09 handshake done sender: [3:7577470931513266775:2113] self: [3:7577470931513266774:0] peer: [6:7577470938917372163:0] socket: 140 qp: -1
2025-11-27T18:17:15.146502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470935730822558:2109] [node 6] ICS09 handshake done sender: [4:7577470935730822549:2113] self: [4:7577470935730822547:0] peer: [6:7577470938917372166:0] socket: 163 qp: -1
2025-11-27T18:17:15.109938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470937218757785:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:15.149028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470938727551759:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.112787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470937218757792:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.121633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470935808234086:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:15.147948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470935730822558:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:15.152665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:15.158283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470936125238348:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27262
2025-11-27T18:17:15.153142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470934432584386:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:17:15.112924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470932923790482:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:17:15.153078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470934622404787:2048] [node 1] ICP22 created new session: [6:7577470938917372172:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.165650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470936125238346:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.121655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:15.147994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:15.112942Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470932923790482:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.149624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470935730822556:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.165820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470931830270970:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.127073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470935808234076:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:15.155443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470938727551751:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.155178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470938917372172:2048] [node 1] ICS09 handshake done sender: [6:7577470938917372159:2116] self: [6:7577470938917372164:0] peer: [1:7577470937218757784:0] socket: 150 qp: -1
2025-11-27T18:17:15.155744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470938917372172:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:15.158335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:15.168124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470938917372165:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:15.184966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470938917372168:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-11-27T18:17:15.177472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470936125238348:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.155470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470938727551751:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.130726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470931513266704:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.113753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470937218757779:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.149928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470935730822560:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59981
2025-11-27T18:17:15.113813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:15.113860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470937218757779:2109] [node 6] ICP22 created new session: [1:7577470937218757800:2109]
2025-11-27T18:17:15.113901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470937218757800:2109] [node 6] ICS09 handshake done sender: [1:7577470937218757785:2113] self: [1:7577470937218757784:0] peer: [6:7577470938917372164:0] socket: 147 qp: -1
2025-11-27T18:17:15.113934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470937218757800:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:15.113993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:15.117458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470937218757779:2109] [node 6] ICP18 handshake (actor [1:7577470937218757792:2115]) is too old
2025-11-27T18:17:15.118138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470932923790482:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:15.132214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470932923790417:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.132310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:15.132366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470932923790417:2048] [node 2] ICP22 created new session: [1:7577470937218757805:2048]
2025-11-27T18:17:15.132408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470937218757805:2048] [node 2] ICS09 handshake done sender: [1:7577470932923790482:2110] self: [1:7577470932923790481:0] peer: [2:7577470938727551765:0] socket: 145 qp: -1
2025-11-27T18:17:15.132430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470937218757805:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:15.185172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470938917372149:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19946) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470938917372165:2117] held: no
2025-11-27T18:17:15.132459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:15.130806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:15.150179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470931435855181:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.184390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470936125238328:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.156462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470938727551749:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.130853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470931513266704:2048] [node 1] ICP22 created new session: [3:7577470935808234089:2048]
2025-11-27T18:17:15.185337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470938917372149:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.133089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470937218757804:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2372
2025-11-27T18:17:15.154794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470935730822560:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.156498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470938727551749:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.157203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470935730822559:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59981
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.138938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470932923790418:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(23280227ns) {HandleHandshakeStatus:329(23276415ns) {DropIncomingHandshake:442(15988ns) DropOutgoingHandshake:457(386ns) SwitchToState:179(791ns) ProcessPendingSessionEvents:495(10979228ns) {ForwardSessionEventToSession:612(5483783ns) ForwardSessionEventToSession:612(160023ns) ForwardSessionEventToSession:612(7655ns) ForwardSessionEventToSession:612(3264650ns) ForwardSessionEventToSession:612(7793ns) ForwardSessionEventToSession:612(2682ns) ForwardSessionEventToSession:612(1997148ns) ForwardSessionEventToSession:612(3574ns) ForwardSessionEventToSession:612(2599ns) ForwardSessionEventToSession:612(6892ns)}}}
2025-11-27T18:17:15.140173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470937218757804:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.152600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470937218757787:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.152640Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470937218757787:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.169820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470937218757787:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:15.189742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470937218757780:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.197362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:15.197419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470937218757780:2108] [node 5] ICP22 created new session: [1:7577470937218757809:2108]
2025-11-27T18:17:15.197470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470937218757809:2108] [node 5] ICS09 handshake done sender: [1:7577470937218757787:2114] self: [1:7577470937218757786:0] peer: [5:7577470936125238347:0] socket: 176 qp: -1
2025-11-27T18:17:15.197585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470937218757809:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:15.197620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:15.200098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470937218757795:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.200136Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470937218757795:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.209503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470937218757795:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:15.222124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470932923790480:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.225437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:15.226064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470932923790480:2107] [node 4] ICP22 created new session: [1:7577470937218757812:2107]
2025-11-27T18:17:15.237417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470937218757812:2107] [node 4] ICS09 handshake done sender: [1:7577470937218757795:2116] self: [1:7577470937218757794:0] peer: [4:7577470935730822566:0] socket: 189 qp: -1
2025-11-27T18:17:15.241762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470937218757812:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:15.185449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:15.130894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470935808234089:2048] [node 1] ICS09 handshake done sender: [3:7577470935808234076:2114] self: [3:7577470935808234080:0] peer: [1:7577470937218757782:0] socket: 148 qp: -1
2025-11-27T18:17:15.157173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:15.158626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470934432584386:2048] [node 1] ICP22 created new session: [2:7577470938727551770:2048]
2025-11-27T18:17:15.162609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470938727551770:2048] [node 1] ICS09 handshake done sender: [2:7577470938727551759:2116] self: [2:7577470938727551765:0] peer: [1:7577470932923790481:0] socket: 158 qp: -1
2025-11-27T18:17:15.168261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470938727551749:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:15.169475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470938727551770:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:15.170715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:15.206835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470934432584387:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.207344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:15.209382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470934432584387:2049] [node 3] ICP22 created new session: [2:7577470938727551775:2049]
2025-11-27T18:17:15.213493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470938727551775:2049] [node 3] ICS09 handshake done sender: [2:7577470938727551749:2110] self: [2:7577470938727551748:0] peer: [3:7577470935808234093:0] socket: 172 qp: -1
2025-11-27T18:17:15.225067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470938727551775:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:15.228788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:15.247844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470934432584386:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(75635781ns) {HandleHandshakeStatus:329(75631868ns) {DropIncomingHandshake:442(506ns) DropOutgoingHandshake:457(6176ns) SwitchToState:179(400ns) ProcessPendingSessionEvents:495(21370735ns) {ForwardSessionEventToSession:612(6287494ns) ForwardSessionEventToSession:612(9084ns) ForwardSessionEventToSession:612(2766ns) ForwardSessionEventToSession:612(8570512ns) ForwardSessionEventToSession:612(2207ns) ForwardSessionEventToSession:612(3302271ns) ForwardSessionEventToSession:612(6532ns) ForwardSessionEventToSession:612(3159ns) ForwardSessionEventToSession:612(3143886ns) ForwardSessionEventToSession:612(15281ns)}}}
2025-11-27T18:17:15.248030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470938727551751:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:15.157766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470931435855182:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.228604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470931830270968:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17417) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470936125238340:2116] held: no
2025-11-27T18:17:15.160767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470935730822559:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.185500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470938917372149:2106] [node 4] ICP22 created new session: [6:7577470938917372174:2106]
2025-11-27T18:17:15.248921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.230180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470936125238351:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27262
2025-11-27T18:17:15.130925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470935808234089:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.185543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470938917372174:2106] [node 4] ICS09 handshake done sender: [6:7577470938917372165:2117] self: [6:7577470938917372166:0] peer: [4:7577470935730822547:0] socket: 165 qp: -1
2025-11-27T18:17:15.170486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470931435855180:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.130955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:15.233183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470936125238340:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:15.185578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470938917372174:2106] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.233683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470931830270968:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.265610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470938727551753:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.191557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470935730822565:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59981
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.185616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:15.265646Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470938727551753:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.233748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:15.131417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470935808234088:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6203
2025-11-27T18:17:15.185977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470938917372171:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-11-27T18:17:15.153410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470935808234088:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.267630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470938727551776:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3581
2025-11-27T18:17:15.233793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470931830270968:2048] [node 1] ICP22 created new session: [5:7577470936125238354:2048]
2025-11-27T18:17:15.193582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470935730822550:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.267765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470934432584449:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.153686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470931513266705:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.233841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470936125238354:2048] [node 1] ICS09 handshake done sender: [5:7577470936125238340:2116] self: [5:7577470936125238347:0] peer: [1:7577470937218757786:0] socket: 179 qp: -1
2025-11-27T18:17:15.186107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470938917372151:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:15.193622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470935730822550:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.297230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470932923790480:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(62558516ns) {HandleHandshakeStatus:329(62553708ns) {DropIncomingHandshake:442(525ns) DropOutgoingHandshake:457(547ns) SwitchToState:179(908ns) ProcessPendingSessionEvents:495(22782596ns) {ForwardSessionEventToSession:612(3222605ns) ForwardSessionEventToSession:612(4449ns) ForwardSessionEventToSession:612(13500934ns) ForwardSessionEventToSession:612(3108ns) ForwardSessionEventToSession:612(5220659ns) ForwardSessionEventToSession:612(3347ns) ForwardSessionEventToSession:612(2747ns) ForwardSessionEventToSession:612(801145ns) ForwardSessionEventToSession:612(4106ns) ForwardSessionEventToSession:612(2086ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.267840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:15.193773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470934622404789:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.193820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:15.206073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470934622404789:2050] [node 3] ICP22 created new session: [6:7577470938917372178:2050]
2025-11-27T18:17:15.207534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470938917372178:2050] [node 3] ICS09 handshake done sender: [6:7577470938917372151:2111] self: [6:7577470938917372163:0] peer: [3:7577470931513266774:0] socket: 144 qp: -1
2025-11-27T18:17:15.208329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470938917372178:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:15.210396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:15.218998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470938917372171:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.219100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470938917372168:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.238549Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470934622404787:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(62962958ns) {HandleHandshakeStatus:329(62961212ns) {DropIncomingHandshake:442(1353ns) DropOutgoingHandshake:457(7097ns) SwitchToState:179(241ns) ProcessPendingSessionEvents:495(32599844ns) {ForwardSessionEventToSession:612(16019059ns) ForwardSessionEventToSession:612(8092803ns) ForwardSessionEventToSession:612(4868ns) ForwardSessionEventToSession:612(3019ns) ForwardSessionEventToSession:612(5611829ns) ForwardSessionEventToSession:612(7061ns) ForwardSessionEventToSession:612(2040873ns) ForwardSessionEventToSession:612(3980ns) ForwardSessionEventToSession:612(3724ns) ForwardSessionEventToSession:612(791354ns)}}}
2025-11-27T18:17:15.248395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470934622404845:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.248474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470934622404788:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.249563Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577470934622404787:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:15.208218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470935730822540:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(28514529ns) {HandleHandshakeStatus:329(28511991ns) {DropIncomingHandshake:442(4005954ns) DropOutgoingHandshake:457(1733ns) SwitchToState:179(373ns) ProcessPendingSessionEvents:495(8158860ns) {ForwardSessionEventToSession:612(930001ns) ForwardSessionEventToSession:612(3483ns) ForwardSessionEventToSession:612(2182ns) ForwardSessionEventToSession:612(1775627ns) ForwardSessionEventToSession:612(3066ns) ForwardSessionEventToSession:612(3505666ns) ForwardSessionEventToSession:612(4233ns) ForwardSessionEventToSession:612(2528ns) ForwardSessionEventToSession:612(1914967ns) ForwardSessionEventToSession:612(3188ns)}}}
2025-11-27T18:17:15.154199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470931513266769:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(24242344ns) {HandleHandshakeStatus:329(24239947ns) {DropIncomingHandshake:442(472ns) DropOutgoingHandshake:457(143ns) SwitchToState:179(209ns) ProcessPendingSessionEvents:495(18747414ns) {ForwardSessionEventToSession:612(7315356ns) ForwardSessionEventToSession:612(4627ns) ForwardSessionEventToSession:612(2798ns) ForwardSessionEventToSession:612(4125703ns) ForwardSessionEventToSession:612(2823ns) ForwardSessionEventToSession:612(766947ns) ForwardSessionEventToSession:612(6118ns) ForwardSessionEventToSession:612(2301ns) ForwardSessionEventToSession:612(6491847ns) ForwardSessionEventToSession:612(5070ns)}}}
2025-11-27T18:17:15.233875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470936125238354:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:15.267886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470934432584449:2107] [node 4] ICP22 created new session: [2:7577470938727551778:2107]
2025-11-27T18:17:15.233910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:15.262957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [6:7577470934622404787:2048] [node 1] ICP35 event processing took too much time StateWork:294(18837210ns) {HandleHandshakeStatus:395(15311213ns) {LogHandshakeFail:472(7344857ns) {UpdateErrorStateLog:545(5486ns)}}}
2025-11-27T18:17:15.159096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470935808234079:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.213724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470935730822560:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.159116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470935808234079:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.267947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470938727551778:2107] [node 4] ICS09 handshake done sender: [2:7577470938727551751:2111] self: [2:7577470938727551750:0] peer: [4:7577470935730822563:0] socket: 168 qp: -1
2025-11-27T18:17:15.272878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470938917372182:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-11-27T18:17:15.234463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470936125238353:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27262
2025-11-27T18:17:15.274659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470931435855182:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.288961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470934622404789:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(63952704ns) {HandleHandshakeStatus:329(63950034ns) {DropIncomingHandshake:442(665ns) DropOutgoingHandshake:457(4215ns) SwitchToState:179(553ns) ProcessPendingSessionEvents:495(35369167ns) {ForwardSessionEventToSession:612(6382981ns) ForwardSessionEventToSession:612(8770907ns) ForwardSessionEventToSession:612(19049ns) ForwardSessionEventToSession:612(4228480ns) ForwardSessionEventToSession:612(4083ns) ForwardSessionEventToSession:612(2120ns) ForwardSessionEventToSession:612(2740194ns) ForwardSessionEventToSession:612(4245ns) ForwardSessionEventToSession:612(2944ns) ForwardSessionEventToSession:612(8814618ns)}}}
2025-11-27T18:17:15.290261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470938917372171:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:15.292542Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470934622404845:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32362) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470938917372171:2122] held: no
2025-11-27T18:17:15.292735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470938917372183:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-11-27T18:17:15.297078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470934622404845:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.297358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:15.297397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470934622404845:2104] [node 5] ICP22 created new session: [6:7577470938917372184:2104]
2025-11-27T18:17:15.298059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470938917372168:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:15.298582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470938917372184:2104] [node 5] ICS09 handshake done sender: [6:7577470938917372171:2122] self: [6:7577470938917372180:0] peer: [5:7577470936125238333:0] socket: 188 qp: -1
2025-11-27T18:17:15.299493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470938917372184:2104] [node 5] ICS10 traffic start
2025-11-27T18:17:15.299555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:15.303602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470934622404788:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.303675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:15.303722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470934622404788:2049] [node 2] ICP22 created new session: [6:7577470938917372186:2049]
2025-11-27T18:17:15.303778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470938917372186:2049] [node 2] ICS09 handshake done sender: [6:7577470938917372168:2119] self: [6:7577470938917372181:0] peer: [2:7577470938727551752:0] socket: 143 qp: -1
2025-11-27T18:17:15.303804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470938917372186:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:15.303858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:15.267979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470938727551778:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.315277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470937218757814:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2372
2025-11-27T18:17:15.169788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470935808234094:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6203
2025-11-27T18:17:15.249915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470936125238350:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27262
2025-11-27T18:17:15.286540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470935730822567:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59981
2025-11-27T18:17:15.263041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470936125238335:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.263082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470936125238335:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.268007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:15.282464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470938727551776:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.282557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470938727551758:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.282579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470938727551758:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.285062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470938727551753:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:15.293108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470938727551758:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:15.296097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470938727551747:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.296165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:15.296214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470938727551747:2108] [node 5] ICP22 created new session: [2:7577470938727551783:2108]
2025-11-27T18:17:15.186710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470935808234081:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.186744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470935808234081:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:15.190667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470935808234079:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:15.207564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470935808234088:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:15.210061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470931513266779:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.210110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:15.210144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470931513266779:2107] [node 4] ICP22 created new session: [3:7577470935808234097:2107]
2025-11-27T18:17:15.211728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470931513266705:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.213509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470935808234097:2107] [node 4] ICS09 handshake done sender: [3:7577470935808234079:2115] self: [3:7577470935808234078:0] peer: [4:7577470935730822564:0] socket: 190 qp: -1
2025-11-27T18:17:15.221366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470935808234097:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:15.221417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:15.268913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470936125238346:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:15.270009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470936125238350:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.271604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470931830270969:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:15.272207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470936125238335:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:15.278125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470931830270970:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.285204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470936125238329:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.285312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:15.285363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470936125238329:2109] [node 6] ICP22 created new session: [5:7577470936125238360:2109]
2025-11-27T18:17:15.285413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470936125238360:2109] [node 6] ICS09 handshake done sender: [5:7577470936125238335:2113] self: [5:7577470936125238333:0] peer: [6:7577470938917372180:0] socket: 185 qp: -1
2025-11-27T18:17:15.285441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470936125238360:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:15.285478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:15.293390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:15.293759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470931830270970:2050] [node 3] ICP22 created new session: [5:7577470936125238363:2050]
2025-11-27T18:17:15.297100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470936125238363:2050] [node 3] ICS09 handshake done sender: [5:7577470936125238346:2118] self: [5:7577470936125238349:0] peer: [3:7577470935808234077:0] socket: 193 qp: -1
2025-11-27T18:17:15.297144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470936125238363:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:15.297193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:15.305460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470936125238359:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27262
2025-11-27T18:17:15.310894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470936125238362:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27262
2025-11-27T18:17:15.311318Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470931830270969:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64781) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470936125238350:2120] held: no
2025-11-27T18:17:15.311431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470936125238348:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:15.311676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470936125238350:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:15.311826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470936125238328:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.311888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:15.311934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470936125238328:2108] [node 4] ICP22 created new session: [5:7577470936125238366:2108]
2025-11-27T18:17:15.311976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470936125238366:2108] [node 4] ICS09 handshake done sender: [5:7577470936125238348:2119] self: [5:7577470936125238352:0] peer: [4:7577470935730822546:0] socket: 194 qp: -1
2025-11-27T18:17:15.312006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470936125238366:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:15.312049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:15.315992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470931830270969:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.316834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470931830270970:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(34812971ns) {HandleHandshakeStatus:329(34809995ns) {DropIncomingHandshake:442(708ns) DropOutgoingHandshake:457(4766ns) SwitchToState:179(1094ns) ProcessPendingSessionEvents:495(14405170ns) {ForwardSessionEventToSession:612(4177352ns) ForwardSessionEventToSession:612(2824330ns) ForwardSessionEventToSession:612(1066316ns) ForwardSessionEventToSession:612(798133ns) ForwardSessionEventToSession:612(3861ns) ForwardSessionEventToSession:612(835078ns) ForwardSessionEventToSession:612(1357171ns) ForwardSessionEventToSession:612(2680ns) ForwardSessionEventToSession:612(2291ns) ForwardSessionEventToSession:612(3322070ns)}}}
2025-11-27T18:17:15.318126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:15.318166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470931830270969:2049] [node 2] ICP22 created new session: [5:7577470936125238368:2049]
2025-11-27T18:17:15.288104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:15.321118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470937218757814:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.225647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:15.296258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470938727551783:2108] [node 5] ICS09 handshake done sender: [2:7577470938727551758:2115] self: [2:7577470938727551755:0] peer: [5:7577470936125238358:0] socket: 196 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:15.318207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470936125238368:2049] [node 2] ICS09 handshake done sender: [5:7577470936125238350:2120] self: [5:7577470936125238358:0] peer: [2:7577470938727551755:0] socket: 197 qp: -1
2025-11-27T18:17:15.318237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470936125238368:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:15.318295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:15.288119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470935730822550:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:15.288145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470931435855182:2050] [node 3] ICP22 created new session: [4:7577470935730822570:2050]
2025-11-27T18:17:15.288682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470935730822568:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59981
2025-11-27T18:17:15.293595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470935730822570:2050] [node 3] ICS09 handshake done sender: [4:7577470935730822560:2118] self: [4:7577470935730822564:0] peer: [3:7577470935808234078:0] socket: 153 qp: -1
2025-11-27T18:17:15.294476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470935730822570:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:15.296539Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470931435855181:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64781) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470935730822556:2116] held: no
2025-11-27T18:17:15.296950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:15.298614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470935730822539:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.298668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:15.298711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470935730822539:2108] [node 5] ICP22 created new session: [4:7577470935730822572:2108]
2025-11-27T18:17:15.298743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470935730822572:2108] [node 5] ICS09 handshake done sender: [4:7577470935730822550:2114] self: [4:7577470935730822546:0] peer: [5:7577470936125238352:0] socket: 159 qp: -1
2025-11-27T18:17:15.298770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470935730822572:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:15.298799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:15.299238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470935730822556:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:15.306060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470935730822559:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:15.310406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470931435855181:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.314631Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470931435855182:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(32258558ns) {HandleHandshakeStatus:329(30733790ns) {DropIncomingHandshake:442(737ns) DropOutgoingHandshake:457(10669ns) SwitchToState:179(304ns) ProcessPendingSessionEvents:495(6006077ns) {ForwardSessionEventToSession:612(5902840ns) ForwardSessionEventToSession:612(3839ns) ForwardSessionEventToSession:612(43338ns) ForwardSessionEventToSession:612(2768ns) ForwardSessionEventToSession:612(6567ns) ForwardSessionEventToSession:612(12708ns) ForwardSessionEventToSession:612(2877ns) ForwardSessionEventToSession:612(5442ns) ForwardSessionEventToSession:612(12563ns) ForwardSessionEventToSession:612(2991ns)}}}
2025-11-27T18:17:15.321296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470932923790480:2107] [node 4] ICP18 handshake (actor [1:7577470937218757814:2118]) is too old
2025-11-27T18:17:15.315670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:15.316031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470931435855181:2049] [node 2] ICP22 created new session: [4:7577470935730822575:2049]
2025-11-27T18:17:15.317799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470935730822575:2049] [node 2] ICS09 handshake done sender: [4:7577470935730822556:2116] self: [4:7577470935730822563:0] peer: [2:7577470938727551750:0] socket: 170 qp: -1
2025-11-27T18:17:15.317844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470935730822575:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:15.317883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:15.318713Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470931435855182:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:15.325661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470931435855180:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:15.331116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470935730822577:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59981
2025-11-27T18:17:15.331173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:15.337838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470931435855180:2048] [node 1] ICP22 created new session: [4:7577470935730822578:2048]
2025-11-27T18:17:15.337913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470935730822578:2048] [node 1] ICS09 handshake done sender: [4:7577470935730822559:2117] self: [4:7577470935730822566:0] peer: [1:7577470937218757794:0] socket: 138 qp: -1
2025-11-27T18:17:15.337951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470935730822578:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:15.338013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:15.228884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470931513266705:2049] [node 2] ICP22 created new session: [3:7577470935808234100:2049]
2025-11-27T18:17:15.231412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470935808234096:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6203
2025-11-27T18:17:15.241494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470935808234081:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:15.242227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470935808234100:2049] [node 2] ICS09 handshake done sender: [3:7577470935808234088:2119] self: [3:7577470935808234093:0] peer: [2:7577470938727551748:0] socket: 178 qp: -1
2025-11-27T18:17:15.246721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470935808234100:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:15.246771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:15.262377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470931513266768:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.262977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:15.264708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470935808234103:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6203
2025-11-27T18:17:15.264937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470931513266768:2108] [node 5] ICP22 created new session: [3:7577470935808234102:2108]
2025-11-27T18:17:15.278324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470935808234102:2108] [node 5] ICS09 handshake done sender: [3:7577470935808234081:2116] self: [3:7577470935808234077:0] peer: [5:7577470936125238349:0] socket: 191 qp: -1
2025-11-27T18:17:15.283974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470935808234102:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:15.286776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:15.287078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470935808234103:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:15.288872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470931513266705:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(53530217ns) {HandleHandshakeStatus:329(53526691ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(4128ns) SwitchToState:179(332ns) ProcessPendingSessionEvents:495(3723702ns) {ForwardSessionEventToSession:612(3666829ns) ForwardSessionEventToSession:612(3959ns) ForwardSessionEventToSession:612(3077ns) ForwardSessionEventToSession:612(6698ns) ForwardSessionEventToSession:612(2005ns) ForwardSessionEventToSession:612(5614ns) ForwardSessionEventToSession:612(2828ns) ForwardSessionEventToSession:612(2596ns) ForwardSessionEventToSession:612(7407ns) ForwardSessionEventToSession:612(3006ns)}}}
2025-11-27T18:17:15.293681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577470931513266705:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:15.308310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470931513266768:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(36734883ns) {HandleHandshakeStatus:329(36731673ns) {DropIncomingHandshake:442(624ns) DropOutgoingHandshake:457(829ns) SwitchToState:179(423ns) ProcessPendingSessionEvents:495(4735315ns) {ForwardSessionEventToSession:612(3338280ns) ForwardSessionEventToSession:612(5594ns) ForwardSessionEventToSession:612(261416ns) ForwardSessionEventToSession:612(3816ns) ForwardSessionEventToSession:612(586446ns) ForwardSessionEventToSession:612(279960ns) ForwardSessionEventToSession:612(2356ns) ForwardSessionEventToSession:612(2392ns) ForwardSessionEventToSession:612(227510ns) ForwardSessionEventToSession:612(2678ns)}}}
2025-11-27T18:17:15.309834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577470931513266768:2108] [node 5] ICP18 handshake (actor [3:7577470935808234103:2122]) is too old
2025-11-27T18:17:15.296285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470938727551783:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:15.296938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470934432584450:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:15.297363Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470934432584387:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(77929421ns) {HandleHandshakeStatus:329(77926693ns) {DropIncomingHandshake:442(12427ns) DropOutgoingHandshake:457(637ns) SwitchToState:179(477ns) ProcessPendingSessionEvents:495(29326959ns) {ForwardSessionEventToSession:612(6993558ns) ForwardSessionEventToSession:612(546514ns) ForwardSessionEventToSession:612(4128ns) ForwardSessionEventToSession:612(2079286ns) ForwardSessionEventToSession:612(8115988ns) ForwardSessionEventToSession:612(3395ns) ForwardSessionEventToSession:612(2937ns) ForwardSessionEventToSession:612(5385630ns) ForwardSessionEventToSession:612(4658ns) ForwardSessionEventToSession:612(6168997ns)}}}
2025-11-27T18:17:15.297956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:15.298363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:15.301093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470934432584450:2109] [node 6] ICP22 created new session: [2:7577470938727551785:2109]
2025-11-27T18:17:15.305421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470938727551785:2109] [node 6] ICS09 handshake done sender: [2:7577470938727551753:2112] self: [2:7577470938727551752:0] peer: [6:7577470938917372181:0] socket: 171 qp: -1
2025-11-27T18:17:15.306120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470938727551785:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:15.306176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:15.343000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470931435855180:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16705375ns) {HandleHandshakeStatus:329(16702026ns) {DropIncomingHandshake:442(799ns) DropOutgoingHandshake:457(5033ns) SwitchToState:179(503ns) ProcessPendingSessionEvents:495(4170837ns) {ForwardSessionEventToSession:612(1752939ns) ForwardSessionEventToSession:612(1498937ns) ForwardSessionEventToSession:612(389992ns) ForwardSessionEventToSession:612(6138ns) ForwardSessionEventToSession:612(489430ns) ForwardSessionEventToSession:612(3423ns) ForwardSessionEventToSession:612(2355ns) ForwardSessionEventToSession:612(4580ns) ForwardSessionEventToSession:612(2762ns) ForwardSessionEventToSession:612(5598ns)}}}
2025-11-27T18:17:15.343078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470931435855180:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:15.387500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:15.387732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470935808234089:2048] [node 1] ICS01 socket: 148 reason# EndOfStream
2025-11-27T18:17:15.387775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470931513266704:2048] [node 1] ICP30 unregister session Session# [3:7577470935808234089:2048] VirtualId# [3:7577470935808234080:0]
2025-11-27T18:17:15.387808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470935808234089:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.387887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:15.387940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:15.389637Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:15.389849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470938727551770:2048] [node 1] ICS07 socket disconnect 158 reason# EndOfStream
2025-11-27T18:17:15.389876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470938727551770:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.389976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:15.390019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470938727551770:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:15.392334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:15.392538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470936125238354:2048] [node 1] ICS01 socket: 179 reason# EndOfStream
2025-11-27T18:17:15.392572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470931830270968:2048] [node 1] ICP30 unregister session Session# [5:7577470936125238354:2048] VirtualId# [5:7577470936125238347:0]
2025-11-27T18:17:15.392602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470936125238354:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.392677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:15.392717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:15.392810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:15.392996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470938917372172:2048] [node 1] ICS07 socket disconnect 150 reason# EndOfStream
2025-11-27T18:17:15.393021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470938917372172:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.393071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:15.393108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470938917372172:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:15.399005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:15.391074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470934432584386:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17417) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:15.405373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470938727551770:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:15.409912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470934432584386:2048] [node 1] ICP30 unregister session Session# [2:7577470938727551770:2048] VirtualId# [2:7577470938727551765:0]
2025-11-27T18:17:15.409646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470935730822578:2048] [node 1] ICS07 socket disconnect 138 reason# EndOfStream
2025-11-27T18:17:15.409677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470935730822578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.409796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:15.409840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470935730822578:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:15.413512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:15.413563Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470934432584386:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17417) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.414324Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470934622404787:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17417) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:15.414389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470938917372172:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:15.414419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470934622404787:2048] [node 1] ICP30 unregister session Session# [6:7577470938917372172:2048] VirtualId# [6:7577470938917372164:0]
2025-11-27T18:17:15.414459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:15.414488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470934622404787:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17417) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.414500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17417) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.413582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17417) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.413838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [2:7577470934432584386:2048] [node 1] ICP35 event processing took too much time StateWork:294(22587731ns) {HandleHandshakeStatus:395(22583299ns) {LogHandshakeFail:472(8511ns) {UpdateErrorStateLog:545(1701ns)} DropOutgoingHandshake:457(380ns) ProcessPendingSessionEvents:495(379ns) UnregisterSession:538(4542937ns) {ProcessPendingSessionEvents:495(1282ns) SwitchToInitialState:202(2348ns) {SwitchToState:179(1402ns)}} TransitToErrorState:752(65430ns) {SwitchToState:179(7930ns) ProcessPendingSessionEvents:495(332ns) DropHandshakes:470(2250ns) {DropIncomingHandshake:442(456ns) DropOutgoingHandshake:457(391ns)}}}}
2025-11-27T18:17:15.425588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470934622404787:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:15.425644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577470934432584386:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:15.429779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470931435855180:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17417) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:15.429857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470935730822578:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:15.429894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470931435855180:2048] [node 1] ICP30 unregister session Session# [4:7577470935730822578:2048] VirtualId# [4:7577470935730822566:0]
2025-11-27T18:17:15.429961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:15.429991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470931435855180:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17417) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.430005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17417) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.440469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470931435855180:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:15.464609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:15.464831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470935808234100:2049] [node 2] ICS07 socket disconnect 178 reason# EndOfStream
2025-11-27T18:17:15.464860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470935808234100:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.464926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:15.464960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470935808234100:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:15.467503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470931513266705:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64781) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:15.467537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470935808234100:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:15.469375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470931513266705:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64781) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:15.469726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470935808234100:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:15.469764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470931513266705:2049] [node 2] ICP30 unregister session Session# [3:7577470935808234100:2049] VirtualId# [3:7577470935808234093:0]
2025-11-27T18:17:15.469806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:15.469834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470931513266705:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64781) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.469848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64781) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.469679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:15.469898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470935730822575:2049] [node 2] ICS01 socket: 170 reason# EndOfStream
2025-11-27T18:17:15.469931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470931435855181:2049] [node 2] ICP30 unregister session Session# [4:7577470935730822575:2049] VirtualId# [4:7577470935730822563:0]
2025-11-27T18:17:15.469970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470935730822575:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.470037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:15.470069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:15.471307Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:15.471532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470936125238368:2049] [node 2] ICS01 socket: 197 reason# EndOfStream
2025-11-27T18:17:15.471559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470931830270969:2049] [node 2] ICP30 unregister session Session# [5:7577470936125238368:2049] VirtualId# [5:7577470936125238358:0]
2025-11-27T18:17:15.471584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470936125238368:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.471708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:15.471756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:15.473429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:15.473587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470938917372186:2049] [node 2] ICS01 socket: 143 reason# EndOfStream
2025-11-27T18:17:15.473606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470934622404788:2049] [node 2] ICP30 unregister session Session# [6:7577470938917372186:2049] VirtualId# [6:7577470938917372181:0]
2025-11-27T18:17:15.473622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470938917372186:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.473683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:15.473717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:15.479939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470931513266705:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:15.524154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:15.524418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470938917372178:2050] [node 3] ICS07 socket disconnect 144 reason# EndOfStream
2025-11-27T18:17:15.524449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470938917372178:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.524531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:15.524566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470938917372178:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:15.525456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:15.525644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470935730822570:2050] [node 3] ICS07 socket disconnect 153 reason# EndOfStream
2025-11-27T18:17:15.525664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470935730822570:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.525727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:15.525770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470935730822570:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:15.526022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:15.526189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470936125238363:2050] [node 3] ICS07 socket disconnect 193 reason# EndOfStream
2025-11-27T18:17:15.526328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470934622404789:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15128) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:15.526382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470938917372178:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:15.526647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470931435855182:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15128) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:15.526692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470935730822570:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:15.526719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470931435855182:2050] [node 3] ICP30 unregister session Session# [4:7577470935730822570:2050] VirtualId# [4:7577470935730822564:0]
2025-11-27T18:17:15.526742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:15.526763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470931435855182:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.526773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.526411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470934622404789:2050] [node 3] ICP30 unregister session Session# [6:7577470938917372178:2050] VirtualId# [6:7577470938917372163:0]
2025-11-27T18:17:15.526456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:15.526486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470934622404789:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.526499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.527003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470936125238363:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.529793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:15.529871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470936125238363:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:15.532466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470931830270970:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15128) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:15.532707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470936125238363:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:15.533436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470931830270970:2050] [node 3] ICP30 unregister session Session# [5:7577470936125238363:2050] VirtualId# [5:7577470936125238349:0]
2025-11-27T18:17:15.535849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:15.536054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470931435855182:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:15.535784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470934622404789:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:15.536372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470931830270970:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.536435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:15.545937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470931830270970:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:15.624553Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:15.624966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470936125238366:2108] [node 4] ICS01 socket: 194 reason# EndOfStream
2025-11-27T18:17:15.625051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470936125238328:2108] [node 4] ICP30 unregister session Session# [5:7577470936125238366:2108] VirtualId# [5:7577470936125238352:0]
2025-11-27T18:17:15.625082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470936125238366:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.625163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:15.625202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:15.625375Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:15.625570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470938917372174:2106] [node 4] ICS01 socket: 165 reason# EndOfStream
2025-11-27T18:17:15.625603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470938917372149:2106] [node 4] ICP30 unregister session Session# [6:7577470938917372174:2106] VirtualId# [6:7577470938917372166:0]
2025-11-27T18:17:15.625628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470938917372174:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.625690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:15.625801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:15.675229Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:15.675479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470938917372184:2104] [node 5] ICS01 socket: 188 reason# EndOfStream
2025-11-27T18:17:15.675516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470934622404845:2104] [node 5] ICP30 unregister session Session# [6:7577470938917372184:2104] VirtualId# [6:7577470938917372180:0]
2025-11-27T18:17:15.675575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470938917372184:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:15.675650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:15.675685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:15.772186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470937406656345:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:15.772277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470937406656346:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:15.794007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470937160147467:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:15.794007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470937160147468:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:15.815809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470936221213300:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:15.818485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470936221213301:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:15.840827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470938274270040:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:15.840837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470938274270039:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:15.840864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470938274270041:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:15.864297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470938534698437:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:15.864342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470938534698438:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:15.864368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470938534698439:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:15.891172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470938958258466:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:15.891241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470938958258467:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:15.891322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470938958258468:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:16.089343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470943253225822:2105] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.113757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470942829665794:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27227
2025-11-27T18:17:16.225692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470941455114827:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:16.269429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470941455114828:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:16.229896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470937406656346:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(131950784ns) {Configure:104(131945939ns) {ProcessConfigured:135(57358327ns) {StartInitialHandshake:157(57355314ns) {DropHandshakes:470(2352ns) {DropIncomingHandshake:442(528ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(931ns) PrepareNewSessionHandshake:477(3114ns) {SwitchToState:179(1504ns)}}}}}
2025-11-27T18:17:16.245835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470940516180659:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:16.254474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470941701623708:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:16.269783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470940516180660:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:16.262491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470938958258466:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(26979138ns) {Configure:104(26973636ns) {ProcessConfigured:135(54629ns) {StartInitialHandshake:157(52413ns) {DropHandshakes:470(1804ns) {DropIncomingHandshake:442(425ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(827ns) PrepareNewSessionHandshake:477(1202ns) {SwitchToState:179(300ns)}}}}}
2025-11-27T18:17:16.296558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470942569237398:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:16.281768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470941455114829:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:16.273844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470941701623710:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.262793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470943253225831:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:16.273923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470940516180661:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:16.303081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470942569237399:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.305739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470938534698438:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(30629688ns) {Configure:104(30623513ns) {ProcessConfigured:135(29177238ns) {StartInitialHandshake:157(29174535ns) {DropHandshakes:470(2962ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(543ns)} GenerateSessionVirtualId:988(1262ns) PrepareNewSessionHandshake:477(3256ns) {SwitchToState:179(1199ns)}}}}}
2025-11-27T18:17:16.281561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470941701623712:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.395872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470943253225836:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:17:16.357588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470942829665799:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:16.355651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470942569237400:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43327
2025-11-27T18:17:16.381658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470941701623713:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45348
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.390057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470941701623713:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.355991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470942569237405:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43327
2025-11-27T18:17:16.401305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470937160147467:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(291605051ns) {Configure:104(291599197ns) {ProcessConfigured:135(67795ns) {StartInitialHandshake:157(64935ns) {DropHandshakes:470(2758ns) {DropIncomingHandshake:442(519ns) DropOutgoingHandshake:457(369ns)} GenerateSessionVirtualId:988(561ns) PrepareNewSessionHandshake:477(2257ns) {SwitchToState:179(860ns)}}}}}
2025-11-27T18:17:16.401641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470942829665800:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:16.404159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470940516180660:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(115016532ns) {Configure:104(115004607ns) {ProcessConfigured:135(1162981ns) {StartInitialHandshake:157(1157414ns) {DropHandshakes:470(1806ns) {DropIncomingHandshake:442(445ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(908ns) PrepareNewSessionHandshake:477(1842ns) {SwitchToState:179(714ns)}}}}}
2025-11-27T18:17:16.398890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470942569237400:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.410370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470940516180666:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27712
2025-11-27T18:17:16.410528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470940516180667:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27712
2025-11-27T18:17:16.404825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470941455114825:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38214
2025-11-27T18:17:16.418723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470936221213301:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(148705735ns) {Configure:104(138637558ns) {ProcessConfigured:135(99447ns) {StartInitialHandshake:157(97417ns) {DropHandshakes:470(1830ns) {DropIncomingHandshake:442(287ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(971ns) PrepareNewSessionHandshake:477(7482ns) {SwitchToState:179(893ns)}}}}}
2025-11-27T18:17:16.421704Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470936221213300:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(299673188ns) {Configure:104(299669268ns) {ProcessConfigured:135(5676448ns) {StartInitialHandshake:157(5674443ns) {DropHandshakes:470(2180ns) {DropIncomingHandshake:442(438ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(2996ns) {SwitchToState:179(1268ns)}}}}}
2025-11-27T18:17:16.403800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470942829665794:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.430024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470941701623702:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.430053Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470941701623702:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.430887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470941701623702:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:16.431239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470937406656345:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.431296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:16.428556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470941455114829:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22877790ns) {Configure:104(22871713ns) {ProcessConfigured:135(51362ns) {StartInitialHandshake:157(49144ns) {DropHandshakes:470(1333ns) {DropIncomingHandshake:442(352ns) DropOutgoingHandshake:457(271ns)} GenerateSessionVirtualId:988(340ns) PrepareNewSessionHandshake:477(4452ns) {SwitchToState:179(430ns)}}}}}
2025-11-27T18:17:16.425603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470943253225839:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.442869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470938534698437:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(241153289ns) {Configure:104(241149439ns) {ProcessConfigured:135(68425ns) {StartInitialHandshake:157(65795ns) {DropHandshakes:470(1938ns) {DropIncomingHandshake:442(590ns) DropOutgoingHandshake:457(156ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:477(1755ns) {SwitchToState:179(714ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.454590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470942569237412:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43327
2025-11-27T18:17:16.434020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470937406656345:2048] [node 2] ICP22 created new session: [1:7577470941701623720:2048]
2025-11-27T18:17:16.466816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470942569237404:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.466852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470942569237404:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.467584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470942569237404:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:16.465433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470943253225839:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.465534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470943253225836:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.465646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470943253225831:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.468739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470942569237412:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.468776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470942569237398:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.468836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:16.468866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470942569237398:2109] [node 6] ICP22 created new session: [4:7577470942569237417:2109]
2025-11-27T18:17:16.428789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470941455114825:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.433675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470940516180666:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.457685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470941701623718:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45348
2025-11-27T18:17:16.429300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470937160147467:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.458104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470936221213300:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.487204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470938958258468:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(113045500ns) {Configure:104(113038567ns) {ProcessConfigured:135(12042875ns) {StartInitialHandshake:157(12039969ns) {DropHandshakes:470(1783ns) {DropIncomingHandshake:442(383ns) DropOutgoingHandshake:457(218ns)} GenerateSessionVirtualId:988(485ns) PrepareNewSessionHandshake:477(3635ns) {SwitchToState:179(1739ns)}}}}}
2025-11-27T18:17:16.487121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470938274270040:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(279307768ns) {Configure:104(279302689ns) {ProcessConfigured:135(22234201ns) {StartInitialHandshake:157(22231983ns) {DropHandshakes:470(14664973ns) {DropIncomingHandshake:442(308ns) DropOutgoingHandshake:457(2707ns)} GenerateSessionVirtualId:988(3224ns) PrepareNewSessionHandshake:477(2817ns) {SwitchToState:179(1519ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.481436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470941701623720:2048] [node 2] ICS09 handshake done sender: [1:7577470941701623702:2107] self: [1:7577470941701623701:0] peer: [2:7577470941455114837:0] socket: 145 qp: -1
2025-11-27T18:17:16.487990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470941701623710:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(114453558ns) {Configure:104(114448264ns) {ProcessConfigured:135(33886169ns) {StartInitialHandshake:157(33884831ns) {DropHandshakes:470(2503ns) {DropIncomingHandshake:442(700ns) DropOutgoingHandshake:457(311ns)} GenerateSessionVirtualId:988(705ns) PrepareNewSessionHandshake:477(2882ns) {SwitchToState:179(1240ns)}}}}}
2025-11-27T18:17:16.493307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470941701623720:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:16.429384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470941455114834:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38214
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.487191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470942569237417:2109] [node 6] ICS09 handshake done sender: [4:7577470942569237404:2112] self: [4:7577470942569237403:0] peer: [6:7577470943253225841:0] socket: 158 qp: -1
2025-11-27T18:17:16.488060Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470937160147468:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(338683907ns) {Configure:104(338677941ns) {ProcessConfigured:135(24520745ns) {StartInitialHandshake:157(24518575ns) {DropHandshakes:470(1629ns) {DropIncomingHandshake:442(312ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(342ns) PrepareNewSessionHandshake:477(3144ns) {SwitchToState:179(1357ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.461595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470940516180667:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.493377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:16.497855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470938958258468:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.522092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470941701623712:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(232713283ns) {Configure:104(232706379ns) {ProcessConfigured:135(16251354ns) {StartInitialHandshake:157(16248029ns) {DropHandshakes:470(2410ns) {DropIncomingHandshake:442(481ns) DropOutgoingHandshake:457(479ns)} GenerateSessionVirtualId:988(621ns) PrepareNewSessionHandshake:477(3077ns) {SwitchToState:179(1062ns)}}}}}
2025-11-27T18:17:16.478018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470940516180659:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(199906655ns) {Configure:104(199899656ns) {ProcessConfigured:135(84709ns) {StartInitialHandshake:157(81877ns) {DropHandshakes:470(5340ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(434ns)} GenerateSessionVirtualId:988(943ns) PrepareNewSessionHandshake:477(1557ns) {SwitchToState:179(500ns)}}}}}
2025-11-27T18:17:16.526885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470942829665800:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(120713823ns) {Configure:104(120710208ns) {ProcessConfigured:135(64911ns) {StartInitialHandshake:157(63050ns) {DropHandshakes:470(5211ns) {DropIncomingHandshake:442(589ns) DropOutgoingHandshake:457(248ns)} GenerateSessionVirtualId:988(810ns) PrepareNewSessionHandshake:477(1658ns) {SwitchToState:179(832ns)}}}}}
2025-11-27T18:17:16.500239Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470938958258467:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(197362239ns) {Configure:104(197357292ns) {ProcessConfigured:135(63851ns) {StartInitialHandshake:157(61217ns) {DropHandshakes:470(2313ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(309ns)} GenerateSessionVirtualId:988(1169ns) PrepareNewSessionHandshake:477(1544ns) {SwitchToState:179(520ns)}}}}}
2025-11-27T18:17:16.522561Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470941455114828:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(176646580ns) {Configure:104(176640431ns) {ProcessConfigured:135(8976852ns) {StartInitialHandshake:157(8974702ns) {DropHandshakes:470(8408ns) {DropIncomingHandshake:442(401ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(463ns) PrepareNewSessionHandshake:477(2210ns) {SwitchToState:179(1239ns)}}}}}
2025-11-27T18:17:16.509455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470943253225846:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:17:16.523000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470938958258468:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(18314114ns) {IncomingHandshake:261(18312403ns) {DropIncomingHandshake:442(457ns) PrepareNewSessionHandshake:477(1855ns) {SwitchToState:179(579ns)} GenerateSessionVirtualId:988(684ns) IssueIncomingHandshakeReply:193(11686635ns) {SwitchToState:179(3107ns)}}}
2025-11-27T18:17:16.546718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470943253225831:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16297) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470943253225836:2113] held: no
2025-11-27T18:17:16.487231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470942569237417:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:16.529799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470941701623724:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45348
2025-11-27T18:17:16.525398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470941455114827:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(208563570ns) {Configure:104(208554075ns) {ProcessConfigured:135(11974605ns) {StartInitialHandshake:157(11971816ns) {DropHandshakes:470(2346ns) {DropIncomingHandshake:442(469ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(1059ns) PrepareNewSessionHandshake:477(4447ns) {SwitchToState:179(1423ns)}}}}}
2025-11-27T18:17:16.529946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470941701623722:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45348
2025-11-27T18:17:16.493816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470938274270039:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(257713519ns) {Configure:104(257699884ns) {ProcessConfigured:135(30025608ns) {StartInitialHandshake:157(30012914ns) {DropHandshakes:470(651ns) {DropIncomingHandshake:442(84ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(145ns) PrepareNewSessionHandshake:477(3559ns) {SwitchToState:179(1851ns)}}}}}
2025-11-27T18:17:16.534028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470942829665809:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27227
2025-11-27T18:17:16.484080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470936221213300:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24333931ns) {IncomingHandshake:261(24331284ns) {DropIncomingHandshake:442(238ns) PrepareNewSessionHandshake:477(1263ns) {SwitchToState:179(566ns)} GenerateSessionVirtualId:988(256ns) IssueIncomingHandshakeReply:193(62427ns) {SwitchToState:179(521ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.562097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470943253225840:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.561227Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470942829665799:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(175440111ns) {Configure:104(175431188ns) {ProcessConfigured:135(22920224ns) {StartInitialHandshake:157(16367274ns) {DropHandshakes:470(3730ns) {DropIncomingHandshake:442(502ns) DropOutgoingHandshake:457(711ns)} GenerateSessionVirtualId:988(1791ns) PrepareNewSessionHandshake:477(2987ns) {SwitchToState:179(1532ns)}}}}}
2025-11-27T18:17:16.500880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470940516180665:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.512660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:16.530099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470941701623727:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45348
2025-11-27T18:17:16.565938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470941455114834:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.530438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470941701623709:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.566376Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470943253225831:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(25455136ns) {HandleHandshakeStatus:395(25452022ns) {LogHandshakeFail:472(7532880ns) {UpdateErrorStateLog:545(7948ns)} DropOutgoingHandshake:457(2437ns) ProcessPendingSessionEvents:495(9379274ns) {DropSessionEvent:512(11168ns) DropSessionEvent:512(3641ns) DropSessionEvent:512(3129ns) DropSessionEvent:512(2803ns) DropSessionEvent:512(3431ns) DropSessionEvent:512(18810ns) DropSessionEvent:512(3600ns) DropSessionEvent:512(3517ns) DropSessionEvent:512(2937ns) DropSessionEvent:512(2826ns)}}}
2025-11-27T18:17:16.544743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470942569237399:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(184295112ns) {Configure:104(184291518ns) {ProcessConfigured:135(27481858ns) {StartInitialHandshake:157(27479188ns) {DropHandshakes:470(1645ns) {DropIncomingHandshake:442(367ns) DropOutgoingHandshake:457(312ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:477(2726ns) {SwitchToState:179(1011ns)}}}}}
2025-11-27T18:17:16.500904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470940516180665:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.569938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470943253225842:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:17:16.586991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470941455114839:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38214
2025-11-27T18:17:16.593233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470942829665809:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.530472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470941701623709:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.502095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470940516180665:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.553473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470938274270041:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.615827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470941455114840:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38214
2025-11-27T18:17:16.597417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470938534698437:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.573351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470943253225839:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.537725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470941701623722:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.555828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470942569237422:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43327
2025-11-27T18:17:16.502301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470940516180661:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.512495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:16.512557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470940516180661:2109] [node 6] ICP22 created new session: [3:7577470940516180681:2109]
2025-11-27T18:17:16.512620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470940516180681:2109] [node 6] ICS09 handshake done sender: [3:7577470940516180665:2111] self: [3:7577470940516180664:0] peer: [6:7577470943253225843:0] socket: 164 qp: -1
2025-11-27T18:17:16.573757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470938958258468:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.616130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470941455114846:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38214
2025-11-27T18:17:16.605704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470942829665815:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27227
2025-11-27T18:17:16.634456Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470938534698439:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(179677701ns) {Configure:104(179668333ns) {ProcessConfigured:135(8270738ns) {StartInitialHandshake:157(8267584ns) {DropHandshakes:470(3125ns) {DropIncomingHandshake:442(915ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(616ns) PrepareNewSessionHandshake:477(2239ns) {SwitchToState:179(1267ns)}}}}}
2025-11-27T18:17:16.645406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470942829665815:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.645508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470942829665816:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27227
2025-11-27T18:17:16.665523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470938534698439:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.669966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470942829665817:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27227
2025-11-27T18:17:16.556239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470942569237405:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.555580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470941701623724:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.662121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470941455114840:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.518289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470940516180681:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:16.573814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.557411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470941701623709:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:16.662193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470941455114846:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.562014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470942569237398:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(52925498ns) {HandleHandshakeStatus:329(52922938ns) {DropIncomingHandshake:442(9207ns) DropOutgoingHandshake:457(306ns) SwitchToState:179(216ns) ProcessPendingSessionEvents:495(110179ns) {ForwardSessionEventToSession:612(63539ns) ForwardSessionEventToSession:612(3553ns) ForwardSessionEventToSession:612(11536ns) ForwardSessionEventToSession:612(2272ns) ForwardSessionEventToSession:612(2528ns) ForwardSessionEventToSession:612(5979ns) ForwardSessionEventToSession:612(2260ns) ForwardSessionEventToSession:612(4335ns) ForwardSessionEventToSession:612(2743ns) ForwardSessionEventToSession:612(2704ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.573855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470938958258468:2050] [node 3] ICP22 created new session: [6:7577470943253225848:2050]
2025-11-27T18:17:16.522702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:16.562794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470938274270039:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.564169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470941701623727:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.664318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470941455114825:2109] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:16.542383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470940516180683:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27712
2025-11-27T18:17:16.565010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470942569237412:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:16.573899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470943253225848:2050] [node 3] ICS09 handshake done sender: [6:7577470943253225839:2115] self: [6:7577470943253225843:0] peer: [3:7577470940516180664:0] socket: 165 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.664964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470937160147467:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.571294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470941701623715:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.691583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470942829665817:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.693675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470942829665809:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:16.573938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470943253225848:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.571269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470938274270041:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.571324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470941701623715:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.665039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:16.546556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470940516180661:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(44123168ns) {HandleHandshakeStatus:329(44111105ns) {DropIncomingHandshake:442(686ns) DropOutgoingHandshake:457(956ns) SwitchToState:179(3212ns) ProcessPendingSessionEvents:495(118304ns) {ForwardSessionEventToSession:612(68787ns) ForwardSessionEventToSession:612(3335ns) ForwardSessionEventToSession:612(12316ns) ForwardSessionEventToSession:612(2930ns) ForwardSessionEventToSession:612(1767ns) ForwardSessionEventToSession:612(7709ns) ForwardSessionEventToSession:612(2479ns) ForwardSessionEventToSession:612(2112ns) ForwardSessionEventToSession:612(5198ns) ForwardSessionEventToSession:612(2435ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.695145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470942829665806:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.665098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470937160147467:2048] [node 1] ICP22 created new session: [2:7577470941455114852:2048]
2025-11-27T18:17:16.594191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470937406656346:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.581934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.573966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:16.554172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470940516180663:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.695170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470942829665806:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.554196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470940516180663:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.600666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470943253225847:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:17:16.594270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:16.665146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470941455114852:2048] [node 1] ICS09 handshake done sender: [2:7577470941455114825:2109] self: [2:7577470941455114837:0] peer: [1:7577470941701623701:0] socket: 147 qp: -1
2025-11-27T18:17:16.581998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470938274270041:2050] [node 3] ICP22 created new session: [4:7577470942569237424:2050]
2025-11-27T18:17:16.557936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470940516180663:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:16.701692Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470938534698437:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9393) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470942829665809:2114] held: no
2025-11-27T18:17:16.601493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470943253225836:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:16.582042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470942569237424:2050] [node 3] ICS09 handshake done sender: [4:7577470942569237412:2116] self: [4:7577470942569237421:0] peer: [3:7577470940516180662:0] socket: 179 qp: -1
2025-11-27T18:17:16.594311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470937406656346:2049] [node 3] ICP22 created new session: [1:7577470941701623734:2049]
2025-11-27T18:17:16.665179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470941455114852:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:16.564663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470940516180660:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.634965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470943253225831:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.582079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470942569237424:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:16.701775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470938534698437:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.594360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470941701623734:2049] [node 3] ICS09 handshake done sender: [1:7577470941701623709:2111] self: [1:7577470941701623707:0] peer: [3:7577470940516180675:0] socket: 150 qp: -1
2025-11-27T18:17:16.665212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:16.564923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:16.635027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:16.582111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.678374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470941455114849:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38214
2025-11-27T18:17:16.594398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470941701623734:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:16.701849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:16.565003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470940516180666:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:16.635078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470943253225831:2110] [node 4] ICP22 created new session: [6:7577470943253225851:2110]
2025-11-27T18:17:16.594428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:16.613446Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470937406656345:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(134662814ns) {HandleHandshakeStatus:329(134660384ns) {DropIncomingHandshake:442(227ns) DropOutgoingHandshake:457(511ns) SwitchToState:179(405ns) ProcessPendingSessionEvents:495(47669569ns) {ForwardSessionEventToSession:612(37787824ns) ForwardSessionEventToSession:612(1623812ns) ForwardSessionEventToSession:612(3442ns) ForwardSessionEventToSession:612(3584ns) ForwardSessionEventToSession:612(5092282ns) ForwardSessionEventToSession:612(23357ns) ForwardSessionEventToSession:612(546376ns) ForwardSessionEventToSession:612(3840ns) ForwardSessionEventToSession:612(537055ns) ForwardSessionEventToSession:612(923078ns)}}}
2025-11-27T18:17:16.578248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470940516180660:2107] [node 4] ICP22 created new session: [3:7577470940516180685:2107]
2025-11-27T18:17:16.587052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470936221213300:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.589556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470940516180685:2107] [node 4] ICS09 handshake done sender: [3:7577470940516180663:2110] self: [3:7577470940516180662:0] peer: [4:7577470942569237421:0] socket: 163 qp: -1
2025-11-27T18:17:16.597666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470940516180685:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:16.617200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470940516180687:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27712
2025-11-27T18:17:16.621603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:16.637769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:16.648634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470936221213300:2048] [node 1] ICP22 created new session: [3:7577470940516180689:2048]
2025-11-27T18:17:16.657450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470940516180660:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(92550392ns) {HandleHandshakeStatus:329(92548268ns) {DropIncomingHandshake:442(11924ns) DropOutgoingHandshake:457(362ns) SwitchToState:179(373ns) ProcessPendingSessionEvents:495(8540772ns) {ForwardSessionEventToSession:612(8487687ns) ForwardSessionEventToSession:612(8951ns) ForwardSessionEventToSession:612(2630ns) ForwardSessionEventToSession:612(5055ns) ForwardSessionEventToSession:612(3136ns) ForwardSessionEventToSession:612(2055ns) ForwardSessionEventToSession:612(8212ns) ForwardSessionEventToSession:612(2841ns) ForwardSessionEventToSession:612(5958ns) ForwardSessionEventToSession:612(4403ns)}}}
2025-11-27T18:17:16.661550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470940516180689:2048] [node 1] ICS09 handshake done sender: [3:7577470940516180666:2112] self: [3:7577470940516180675:0] peer: [1:7577470941701623707:0] socket: 161 qp: -1
2025-11-27T18:17:16.665531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470940516180689:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:16.665573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:16.672021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470936221213300:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(89017949ns) {HandleHandshakeStatus:329(89015378ns) {DropIncomingHandshake:442(1719ns) DropOutgoingHandshake:457(14417ns) SwitchToState:179(476ns) ProcessPendingSessionEvents:495(125428ns) {ForwardSessionEventToSession:612(57658ns) ForwardSessionEventToSession:612(2434ns) ForwardSessionEventToSession:612(9370ns) ForwardSessionEventToSession:612(2608ns) ForwardSessionEventToSession:612(14920ns) ForwardSessionEventToSession:612(3027ns) ForwardSessionEventToSession:612(3144ns) ForwardSessionEventToSession:612(20193ns) ForwardSessionEventToSession:612(2296ns) ForwardSessionEventToSession:612(2906ns)}}}
2025-11-27T18:17:16.673480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470940516180687:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.680825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470940516180678:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.680861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470940516180678:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.681994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470936221213301:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.688028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470940516180678:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:16.689102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470940516180659:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.689188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:16.689234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470940516180659:2108] [node 5] ICP22 created new session: [3:7577470940516180693:2108]
2025-11-27T18:17:16.691892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470940516180693:2108] [node 5] ICS09 handshake done sender: [3:7577470940516180678:2116] self: [3:7577470940516180677:0] peer: [5:7577470942829665818:0] socket: 162 qp: -1
2025-11-27T18:17:16.693292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470940516180693:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:16.697569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470940516180694:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27712
2025-11-27T18:17:16.707296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:16.712664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470940516180687:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:16.726299Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470940516180659:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(37907053ns) {HandleHandshakeStatus:329(37905120ns) {DropIncomingHandshake:442(1222ns) DropOutgoingHandshake:457(1130ns) SwitchToState:179(138ns) ProcessPendingSessionEvents:495(6592554ns) {ForwardSessionEventToSession:612(6539193ns) ForwardSessionEventToSession:612(9122ns) ForwardSessionEventToSession:612(4207ns) ForwardSessionEventToSession:612(2919ns) ForwardSessionEventToSession:612(4735ns) ForwardSessionEventToSession:612(5914ns) ForwardSessionEventToSession:612(2196ns) ForwardSessionEventToSession:612(4378ns) ForwardSessionEventToSession:612(1647ns) ForwardSessionEventToSession:612(4493ns)}}}
2025-11-27T18:17:16.729754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470936221213301:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.701893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470938534698437:2048] [node 1] ICP22 created new session: [5:7577470942829665819:2048]
2025-11-27T18:17:16.741382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:16.687269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470941455114841:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.668222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470938274270039:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9393) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470942569237405:2113] held: no
2025-11-27T18:17:16.741440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470936221213301:2049] [node 2] ICP22 created new session: [3:7577470940516180696:2049]
2025-11-27T18:17:16.741632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470940516180696:2049] [node 2] ICS09 handshake done sender: [3:7577470940516180687:2118] self: [3:7577470940516180691:0] peer: [2:7577470941455114838:0] socket: 200 qp: -1
2025-11-27T18:17:16.741671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470940516180696:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:16.741725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:16.741978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577470936221213301:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:16.635121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470943253225851:2110] [node 4] ICS09 handshake done sender: [6:7577470943253225836:2113] self: [6:7577470943253225841:0] peer: [4:7577470942569237403:0] socket: 160 qp: -1
2025-11-27T18:17:16.613643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470941701623721:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.686572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470942569237428:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43327
2025-11-27T18:17:16.687293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470941455114841:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.701937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470942829665819:2048] [node 1] ICS09 handshake done sender: [5:7577470942829665809:2114] self: [5:7577470942829665814:0] peer: [1:7577470941701623717:0] socket: 191 qp: -1
2025-11-27T18:17:16.613665Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470941701623721:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.701977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470942829665819:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:16.635161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470943253225851:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:16.689294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470941455114833:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.691933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470942569237427:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43327
2025-11-27T18:17:16.689315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470941455114833:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.634632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470941701623715:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:16.699077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470942569237405:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:16.635190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:16.702012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:16.691106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470941455114833:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:16.637896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470941701623721:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:16.648074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470943253225853:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:17:16.702451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470942829665799:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.646571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470941701623708:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.693087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470941455114841:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:16.646632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:16.646699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470941701623708:2108] [node 4] ICP22 created new session: [1:7577470941701623739:2108]
2025-11-27T18:17:16.646738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470941701623739:2108] [node 4] ICS09 handshake done sender: [1:7577470941701623715:2113] self: [1:7577470941701623714:0] peer: [4:7577470942569237423:0] socket: 155 qp: -1
2025-11-27T18:17:16.705540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470941455114829:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.646770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470941701623739:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:16.646805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:16.705609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:16.648494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470943253225847:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.658054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470941701623710:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.705652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470941455114829:2110] [node 6] ICP22 created new session: [2:7577470941455114857:2110]
2025-11-27T18:17:16.670257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470943253225822:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.674610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470941701623718:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.705698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470941455114857:2110] [node 6] ICS09 handshake done sender: [2:7577470941455114833:2112] self: [2:7577470941455114832:0] peer: [6:7577470943253225856:0] socket: 169 qp: -1
2025-11-27T18:17:16.679591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470943253225840:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.687448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:16.705726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470941455114857:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:16.687995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470938958258467:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.691935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470941701623710:2109] [node 5] ICP22 created new session: [1:7577470941701623741:2109]
2025-11-27T18:17:16.705755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:16.688025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470943253225853:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.693333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470941701623741:2109] [node 5] ICS09 handshake done sender: [1:7577470941701623721:2115] self: [1:7577470941701623717:0] peer: [5:7577470942829665814:0] socket: 189 qp: -1
2025-11-27T18:17:16.706208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470941455114849:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.692246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470943253225857:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:17:16.693833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470941701623726:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.706555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470941455114829:2110] [node 6] ICP18 handshake (actor [2:7577470941455114849:2120]) is too old
2025-11-27T18:17:16.693366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470938958258466:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.713039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470937160147468:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.693853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470941701623726:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.695109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470941701623741:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:16.697349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470943253225840:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:16.713114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:16.700978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:16.703155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470943253225859:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:17:16.701135Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470941701623726:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:16.713165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470937160147468:2049] [node 3] ICP22 created new session: [2:7577470941455114860:2049]
2025-11-27T18:17:16.707559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470938958258467:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.712481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470937406656345:2048] [node 2] ICP18 handshake (actor [1:7577470941701623718:2114]) is too old
2025-11-27T18:17:16.713209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470941455114860:2049] [node 3] ICS09 handshake done sender: [2:7577470941455114841:2116] self: [2:7577470941455114838:0] peer: [3:7577470940516180691:0] socket: 199 qp: -1
2025-11-27T18:17:16.712190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470943253225853:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:16.734062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470941701623712:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.714786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470941455114860:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:16.719501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:16.734137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:16.727689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470938958258466:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.734175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470941701623712:2110] [node 6] ICP22 created new session: [1:7577470941701623744:2110]
2025-11-27T18:17:16.719963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470937160147467:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32826956ns) {HandleHandshakeStatus:329(32821503ns) {DropIncomingHandshake:442(241ns) DropOutgoingHandshake:457(4436ns) SwitchToState:179(271ns) ProcessPendingSessionEvents:495(32024030ns) {ForwardSessionEventToSession:612(21571927ns) ForwardSessionEventToSession:612(16025ns) ForwardSessionEventToSession:612(1312127ns) ForwardSessionEventToSession:612(2978ns) ForwardSessionEventToSession:612(2351ns) ForwardSessionEventToSession:612(2869862ns) ForwardSessionEventToSession:612(2786ns) ForwardSessionEventToSession:612(348018ns) ForwardSessionEventToSession:612(1513535ns) ForwardSessionEventToSession:612(20241ns)}}}
2025-11-27T18:17:16.727742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:16.734221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470941701623744:2110] [node 6] ICS09 handshake done sender: [1:7577470941701623726:2118] self: [1:7577470941701623725:0] peer: [6:7577470943253225858:0] socket: 202 qp: -1
2025-11-27T18:17:16.723998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:16.727789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470938958258466:2048] [node 1] ICP22 created new session: [6:7577470943253225861:2048]
2025-11-27T18:17:16.734253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470941701623744:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:16.730343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470941455114843:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.727830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470943253225861:2048] [node 1] ICS09 handshake done sender: [6:7577470943253225853:2121] self: [6:7577470943253225858:0] peer: [1:7577470941701623725:0] socket: 203 qp: -1
2025-11-27T18:17:16.734282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:16.730376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470941455114843:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.727855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470943253225861:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:16.727887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:16.758815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470941701623710:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(95301089ns) {HandleHandshakeStatus:329(95298217ns) {DropIncomingHandshake:442(13816ns) DropOutgoingHandshake:457(2135ns) SwitchToState:179(198ns) ProcessPendingSessionEvents:495(23686473ns) {ForwardSessionEventToSession:612(13478790ns) ForwardSessionEventToSession:612(10282ns) ForwardSessionEventToSession:612(2463071ns) ForwardSessionEventToSession:612(3483ns) ForwardSessionEventToSession:612(2178ns) ForwardSessionEventToSession:612(1748681ns) ForwardSessionEventToSession:612(2349ns) ForwardSessionEventToSession:612(1431837ns) ForwardSessionEventToSession:612(4515056ns) ForwardSessionEventToSession:612(9823ns)}}}
2025-11-27T18:17:16.729465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470938958258467:2049] [node 2] ICP22 created new session: [6:7577470943253225860:2049]
2025-11-27T18:17:16.742384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470943253225863:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:17:16.743009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470943253225860:2049] [node 2] ICS09 handshake done sender: [6:7577470943253225840:2116] self: [6:7577470943253225856:0] peer: [2:7577470941455114832:0] socket: 172 qp: -1
2025-11-27T18:17:16.737743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470937160147468:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(24535047ns) {HandleHandshakeStatus:329(24533578ns) {DropIncomingHandshake:442(14991ns) DropOutgoingHandshake:457(433ns) SwitchToState:179(389ns) ProcessPendingSessionEvents:495(102285ns) {ForwardSessionEventToSession:612(51714ns) ForwardSessionEventToSession:612(6841ns) ForwardSessionEventToSession:612(2946ns) ForwardSessionEventToSession:612(2752ns) ForwardSessionEventToSession:612(5080ns) ForwardSessionEventToSession:612(2387ns) ForwardSessionEventToSession:612(13487ns) ForwardSessionEventToSession:612(2297ns) ForwardSessionEventToSession:612(2046ns) ForwardSessionEventToSession:612(5189ns)}}}
2025-11-27T18:17:16.743048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470943253225860:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:16.753296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:16.748254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470941455114843:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:16.754943Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470943253225847:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:16.753790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470941455114827:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.758273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470943253225822:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23402) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470943253225847:2120] held: no
2025-11-27T18:17:16.758683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:16.758359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470943253225822:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.760402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470941455114845:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.758449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:16.760423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470941455114845:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.758754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470943253225822:2105] [node 5] ICP22 created new session: [6:7577470943253225865:2105]
2025-11-27T18:17:16.762753Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470941455114845:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:16.761212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470943253225865:2105] [node 5] ICS09 handshake done sender: [6:7577470943253225847:2120] self: [6:7577470943253225855:0] peer: [5:7577470942829665805:0] socket: 197 qp: -1
2025-11-27T18:17:16.763283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470941455114827:2107] [node 4] ICP22 created new session: [2:7577470941455114863:2107]
2025-11-27T18:17:16.762859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470943253225865:2105] [node 5] ICS10 traffic start
2025-11-27T18:17:16.763587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470941455114863:2107] [node 4] ICS09 handshake done sender: [2:7577470941455114843:2117] self: [2:7577470941455114842:0] peer: [4:7577470942569237431:0] socket: 168 qp: -1
2025-11-27T18:17:16.762915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:16.763758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470941455114828:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.774739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470938958258467:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(56303286ns) {HandleHandshakeStatus:329(56301092ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(10575ns) SwitchToState:179(347ns) ProcessPendingSessionEvents:495(5209138ns) {ForwardSessionEventToSession:612(2274798ns) ForwardSessionEventToSession:612(3634ns) ForwardSessionEventToSession:612(545965ns) ForwardSessionEventToSession:612(2543ns) ForwardSessionEventToSession:612(1900428ns) ForwardSessionEventToSession:612(3328ns) ForwardSessionEventToSession:612(2402ns) ForwardSessionEventToSession:612(248322ns) ForwardSessionEventToSession:612(210666ns) ForwardSessionEventToSession:612(4377ns)}}}
2025-11-27T18:17:16.765877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470941455114863:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:16.765918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:16.769488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:16.774663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470941455114828:2108] [node 5] ICP22 created new session: [2:7577470941455114866:2108]
2025-11-27T18:17:16.714562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470942829665823:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27227
2025-11-27T18:17:16.714946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470942829665815:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:16.775655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470941455114866:2108] [node 5] ICS09 handshake done sender: [2:7577470941455114845:2118] self: [2:7577470941455114844:0] peer: [5:7577470942829665833:0] socket: 215 qp: -1
2025-11-27T18:17:16.715093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470938534698439:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.775986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470941455114866:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:16.776044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:16.715147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:16.781055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470941455114827:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(23653263ns) {HandleHandshakeStatus:329(23650907ns) {DropIncomingHandshake:442(21322ns) DropOutgoingHandshake:457(364ns) SwitchToState:179(386ns) ProcessPendingSessionEvents:495(3562259ns) {ForwardSessionEventToSession:612(1653976ns) ForwardSessionEventToSession:612(4310ns) ForwardSessionEventToSession:612(510333ns) ForwardSessionEventToSession:612(4126ns) ForwardSessionEventToSession:612(172570ns) ForwardSessionEventToSession:612(930175ns) ForwardSessionEventToSession:612(2694ns) ForwardSessionEventToSession:612(2341ns) ForwardSessionEventToSession:612(268105ns) ForwardSessionEventToSession:612(2826ns)}}}
2025-11-27T18:17:16.715190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470938534698439:2050] [node 3] ICP22 created new session: [5:7577470942829665825:2050]
2025-11-27T18:17:16.715236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470942829665825:2050] [node 3] ICS09 handshake done sender: [5:7577470942829665815:2117] self: [5:7577470942829665818:0] peer: [3:7577470940516180677:0] socket: 198 qp: -1
2025-11-27T18:17:16.715259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470942829665825:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:16.715285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:16.720518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470942829665806:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:16.733898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470942829665827:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27227
2025-11-27T18:17:16.735775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470942829665828:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27227
2025-11-27T18:17:16.745816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470942829665800:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.745893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:16.748980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470942829665800:2111] [node 6] ICP22 created new session: [5:7577470942829665829:2111]
2025-11-27T18:17:16.749043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470942829665817:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:16.753448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470942829665829:2111] [node 6] ICS09 handshake done sender: [5:7577470942829665806:2113] self: [5:7577470942829665805:0] peer: [6:7577470943253225855:0] socket: 188 qp: -1
2025-11-27T18:17:16.753486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470942829665829:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:16.753533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:16.755712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470942829665799:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.755776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:16.755817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470942829665799:2110] [node 4] ICP22 created new session: [5:7577470942829665831:2110]
2025-11-27T18:17:16.755850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470942829665831:2110] [node 4] ICS09 handshake done sender: [5:7577470942829665817:2119] self: [5:7577470942829665821:0] peer: [4:7577470942569237415:0] socket: 205 qp: -1
2025-11-27T18:17:16.755880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470942829665831:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:16.707467Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470938274270041:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(107242792ns) {HandleHandshakeStatus:329(107239448ns) {DropIncomingHandshake:442(572ns) DropOutgoingHandshake:457(10561853ns) SwitchToState:179(1665ns) ProcessPendingSessionEvents:495(56045000ns) {ForwardSessionEventToSession:612(31272829ns) ForwardSessionEventToSession:612(4178961ns) ForwardSessionEventToSession:612(2759ns) ForwardSessionEventToSession:612(7607338ns) ForwardSessionEventToSession:612(4828011ns) ForwardSessionEventToSession:612(8601ns) ForwardSessionEventToSession:612(4329918ns) ForwardSessionEventToSession:612(2928ns) ForwardSessionEventToSession:612(3251ns) ForwardSessionEventToSession:612(3783594ns)}}}
2025-11-27T18:17:16.755921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:16.760016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470942829665827:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.760143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470938534698438:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.714293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470938274270039:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.769541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470938534698438:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18069) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470942829665827:2121] held: no
2025-11-27T18:17:16.714324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:16.774354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470942829665834:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27227
2025-11-27T18:17:16.719533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470942569237418:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.777847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470942829665827:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:16.719571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470942569237418:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:16.779214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470938534698438:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.720407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470942569237428:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.784632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:16.784677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470938534698438:2049] [node 2] ICP22 created new session: [5:7577470942829665835:2049]
2025-11-27T18:17:16.724105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470938274270039:2048] [node 1] ICP22 created new session: [4:7577470942569237429:2048]
2025-11-27T18:17:16.784721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470942829665835:2049] [node 2] ICS09 handshake done sender: [5:7577470942829665827:2121] self: [5:7577470942829665833:0] peer: [2:7577470941455114844:0] socket: 180 qp: -1
2025-11-27T18:17:16.724166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470942569237429:2048] [node 1] ICS09 handshake done sender: [4:7577470942569237405:2113] self: [4:7577470942569237423:0] peer: [1:7577470941701623714:0] socket: 156 qp: -1
2025-11-27T18:17:16.784759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470942829665835:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:16.724206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470942569237429:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:16.784817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:16.724257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:16.724932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470938274270040:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:16.729481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470942569237418:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:16.750971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470942569237399:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:16.751055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:16.751101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470942569237399:2108] [node 5] ICP22 created new session: [4:7577470942569237435:2108]
2025-11-27T18:17:16.751141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470942569237435:2108] [node 5] ICS09 handshake done sender: [4:7577470942569237418:2117] self: [4:7577470942569237415:0] peer: [5:7577470942829665821:0] socket: 157 qp: -1
2025-11-27T18:17:16.753365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470942569237433:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43327
2025-11-27T18:17:16.753413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470942569237435:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:16.753475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:16.757091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470942569237433:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:16.759226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577470942569237399:2108] [node 5] ICP18 handshake (actor [4:7577470942569237433:2121]) is too old
2025-11-27T18:17:16.765580Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470938274270040:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18069) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470942569237428:2120] held: no
2025-11-27T18:17:16.774382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470942569237434:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43327
2025-11-27T18:17:16.778623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470942569237428:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:16.781305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470938274270040:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:16.781358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:16.781399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470938274270040:2049] [node 2] ICP22 created new session: [4:7577470942569237437:2049]
2025-11-27T18:17:16.781454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470942569237437:2049] [node 2] ICS09 handshake done sender: [4:7577470942569237428:2120] self: [4:7577470942569237431:0] peer: [2:7577470941455114842:0] socket: 207 qp: -1
2025-11-27T18:17:16.781486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470942569237437:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:16.781539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:16.834826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:16.835096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470941455114852:2048] [node 1] ICS07 socket disconnect 147 reason# EndOfStream
2025-11-27T18:17:16.835122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470941455114852:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.835210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:16.835250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470941455114852:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:16.838942Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470937160147467:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9393) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:16.839000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470941455114852:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:16.839030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470937160147467:2048] [node 1] ICP30 unregister session Session# [2:7577470941455114852:2048] VirtualId# [2:7577470941455114837:0]
2025-11-27T18:17:16.839077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:16.839104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470937160147467:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:16.839118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:16.841440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:16.841682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470940516180689:2048] [node 1] ICS01 socket: 161 reason# EndOfStream
2025-11-27T18:17:16.841720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470936221213300:2048] [node 1] ICP30 unregister session Session# [3:7577470940516180689:2048] VirtualId# [3:7577470940516180675:0]
2025-11-27T18:17:16.841754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470940516180689:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.841834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:16.841875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:16.846957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577470937160147467:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:16.846979Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:16.847202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470943253225861:2048] [node 1] ICS01 socket: 203 reason# EndOfStream
2025-11-27T18:17:16.847241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470938958258466:2048] [node 1] ICP30 unregister session Session# [6:7577470943253225861:2048] VirtualId# [6:7577470943253225858:0]
2025-11-27T18:17:16.847273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470943253225861:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.847341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:16.847420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:16.847512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:16.848418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470942569237429:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-27T18:17:16.848446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470938274270039:2048] [node 1] ICP30 unregister session Session# [4:7577470942569237429:2048] VirtualId# [4:7577470942569237423:0]
2025-11-27T18:17:16.848469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470942569237429:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.848558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:16.848617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:16.849405Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:16.849611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470942829665819:2048] [node 1] ICS01 socket: 191 reason# EndOfStream
2025-11-27T18:17:16.849642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470938534698437:2048] [node 1] ICP30 unregister session Session# [5:7577470942829665819:2048] VirtualId# [5:7577470942829665814:0]
2025-11-27T18:17:16.849664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470942829665819:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.849735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:16.849769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:16.874084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:16.874288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470940516180696:2049] [node 2] ICS07 socket disconnect 200 reason# EndOfStream
2025-11-27T18:17:16.874313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470940516180696:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.874384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:16.874425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470940516180696:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:16.878410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470936221213301:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18069) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:16.878451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470940516180696:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:16.879743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470936221213301:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18069) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:16.879792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470940516180696:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:16.879820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470936221213301:2049] [node 2] ICP30 unregister session Session# [3:7577470940516180696:2049] VirtualId# [3:7577470940516180691:0]
2025-11-27T18:17:16.879861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:16.879885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470936221213301:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18069) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:16.879896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18069) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:16.885470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:16.885671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470942569237437:2049] [node 2] ICS01 socket: 207 reason# EndOfStream
2025-11-27T18:17:16.885703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470938274270040:2049] [node 2] ICP30 unregister session Session# [4:7577470942569237437:2049] VirtualId# [4:7577470942569237431:0]
2025-11-27T18:17:16.885733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470942569237437:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.885816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:16.885855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:16.885936Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:16.886089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470942829665835:2049] [node 2] ICS01 socket: 180 reason# EndOfStream
2025-11-27T18:17:16.886114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470938534698438:2049] [node 2] ICP30 unregister session Session# [5:7577470942829665835:2049] VirtualId# [5:7577470942829665833:0]
2025-11-27T18:17:16.886132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470942829665835:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.886180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:16.886208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:16.886283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:16.886451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470943253225860:2049] [node 2] ICS07 socket disconnect 172 reason# EndOfStream
2025-11-27T18:17:16.886472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470943253225860:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.886523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:16.886557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470943253225860:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:16.888309Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470938958258467:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18069) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:16.888363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470943253225860:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:16.888388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470938958258467:2049] [node 2] ICP30 unregister session Session# [6:7577470943253225860:2049] VirtualId# [6:7577470943253225856:0]
2025-11-27T18:17:16.888427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:16.888454Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470938958258467:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18069) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:16.888467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18069) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:16.894056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470936221213301:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:16.901374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470938958258467:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:16.945521Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:16.945769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470942569237424:2050] [node 3] ICS07 socket disconnect 179 reason# EndOfStream
2025-11-27T18:17:16.945796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470942569237424:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.945893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:16.949538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:16.949998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:16.950236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470942829665825:2050] [node 3] ICS01 socket: 198 reason# EndOfStream
2025-11-27T18:17:16.945937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470942569237424:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:16.950279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470938534698439:2050] [node 3] ICP30 unregister session Session# [5:7577470942829665825:2050] VirtualId# [5:7577470942829665818:0]
2025-11-27T18:17:16.953228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470943253225848:2050] [node 3] ICS01 socket: 165 reason# EndOfStream
2025-11-27T18:17:16.955490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470938958258468:2050] [node 3] ICP30 unregister session Session# [6:7577470943253225848:2050] VirtualId# [6:7577470943253225843:0]
2025-11-27T18:17:16.953109Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470938274270041:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16492) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:16.955021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470942569237424:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:16.955061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470938274270041:2050] [node 3] ICP30 unregister session Session# [4:7577470942569237424:2050] VirtualId# [4:7577470942569237421:0]
2025-11-27T18:17:16.955113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:16.955142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470938274270041:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16492) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:16.955155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16492) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:16.955531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470943253225848:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.955611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:16.955648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:16.950307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470942829665825:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:16.950388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:16.950431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:16.965217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470938274270041:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:17.026006Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:17.026199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470943253225851:2110] [node 4] ICS01 socket: 160 reason# EndOfStream
2025-11-27T18:17:17.026231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470943253225831:2110] [node 4] ICP30 unregister session Session# [6:7577470943253225851:2110] VirtualId# [6:7577470943253225841:0]
2025-11-27T18:17:17.026254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470943253225851:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:17.026296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:17.026332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:17.025489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:17.025735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470942829665831:2110] [node 4] ICS01 socket: 205 reason# EndOfStream
2025-11-27T18:17:17.025772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470942829665799:2110] [node 4] ICP30 unregister session Session# [5:7577470942829665831:2110] VirtualId# [5:7577470942829665821:0]
2025-11-27T18:17:17.025800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470942829665831:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:17.025877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:17.025926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:17.062841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:17.063083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470943253225865:2105] [node 5] ICS01 socket: 197 reason# EndOfStream
2025-11-27T18:17:17.063219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470943253225822:2105] [node 5] ICP30 unregister session Session# [6:7577470943253225865:2105] VirtualId# [6:7577470943253225855:0]
2025-11-27T18:17:17.063252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470943253225865:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:17.063460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:17.063506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:17.141097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470945339082657:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:17.141135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470945339082658:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:17.169539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470946456398033:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:17.173846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470946456398032:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:17.191287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470945497382924:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:17.191329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470945497382925:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:17.216537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470945820442126:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:17.216585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470945820442128:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:17.216863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470945820442127:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:17.246655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470944860741909:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:17.246695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470944860741910:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:17.246722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470944860741911:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:17.272817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470944296551452:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:17.273012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470944296551454:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:17.273029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470944296551453:2049] [node 2] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.348865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470944860741970:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47410
2025-11-27T18:17:17.346930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470944296551515:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:17.396599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470945497382986:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:17.396927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470945497382987:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:17.404724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470945497382989:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:17.481848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470946456398095:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:17.462412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470945339082723:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:17.541600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470946456398097:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:17.529359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470945820442189:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.566997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470945497382986:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(145961899ns) {Configure:104(145950973ns) {ProcessConfigured:135(72706ns) {StartInitialHandshake:157(69966ns) {DropHandshakes:470(3697ns) {DropIncomingHandshake:442(851ns) DropOutgoingHandshake:457(676ns)} GenerateSessionVirtualId:988(1140ns) PrepareNewSessionHandshake:477(1861ns) {SwitchToState:179(581ns)}}}}}
2025-11-27T18:17:17.465488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470945339082722:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:17.550156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470946456398096:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:17.528977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470944860741970:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.595159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470944296551453:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(73185512ns) {Configure:104(73179956ns) {ProcessConfigured:135(95922ns) {StartInitialHandshake:157(92714ns) {DropHandshakes:470(5940ns) {DropIncomingHandshake:442(836ns) DropOutgoingHandshake:457(423ns)} GenerateSessionVirtualId:988(1446ns) PrepareNewSessionHandshake:477(2345ns) {SwitchToState:179(945ns)}}}}}
2025-11-27T18:17:17.553443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470944860741974:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:17.567484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470945497382992:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8006
2025-11-27T18:17:17.541521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470945339082724:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:17.588459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470945820442190:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:17.621475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470945497382992:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.580692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470944860741975:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:17.642815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470945820442127:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(229048622ns) {Configure:104(229038787ns) {ProcessConfigured:135(72870ns) {StartInitialHandshake:157(68974ns) {DropHandshakes:470(3276ns) {DropIncomingHandshake:442(744ns) DropOutgoingHandshake:457(495ns)} GenerateSessionVirtualId:988(1518ns) PrepareNewSessionHandshake:477(2299ns) {SwitchToState:179(815ns)}}}}}
2025-11-27T18:17:17.643356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470945820442191:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37686
2025-11-27T18:17:17.643515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470945820442192:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37686
2025-11-27T18:17:17.650921Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470945820442189:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(88561017ns) {Configure:104(88549136ns) {ProcessConfigured:135(68652ns) {StartInitialHandshake:157(64000ns) {DropHandshakes:470(3188ns) {DropIncomingHandshake:442(688ns) DropOutgoingHandshake:457(415ns)} GenerateSessionVirtualId:988(948ns) PrepareNewSessionHandshake:477(2003ns) {SwitchToState:179(444ns)}}}}}
2025-11-27T18:17:17.611283Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470944860741974:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22407512ns) {Configure:104(12829455ns) {ProcessConfigured:135(79554ns) {StartInitialHandshake:157(76398ns) {DropHandshakes:470(5962ns) {DropIncomingHandshake:442(578ns) DropOutgoingHandshake:457(501ns)} GenerateSessionVirtualId:988(1065ns) PrepareNewSessionHandshake:477(1929ns) {SwitchToState:179(683ns)}}}}}
2025-11-27T18:17:17.683164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470945820442191:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.684054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470944296551525:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.683367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470945820442192:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.701066Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470945339082658:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(255390232ns) {Configure:104(255380306ns) {ProcessConfigured:135(36380304ns) {StartInitialHandshake:157(36377018ns) {DropHandshakes:470(3314ns) {DropIncomingHandshake:442(735ns) DropOutgoingHandshake:457(411ns)} GenerateSessionVirtualId:988(859ns) PrepareNewSessionHandshake:477(2994ns) {SwitchToState:179(1157ns)}}}}}
2025-11-27T18:17:17.702058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470945339082723:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(161776015ns) {Configure:104(161769390ns) {ProcessConfigured:135(38309ns) {StartInitialHandshake:157(36996ns) {DropHandshakes:470(668ns) {DropIncomingHandshake:442(229ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(111ns) PrepareNewSessionHandshake:477(628ns) {SwitchToState:179(173ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.710661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470945820442200:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37686
2025-11-27T18:17:17.714396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470945339082657:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(260518908ns) {Configure:104(260514312ns) {ProcessConfigured:135(30320158ns) {StartInitialHandshake:157(30317478ns) {DropHandshakes:470(1652ns) {DropIncomingHandshake:442(367ns) DropOutgoingHandshake:457(422ns)} GenerateSessionVirtualId:988(512ns) PrepareNewSessionHandshake:477(4299ns) {SwitchToState:179(2188ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.705606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470944296551528:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24940
2025-11-27T18:17:17.694445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470946456398097:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(120046635ns) {Configure:104(120039473ns) {ProcessConfigured:135(72505ns) {StartInitialHandshake:157(69084ns) {DropHandshakes:470(6788ns) {DropIncomingHandshake:442(979ns) DropOutgoingHandshake:457(755ns)} GenerateSessionVirtualId:988(1024ns) PrepareNewSessionHandshake:477(1941ns) {SwitchToState:179(778ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.706889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470944296551526:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24940
2025-11-27T18:17:17.694949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470946456398100:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2354
2025-11-27T18:17:17.738463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470945497382924:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(273115109ns) {Configure:104(273106098ns) {ProcessConfigured:135(11174643ns) {StartInitialHandshake:157(11171019ns) {DropHandshakes:470(2962ns) {DropIncomingHandshake:442(922ns) DropOutgoingHandshake:457(390ns)} GenerateSessionVirtualId:988(1319ns) PrepareNewSessionHandshake:477(5443ns) {SwitchToState:179(3175ns)}}}}}
2025-11-27T18:17:17.727074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470944296551528:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.746852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470944860741986:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47410
2025-11-27T18:17:17.725961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470946456398105:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2354
2025-11-27T18:17:17.743653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470945497383001:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8006
2025-11-27T18:17:17.775130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470945339082722:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(236256664ns) {Configure:104(236248129ns) {ProcessConfigured:135(17112332ns) {StartInitialHandshake:157(17109510ns) {DropHandshakes:470(3625ns) {DropIncomingHandshake:442(869ns) DropOutgoingHandshake:457(416ns)} GenerateSessionVirtualId:988(1027ns) PrepareNewSessionHandshake:477(4152ns) {SwitchToState:179(1621ns)}}}}}
2025-11-27T18:17:17.790550Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470945820442128:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(303984731ns) {Configure:104(303977789ns) {ProcessConfigured:135(36786771ns) {StartInitialHandshake:157(36784106ns) {DropHandshakes:470(1664ns) {DropIncomingHandshake:442(709ns) DropOutgoingHandshake:457(231ns)} GenerateSessionVirtualId:988(422ns) PrepareNewSessionHandshake:477(4848ns) {SwitchToState:179(1704ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.727205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470944296551453:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:17.801491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470945820442126:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(287044860ns) {Configure:104(287037417ns) {ProcessConfigured:135(16189448ns) {StartInitialHandshake:157(16187124ns) {DropHandshakes:470(2395ns) {DropIncomingHandshake:442(388ns) DropOutgoingHandshake:457(519ns)} GenerateSessionVirtualId:988(1495ns) PrepareNewSessionHandshake:477(1909ns) {SwitchToState:179(234ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.769517Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470944860741975:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(136813489ns) {Configure:104(136812428ns) {ProcessConfigured:135(16520894ns) {StartInitialHandshake:157(16463134ns) {DropHandshakes:470(2563ns) {DropIncomingHandshake:442(373ns) DropOutgoingHandshake:457(441ns)} GenerateSessionVirtualId:988(5055ns) PrepareNewSessionHandshake:477(4171ns) {SwitchToState:179(1576ns)}} IncomingHandshake:261(27085ns) {DropIncomingHandshake:442(358ns) PrepareNewSessionHandshake:477(343ns) {SwitchToState:179(153ns)} GenerateSessionVirtualId:988(332ns) IssueIncomingHandshakeReply:193(1230ns)}}}}
2025-11-27T18:17:17.809363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470945820442128:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:17.726022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470946456398106:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2354
2025-11-27T18:17:17.777303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470945497382989:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(278926616ns) {Configure:104(278921388ns) {ProcessConfigured:135(11214285ns) {StartInitialHandshake:157(11213128ns) {DropHandshakes:470(1533ns) {DropIncomingHandshake:442(235ns) DropOutgoingHandshake:457(291ns)} GenerateSessionVirtualId:988(510ns) PrepareNewSessionHandshake:477(3155ns) {SwitchToState:179(943ns)}}}}}
2025-11-27T18:17:17.747522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470944296551530:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24940
2025-11-27T18:17:17.781688Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470944860741911:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(241093996ns) {Configure:104(241087754ns) {ProcessConfigured:135(11994560ns) {StartInitialHandshake:157(11991742ns) {DropHandshakes:470(3984ns) {DropIncomingHandshake:442(498ns) DropOutgoingHandshake:457(512ns)} GenerateSessionVirtualId:988(881ns) PrepareNewSessionHandshake:477(4770ns) {SwitchToState:179(2802ns)}}}}}
2025-11-27T18:17:17.747631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470944296551531:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24940
2025-11-27T18:17:17.803995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470945339082738:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48710
2025-11-27T18:17:17.783295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470945497382987:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(307359865ns) {Configure:104(307355251ns) {ProcessConfigured:135(20823700ns) {StartInitialHandshake:157(20821306ns) {DropHandshakes:470(4154837ns) {DropIncomingHandshake:442(1628ns) DropOutgoingHandshake:457(592ns)} GenerateSessionVirtualId:988(1118ns) PrepareNewSessionHandshake:477(3437ns) {SwitchToState:179(1493ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.789503Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470944860741909:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(307059134ns) {Configure:104(307054102ns) {ProcessConfigured:135(11610449ns) {StartInitialHandshake:157(11606743ns) {DropHandshakes:470(3226ns) {DropIncomingHandshake:442(764ns) DropOutgoingHandshake:457(364ns)} GenerateSessionVirtualId:988(1318ns) PrepareNewSessionHandshake:477(3218ns) {SwitchToState:179(1093ns)}}}}}
2025-11-27T18:17:17.737396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470946456398099:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.846666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470945820442208:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37686
2025-11-27T18:17:17.797691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470945497383004:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8006
2025-11-27T18:17:17.827031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470945339082724:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(240697430ns) {Configure:104(240689428ns) {ProcessConfigured:135(18316818ns) {StartInitialHandshake:157(12085633ns) {DropHandshakes:470(1954ns) {DropIncomingHandshake:442(281ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(1435ns) PrepareNewSessionHandshake:477(3862ns) {SwitchToState:179(1701ns)}}}}}
2025-11-27T18:17:17.737438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470946456398099:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.765571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470944296551525:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(59781363ns) {Configure:104(59775126ns) {ProcessConfigured:135(72740ns) {StartInitialHandshake:157(70394ns) {DropHandshakes:470(3258ns) {DropIncomingHandshake:442(820ns) DropOutgoingHandshake:457(380ns)} GenerateSessionVirtualId:988(607ns) PrepareNewSessionHandshake:477(1136ns) {SwitchToState:179(571ns)}}}}}
2025-11-27T18:17:17.850005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470945820442197:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.810220Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470945497382925:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(336931603ns) {Configure:104(330934381ns) {ProcessConfigured:135(13999488ns) {StartInitialHandshake:157(13996618ns) {DropHandshakes:470(3079ns) {DropIncomingHandshake:442(749ns) DropOutgoingHandshake:457(524ns)} GenerateSessionVirtualId:988(955ns) PrepareNewSessionHandshake:477(2547ns) {SwitchToState:179(1031ns)}}}}}
2025-11-27T18:17:17.792261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470944860741986:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.797921Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470944860741910:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(295072011ns) {Configure:104(295066746ns) {ProcessConfigured:135(18376487ns) {StartInitialHandshake:157(18373235ns) {DropHandshakes:470(3233ns) {DropIncomingHandshake:442(582ns) DropOutgoingHandshake:457(465ns)} GenerateSessionVirtualId:988(809ns) PrepareNewSessionHandshake:477(6004ns) {SwitchToState:179(2523ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.740409Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470946456398032:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(347589093ns) {Configure:104(347582387ns) {ProcessConfigured:135(17793317ns) {StartInitialHandshake:157(17790195ns) {DropHandshakes:470(1996ns) {DropIncomingHandshake:442(477ns) DropOutgoingHandshake:457(484ns)} GenerateSessionVirtualId:988(442ns) PrepareNewSessionHandshake:477(3660ns) {SwitchToState:179(1757ns)}}}}}
2025-11-27T18:17:17.806007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470944296551530:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.813273Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470944296551528:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:17.825078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470944296551526:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.850022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470945820442200:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.835739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470945339082738:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.810241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470945497383004:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.806612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470944860741911:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:17.826044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470944296551525:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:17.742946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470946456398099:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:17.790984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470946456398033:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(300992516ns) {Configure:104(299866580ns) {ProcessConfigured:135(20577254ns) {StartInitialHandshake:157(20574506ns) {DropHandshakes:470(3473ns) {DropIncomingHandshake:442(1174ns) DropOutgoingHandshake:457(515ns)} GenerateSessionVirtualId:988(620ns) PrepareNewSessionHandshake:477(2774ns) {SwitchToState:179(803ns)}}}}}
2025-11-27T18:17:17.805938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470946456398097:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:17.809766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470946456398100:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.810449Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470946456398096:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(227958774ns) {Configure:104(227947101ns) {ProcessConfigured:135(46639ns) {StartInitialHandshake:157(44253ns) {DropHandshakes:470(1806ns) {DropIncomingHandshake:442(331ns) DropOutgoingHandshake:457(496ns)} GenerateSessionVirtualId:988(797ns) PrepareNewSessionHandshake:477(472ns) {SwitchToState:179(143ns)}}}}}
2025-11-27T18:17:17.825183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:17.827195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470946456398097:2109] [node 6] ICP22 created new session: [2:7577470946456398116:2109]
2025-11-27T18:17:17.829724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470946456398105:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.829802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470946456398106:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.830060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470946456398115:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2354
2025-11-27T18:17:17.830306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470946456398032:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:17.839995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470946456398115:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.840112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470946456398095:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(232121548ns) {Configure:104(232111759ns) {ProcessConfigured:135(5322003ns) {StartInitialHandshake:157(5319369ns) {DropHandshakes:470(10226ns) {DropIncomingHandshake:442(549ns) DropOutgoingHandshake:457(612ns)} GenerateSessionVirtualId:988(780ns) PrepareNewSessionHandshake:477(3553ns) {SwitchToState:179(1663ns)}}}}}
2025-11-27T18:17:17.850250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470946456398116:2109] [node 6] ICS09 handshake done sender: [2:7577470946456398099:2110] self: [2:7577470946456398098:0] peer: [6:7577470944296551529:0] socket: 166 qp: -1
2025-11-27T18:17:17.862876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470946456398116:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:17.862940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:17.888753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470946456398097:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(82737713ns) {HandleHandshakeStatus:329(82734429ns) {DropIncomingHandshake:442(489ns) DropOutgoingHandshake:457(665ns) SwitchToState:179(748ns) ProcessPendingSessionEvents:495(10381644ns) {ForwardSessionEventToSession:612(10320510ns) ForwardSessionEventToSession:612(16505ns) ForwardSessionEventToSession:612(2453ns) ForwardSessionEventToSession:612(5207ns) ForwardSessionEventToSession:612(1998ns) ForwardSessionEventToSession:612(6110ns) ForwardSessionEventToSession:612(5094ns) ForwardSessionEventToSession:612(2174ns) ForwardSessionEventToSession:612(2183ns) ForwardSessionEventToSession:612(4559ns)}}}
2025-11-27T18:17:17.888838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470946456398097:2109] [node 6] ICP18 handshake (actor [2:7577470946456398105:2113]) is too old
2025-11-27T18:17:17.859822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470944860741980:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.865508Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470944860741980:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.873034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470944860741980:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:17.873391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470944860741975:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:17.873452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:17.873507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470944860741975:2110] [node 6] ICP22 created new session: [5:7577470944860741995:2110]
2025-11-27T18:17:17.873564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470944860741995:2110] [node 6] ICS09 handshake done sender: [5:7577470944860741980:2113] self: [5:7577470944860741979:0] peer: [6:7577470944296551536:0] socket: 175 qp: -1
2025-11-27T18:17:17.873605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470944860741995:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:17.873645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.850046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470945339082728:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.810357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470945497383000:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.850074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470945820442197:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.810379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470945497383000:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.811182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470945497383000:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:17.817482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470945497382991:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.817522Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470945497382991:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.865719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470945497382987:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:17.870100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470945497382991:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:17.882126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470945497383007:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8006
2025-11-27T18:17:17.882430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:17.889749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470945497383001:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.897532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470945497382987:2108] [node 5] ICP22 created new session: [3:7577470945497383012:2108]
2025-11-27T18:17:17.906354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470945497382924:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:17.912088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470945497383007:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.911542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470945497382986:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:17.912184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470945497383012:2108] [node 5] ICS09 handshake done sender: [3:7577470945497383000:2114] self: [3:7577470945497382999:0] peer: [5:7577470944860741991:0] socket: 182 qp: -1
2025-11-27T18:17:17.912225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470945497383012:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.826658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470944296551515:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.921582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470944860741993:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47410
2025-11-27T18:17:17.850089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470945339082728:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.856659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470945820442190:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(161432618ns) {Configure:104(161422915ns) {ProcessConfigured:135(11749356ns) {StartInitialHandshake:157(11746136ns) {DropHandshakes:470(1162ns) {DropIncomingHandshake:442(274ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(253ns) PrepareNewSessionHandshake:477(5009ns) {SwitchToState:179(2020ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.929476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470944860741998:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47410
2025-11-27T18:17:17.923504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470945497382925:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.860081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470945820442126:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:17.890883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470945820442197:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:17.893461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470945820442212:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37686
2025-11-27T18:17:17.893514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470945820442213:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37686
2025-11-27T18:17:17.901696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470945820442191:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:17.919587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470945820442189:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:17.919642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:17.922292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470945820442128:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:17.922372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:17.922426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470945820442128:2050] [node 3] ICP22 created new session: [4:7577470945820442215:2050]
2025-11-27T18:17:17.922475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470945820442215:2050] [node 3] ICS09 handshake done sender: [4:7577470945820442191:2110] self: [4:7577470945820442205:0] peer: [3:7577470945497382990:0] socket: 157 qp: -1
2025-11-27T18:17:17.922510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470945820442215:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:17.922548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:17.924201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470945820442212:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.930320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470945820442189:2109] [node 6] ICP22 created new session: [4:7577470945820442214:2109]
2025-11-27T18:17:17.910877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470946456398122:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2354
2025-11-27T18:17:17.911632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470946456398106:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:17.911882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470946456398032:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:17.911949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:17.934496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470944860741992:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47410
2025-11-27T18:17:17.831099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470944296551453:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:17.874908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470945339082730:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.911992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470946456398032:2048] [node 1] ICP22 created new session: [2:7577470946456398123:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.923593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470945497383014:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8006
2025-11-27T18:17:17.934513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470945820442214:2109] [node 6] ICS09 handshake done sender: [4:7577470945820442197:2113] self: [4:7577470945820442196:0] peer: [6:7577470944296551537:0] socket: 162 qp: -1
2025-11-27T18:17:17.831182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:17.935587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470944860741998:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.874946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470945339082730:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.930283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470946456398123:2048] [node 1] ICS09 handshake done sender: [2:7577470946456398106:2114] self: [2:7577470946456398119:0] peer: [1:7577470945339082726:0] socket: 177 qp: -1
2025-11-27T18:17:17.931689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:17.831246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470944296551453:2049] [node 2] ICP22 created new session: [6:7577470944296551538:2049]
2025-11-27T18:17:17.878463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470945339082728:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:17.935999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470945820442214:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.939694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470944860741993:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.933792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470946456398123:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:17.831295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470944296551538:2049] [node 2] ICS09 handshake done sender: [6:7577470944296551528:2115] self: [6:7577470944296551529:0] peer: [2:7577470946456398098:0] socket: 167 qp: -1
2025-11-27T18:17:17.939275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470945820442219:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37686
2025-11-27T18:17:17.932019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:17.945115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470944860741974:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:17.953379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470944860741986:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:17.956580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470944860741910:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:17.968075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470944860741911:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:17.905648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470945339082736:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48710
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.933830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:17.935838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470945497382986:2107] [node 4] ICP22 created new session: [3:7577470945497383018:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.931192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470945339082730:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:17.946514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470945820442200:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:17.831323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470944296551538:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:17.990638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:17.935597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470946456398108:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.936925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470945497383018:2107] [node 4] ICS09 handshake done sender: [3:7577470945497382991:2110] self: [3:7577470945497382990:0] peer: [4:7577470945820442205:0] socket: 155 qp: -1
2025-11-27T18:17:17.949896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:17.831360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:17.939696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470945339082727:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.990704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470944860741911:2050] [node 3] ICP22 created new session: [5:7577470944860742002:2050]
2025-11-27T18:17:17.935628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470946456398108:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.936954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470945497383018:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:17.956691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470945820442126:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:18.008224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470944860742002:2050] [node 3] ICS09 handshake done sender: [5:7577470944860741986:2115] self: [5:7577470944860741991:0] peer: [3:7577470945497382999:0] socket: 184 qp: -1
2025-11-27T18:17:17.939727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470945339082727:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.856366Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470944296551515:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(16483138ns) {IncomingHandshake:261(16481209ns) {DropIncomingHandshake:442(1418ns) PrepareNewSessionHandshake:477(5721ns) {SwitchToState:179(2751ns)} GenerateSessionVirtualId:988(434ns) IssueIncomingHandshakeReply:193(47320ns) {SwitchToState:179(299ns)}}}
2025-11-27T18:17:17.945439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470946456398124:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2354
2025-11-27T18:17:17.945334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470945497382998:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.961379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470945820442207:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.939835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470945339082736:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:18.018576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470944860742002:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:17.878446Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470944296551452:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(273887147ns) {Configure:104(273878125ns) {ProcessConfigured:135(10629837ns) {StartInitialHandshake:157(10627672ns) {DropHandshakes:470(1774ns) {DropIncomingHandshake:442(405ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(766ns) PrepareNewSessionHandshake:477(2481ns) {SwitchToState:179(1070ns)}}}}}
2025-11-27T18:17:17.945372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470945497382998:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.954181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470946456398124:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:18.020764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:17.961422Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470945820442207:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.942534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470945339082657:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:17.883157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470944296551515:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12985) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470944296551530:2116] held: no
2025-11-27T18:17:17.971126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:18.022715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470944860741974:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13971) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470944860741998:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.961679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470946456398108:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:17.971500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470945820442219:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.953799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470945497382987:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(87826847ns) {HandleHandshakeStatus:329(87823328ns) {DropIncomingHandshake:442(2203ns) DropOutgoingHandshake:457(3236ns) SwitchToState:179(802ns) ProcessPendingSessionEvents:495(124397ns) {ForwardSessionEventToSession:612(73780ns) ForwardSessionEventToSession:612(3245ns) ForwardSessionEventToSession:612(5803ns) ForwardSessionEventToSession:612(2839ns) ForwardSessionEventToSession:612(2401ns) ForwardSessionEventToSession:612(5392ns) ForwardSessionEventToSession:612(3479ns) ForwardSessionEventToSession:612(10737ns) ForwardSessionEventToSession:612(2936ns) ForwardSessionEventToSession:612(2762ns)}}}
2025-11-27T18:17:17.955420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.961450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470945497383014:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.942595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:17.888125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470944296551541:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24940
2025-11-27T18:17:17.982850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470946456398112:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:18.027724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470949155709299:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47410
2025-11-27T18:17:17.970760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470945497383020:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8006
2025-11-27T18:17:17.942645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470945339082657:2048] [node 2] ICP22 created new session: [1:7577470945339082743:2048]
2025-11-27T18:17:17.982801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470945820442126:2048] [node 1] ICP22 created new session: [4:7577470945820442221:2048]
2025-11-27T18:17:18.037430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470949155709302:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47410
2025-11-27T18:17:17.982893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470946456398112:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.918166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470944296551542:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24940
2025-11-27T18:17:17.988762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470945820442207:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:17.971179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577470945497382987:2108] [node 5] ICP18 handshake (actor [3:7577470945497383014:2119]) is too old
2025-11-27T18:17:18.038169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470944860741998:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:17.985635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470946456398033:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:17.921940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470944296551543:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24940
2025-11-27T18:17:17.989036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470945820442190:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:17.942689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470945339082743:2048] [node 2] ICS09 handshake done sender: [1:7577470945339082728:2113] self: [1:7577470945339082726:0] peer: [2:7577470946456398119:0] socket: 170 qp: -1
2025-11-27T18:17:17.993378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470945497382998:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:17.922086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470944296551526:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:17.932451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470944296551543:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:17.932507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470944296551525:2112] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:17.933981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:17.934848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470944296551454:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:17.935743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470944296551525:2112] [node 4] ICP22 created new session: [6:7577470944296551544:2112]
2025-11-27T18:17:18.039900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470944860742000:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47410
2025-11-27T18:17:17.989126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:17.989177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470945820442190:2108] [node 5] ICP22 created new session: [4:7577470945820442223:2108]
2025-11-27T18:17:17.989227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470945820442223:2108] [node 5] ICS09 handshake done sender: [4:7577470945820442207:2117] self: [4:7577470945820442206:0] peer: [5:7577470944860741999:0] socket: 183 qp: -1
2025-11-27T18:17:17.989286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470945820442223:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:17.989351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:17.990056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470945820442221:2048] [node 1] ICS09 handshake done sender: [4:7577470945820442200:2114] self: [4:7577470945820442209:0] peer: [1:7577470945339082729:0] socket: 172 qp: -1
2025-11-27T18:17:17.997625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470945820442127:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:18.009421Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470945820442190:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(20106014ns) {HandleHandshakeStatus:329(20098899ns) {DropIncomingHandshake:442(13405ns) DropOutgoingHandshake:457(677ns) SwitchToState:179(812ns) ProcessPendingSessionEvents:495(19648645ns) {ForwardSessionEventToSession:612(53398ns) ForwardSessionEventToSession:612(10577ns) ForwardSessionEventToSession:612(2502ns) ForwardSessionEventToSession:612(657026ns) ForwardSessionEventToSession:612(7737ns) ForwardSessionEventToSession:612(2621ns) ForwardSessionEventToSession:612(4235856ns) ForwardSessionEventToSession:612(2644ns) ForwardSessionEventToSession:612(14645156ns) ForwardSessionEventToSession:612(14039ns)}}}
2025-11-27T18:17:18.018635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:18.045755Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470944860741993:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:18.053653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470944860741974:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:18.053719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:18.053764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470944860741974:2109] [node 4] ICP22 created new session: [5:7577470949155709303:2109]
2025-11-27T18:17:17.942718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470945339082743:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:18.010067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470945497382989:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:18.020906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470946456398033:2049] [node 3] ICP22 created new session: [2:7577470950751365425:2049]
2025-11-27T18:17:18.023364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470950751365425:2049] [node 3] ICS09 handshake done sender: [2:7577470946456398108:2115] self: [2:7577470946456398104:0] peer: [3:7577470945497383016:0] socket: 188 qp: -1
2025-11-27T18:17:18.024868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470946456398114:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:18.024918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470946456398114:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:17.936518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470944296551515:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(22342893ns) {HandleHandshakeStatus:395(22339205ns) {LogHandshakeFail:472(7966955ns) {UpdateErrorStateLog:545(10788ns)} DropOutgoingHandshake:457(908ns) ProcessPendingSessionEvents:495(14302728ns) {DropSessionEvent:512(10845ns) DropSessionEvent:512(3868ns) DropSessionEvent:512(3281ns) DropSessionEvent:512(4093ns) DropSessionEvent:512(4646ns) DropSessionEvent:512(6427008ns) DropSessionEvent:512(3520ns) DropSessionEvent:512(7809026ns) DropSessionEvent:512(20090ns) DropSessionEvent:512(3066ns)}}}
2025-11-27T18:17:18.026334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470946456398112:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:17.940150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470944296551530:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:18.026800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470946456398032:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(78176642ns) {HandleHandshakeStatus:329(78175068ns) {DropIncomingHandshake:442(1290ns) DropOutgoingHandshake:457(9526ns) SwitchToState:179(749ns) ProcessPendingSessionEvents:495(48932743ns) {ForwardSessionEventToSession:612(20264458ns) ForwardSessionEventToSession:612(6775088ns) ForwardSessionEventToSession:612(4203ns) ForwardSessionEventToSession:612(4815110ns) ForwardSessionEventToSession:612(9853897ns) ForwardSessionEventToSession:612(2658ns) ForwardSessionEventToSession:612(2151518ns) ForwardSessionEventToSession:612(6634ns) ForwardSessionEventToSession:612(2941ns) ForwardSessionEventToSession:612(5025349ns)}}}
2025-11-27T18:17:17.940219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470944296551544:2112] [node 4] ICS09 handshake done sender: [6:7577470944296551526:2114] self: [6:7577470944296551537:0] peer: [4:7577470945820442196:0] socket: 163 qp: -1
2025-11-27T18:17:18.028074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470950751365425:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:17.940249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470944296551544:2112] [node 4] ICS10 traffic start
2025-11-27T18:17:18.053812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470949155709303:2109] [node 4] ICS09 handshake done sender: [5:7577470944860741998:2120] self: [5:7577470944860741999:0] peer: [4:7577470945820442206:0] socket: 212 qp: -1
2025-11-27T18:17:18.010124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:17.956728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:17.961462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470944296551515:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:17.961526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:17.971903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470944296551515:2107] [node 5] ICP22 created new session: [6:7577470944296551548:2107]
2025-11-27T18:17:17.987827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470944296551548:2107] [node 5] ICS09 handshake done sender: [6:7577470944296551530:2116] self: [6:7577470944296551536:0] peer: [5:7577470944860741979:0] socket: 179 qp: -1
2025-11-27T18:17:18.008280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470944296551548:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:18.014899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:18.018288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470944296551549:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24940
2025-11-27T18:17:18.021588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470944296551454:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17951) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470944296551543:2122] held: no
2025-11-27T18:17:18.026353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470944296551549:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:18.027794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470944296551452:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:18.026852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470944296551515:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(66327786ns) {HandleHandshakeStatus:329(66325254ns) {DropIncomingHandshake:442(2516ns) DropOutgoingHandshake:457(618ns) SwitchToState:179(624ns) ProcessPendingSessionEvents:495(265ns)}}
2025-11-27T18:17:18.030314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470944296551525:2112] [node 4] ICP35 event processing took too much time PendingConnection:279(72025847ns) {HandleHandshakeStatus:329(72023587ns) {DropIncomingHandshake:442(1170ns) DropOutgoingHandshake:457(4619ns) SwitchToState:179(615ns) ProcessPendingSessionEvents:495(19322242ns) {ForwardSessionEventToSession:612(19060792ns) ForwardSessionEventToSession:612(5517ns) ForwardSessionEventToSession:612(2482ns) ForwardSessionEventToSession:612(6008ns) ForwardSessionEventToSession:612(3161ns) ForwardSessionEventToSession:612(4979ns) ForwardSessionEventToSession:612(5455ns) ForwardSessionEventToSession:612(2580ns) ForwardSessionEventToSession:612(213196ns) ForwardSessionEventToSession:612(3083ns)}}}
2025-11-27T18:17:18.062721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470944296551550:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24940
2025-11-27T18:17:18.070051Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470944296551452:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18110) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470944296551549:2123] held: no
2025-11-27T18:17:18.075857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470944296551543:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:18.076623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470944296551454:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:18.076755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:18.076812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470944296551454:2050] [node 3] ICP22 created new session: [6:7577470948591518850:2050]
2025-11-27T18:17:18.076873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470948591518850:2050] [node 3] ICS09 handshake done sender: [6:7577470944296551543:2122] self: [6:7577470944296551545:0] peer: [3:7577470945497382996:0] socket: 176 qp: -1
2025-11-27T18:17:18.076909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470948591518850:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:18.076971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:18.078306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470948591518849:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24940
2025-11-27T18:17:18.013136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470945820442221:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:18.028125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:17.942746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:17.944102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470945339082723:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:17.944209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:17.944255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470945339082723:2107] [node 4] ICP22 created new session: [1:7577470945339082747:2107]
2025-11-27T18:17:17.944294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470945339082747:2107] [node 4] ICS09 handshake done sender: [1:7577470945339082730:2114] self: [1:7577470945339082729:0] peer: [4:7577470945820442209:0] socket: 171 qp: -1
2025-11-27T18:17:17.944324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470945339082747:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:17.944367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:17.968298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470945339082727:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:17.984584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470945339082745:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48710
2025-11-27T18:17:17.985101Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577470945339082723:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:18.001407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470945339082746:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48710
2025-11-27T18:17:18.001855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470945339082658:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:18.001963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:18.002010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470945339082658:2049] [node 3] ICP22 created new session: [1:7577470945339082753:2049]
2025-11-27T18:17:18.002060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470945339082753:2049] [node 3] ICS09 handshake done sender: [1:7577470945339082727:2112] self: [1:7577470945339082725:0] peer: [3:7577470945497383013:0] socket: 174 qp: -1
2025-11-27T18:17:18.002099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470945339082753:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:18.002130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:18.021655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470945339082746:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:18.032798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470945339082739:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:18.032834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470945339082739:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:18.051203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470945339082739:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:18.063849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470945339082735:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:18.063892Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470945339082735:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:18.064217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470945339082724:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:18.066229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470949634050052:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48710
2025-11-27T18:17:18.073918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:18.075535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470945339082724:2110] [node 6] ICP22 created new session: [1:7577470949634050054:2110]
2025-11-27T18:17:18.075755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470949634050052:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:18.076747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470949634050054:2110] [node 6] ICS09 handshake done sender: [1:7577470945339082739:2118] self: [1:7577470945339082737:0] peer: [6:7577470948591518848:0] socket: 217 qp: -1
2025-11-27T18:17:18.076809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470949634050054:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:18.076854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:18.079475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470945339082735:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:18.080145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470945339082722:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:18.080194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:18.081106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470945339082722:2109] [node 5] ICP22 created new session: [1:7577470949634050058:2109]
2025-11-27T18:17:18.053845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470949155709303:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:18.033006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470946456398114:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:18.018590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:18.010178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470945497382989:2109] [node 6] ICP22 created new session: [3:7577470949792350319:2109]
2025-11-27T18:17:18.053906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:18.010224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470949792350319:2109] [node 6] ICS09 handshake done sender: [3:7577470945497382998:2113] self: [3:7577470945497382996:0] peer: [6:7577470944296551545:0] socket: 205 qp: -1
2025-11-27T18:17:18.010253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470949792350319:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:18.010284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:18.011387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470945497383022:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8006
2025-11-27T18:17:18.013116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470945497383001:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:18.019273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470945497382924:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:18.019316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:18.019359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470945497382924:2048] [node 1] ICP22 created new session: [3:7577470949792350321:2048]
2025-11-27T18:17:18.019411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470949792350321:2048] [node 1] ICS09 handshake done sender: [3:7577470945497383001:2115] self: [3:7577470945497383013:0] peer: [1:7577470945339082725:0] socket: 181 qp: -1
2025-11-27T18:17:18.019443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470949792350321:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:18.019474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:18.022232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470945497383007:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:18.025911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577470945497382924:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:18.026695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470945497382925:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:18.027619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:18.028396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470945497382925:2049] [node 2] ICP22 created new session: [3:7577470949792350323:2049]
2025-11-27T18:17:18.038675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470949792350323:2049] [node 2] ICS09 handshake done sender: [3:7577470945497383007:2118] self: [3:7577470945497383016:0] peer: [2:7577470946456398104:0] socket: 190 qp: -1
2025-11-27T18:17:18.044965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470949792350323:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:18.049000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:18.056889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470945497382986:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(116046370ns) {HandleHandshakeStatus:329(116037379ns) {DropIncomingHandshake:442(16273ns) DropOutgoingHandshake:457(268ns) SwitchToState:179(542ns) ProcessPendingSessionEvents:495(29963209ns) {ForwardSessionEventToSession:612(23216580ns) ForwardSessionEventToSession:612(5065ns) ForwardSessionEventToSession:612(1988379ns) ForwardSessionEventToSession:612(4525ns) ForwardSessionEventToSession:612(2612ns) ForwardSessionEventToSession:612(544462ns) ForwardSessionEventToSession:612(3327072ns) ForwardSessionEventToSession:612(5543ns) ForwardSessionEventToSession:612(848660ns) ForwardSessionEventToSession:612(2398ns)}}}
2025-11-27T18:17:18.075718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470945497382925:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(33527350ns) {HandleHandshakeStatus:329(33525230ns) {DropIncomingHandshake:442(1513ns) DropOutgoingHandshake:457(3703ns) SwitchToState:179(2778ns) ProcessPendingSessionEvents:495(2726480ns) {ForwardSessionEventToSession:612(2661407ns) ForwardSessionEventToSession:612(6099ns) ForwardSessionEventToSession:612(13460ns) ForwardSessionEventToSession:612(2661ns) ForwardSessionEventToSession:612(9493ns) ForwardSessionEventToSession:612(9257ns) ForwardSessionEventToSession:612(3665ns) ForwardSessionEventToSession:612(2465ns) ForwardSessionEventToSession:612(5174ns) ForwardSessionEventToSession:612(2123ns)}}}
2025-11-27T18:17:18.075788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577470945497382925:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:18.088527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470944296551549:2123] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:18.038440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470950115409523:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37686
2025-11-27T18:17:18.044312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470945820442126:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(87412759ns) {HandleHandshakeStatus:329(87410891ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(4362ns) SwitchToState:179(274ns) ProcessPendingSessionEvents:495(13703920ns) {ForwardSessionEventToSession:612(12638397ns) ForwardSessionEventToSession:612(5086ns) ForwardSessionEventToSession:612(21636ns) ForwardSessionEventToSession:612(5078ns) ForwardSessionEventToSession:612(3041ns) ForwardSessionEventToSession:612(2363ns) ForwardSessionEventToSession:612(8002ns) ForwardSessionEventToSession:612(2994ns) ForwardSessionEventToSession:612(5799ns) ForwardSessionEventToSession:612(2444ns)}}}
2025-11-27T18:17:18.052282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470945820442189:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(115095383ns) {HandleHandshakeStatus:329(115093184ns) {DropIncomingHandshake:442(1939ns) DropOutgoingHandshake:457(2070ns) SwitchToState:179(655ns) ProcessPendingSessionEvents:495(37123315ns) {ForwardSessionEventToSession:612(30479761ns) ForwardSessionEventToSession:612(1879660ns) ForwardSessionEventToSession:612(2330ns) ForwardSessionEventToSession:612(503034ns) ForwardSessionEventToSession:612(3128ns) ForwardSessionEventToSession:612(2864ns) ForwardSessionEventToSession:612(2772738ns) ForwardSessionEventToSession:612(3089ns) ForwardSessionEventToSession:612(600537ns) ForwardSessionEventToSession:612(852016ns)}}}
2025-11-27T18:17:18.052765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577470945820442189:2109] [node 6] ICP18 handshake (actor [4:7577470945820442212:2119]) is too old
2025-11-27T18:17:18.053084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470945820442219:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:18.057004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470945820442127:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12809) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470945820442219:2121] held: no
2025-11-27T18:17:18.064728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470945820442127:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:18.064770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:18.064829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470945820442127:2049] [node 2] ICP22 created new session: [4:7577470950115409524:2049]
2025-11-27T18:17:18.064868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470950115409524:2049] [node 2] ICS09 handshake done sender: [4:7577470945820442219:2121] self: [4:7577470945820442225:0] peer: [2:7577470946456398113:0] socket: 178 qp: -1
2025-11-27T18:17:18.064902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470950115409524:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:18.064950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:18.086433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470945339082724:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(18130902ns) {HandleHandshakeStatus:329(18128482ns) {DropIncomingHandshake:442(16114ns) DropOutgoingHandshake:457(457ns) SwitchToState:179(714ns) ProcessPendingSessionEvents:495(947893ns) {ForwardSessionEventToSession:612(898259ns) ForwardSessionEventToSession:612(4703ns) ForwardSessionEventToSession:612(5943ns) ForwardSessionEventToSession:612(2234ns) ForwardSessionEventToSession:612(7815ns) ForwardSessionEventToSession:612(5098ns) ForwardSessionEventToSession:612(2922ns) ForwardSessionEventToSession:612(3482ns) ForwardSessionEventToSession:612(4574ns) ForwardSessionEventToSession:612(2117ns)}}}
2025-11-27T18:17:18.038892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470946456398095:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:18.042134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:18.042201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470946456398095:2108] [node 5] ICP22 created new session: [2:7577470950751365429:2108]
2025-11-27T18:17:18.042248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470950751365429:2108] [node 5] ICS09 handshake done sender: [2:7577470946456398112:2116] self: [2:7577470946456398111:0] peer: [5:7577470944860742001:0] socket: 196 qp: -1
2025-11-27T18:17:18.042305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470950751365429:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:18.042344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:18.045355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470946456398096:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:18.051142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:18.055713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470946456398033:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(69877713ns) {HandleHandshakeStatus:329(69874449ns) {DropIncomingHandshake:442(12687ns) DropOutgoingHandshake:457(674ns) SwitchToState:179(6232ns) ProcessPendingSessionEvents:495(9935801ns) {ForwardSessionEventToSession:612(9892081ns) ForwardSessionEventToSession:612(4347ns) ForwardSessionEventToSession:612(5374ns) ForwardSessionEventToSession:612(2434ns) ForwardSessionEventToSession:612(2291ns) ForwardSessionEventToSession:612(8310ns) ForwardSessionEventToSession:612(3249ns) ForwardSessionEventToSession:612(2683ns) ForwardSessionEventToSession:612(4729ns) ForwardSessionEventToSession:612(1927ns)}}}
2025-11-27T18:17:18.056624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470946456398096:2107] [node 4] ICP22 created new session: [2:7577470950751365431:2107]
2025-11-27T18:17:18.058470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470950751365431:2107] [node 4] ICS09 handshake done sender: [2:7577470946456398114:2117] self: [2:7577470946456398113:0] peer: [4:7577470945820442225:0] socket: 150 qp: -1
2025-11-27T18:17:18.059218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470950751365431:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:18.059875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:18.068831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470946456398096:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(25352597ns) {HandleHandshakeStatus:329(25349712ns) {DropIncomingHandshake:442(740438ns) DropOutgoingHandshake:457(815ns) SwitchToState:179(954ns) ProcessPendingSessionEvents:495(2316918ns) {ForwardSessionEventToSession:612(2270337ns) ForwardSessionEventToSession:612(5700ns) ForwardSessionEventToSession:612(2711ns) ForwardSessionEventToSession:612(5511ns) ForwardSessionEventToSession:612(5622ns) ForwardSessionEventToSession:612(2610ns) ForwardSessionEventToSession:612(7037ns) ForwardSessionEventToSession:612(2113ns) ForwardSessionEventToSession:612(4973ns) ForwardSessionEventToSession:612(1887ns)}}}
2025-11-27T18:17:18.087027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470949634050058:2109] [node 5] ICS09 handshake done sender: [1:7577470945339082735:2115] self: [1:7577470945339082734:0] peer: [5:7577470949155709307:0] socket: 218 qp: -1
2025-11-27T18:17:18.054286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470944860741910:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12809) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470944860741993:2119] held: no
2025-11-27T18:17:18.110099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470944296551452:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:18.097410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470949634050058:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:18.101383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:18.110156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:18.117630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470944296551452:2048] [node 1] ICP22 created new session: [6:7577470948591518852:2048]
2025-11-27T18:17:18.120470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470945339082722:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(40384340ns) {HandleHandshakeStatus:329(40382199ns) {DropIncomingHandshake:442(15642ns) DropOutgoingHandshake:457(463ns) SwitchToState:179(251ns) ProcessPendingSessionEvents:495(155652ns) {ForwardSessionEventToSession:612(99734ns) ForwardSessionEventToSession:612(5747ns) ForwardSessionEventToSession:612(3242ns) ForwardSessionEventToSession:612(5290ns) ForwardSessionEventToSession:612(10221ns) ForwardSessionEventToSession:612(3139ns) ForwardSessionEventToSession:612(8362ns) ForwardSessionEventToSession:612(2388ns) ForwardSessionEventToSession:612(3475ns) ForwardSessionEventToSession:612(5321ns)}}}
2025-11-27T18:17:18.121532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470948591518852:2048] [node 1] ICS09 handshake done sender: [6:7577470944296551549:2123] self: [6:7577470948591518848:0] peer: [1:7577470945339082737:0] socket: 192 qp: -1
2025-11-27T18:17:18.054366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470944860741910:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:18.054395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:18.054433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470944860741910:2049] [node 2] ICP22 created new session: [5:7577470949155709305:2049]
2025-11-27T18:17:18.054485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470949155709305:2049] [node 2] ICS09 handshake done sender: [5:7577470944860741993:2119] self: [5:7577470944860742001:0] peer: [2:7577470946456398111:0] socket: 197 qp: -1
2025-11-27T18:17:18.054514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470949155709305:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:18.054557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:18.054935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470944860742000:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:18.055263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470944860741909:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:18.069311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470944860741911:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(93011461ns) {HandleHandshakeStatus:329(93007714ns) {DropIncomingHandshake:442(1259ns) DropOutgoingHandshake:457(13561ns) SwitchToState:179(641ns) ProcessPendingSessionEvents:495(27387684ns) {ForwardSessionEventToSession:612(8776842ns) ForwardSessionEventToSession:612(4730ns) ForwardSessionEventToSession:612(6949631ns) ForwardSessionEventToSession:612(7128ns) ForwardSessionEventToSession:612(3096ns) ForwardSessionEventToSession:612(4549118ns) ForwardSessionEventToSession:612(7306ns) ForwardSessionEventToSession:612(3492ns) ForwardSessionEventToSession:612(673789ns) ForwardSessionEventToSession:612(2883ns)}}}
2025-11-27T18:17:18.079521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470949155709308:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47410
2025-11-27T18:17:18.086927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470944860742000:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:18.093537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470944860741909:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18110) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470944860742000:2121] held: no
2025-11-27T18:17:18.104707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470944860741909:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:18.121318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:18.121783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470944860741909:2048] [node 1] ICP22 created new session: [5:7577470949155709309:2048]
2025-11-27T18:17:18.121826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470949155709309:2048] [node 1] ICS09 handshake done sender: [5:7577470944860742000:2121] self: [5:7577470949155709307:0] peer: [1:7577470945339082734:0] socket: 219 qp: -1
2025-11-27T18:17:18.121864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470949155709309:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:18.121914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:18.122102Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470944860741909:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17334717ns) {HandleHandshakeStatus:329(17332283ns) {DropIncomingHandshake:442(1128ns) DropOutgoingHandshake:457(347ns) SwitchToState:179(891ns) ProcessPendingSessionEvents:495(1424ns)}}
2025-11-27T18:17:18.128866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470948591518852:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:18.128932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:18.129141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470944296551452:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24067967ns) {HandleHandshakeStatus:329(23136410ns) {DropIncomingHandshake:442(2030ns) DropOutgoingHandshake:457(336ns) SwitchToState:179(678ns) ProcessPendingSessionEvents:495(1262ns)}}
2025-11-27T18:17:18.162785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:18.162998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470949792350321:2048] [node 1] ICS07 socket disconnect 181 reason# EndOfStream
2025-11-27T18:17:18.163023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470949792350321:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.163088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:18.163123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470949792350321:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:18.163993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470945497382924:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18110) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:18.164037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470949792350321:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:18.164071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470945497382924:2048] [node 1] ICP30 unregister session Session# [3:7577470949792350321:2048] VirtualId# [3:7577470945497383013:0]
2025-11-27T18:17:18.164115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:18.164140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470945497382924:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.164155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.165373Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:18.165578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470946456398123:2048] [node 1] ICS07 socket disconnect 177 reason# EndOfStream
2025-11-27T18:17:18.165602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470946456398123:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.165682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:18.169337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470946456398123:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:18.173492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470945497382924:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:18.173437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:18.173614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470945820442221:2048] [node 1] ICS07 socket disconnect 172 reason# EndOfStream
2025-11-27T18:17:18.173645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470945820442221:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.173711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:18.173736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470945820442221:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:18.174757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470945820442126:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18110) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:18.174811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470945820442221:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:18.174844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470945820442126:2048] [node 1] ICP30 unregister session Session# [4:7577470945820442221:2048] VirtualId# [4:7577470945820442209:0]
2025-11-27T18:17:18.174886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:18.174915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470945820442126:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.174926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.176989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:18.177159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470948591518852:2048] [node 1] ICS01 socket: 192 reason# EndOfStream
2025-11-27T18:17:18.177187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470944296551452:2048] [node 1] ICP30 unregister session Session# [6:7577470948591518852:2048] VirtualId# [6:7577470948591518848:0]
2025-11-27T18:17:18.177211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470948591518852:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.177407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:18.177666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470949155709309:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-11-27T18:17:18.177699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470944860741909:2048] [node 1] ICP30 unregister session Session# [5:7577470949155709309:2048] VirtualId# [5:7577470949155709307:0]
2025-11-27T18:17:18.177730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470949155709309:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.177806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:18.177844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:18.177441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:18.178714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:18.183840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470945820442126:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:18.183058Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470946456398032:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18110) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:18.183130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470946456398123:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:18.183160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470946456398032:2048] [node 1] ICP30 unregister session Session# [2:7577470946456398123:2048] VirtualId# [2:7577470946456398119:0]
2025-11-27T18:17:18.183207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:18.183243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470946456398032:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.183257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.193697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577470946456398032:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:18.239591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:18.239823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470949792350323:2049] [node 2] ICS07 socket disconnect 190 reason# EndOfStream
2025-11-27T18:17:18.239852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470949792350323:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.239924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:18.239960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470949792350323:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:18.243499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470945497382925:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12809) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:18.243543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470949792350323:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:18.244350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470945497382925:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12809) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:18.244395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470949792350323:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:18.244431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470945497382925:2049] [node 2] ICP30 unregister session Session# [3:7577470949792350323:2049] VirtualId# [3:7577470945497383016:0]
2025-11-27T18:17:18.244479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:18.244507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470945497382925:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12809) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.244520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12809) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.247998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:18.248195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470944296551538:2049] [node 2] ICS01 socket: 167 reason# EndOfStream
2025-11-27T18:17:18.248232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470944296551453:2049] [node 2] ICP30 unregister session Session# [6:7577470944296551538:2049] VirtualId# [6:7577470944296551529:0]
2025-11-27T18:17:18.248260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470944296551538:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.248340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:18.248386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:18.249392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:18.249580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470949155709305:2049] [node 2] ICS01 socket: 197 reason# EndOfStream
2025-11-27T18:17:18.249622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470944860741910:2049] [node 2] ICP30 unregister session Session# [5:7577470949155709305:2049] VirtualId# [5:7577470944860742001:0]
2025-11-27T18:17:18.249649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470949155709305:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.249727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:18.249766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:18.249820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:18.250023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470950115409524:2049] [node 2] ICS01 socket: 178 reason# EndOfStream
2025-11-27T18:17:18.250058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470945820442127:2049] [node 2] ICP30 unregister session Session# [4:7577470950115409524:2049] VirtualId# [4:7577470945820442225:0]
2025-11-27T18:17:18.250088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470950115409524:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.250166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:18.250205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:18.257552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470945497382925:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:18.306975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:18.307210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470948591518850:2050] [node 3] ICS01 socket: 176 reason# EndOfStream
2025-11-27T18:17:18.307256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470944296551454:2050] [node 3] ICP30 unregister session Session# [6:7577470948591518850:2050] VirtualId# [6:7577470944296551545:0]
2025-11-27T18:17:18.307286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470948591518850:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.307375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:18.307422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:18.309634Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:18.309819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470944860742002:2050] [node 3] ICS07 socket disconnect 184 reason# EndOfStream
2025-11-27T18:17:18.309847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470944860742002:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.309919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:18.309956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470944860742002:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:18.311015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470944860741911:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17951) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:18.311070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470944860742002:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:18.311098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470944860741911:2050] [node 3] ICP30 unregister session Session# [5:7577470944860742002:2050] VirtualId# [5:7577470944860741991:0]
2025-11-27T18:17:18.311140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:18.311168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470944860741911:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.311182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.311970Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:18.312143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470945820442215:2050] [node 3] ICS01 socket: 157 reason# EndOfStream
2025-11-27T18:17:18.312194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470945820442128:2050] [node 3] ICP30 unregister session Session# [4:7577470945820442215:2050] VirtualId# [4:7577470945820442205:0]
2025-11-27T18:17:18.312221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470945820442215:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.312286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:18.312324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:18.318899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470944860741911:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:18.365547Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:18.365768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470949155709303:2109] [node 4] ICS01 socket: 212 reason# EndOfStream
2025-11-27T18:17:18.365809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470944860741974:2109] [node 4] ICP30 unregister session Session# [5:7577470949155709303:2109] VirtualId# [5:7577470944860741999:0]
2025-11-27T18:17:18.365841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470949155709303:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.365917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:18.365958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:18.366307Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:18.366491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470944296551544:2112] [node 4] ICS07 socket disconnect 163 reason# EndOfStream
2025-11-27T18:17:18.366515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470944296551544:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.366587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:18.366625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470944296551544:2112] [node 4] ICS15 start handshake
2025-11-27T18:17:18.367780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470944296551525:2112] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13971) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:18.367829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470944296551544:2112] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:18.367864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470944296551525:2112] [node 4] ICP30 unregister session Session# [6:7577470944296551544:2112] VirtualId# [6:7577470944296551537:0]
2025-11-27T18:17:18.367901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:18.367944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470944296551525:2112] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.367975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13971) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:18.381362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470944296551525:2112] [node 4] ICP33 wake up from error state
2025-11-27T18:17:18.445431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:18.445647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470944296551548:2107] [node 5] ICS01 socket: 179 reason# EndOfStream
2025-11-27T18:17:18.445693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470944296551515:2107] [node 5] ICP30 unregister session Session# [6:7577470944296551548:2107] VirtualId# [6:7577470944296551536:0]
2025-11-27T18:17:18.445727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470944296551548:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:18.445806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:18.445852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:18.573532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470948265475026:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:18.573591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470948265475027:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:18.591485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470949822842973:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:18.591530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470949822842974:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:18.632958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470948924591454:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:18.633015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470948924591455:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:18.673541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470947800732630:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:18.673599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470947800732631:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:18.673629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470947800732632:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:18.697747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470951568604489:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:18.697837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470951568604490:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:18.697868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470951568604491:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:18.769946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470950134452813:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:18.770001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470950134452814:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:18.770029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470950134452815:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:19.115449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470955863571812:2064] [node 6] ICP01 ready to work
2025-11-27T18:17:19.115133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470953219558821:2099] [node 6] ICP01 ready to work
2025-11-27T18:17:19.170059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470955863571834:2079] [node 4] ICP01 ready to work
2025-11-27T18:17:19.167115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470954117810344:2106] [node 5] ICP01 ready to work
2025-11-27T18:17:19.173748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470953219558822:2106] [node 5] ICP01 ready to work
2025-11-27T18:17:19.214421Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470955863571812:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(43987359ns) {Configure:104(43981078ns) {ProcessConfigured:135(5423269ns) {StartInitialHandshake:157(5419605ns) {DropHandshakes:470(3748ns) {DropIncomingHandshake:442(919ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(1184ns) PrepareNewSessionHandshake:477(4239ns) {SwitchToState:179(1802ns)}}}}}
2025-11-27T18:17:19.214037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470952560442411:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:19.214205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470954117810345:2107] [node 6] ICP01 ready to work
2025-11-27T18:17:19.218569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470952560442412:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:19.218869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470952560442413:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:19.227391Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470947800732630:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(189848519ns) {Configure:104(189840550ns) {ProcessConfigured:135(42240ns) {StartInitialHandshake:157(40625ns) {DropHandshakes:470(1874ns) {DropIncomingHandshake:442(440ns) DropOutgoingHandshake:457(614ns)} GenerateSessionVirtualId:988(579ns) PrepareNewSessionHandshake:477(1933ns) {SwitchToState:179(1034ns)}}}}}
2025-11-27T18:17:19.227964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470952095699991:2084] [node 6] ICP01 ready to work
2025-11-27T18:17:19.228244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470952095699992:2088] [node 5] ICP01 ready to work
2025-11-27T18:17:19.219304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470954117810350:2117] [node 4] ICP01 ready to work
2025-11-27T18:17:19.233928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470953219558823:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.251224Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470950134452815:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(159902555ns) {Configure:104(159894085ns) {ProcessConfigured:135(3131428ns) {StartInitialHandshake:157(3129653ns) {DropHandshakes:470(1452ns) {DropIncomingHandshake:442(298ns) DropOutgoingHandshake:457(416ns)} GenerateSessionVirtualId:988(530ns) PrepareNewSessionHandshake:477(3651ns) {SwitchToState:179(2120ns)}}}}}
2025-11-27T18:17:19.256830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470953219558834:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
2025-11-27T18:17:19.252317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470951568604490:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(29352409ns) {Configure:104(29344794ns) {ProcessConfigured:135(5937441ns) {StartInitialHandshake:157(5935285ns) {DropHandshakes:470(1382ns) {DropIncomingHandshake:442(296ns) DropOutgoingHandshake:457(352ns)} GenerateSessionVirtualId:988(427ns) PrepareNewSessionHandshake:477(2961ns) {SwitchToState:179(1278ns)}}}}}
2025-11-27T18:17:19.256981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470953219558835:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.262090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470954117810360:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19308
2025-11-27T18:17:19.262163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470954117810361:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19308
2025-11-27T18:17:19.251932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470954429420174:2065] [node 5] ICP01 ready to work
2025-11-27T18:17:19.254155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470954429420175:2068] [node 4] ICP01 ready to work
2025-11-27T18:17:19.263833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470953219558834:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.267746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470954117810360:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.269925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470948265475026:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22527738ns) {Configure:104(22519824ns) {ProcessConfigured:135(5777794ns) {StartInitialHandshake:157(5775882ns) {DropHandshakes:470(2756ns) {DropIncomingHandshake:442(712ns) DropOutgoingHandshake:457(371ns)} GenerateSessionVirtualId:988(468ns) PrepareNewSessionHandshake:477(3471ns) {SwitchToState:179(1761ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.278033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470955863571860:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64612
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.278213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470955863571861:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64612
2025-11-27T18:17:19.280351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470955863571860:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.289430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470954117810363:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19308
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.301172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470954429420176:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-11-27T18:17:19.305535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470954429420177:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-11-27T18:17:19.307826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470951568604490:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.310240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470955863571861:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.310371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470951568604491:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.309735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470953219558835:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.310364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470948924591455:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.310745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470953219558839:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
2025-11-27T18:17:19.310591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470954429420177:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:17:19.310853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470953219558840:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
::ffff:127.1.0.0
2025-11-27T18:17:19.311227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470955863571867:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64612
2025-11-27T18:17:19.311396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470954117810352:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.311288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470953219558839:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.311383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470953219558827:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.311402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470953219558827:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.311431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470954117810352:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.312457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470954117810349:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.312477Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470954117810349:2123] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.313169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470954429420174:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.316558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470952560442418:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53359
2025-11-27T18:17:19.319695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470955863571851:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.313313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470954429420176:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.316713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470952560442419:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53359
2025-11-27T18:17:19.323011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470952560442419:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.324098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470954117810363:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.319734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470955863571851:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.324970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470954117810361:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.326039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470950134452815:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.327026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470953219558827:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:19.326881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470955863571868:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64612
2025-11-27T18:17:19.326990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470955863571869:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64612
2025-11-27T18:17:19.327851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470953219558825:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.329013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470954429420191:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.329499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470955863571851:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:19.329770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470955863571867:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.329854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470955863571812:2064] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.329375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470954117810364:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19308
2025-11-27T18:17:19.327883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470953219558825:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.330678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:19.330729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470954117810365:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19308
2025-11-27T18:17:19.328445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470953219558840:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.330992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470954117810352:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:19.334527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470952560442418:2138] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.328566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470953219558822:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.328622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:19.328671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470953219558822:2106] [node 5] ICP22 created new session: [3:7577470953219558847:2106]
2025-11-27T18:17:19.328745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470953219558847:2106] [node 5] ICS09 handshake done sender: [3:7577470953219558827:2119] self: [3:7577470953219558826:0] peer: [5:7577470955863571866:0] socket: 168 qp: -1
2025-11-27T18:17:19.328786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470953219558847:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:19.330737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470955863571812:2064] [node 6] ICP22 created new session: [5:7577470955863571872:2064]
2025-11-27T18:17:19.336081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470954117810349:2123] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.328828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:19.330449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470953219558845:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
2025-11-27T18:17:19.330835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470955863571872:2064] [node 6] ICS09 handshake done sender: [5:7577470955863571851:2110] self: [5:7577470955863571849:0] peer: [6:7577470954429420189:0] socket: 175 qp: -1
2025-11-27T18:17:19.331946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470955863571872:2064] [node 6] ICS10 traffic start
2025-11-27T18:17:19.332022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:19.332739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470955863571861:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:19.333802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470951568604491:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.333867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:19.333913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470951568604491:2050] [node 3] ICP22 created new session: [5:7577470955863571874:2050]
2025-11-27T18:17:19.335251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470955863571874:2050] [node 3] ICS09 handshake done sender: [5:7577470955863571861:2116] self: [5:7577470955863571866:0] peer: [3:7577470953219558826:0] socket: 174 qp: -1
2025-11-27T18:17:19.335413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470955863571874:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:19.335447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:19.337222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577470955863571812:2064] [node 6] ICP18 handshake (actor [5:7577470955863571867:2117]) is too old
2025-11-27T18:17:19.337382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470955863571871:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64612
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.336262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470954429420177:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:19.338857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470953219558845:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.338936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470953219558846:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
2025-11-27T18:17:19.339936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470953219558825:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:19.340280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470949822842974:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.340342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:19.340392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470949822842974:2049] [node 3] ICP22 created new session: [2:7577470954117810370:2049]
2025-11-27T18:17:19.340453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470954117810370:2049] [node 3] ICS09 handshake done sender: [2:7577470954117810349:2123] self: [2:7577470954117810348:0] peer: [3:7577470953219558841:0] socket: 157 qp: -1
2025-11-27T18:17:19.340485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470954117810370:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:19.340526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:19.341701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470954429420174:2065] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.341748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:19.341795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470954429420174:2065] [node 5] ICP22 created new session: [6:7577470954429420193:2065]
2025-11-27T18:17:19.341840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470954429420193:2065] [node 5] ICS09 handshake done sender: [6:7577470954429420177:2112] self: [6:7577470954429420189:0] peer: [5:7577470955863571849:0] socket: 181 qp: -1
2025-11-27T18:17:19.341862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470954429420193:2065] [node 5] ICS10 traffic start
2025-11-27T18:17:19.341914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.340890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470954117810344:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.345863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470952560442431:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53359
2025-11-27T18:17:19.340932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:19.340974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470954117810344:2106] [node 5] ICP22 created new session: [2:7577470954117810372:2106]
2025-11-27T18:17:19.341018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470954117810372:2106] [node 5] ICS09 handshake done sender: [2:7577470954117810352:2124] self: [2:7577470954117810351:0] peer: [5:7577470955863571865:0] socket: 158 qp: -1
2025-11-27T18:17:19.341050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470954117810372:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:19.341115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:19.346269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470954117810364:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.347087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470953219558834:2123] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:19.345521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470954429420192:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-11-27T18:17:19.350377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470948924591454:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.350463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470953219558821:2099] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.350512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:19.350553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470953219558821:2099] [node 6] ICP22 created new session: [3:7577470953219558853:2099]
2025-11-27T18:17:19.351855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470954429420176:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:19.351554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470953219558853:2099] [node 6] ICS09 handshake done sender: [3:7577470953219558825:2118] self: [3:7577470953219558824:0] peer: [6:7577470954429420190:0] socket: 167 qp: -1
2025-11-27T18:17:19.351640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470953219558853:2099] [node 6] ICS10 traffic start
2025-11-27T18:17:19.351712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:19.358521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470954117810365:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.359090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470949822842973:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.357926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470948924591455:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.359645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.361814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470952095700004:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9567
2025-11-27T18:17:19.361879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470952095700005:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9567
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.368413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470954117810378:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19308
2025-11-27T18:17:19.368964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470954117810365:2131] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:19.369184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470949822842973:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.369435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470952095700004:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.359740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470948924591455:2049] [node 2] ICP22 created new session: [3:7577470953219558855:2049]
2025-11-27T18:17:19.366276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470952560442417:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.377414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470947800732631:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.377926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470952095700007:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9567
2025-11-27T18:17:19.369231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:19.359817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470950134452815:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.369415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470949822842973:2048] [node 1] ICP22 created new session: [2:7577470954117810379:2048]
2025-11-27T18:17:19.369456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470954117810379:2048] [node 1] ICS09 handshake done sender: [2:7577470954117810365:2131] self: [2:7577470954117810377:0] peer: [1:7577470952560442416:0] socket: 195 qp: -1
2025-11-27T18:17:19.369492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470954117810379:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:19.369526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:19.359785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470953219558855:2049] [node 2] ICS09 handshake done sender: [3:7577470953219558834:2123] self: [3:7577470953219558841:0] peer: [2:7577470954117810348:0] socket: 165 qp: -1
2025-11-27T18:17:19.359880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:19.359928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470950134452815:2050] [node 3] ICP22 created new session: [6:7577470954429420196:2050]
2025-11-27T18:17:19.359973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470954429420196:2050] [node 3] ICS09 handshake done sender: [6:7577470954429420176:2111] self: [6:7577470954429420190:0] peer: [3:7577470953219558824:0] socket: 172 qp: -1
2025-11-27T18:17:19.360005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470954429420196:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:19.360058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:19.383321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470954117810356:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.383354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470954117810356:2126] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.384829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470954117810356:2126] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:19.385878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470950134452815:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(25849996ns) {HandleHandshakeStatus:329(25847168ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(5231ns) SwitchToState:179(307ns) ProcessPendingSessionEvents:495(127075ns) {ForwardSessionEventToSession:612(58224ns) ForwardSessionEventToSession:612(8511ns) ForwardSessionEventToSession:612(4490ns) ForwardSessionEventToSession:612(1864ns) ForwardSessionEventToSession:612(3270ns) ForwardSessionEventToSession:612(5354ns) ForwardSessionEventToSession:612(21635ns) ForwardSessionEventToSession:612(2768ns) ForwardSessionEventToSession:612(7343ns) ForwardSessionEventToSession:612(4138ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.401914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470955863571868:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.366312Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470952560442417:2137] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.409410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470955863571860:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:19.359823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470953219558855:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:19.385094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470954117810350:2117] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.397251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470954429420198:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-11-27T18:17:19.413683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470951568604490:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27723) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470955863571860:2115] held: no
2025-11-27T18:17:19.413793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470951568604490:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.413840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:19.413893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470951568604490:2049] [node 2] ICP22 created new session: [5:7577470955863571877:2049]
2025-11-27T18:17:19.413934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470955863571877:2049] [node 2] ICS09 handshake done sender: [5:7577470955863571860:2115] self: [5:7577470955863571865:0] peer: [2:7577470954117810351:0] socket: 173 qp: -1
2025-11-27T18:17:19.413968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470955863571877:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:19.414014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:19.414894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470951568604489:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.385154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:19.385198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470954117810350:2117] [node 4] ICP22 created new session: [2:7577470954117810382:2117]
2025-11-27T18:17:19.385330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470954117810382:2117] [node 4] ICS09 handshake done sender: [2:7577470954117810356:2126] self: [2:7577470954117810355:0] peer: [4:7577470952095700006:0] socket: 171 qp: -1
2025-11-27T18:17:19.385375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470954117810382:2117] [node 4] ICS10 traffic start
2025-11-27T18:17:19.416346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470952095700008:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9567
2025-11-27T18:17:19.385409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:19.359870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:19.393665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577470948924591455:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:19.367164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470952560442417:2137] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:19.419583Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470954117810350:2117] [node 4] ICP35 event processing took too much time PendingConnection:279(34271198ns) {HandleHandshakeStatus:329(34264826ns) {DropIncomingHandshake:442(13182ns) DropOutgoingHandshake:457(576ns) SwitchToState:179(605ns) ProcessPendingSessionEvents:495(1957851ns) {ForwardSessionEventToSession:612(1907103ns) ForwardSessionEventToSession:612(4914ns) ForwardSessionEventToSession:612(10020ns) ForwardSessionEventToSession:612(2198ns) ForwardSessionEventToSession:612(3149ns) ForwardSessionEventToSession:612(4726ns) ForwardSessionEventToSession:612(2813ns) ForwardSessionEventToSession:612(3044ns) ForwardSessionEventToSession:612(7703ns) ForwardSessionEventToSession:612(2030ns)}}}
2025-11-27T18:17:19.367512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470948265475026:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.367606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:19.433486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470954429420198:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.433627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470950134452814:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.433926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470954117810354:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.367658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470948265475026:2048] [node 2] ICP22 created new session: [1:7577470952560442433:2048]
2025-11-27T18:17:19.433954Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470954117810354:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.367699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470952560442433:2048] [node 2] ICS09 handshake done sender: [1:7577470952560442417:2137] self: [1:7577470952560442416:0] peer: [2:7577470954117810377:0] socket: 191 qp: -1
2025-11-27T18:17:19.367733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470952560442433:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:19.367769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:19.415498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470948265475026:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(47748614ns) {HandleHandshakeStatus:329(47744193ns) {DropIncomingHandshake:442(13834ns) DropOutgoingHandshake:457(559ns) SwitchToState:179(752ns) ProcessPendingSessionEvents:495(2747431ns) {ForwardSessionEventToSession:612(2663439ns) ForwardSessionEventToSession:612(6588ns) ForwardSessionEventToSession:612(39986ns) ForwardSessionEventToSession:612(3103ns) ForwardSessionEventToSession:612(2579ns) ForwardSessionEventToSession:612(10210ns) ForwardSessionEventToSession:612(2049ns) ForwardSessionEventToSession:612(4382ns) ForwardSessionEventToSession:612(2239ns) ForwardSessionEventToSession:612(2967ns)}}}
2025-11-27T18:17:19.422324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470952560442415:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.422351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470952560442415:2136] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.435830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470952095700009:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9567
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.436009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470952095700005:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.424662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470952560442421:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.437996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470955863571881:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64612
2025-11-27T18:17:19.424684Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470952560442421:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.434650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470952560442431:2143] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.435135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470952560442435:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53359
2025-11-27T18:17:19.435245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470952560442436:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53359
2025-11-27T18:17:19.438393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470952560442415:2136] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.443411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470953219558858:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
2025-11-27T18:17:19.442812Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470952095700004:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:19.449502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470953219558845:2127] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:19.443797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470947800732631:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27723) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470952095700004:2115] held: no
2025-11-27T18:17:19.443886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470947800732631:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.443950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:19.444002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470947800732631:2049] [node 2] ICP22 created new session: [4:7577470952095700010:2049]
2025-11-27T18:17:19.444042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470952095700010:2049] [node 2] ICS09 handshake done sender: [4:7577470952095700004:2115] self: [4:7577470952095700006:0] peer: [2:7577470954117810355:0] socket: 179 qp: -1
2025-11-27T18:17:19.444076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470952095700010:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:19.444124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:19.444665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470952095700009:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.444765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470952095700007:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.445182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470947800732632:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.453478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470954117810354:2125] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:19.466172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470955863571868:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.461663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470952560442421:2140] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:19.467006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470952560442436:2145] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.467088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470952560442435:2144] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.467123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470948265475027:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.467200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:19.467257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470948265475027:2049] [node 3] ICP22 created new session: [1:7577470952560442441:2049]
2025-11-27T18:17:19.466338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470948924591454:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.470446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470951568604489:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.467367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470952560442441:2049] [node 3] ICS09 handshake done sender: [1:7577470952560442415:2136] self: [1:7577470952560442414:0] peer: [3:7577470953219558852:0] socket: 190 qp: -1
2025-11-27T18:17:19.467396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470952560442441:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:19.467460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:19.467990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470952560442412:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.468045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:19.468087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470952560442412:2109] [node 5] ICP22 created new session: [1:7577470952560442443:2109]
2025-11-27T18:17:19.468124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470952560442443:2109] [node 5] ICS09 handshake done sender: [1:7577470952560442421:2140] self: [1:7577470952560442420:0] peer: [5:7577470955863571880:0] socket: 192 qp: -1
2025-11-27T18:17:19.468153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470952560442443:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:19.468181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:19.469577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470954429420201:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-11-27T18:17:19.469682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470954429420202:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-11-27T18:17:19.470381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470954429420203:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-11-27T18:17:19.466391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:19.470559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470954429420198:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:19.472498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470950134452814:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.472557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:19.472601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470950134452814:2049] [node 2] ICP22 created new session: [6:7577470954429420204:2049]
2025-11-27T18:17:19.472640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470954429420204:2049] [node 2] ICS09 handshake done sender: [6:7577470954429420198:2119] self: [6:7577470954429420200:0] peer: [2:7577470954117810353:0] socket: 185 qp: -1
2025-11-27T18:17:19.472672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470954429420204:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:19.472730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:19.470578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470954117810345:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.470637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:19.470677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470954117810345:2107] [node 6] ICP22 created new session: [2:7577470954117810385:2107]
2025-11-27T18:17:19.471076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470954117810385:2107] [node 6] ICS09 handshake done sender: [2:7577470954117810354:2125] self: [2:7577470954117810353:0] peer: [6:7577470954429420200:0] socket: 169 qp: -1
2025-11-27T18:17:19.471111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470954117810385:2107] [node 6] ICS10 traffic start
2025-11-27T18:17:19.471146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:19.470516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:19.470570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470951568604489:2048] [node 1] ICP22 created new session: [5:7577470955863571882:2048]
2025-11-27T18:17:19.470607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470955863571882:2048] [node 1] ICS09 handshake done sender: [5:7577470955863571868:2118] self: [5:7577470955863571880:0] peer: [1:7577470952560442420:0] socket: 193 qp: -1
2025-11-27T18:17:19.470633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470955863571882:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:19.470661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:19.466427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470948924591454:2048] [node 1] ICP22 created new session: [3:7577470953219558859:2048]
2025-11-27T18:17:19.475619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470954429420203:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.477049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470952560442412:2109] [node 5] ICP18 handshake (actor [1:7577470952560442436:2145]) is too old
2025-11-27T18:17:19.466477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470953219558859:2048] [node 1] ICS09 handshake done sender: [3:7577470953219558845:2127] self: [3:7577470953219558852:0] peer: [1:7577470952560442414:0] socket: 198 qp: -1
2025-11-27T18:17:19.466514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470953219558859:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:19.466566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:19.473675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470953219558833:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.473698Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470953219558833:2122] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.479612Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470953219558833:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:19.479625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470952095700016:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9567
2025-11-27T18:17:19.479913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470953219558823:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.480810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470954429420175:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.480034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:19.480084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470953219558823:2109] [node 4] ICP22 created new session: [3:7577470953219558862:2109]
2025-11-27T18:17:19.480228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470953219558862:2109] [node 4] ICS09 handshake done sender: [3:7577470953219558833:2122] self: [3:7577470953219558832:0] peer: [4:7577470952095700014:0] socket: 213 qp: -1
2025-11-27T18:17:19.480259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470953219558862:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:19.480309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:19.502254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470954429420201:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.502727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470950134452813:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.503140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470952560442423:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.503177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470952560442423:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.503901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470952560442423:2141] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:19.504128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470952560442413:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.504204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:19.504253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470952560442413:2111] [node 6] ICP22 created new session: [1:7577470952560442448:2111]
2025-11-27T18:17:19.504295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470952560442448:2111] [node 6] ICS09 handshake done sender: [1:7577470952560442423:2141] self: [1:7577470952560442422:0] peer: [6:7577470954429420207:0] socket: 216 qp: -1
2025-11-27T18:17:19.504325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470952560442448:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:19.504349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.505428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470954429420208:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-11-27T18:17:19.505872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470954429420201:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:19.506025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470950134452813:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.506096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:19.506140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470950134452813:2048] [node 1] ICP22 created new session: [6:7577470954429420209:2048]
2025-11-27T18:17:19.506190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470954429420209:2048] [node 1] ICS09 handshake done sender: [6:7577470954429420201:2120] self: [6:7577470954429420207:0] peer: [1:7577470952560442422:0] socket: 220 qp: -1
2025-11-27T18:17:19.506229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470954429420209:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:19.506276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:19.509711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470952095699998:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.509758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470952095699998:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.522164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470952095700009:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:19.525213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470954429420211:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-11-27T18:17:19.522948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470947800732632:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470952095700009:2119] held: no
2025-11-27T18:17:19.523045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470947800732632:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.523112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:19.523167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470947800732632:2050] [node 3] ICP22 created new session: [4:7577470952095700018:2050]
2025-11-27T18:17:19.523214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470952095700018:2050] [node 3] ICS09 handshake done sender: [4:7577470952095700009:2119] self: [4:7577470952095700014:0] peer: [3:7577470953219558832:0] socket: 176 qp: -1
2025-11-27T18:17:19.523248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470952095700018:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:19.523280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:19.524145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470952095700017:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9567
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.525726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470955863571884:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64612
2025-11-27T18:17:19.529496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470952095700017:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.529680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470952095699998:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:19.530381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470954429420203:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:19.530577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470954429420175:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.530674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:19.530730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470954429420175:2068] [node 4] ICP22 created new session: [6:7577470954429420212:2068]
2025-11-27T18:17:19.530782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470954429420212:2068] [node 4] ICS09 handshake done sender: [6:7577470954429420203:2122] self: [6:7577470954429420206:0] peer: [4:7577470952095699997:0] socket: 223 qp: -1
2025-11-27T18:17:19.530815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470954429420212:2068] [node 4] ICS10 traffic start
2025-11-27T18:17:19.530861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:19.532707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470952095699991:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.532820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:19.532866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470952095699991:2084] [node 6] ICP22 created new session: [4:7577470952095700022:2084]
2025-11-27T18:17:19.532903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470952095700022:2084] [node 6] ICS09 handshake done sender: [4:7577470952095699998:2113] self: [4:7577470952095699997:0] peer: [6:7577470954429420206:0] socket: 221 qp: -1
2025-11-27T18:17:19.532934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470952095700022:2084] [node 6] ICS10 traffic start
2025-11-27T18:17:19.532982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:19.535625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470947800732630:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.536114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470952560442425:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.536143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470952560442425:2142] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.536982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470952560442425:2142] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:19.537285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470952560442411:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.537374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:19.537421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470952560442411:2107] [node 4] ICP22 created new session: [1:7577470952560442451:2107]
2025-11-27T18:17:19.537447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470952560442451:2107] [node 4] ICS09 handshake done sender: [1:7577470952560442425:2142] self: [1:7577470952560442424:0] peer: [4:7577470952095700024:0] socket: 206 qp: -1
2025-11-27T18:17:19.537473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470952560442451:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:19.537496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.540162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470947800732630:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28624) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470952095700017:2121] held: no
2025-11-27T18:17:19.540364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470952095700025:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9567
2025-11-27T18:17:19.540668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470955863571884:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.540809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470955863571834:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:19.541102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470952095700000:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:19.541127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470952095700000:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:19.541852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470952095700017:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:19.542149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470947800732630:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.542224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:19.542271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470947800732630:2048] [node 1] ICP22 created new session: [4:7577470952095700027:2048]
2025-11-27T18:17:19.542312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470952095700027:2048] [node 1] ICS09 handshake done sender: [4:7577470952095700017:2121] self: [4:7577470952095700024:0] peer: [1:7577470952560442424:0] socket: 166 qp: -1
2025-11-27T18:17:19.542345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470952095700027:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:19.542381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:19.543403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470952095700000:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:19.543906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470955863571886:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64612
2025-11-27T18:17:19.544353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470955863571884:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:19.544554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470955863571834:2079] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:19.544596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:19.544641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470955863571834:2079] [node 4] ICP22 created new session: [5:7577470955863571887:2079]
2025-11-27T18:17:19.544678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470955863571887:2079] [node 4] ICS09 handshake done sender: [5:7577470955863571884:2122] self: [5:7577470955863571885:0] peer: [4:7577470952095699999:0] socket: 215 qp: -1
2025-11-27T18:17:19.544709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470955863571887:2079] [node 4] ICS10 traffic start
2025-11-27T18:17:19.544737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:19.546408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470952095699992:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:19.546483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:19.546518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470952095699992:2088] [node 5] ICP22 created new session: [4:7577470952095700029:2088]
2025-11-27T18:17:19.546564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470952095700029:2088] [node 5] ICS09 handshake done sender: [4:7577470952095700000:2114] self: [4:7577470952095699999:0] peer: [5:7577470955863571885:0] socket: 183 qp: -1
2025-11-27T18:17:19.546591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470952095700029:2088] [node 5] ICS10 traffic start
2025-11-27T18:17:19.546616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:19.564783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:19.565019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470954117810379:2048] [node 1] ICS01 socket: 195 reason# EndOfStream
2025-11-27T18:17:19.565057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470949822842973:2048] [node 1] ICP30 unregister session Session# [2:7577470954117810379:2048] VirtualId# [2:7577470954117810377:0]
2025-11-27T18:17:19.565087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470954117810379:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.565166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:19.565217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:19.565988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:19.566175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470953219558859:2048] [node 1] ICS07 socket disconnect 198 reason# EndOfStream
2025-11-27T18:17:19.566207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470953219558859:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.566268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:19.566316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470953219558859:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:19.570920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:19.571146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470952095700027:2048] [node 1] ICS01 socket: 166 reason# EndOfStream
2025-11-27T18:17:19.571188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470947800732630:2048] [node 1] ICP30 unregister session Session# [4:7577470952095700027:2048] VirtualId# [4:7577470952095700024:0]
2025-11-27T18:17:19.571224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470952095700027:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.571296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:19.571333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:19.577444Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:19.577649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470955863571882:2048] [node 1] ICS01 socket: 193 reason# EndOfStream
2025-11-27T18:17:19.577683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470951568604489:2048] [node 1] ICP30 unregister session Session# [5:7577470955863571882:2048] VirtualId# [5:7577470955863571880:0]
2025-11-27T18:17:19.577726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470955863571882:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.577795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:19.577835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:19.577923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:19.578066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470954429420209:2048] [node 1] ICS01 socket: 220 reason# EndOfStream
2025-11-27T18:17:19.578094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470950134452813:2048] [node 1] ICP30 unregister session Session# [6:7577470954429420209:2048] VirtualId# [6:7577470954429420207:0]
2025-11-27T18:17:19.578111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470954429420209:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.578147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:19.578177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:19.579270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470948924591454:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28624) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:19.579328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470953219558859:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:19.580215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470948924591454:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28624) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:19.580262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470953219558859:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:19.580289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470948924591454:2048] [node 1] ICP30 unregister session Session# [3:7577470953219558859:2048] VirtualId# [3:7577470953219558852:0]
2025-11-27T18:17:19.580326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:19.580349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470948924591454:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:19.580376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:19.591680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470948924591454:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:19.600182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:19.600425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470953219558855:2049] [node 2] ICS07 socket disconnect 165 reason# EndOfStream
2025-11-27T18:17:19.600454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470953219558855:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.600529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:19.600571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470953219558855:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:19.604644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470948924591455:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27723) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:19.604690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470953219558855:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:19.605493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470948924591455:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27723) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:19.605576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470953219558855:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:19.605611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470948924591455:2049] [node 2] ICP30 unregister session Session# [3:7577470953219558855:2049] VirtualId# [3:7577470953219558841:0]
2025-11-27T18:17:19.605657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:19.605688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470948924591455:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27723) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:19.605705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27723) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:19.607733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:19.607969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470955863571877:2049] [node 2] ICS01 socket: 173 reason# EndOfStream
2025-11-27T18:17:19.608014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470951568604490:2049] [node 2] ICP30 unregister session Session# [5:7577470955863571877:2049] VirtualId# [5:7577470955863571865:0]
2025-11-27T18:17:19.608048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470955863571877:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.608112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:19.608148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:19.610099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:19.610287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470952095700010:2049] [node 2] ICS01 socket: 179 reason# EndOfStream
2025-11-27T18:17:19.610339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470947800732631:2049] [node 2] ICP30 unregister session Session# [4:7577470952095700010:2049] VirtualId# [4:7577470952095700006:0]
2025-11-27T18:17:19.610367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470952095700010:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.610427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:19.610462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:19.613666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:19.613889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470954429420204:2049] [node 2] ICS07 socket disconnect 185 reason# EndOfStream
2025-11-27T18:17:19.613916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470954429420204:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.613991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:19.614033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470954429420204:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:19.614964Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470950134452814:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27723) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:19.615014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470954429420204:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:19.615045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470950134452814:2049] [node 2] ICP30 unregister session Session# [6:7577470954429420204:2049] VirtualId# [6:7577470954429420200:0]
2025-11-27T18:17:19.615085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:19.615114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470950134452814:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27723) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:19.615129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27723) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:19.615685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470948924591455:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:19.624896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470950134452814:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:19.665383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:19.664299Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:19.664537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470952095700018:2050] [node 3] ICS01 socket: 176 reason# EndOfStream
2025-11-27T18:17:19.664577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470947800732632:2050] [node 3] ICP30 unregister session Session# [4:7577470952095700018:2050] VirtualId# [4:7577470952095700014:0]
2025-11-27T18:17:19.664620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470952095700018:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.664704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:19.664750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:19.665389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:19.665604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470954429420196:2050] [node 3] ICS07 socket disconnect 172 reason# EndOfStream
2025-11-27T18:17:19.665643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470954429420196:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.665716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:19.665765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470954429420196:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:19.665603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470955863571874:2050] [node 3] ICS01 socket: 174 reason# EndOfStream
2025-11-27T18:17:19.665650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470951568604491:2050] [node 3] ICP30 unregister session Session# [5:7577470955863571874:2050] VirtualId# [5:7577470955863571866:0]
2025-11-27T18:17:19.665679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470955863571874:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.665750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:19.665794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:19.666526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470950134452815:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7482) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:19.666575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470954429420196:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:19.666602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470950134452815:2050] [node 3] ICP30 unregister session Session# [6:7577470954429420196:2050] VirtualId# [6:7577470954429420190:0]
2025-11-27T18:17:19.666665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:19.666692Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470950134452815:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7482) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:19.666706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7482) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:19.678340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470950134452815:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:19.689429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:19.690143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470954429420212:2068] [node 4] ICS01 socket: 223 reason# EndOfStream
2025-11-27T18:17:19.690182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470954429420175:2068] [node 4] ICP30 unregister session Session# [6:7577470954429420212:2068] VirtualId# [6:7577470954429420206:0]
2025-11-27T18:17:19.690214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470954429420212:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.690276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:19.690317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:19.689430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:19.689746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470955863571887:2079] [node 4] ICS01 socket: 215 reason# EndOfStream
2025-11-27T18:17:19.689787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470955863571834:2079] [node 4] ICP30 unregister session Session# [5:7577470955863571887:2079] VirtualId# [5:7577470955863571885:0]
2025-11-27T18:17:19.689832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470955863571887:2079] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.689915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:19.689968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:19.730054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:19.730245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470954429420193:2065] [node 5] ICS01 socket: 181 reason# EndOfStream
2025-11-27T18:17:19.730292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470954429420174:2065] [node 5] ICP30 unregister session Session# [6:7577470954429420193:2065] VirtualId# [6:7577470954429420189:0]
2025-11-27T18:17:19.730317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470954429420193:2065] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:19.730387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:19.730446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:19.799375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470954557042142:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:19.799427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470954557042143:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:19.820881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470956108548215:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:19.821124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470956108548216:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:19.843446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470954250163573:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:19.843471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470954250163574:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:19.866625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470953387485549:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:19.866679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470953387485550:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:19.868473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470953387485551:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:19.887696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470952165848824:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:19.887722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470952165848825:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:19.887743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470952165848826:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:19.908600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470954821910434:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:19.908653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470954821910436:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:19.908794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470954821910435:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:20.106995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470959116877782:2092] [node 4] ICP01 ready to work
2025-11-27T18:17:20.167885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470958852009501:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:20.175937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470959116877791:2095] [node 5] ICP01 ready to work
2025-11-27T18:17:20.219472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470960403515575:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:20.262452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470958545130932:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:20.251774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470958852009502:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:20.293668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470956108548215:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(188741717ns) {Configure:104(188734085ns) {ProcessConfigured:135(80220ns) {StartInitialHandshake:157(77065ns) {DropHandshakes:470(2530ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(986ns) PrepareNewSessionHandshake:477(1572ns) {SwitchToState:179(487ns)}}}}}
2025-11-27T18:17:20.294179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470960403515578:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:20.294517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470960403515579:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:20.283829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470958545130934:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:20.309410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470958852009503:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:20.322588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470954557042142:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(240845031ns) {Configure:104(240837849ns) {ProcessConfigured:135(81768ns) {StartInitialHandshake:157(78312ns) {DropHandshakes:470(2385ns) {DropIncomingHandshake:442(537ns) DropOutgoingHandshake:457(608ns)} GenerateSessionVirtualId:988(1244ns) PrepareNewSessionHandshake:477(1613ns) {SwitchToState:179(576ns)}}}}}
2025-11-27T18:17:20.330276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470956460816184:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:20.330536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470957682452916:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:20.332358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470954821910436:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(156195799ns) {Configure:104(156191804ns) {ProcessConfigured:135(67159ns) {StartInitialHandshake:157(64413ns) {DropHandshakes:470(4963ns) {DropIncomingHandshake:442(770ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(1128ns) PrepareNewSessionHandshake:477(1315ns) {SwitchToState:179(331ns)}}}}}
2025-11-27T18:17:20.286098Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470954250163574:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(115438402ns) {Configure:104(115429006ns) {ProcessConfigured:135(24962794ns) {StartInitialHandshake:157(24959419ns) {DropHandshakes:470(2188ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(208ns)} GenerateSessionVirtualId:988(893ns) PrepareNewSessionHandshake:477(10586ns) {SwitchToState:179(2508ns)}}}}}
2025-11-27T18:17:20.286579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470958545130936:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:20.330747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470960403515578:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23159045ns) {Configure:104(23154262ns) {ProcessConfigured:135(22054194ns) {StartInitialHandshake:157(22037509ns) {DropHandshakes:470(2403ns) {DropIncomingHandshake:442(623ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(574ns) PrepareNewSessionHandshake:477(1890554ns) {SwitchToState:179(5296ns)}}}}}
2025-11-27T18:17:20.341473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470952165848825:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(75429247ns) {Configure:104(75421147ns) {ProcessConfigured:135(114242ns) {StartInitialHandshake:157(112395ns) {DropHandshakes:470(2181ns) {DropIncomingHandshake:442(362ns) DropOutgoingHandshake:457(257ns)} GenerateSessionVirtualId:988(874ns) PrepareNewSessionHandshake:477(2068ns) {SwitchToState:179(445ns)}}}}}
2025-11-27T18:17:20.350354Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470959116877782:2092] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(213501007ns) {Configure:104(213494212ns) {ProcessConfigured:135(58173ns) {StartInitialHandshake:157(55454ns) {DropHandshakes:470(1808ns) {DropIncomingHandshake:442(498ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(511ns) PrepareNewSessionHandshake:477(1170ns) {SwitchToState:179(403ns)}}}}}
2025-11-27T18:17:20.349011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470956460816193:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:20.348576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470957682452917:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:20.350184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470953387485549:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(167720901ns) {Configure:104(167712583ns) {ProcessConfigured:135(48450ns) {StartInitialHandshake:157(46307ns) {DropHandshakes:470(1759ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(472ns) PrepareNewSessionHandshake:477(916ns) {SwitchToState:179(269ns)}}}}}
2025-11-27T18:17:20.356478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470958852009503:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31126082ns) {Configure:104(31123566ns) {ProcessConfigured:135(53112ns) {StartInitialHandshake:157(50777ns) {DropHandshakes:470(659ns) {DropIncomingHandshake:442(91ns) DropOutgoingHandshake:457(149ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:477(1561ns) {SwitchToState:179(426ns)}}}}}
2025-11-27T18:17:20.360411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470959116877791:2095] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20703507ns) {Configure:104(20698948ns) {ProcessConfigured:135(4770720ns) {StartInitialHandshake:157(2693005ns) {DropHandshakes:470(1429ns) {DropIncomingHandshake:442(703ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(860ns) PrepareNewSessionHandshake:477(1866ns) {SwitchToState:179(181ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.361521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470958852009515:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36685
2025-11-27T18:17:20.365859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470956108548216:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(132218980ns) {Configure:104(132213777ns) {ProcessConfigured:135(2564064ns) {StartInitialHandshake:157(2561674ns) {DropHandshakes:470(2298ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(956ns) PrepareNewSessionHandshake:477(3311ns) {SwitchToState:179(1109ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.371124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470960403515579:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(34528655ns) {Configure:104(34522468ns) {ProcessConfigured:135(4378695ns) {StartInitialHandshake:157(4376443ns) {DropHandshakes:470(482ns) {DropIncomingHandshake:442(94ns) DropOutgoingHandshake:457(87ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(3102ns) {SwitchToState:179(1045ns)}}}}}
2025-11-27T18:17:20.366375Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470954250163573:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(77488428ns) {Configure:104(77482926ns) {ProcessConfigured:135(3439956ns) {StartInitialHandshake:157(3436570ns) {DropHandshakes:470(1609ns) {DropIncomingHandshake:442(447ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(1606ns) PrepareNewSessionHandshake:477(2566ns) {SwitchToState:179(1167ns)}}}}}
2025-11-27T18:17:20.371437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470958545130936:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(35367482ns) {Configure:104(35362425ns) {ProcessConfigured:135(1771954ns) {StartInitialHandshake:157(1769104ns) {DropHandshakes:470(1297ns) {DropIncomingHandshake:442(242ns) DropOutgoingHandshake:457(167ns)} GenerateSessionVirtualId:988(412ns) PrepareNewSessionHandshake:477(4775ns) {SwitchToState:179(833ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.383915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470958852009518:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36685
2025-11-27T18:17:20.375188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470953387485551:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21694706ns) {Configure:104(19242097ns) {ProcessConfigured:135(65264ns) {StartInitialHandshake:157(63271ns) {DropHandshakes:470(2556ns) {DropIncomingHandshake:442(512ns) DropOutgoingHandshake:457(265ns)} GenerateSessionVirtualId:988(544ns) PrepareNewSessionHandshake:477(1454ns) {SwitchToState:179(344ns)}}}}}
2025-11-27T18:17:20.373787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470958545130943:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15147
2025-11-27T18:17:20.386337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470954557042143:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(64333971ns) {Configure:104(64322775ns) {ProcessConfigured:135(5328272ns) {StartInitialHandshake:157(5325876ns) {DropHandshakes:470(1400ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(446ns) PrepareNewSessionHandshake:477(8139ns) {SwitchToState:179(3776ns)}}}}}
2025-11-27T18:17:20.387258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470958852009515:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.407351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470954821910435:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(30192955ns) {Configure:104(30187421ns) {ProcessConfigured:135(74056ns) {StartInitialHandshake:157(70810ns) {DropHandshakes:470(5194ns) {DropIncomingHandshake:442(630ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(832ns) PrepareNewSessionHandshake:477(1717ns) {SwitchToState:179(815ns)}}}}}
2025-11-27T18:17:20.396391Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470958545130934:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20150352ns) {Configure:104(20145590ns) {ProcessConfigured:135(1218365ns) {StartInitialHandshake:157(1214453ns) {DropHandshakes:470(3389ns) {DropIncomingHandshake:442(944ns) DropOutgoingHandshake:457(620ns)} GenerateSessionVirtualId:988(800ns) PrepareNewSessionHandshake:477(5123ns) {SwitchToState:179(1322ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.409615Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470954821910434:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33051864ns) {Configure:104(33046315ns) {ProcessConfigured:135(7123191ns) {StartInitialHandshake:157(7120587ns) {DropHandshakes:470(2305ns) {DropIncomingHandshake:442(556ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(1020ns) PrepareNewSessionHandshake:477(3416ns) {SwitchToState:179(1010ns)}}}}}
2025-11-27T18:17:20.413986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470959116877801:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21029
2025-11-27T18:17:20.414126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470959116877802:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21029
2025-11-27T18:17:20.414837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470958545130949:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15147
2025-11-27T18:17:20.415122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470953387485550:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48027257ns) {Configure:104(48020551ns) {ProcessConfigured:135(16748823ns) {StartInitialHandshake:157(16745638ns) {DropHandshakes:470(2047ns) {DropIncomingHandshake:442(474ns) DropOutgoingHandshake:457(326ns)} GenerateSessionVirtualId:988(956ns) PrepareNewSessionHandshake:477(3216ns) {SwitchToState:179(1238ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.416459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470956460816199:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5998
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.415849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470959116877802:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.416524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470958545130943:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.421589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470958852009518:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.415992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470954821910436:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.420570Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470957682452916:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39103400ns) {Configure:104(39098294ns) {ProcessConfigured:135(104877ns) {StartInitialHandshake:157(102960ns) {DropHandshakes:470(3004ns) {DropIncomingHandshake:442(879ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(962ns) PrepareNewSessionHandshake:477(1361ns) {SwitchToState:179(523ns)}}}}}
2025-11-27T18:17:20.417063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470959116877801:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.423528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470954821910434:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.430474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470956460816199:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.422731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470957682452926:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9045
2025-11-27T18:17:20.422867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470957682452927:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9045
2025-11-27T18:17:20.429432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470957682452926:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.421474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470958545130939:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.426878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470958852009509:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.426906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470958852009509:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.421504Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470958545130939:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.430208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470959116877812:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21029
2025-11-27T18:17:20.429616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470958852009521:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36685
2025-11-27T18:17:20.430468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470958852009509:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:20.432964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470958852009502:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.433014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:20.433054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470958852009502:2108] [node 6] ICP22 created new session: [1:7577470958852009523:2108]
2025-11-27T18:17:20.433109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470958852009523:2108] [node 6] ICS09 handshake done sender: [1:7577470958852009509:2112] self: [1:7577470958852009508:0] peer: [6:7577470959116877811:0] socket: 167 qp: -1
2025-11-27T18:17:20.433141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470958852009523:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:20.433171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.429318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470958545130949:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.437376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470957682452927:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.456349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470960403515588:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28428
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.464611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470958852009526:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36685
2025-11-27T18:17:20.441134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470953387485549:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.453792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470956460816201:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5998
2025-11-27T18:17:20.457010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470958545130939:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.457126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470960403515590:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28428
2025-11-27T18:17:20.469383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470956460816201:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.471890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470958545130951:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15147
2025-11-27T18:17:20.472254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470956460816202:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5998
2025-11-27T18:17:20.478157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470952165848824:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:20.481609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470959116877801:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:20.477407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470960403515589:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28428
2025-11-27T18:17:20.478343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470958852009507:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.496073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470957682452936:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9045
2025-11-27T18:17:20.478399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470958852009507:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.486479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470960403515588:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.491111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470958545130951:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.497441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470958545130936:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.497518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:20.497570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470958545130936:2109] [node 6] ICP22 created new session: [3:7577470958545130954:2109]
2025-11-27T18:17:20.497620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470958545130954:2109] [node 6] ICS09 handshake done sender: [3:7577470958545130939:2111] self: [3:7577470958545130937:0] peer: [6:7577470959116877810:0] socket: 173 qp: -1
2025-11-27T18:17:20.481322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470958852009521:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.502338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470960403515589:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.497655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470958545130954:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:20.497707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.510252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470954821910434:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.511437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:20.493301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470958852009511:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.506946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470960403515590:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.493337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470958852009511:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.509590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470960403515594:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28428
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.514654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470957682452935:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9045
2025-11-27T18:17:20.512369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470956108548215:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:20.516577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470960403515594:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.493981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470958852009507:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:20.514468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470954821910434:2048] [node 1] ICP22 created new session: [6:7577470959116877814:2048]
2025-11-27T18:17:20.516303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470956460816204:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5998
2025-11-27T18:17:20.498528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470958852009511:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:20.510345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470958852009501:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.511471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:20.517368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470958545130958:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15147
2025-11-27T18:17:20.516288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470959116877814:2048] [node 1] ICS09 handshake done sender: [6:7577470959116877801:2113] self: [6:7577470959116877811:0] peer: [1:7577470958852009508:0] socket: 175 qp: -1
2025-11-27T18:17:20.512423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470958852009501:2107] [node 4] ICP22 created new session: [1:7577470958852009530:2107]
2025-11-27T18:17:20.512787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470958852009503:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.512856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:20.512901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470958852009503:2109] [node 5] ICP22 created new session: [1:7577470958852009531:2109]
2025-11-27T18:17:20.512942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470958852009531:2109] [node 5] ICS09 handshake done sender: [1:7577470958852009511:2113] self: [1:7577470958852009510:0] peer: [5:7577470956460816203:0] socket: 198 qp: -1
2025-11-27T18:17:20.512972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470958852009531:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:20.513010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:20.514440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470958852009530:2107] [node 4] ICS09 handshake done sender: [1:7577470958852009507:2111] self: [1:7577470958852009506:0] peer: [4:7577470957682452933:0] socket: 166 qp: -1
2025-11-27T18:17:20.514480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470958852009530:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:20.515105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:20.517355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470958852009505:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.517383Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470958852009505:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.523259Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470958852009505:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:20.525874Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470952165848824:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19855) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470956460816201:2117] held: no
2025-11-27T18:17:20.526052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470956460816201:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:20.523585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470954557042142:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.521438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470957682452926:2122] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.530971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470958545130958:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.531202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470954250163573:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:20.530699Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470953387485549:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19855) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470957682452926:2122] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.539225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470960403515599:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28428
2025-11-27T18:17:20.539680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470952165848824:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.539737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:20.539778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470952165848824:2048] [node 1] ICP22 created new session: [5:7577470956460816205:2048]
2025-11-27T18:17:20.539818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470956460816205:2048] [node 1] ICS09 handshake done sender: [5:7577470956460816201:2117] self: [5:7577470956460816203:0] peer: [1:7577470958852009510:0] socket: 199 qp: -1
2025-11-27T18:17:20.539854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470956460816205:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:20.539884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:20.516868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470959116877814:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.546781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470960403515600:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28428
2025-11-27T18:17:20.547293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470960403515589:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:20.537589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470953387485549:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.523628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:20.516907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:20.545918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:20.523680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470954557042142:2048] [node 2] ICP22 created new session: [1:7577470958852009536:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.550250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470956460816202:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.530894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470954821910436:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11067) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470959116877802:2114] held: no
2025-11-27T18:17:20.550562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470956108548215:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.523720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470958852009536:2048] [node 2] ICS09 handshake done sender: [1:7577470958852009505:2110] self: [1:7577470958852009504:0] peer: [2:7577470960403515596:0] socket: 165 qp: -1
2025-11-27T18:17:20.547684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470953387485549:2048] [node 1] ICP22 created new session: [4:7577470957682452938:2048]
2025-11-27T18:17:20.550646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:20.552494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470956108548215:2048] [node 1] ICP22 created new session: [2:7577470960403515601:2048]
2025-11-27T18:17:20.548194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470957682452937:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9045
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.531799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470954821910434:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22075905ns) {HandleHandshakeStatus:329(21356625ns) {DropIncomingHandshake:442(656ns) DropOutgoingHandshake:457(7397ns) SwitchToState:179(460ns) ProcessPendingSessionEvents:495(6922360ns) {ForwardSessionEventToSession:612(6866790ns) ForwardSessionEventToSession:612(4860ns) ForwardSessionEventToSession:612(6217ns) ForwardSessionEventToSession:612(2251ns) ForwardSessionEventToSession:612(2889ns) ForwardSessionEventToSession:612(13929ns) ForwardSessionEventToSession:612(2400ns) ForwardSessionEventToSession:612(4890ns) ForwardSessionEventToSession:612(6216ns) ForwardSessionEventToSession:612(2173ns)}}}
2025-11-27T18:17:20.552508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470957682452938:2048] [node 1] ICS09 handshake done sender: [4:7577470957682452926:2122] self: [4:7577470957682452933:0] peer: [1:7577470958852009506:0] socket: 188 qp: -1
2025-11-27T18:17:20.552634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470957682452939:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9045
2025-11-27T18:17:20.552763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470957682452938:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:20.531983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470959116877813:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21029
2025-11-27T18:17:20.553284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.553216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470960403515601:2048] [node 1] ICS09 handshake done sender: [2:7577470960403515589:2117] self: [2:7577470960403515596:0] peer: [1:7577470958852009504:0] socket: 180 qp: -1
2025-11-27T18:17:20.553282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470960403515601:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:20.553101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470952165848825:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:20.557780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:20.557825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470956460816208:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5998
2025-11-27T18:17:20.523748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470958852009536:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:20.524565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:20.558845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470960403515600:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.557680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470953387485549:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(26477337ns) {HandleHandshakeStatus:329(26475281ns) {DropIncomingHandshake:442(1489ns) DropOutgoingHandshake:457(338ns) SwitchToState:179(710ns) ProcessPendingSessionEvents:495(976ns)}}
2025-11-27T18:17:20.557872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470957682452939:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.543952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470958852009517:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.544972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470959116877802:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:20.557574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470958545130960:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15147
2025-11-27T18:17:20.557878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470958545130958:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:20.559296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470954250163573:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.559954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:20.560754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470954250163573:2048] [node 1] ICP22 created new session: [3:7577470958545130961:2048]
2025-11-27T18:17:20.550772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470954821910436:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.550810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:20.543976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470958852009517:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.558931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470957682452937:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.547699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470958852009517:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:20.559497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470957682452935:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.572963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470958545130961:2048] [node 1] ICS09 handshake done sender: [3:7577470958545130958:2118] self: [3:7577470958545130959:0] peer: [1:7577470958852009516:0] socket: 194 qp: -1
2025-11-27T18:17:20.570006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470953387485551:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:20.551707Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470958852009526:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.570216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470953387485550:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:20.573032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470958545130961:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:20.551748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470954557042143:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.573080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:20.551795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:20.573472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470958545130942:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.551831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470954557042143:2049] [node 3] ICP22 created new session: [1:7577470958852009539:2049]
2025-11-27T18:17:20.551871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470958852009539:2049] [node 3] ICS09 handshake done sender: [1:7577470958852009517:2115] self: [1:7577470958852009516:0] peer: [3:7577470958545130959:0] socket: 168 qp: -1
2025-11-27T18:17:20.573493Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470958545130942:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.551895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470958852009539:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:20.550853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470954821910436:2050] [node 3] ICP22 created new session: [6:7577470959116877817:2050]
2025-11-27T18:17:20.550891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470959116877817:2050] [node 3] ICS09 handshake done sender: [6:7577470959116877802:2114] self: [6:7577470959116877810:0] peer: [3:7577470958545130937:0] socket: 178 qp: -1
2025-11-27T18:17:20.574374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577470954250163573:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:20.574520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470960403515583:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.551921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:20.550927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470959116877817:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:20.550955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:20.557713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470954557042142:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(28701125ns) {HandleHandshakeStatus:329(28697620ns) {DropIncomingHandshake:442(604ns) DropOutgoingHandshake:457(871ns) SwitchToState:179(598ns) ProcessPendingSessionEvents:495(11021003ns) {ForwardSessionEventToSession:612(4389253ns) ForwardSessionEventToSession:612(5699ns) ForwardSessionEventToSession:612(1938287ns) ForwardSessionEventToSession:612(2663ns) ForwardSessionEventToSession:612(1040491ns) ForwardSessionEventToSession:612(535542ns) ForwardSessionEventToSession:612(7234ns) ForwardSessionEventToSession:612(1098916ns) ForwardSessionEventToSession:612(1978274ns) ForwardSessionEventToSession:612(2951ns)}}}
2025-11-27T18:17:20.575859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470956460816208:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.558568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470954557042142:2048] [node 2] ICP18 handshake (actor [1:7577470958852009526:2118]) is too old
2025-11-27T18:17:20.574560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470960403515583:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.575527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470960403515581:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.575558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470960403515581:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.577710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470956460816184:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:20.578037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470957682452925:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.581163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470956460816212:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5998
2025-11-27T18:17:20.578060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470957682452925:2121] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.580222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470957682452925:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:20.582130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470960403515581:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.585768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470956460816211:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5998
2025-11-27T18:17:20.583270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470958545130942:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:20.585487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470960403515575:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.589792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470957682452916:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.590734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470958545130965:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15147
2025-11-27T18:17:20.590215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470956460816213:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5998
2025-11-27T18:17:20.592931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470958545130932:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.590001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:20.591115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470960403515575:2107] [node 4] ICP22 created new session: [2:7577470960403515607:2107]
2025-11-27T18:17:20.592243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470960403515607:2107] [node 4] ICS09 handshake done sender: [2:7577470960403515581:2112] self: [2:7577470960403515580:0] peer: [4:7577470957682452944:0] socket: 207 qp: -1
2025-11-27T18:17:20.592355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470960403515607:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:20.592396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:20.595053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470960403515583:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:20.597629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470960403515578:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.597724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:20.597763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470960403515578:2108] [node 5] ICP22 created new session: [2:7577470960403515610:2108]
2025-11-27T18:17:20.597812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470960403515610:2108] [node 5] ICS09 handshake done sender: [2:7577470960403515583:2113] self: [2:7577470960403515582:0] peer: [5:7577470956460816209:0] socket: 205 qp: -1
2025-11-27T18:17:20.597852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470960403515610:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:20.597909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:20.591988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.591454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470956460816208:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.599370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470959116877819:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21029
2025-11-27T18:17:20.593823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:20.592385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470957682452916:2108] [node 5] ICP22 created new session: [4:7577470957682452948:2108]
2025-11-27T18:17:20.593770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470956460816184:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.594442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470958545130932:2107] [node 4] ICP22 created new session: [3:7577470958545130966:2107]
2025-11-27T18:17:20.594562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:20.592463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470957682452948:2108] [node 5] ICS09 handshake done sender: [4:7577470957682452925:2121] self: [4:7577470957682452924:0] peer: [5:7577470956460816210:0] socket: 220 qp: -1
2025-11-27T18:17:20.598370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470958545130966:2107] [node 4] ICS09 handshake done sender: [3:7577470958545130942:2113] self: [3:7577470958545130941:0] peer: [4:7577470957682452943:0] socket: 184 qp: -1
2025-11-27T18:17:20.598408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470958545130966:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:20.599050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.607224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470959116877820:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21029
2025-11-27T18:17:20.592493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470957682452948:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:20.608402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470958545130965:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.608580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470954250163574:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:20.615637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470958545130948:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.615659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470958545130948:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.616504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470958545130948:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:20.595347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470956460816184:2108] [node 4] ICP22 created new session: [5:7577470956460816214:2108]
2025-11-27T18:17:20.618411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470959116877819:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.618576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470954821910435:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.592544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:20.595382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470956460816212:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.622357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470960403515585:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.598531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470957682452946:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9045
2025-11-27T18:17:20.622399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470960403515585:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.622539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470958545130934:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.604877Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470953387485550:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3183) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470957682452935:2125] held: no
2025-11-27T18:17:20.600300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470956460816214:2108] [node 4] ICS09 handshake done sender: [5:7577470956460816208:2120] self: [5:7577470956460816210:0] peer: [4:7577470957682452924:0] socket: 221 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.624598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470960403515587:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.624615Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470960403515587:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.628493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470960403515585:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:20.626761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:20.627623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470958545130934:2108] [node 5] ICP22 created new session: [3:7577470958545130970:2108]
2025-11-27T18:17:20.629017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470958545130970:2108] [node 5] ICS09 handshake done sender: [3:7577470958545130948:2115] self: [3:7577470958545130947:0] peer: [5:7577470956460816216:0] socket: 225 qp: -1
2025-11-27T18:17:20.629053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470958545130970:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:20.629122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:20.600329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470956460816214:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:20.605028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470957682452947:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9045
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.628646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470959116877823:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21029
2025-11-27T18:17:20.629698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470960403515587:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:20.601964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:20.608908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470952165848825:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3183) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470956460816202:2118] held: no
2025-11-27T18:17:20.609604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470956460816202:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:20.613892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470952165848826:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:20.613981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470952165848825:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.614044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:20.614085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470952165848825:2049] [node 2] ICP22 created new session: [5:7577470956460816218:2049]
2025-11-27T18:17:20.614134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470956460816218:2049] [node 2] ICS09 handshake done sender: [5:7577470956460816202:2118] self: [5:7577470956460816209:0] peer: [2:7577470960403515582:0] socket: 206 qp: -1
2025-11-27T18:17:20.614160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470956460816218:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:20.614192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:20.630692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470959116877822:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21029
2025-11-27T18:17:20.630897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470959116877820:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.630948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470959116877819:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:20.632246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470959116877782:2092] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:20.606556Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470953387485551:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11067) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470957682452937:2127] held: no
2025-11-27T18:17:20.632981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470960403515579:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.632652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470954821910435:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.622312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470956460816220:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5998
2025-11-27T18:17:20.606817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470957682452935:2125] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:20.633089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:20.633441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:20.624172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470956460816212:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:20.634558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470954821910435:2049] [node 2] ICP22 created new session: [6:7577470959116877825:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.635354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470959116877825:2049] [node 2] ICS09 handshake done sender: [6:7577470959116877819:2119] self: [6:7577470959116877821:0] peer: [2:7577470960403515586:0] socket: 201 qp: -1
2025-11-27T18:17:20.610225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470953387485550:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.635866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470959116877825:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:20.636384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:20.610305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:20.610351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470953387485550:2049] [node 2] ICP22 created new session: [4:7577470957682452951:2049]
2025-11-27T18:17:20.610387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470957682452951:2049] [node 2] ICS09 handshake done sender: [4:7577470957682452935:2125] self: [4:7577470957682452944:0] peer: [2:7577470960403515580:0] socket: 208 qp: -1
2025-11-27T18:17:20.610419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470957682452951:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:20.610448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:20.613545Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470957682452937:2127] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:20.640133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470959116877822:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.624387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470952165848826:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.624452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:20.625454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470952165848826:2050] [node 3] ICP22 created new session: [5:7577470956460816221:2050]
2025-11-27T18:17:20.633132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470960403515579:2110] [node 6] ICP22 created new session: [2:7577470960403515614:2110]
2025-11-27T18:17:20.633188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470960403515614:2110] [node 6] ICS09 handshake done sender: [2:7577470960403515587:2115] self: [2:7577470960403515586:0] peer: [6:7577470959116877821:0] socket: 193 qp: -1
2025-11-27T18:17:20.633219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470960403515614:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:20.627670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470956460816221:2050] [node 3] ICS09 handshake done sender: [5:7577470956460816212:2122] self: [5:7577470956460816216:0] peer: [3:7577470958545130947:0] socket: 226 qp: -1
2025-11-27T18:17:20.620504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470953387485551:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.633297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:20.627697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470956460816221:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:20.641922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470958545130971:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15147
2025-11-27T18:17:20.634518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470956108548216:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.627734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:20.620596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:20.644100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470959116877791:2095] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:20.621196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470953387485551:2050] [node 3] ICP22 created new session: [4:7577470957682452953:2050]
2025-11-27T18:17:20.639253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:20.639302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470956108548216:2049] [node 3] ICP22 created new session: [2:7577470960403515616:2049]
2025-11-27T18:17:20.644712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470959116877782:2092] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470959116877820:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.645132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470959116877827:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21029
2025-11-27T18:17:20.645680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470959116877820:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:20.639337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470960403515616:2049] [node 3] ICS09 handshake done sender: [2:7577470960403515585:2114] self: [2:7577470960403515584:0] peer: [3:7577470958545130968:0] socket: 222 qp: -1
2025-11-27T18:17:20.621331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470957682452953:2050] [node 3] ICS09 handshake done sender: [4:7577470957682452937:2127] self: [4:7577470957682452943:0] peer: [3:7577470958545130941:0] socket: 213 qp: -1
2025-11-27T18:17:20.646083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470956460816198:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.646114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470956460816198:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.644706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470954250163574:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3183) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470958545130965:2120] held: no
2025-11-27T18:17:20.639367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470960403515616:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:20.621374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470957682452953:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:20.647316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470956460816198:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:20.646944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470959116877782:2092] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.645051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470958545130965:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:20.648125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:20.621425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:20.647041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470954250163574:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.634577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470957682452932:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:20.648406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470959116877782:2092] [node 4] ICP22 created new session: [6:7577470959116877830:2092]
2025-11-27T18:17:20.634603Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470957682452932:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:20.637984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470957682452932:2124] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:20.640561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470957682452917:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.640613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:20.640651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470957682452917:2109] [node 6] ICP22 created new session: [4:7577470957682452956:2109]
2025-11-27T18:17:20.640679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470957682452956:2109] [node 6] ICS09 handshake done sender: [4:7577470957682452932:2124] self: [4:7577470957682452931:0] peer: [6:7577470959116877824:0] socket: 176 qp: -1
2025-11-27T18:17:20.639426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:20.640697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470957682452956:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:20.648016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470956460816193:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:20.640721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:20.648085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:20.648369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470956460816193:2109] [node 6] ICP22 created new session: [5:7577470956460816224:2109]
2025-11-27T18:17:20.648756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470956460816224:2109] [node 6] ICS09 handshake done sender: [5:7577470956460816198:2115] self: [5:7577470956460816197:0] peer: [6:7577470959116877828:0] socket: 192 qp: -1
2025-11-27T18:17:20.648655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470959116877829:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21029
2025-11-27T18:17:20.648807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470956460816224:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:20.648846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:20.647113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:20.647178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470954250163574:2049] [node 2] ICP22 created new session: [3:7577470958545130973:2049]
2025-11-27T18:17:20.647221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470958545130973:2049] [node 2] ICS09 handshake done sender: [3:7577470958545130965:2120] self: [3:7577470958545130968:0] peer: [2:7577470960403515584:0] socket: 224 qp: -1
2025-11-27T18:17:20.647252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470958545130973:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:20.649010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470959116877830:2092] [node 4] ICS09 handshake done sender: [6:7577470959116877820:2120] self: [6:7577470959116877824:0] peer: [4:7577470957682452931:0] socket: 191 qp: -1
2025-11-27T18:17:20.649111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470959116877830:2092] [node 4] ICS10 traffic start
2025-11-27T18:17:20.647309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:20.649313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:20.650121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470959116877822:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:20.651714Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470959116877791:2095] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28183) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470959116877822:2121] held: no
2025-11-27T18:17:20.651805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470959116877791:2095] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:20.651845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:20.651895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470959116877791:2095] [node 5] ICP22 created new session: [6:7577470959116877832:2095]
2025-11-27T18:17:20.651950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470959116877832:2095] [node 5] ICS09 handshake done sender: [6:7577470959116877822:2121] self: [6:7577470959116877828:0] peer: [5:7577470956460816197:0] socket: 162 qp: -1
2025-11-27T18:17:20.651980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470959116877832:2095] [node 5] ICS10 traffic start
2025-11-27T18:17:20.652017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:20.680200Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:20.680451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470960403515601:2048] [node 1] ICS07 socket disconnect 180 reason# EndOfStream
2025-11-27T18:17:20.680483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470960403515601:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.680562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:20.680615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470960403515601:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:20.681608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:20.681816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470958545130961:2048] [node 1] ICS01 socket: 194 reason# EndOfStream
2025-11-27T18:17:20.681855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470954250163573:2048] [node 1] ICP30 unregister session Session# [3:7577470958545130961:2048] VirtualId# [3:7577470958545130959:0]
2025-11-27T18:17:20.681885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470958545130961:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.681950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:20.681999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:20.683068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470956108548215:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19855) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:20.683121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470960403515601:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:20.705506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:20.705722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470957682452938:2048] [node 1] ICS01 socket: 188 reason# EndOfStream
2025-11-27T18:17:20.707673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:20.707654Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470956108548215:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19855) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:20.705758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470953387485549:2048] [node 1] ICP30 unregister session Session# [4:7577470957682452938:2048] VirtualId# [4:7577470957682452933:0]
2025-11-27T18:17:20.707757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:20.705812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470957682452938:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.711515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470959116877814:2048] [node 1] ICS01 socket: 175 reason# EndOfStream
2025-11-27T18:17:20.705901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:20.705943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:20.712263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470956460816205:2048] [node 1] ICS01 socket: 199 reason# EndOfStream
2025-11-27T18:17:20.718122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470952165848824:2048] [node 1] ICP30 unregister session Session# [5:7577470956460816205:2048] VirtualId# [5:7577470956460816203:0]
2025-11-27T18:17:20.718169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470956460816205:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.718255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:20.718301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:20.711565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470954821910434:2048] [node 1] ICP30 unregister session Session# [6:7577470959116877814:2048] VirtualId# [6:7577470959116877811:0]
2025-11-27T18:17:20.711591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470959116877814:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.711675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:20.711715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:20.748039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:20.748282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470958545130973:2049] [node 2] ICS01 socket: 224 reason# EndOfStream
2025-11-27T18:17:20.748327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470954250163574:2049] [node 2] ICP30 unregister session Session# [3:7577470958545130973:2049] VirtualId# [3:7577470958545130968:0]
2025-11-27T18:17:20.748362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470958545130973:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.748423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:20.748453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:20.757341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:20.757592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470956460816218:2049] [node 2] ICS01 socket: 206 reason# EndOfStream
2025-11-27T18:17:20.757628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470952165848825:2049] [node 2] ICP30 unregister session Session# [5:7577470956460816218:2049] VirtualId# [5:7577470956460816209:0]
2025-11-27T18:17:20.757657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470956460816218:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.757736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:20.757771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:20.757407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:20.757854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:20.757592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470957682452951:2049] [node 2] ICS01 socket: 208 reason# EndOfStream
2025-11-27T18:17:20.758026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470959116877825:2049] [node 2] ICS01 socket: 201 reason# EndOfStream
2025-11-27T18:17:20.757626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470953387485550:2049] [node 2] ICP30 unregister session Session# [4:7577470957682452951:2049] VirtualId# [4:7577470957682452944:0]
2025-11-27T18:17:20.758060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470954821910435:2049] [node 2] ICP30 unregister session Session# [6:7577470959116877825:2049] VirtualId# [6:7577470959116877821:0]
2025-11-27T18:17:20.757653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470957682452951:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.758080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470959116877825:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.757726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:20.758130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:20.757764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:20.758158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:20.805585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:20.805838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470957682452953:2050] [node 3] ICS01 socket: 213 reason# EndOfStream
2025-11-27T18:17:20.808833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470953387485551:2050] [node 3] ICP30 unregister session Session# [4:7577470957682452953:2050] VirtualId# [4:7577470957682452943:0]
2025-11-27T18:17:20.808892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470957682452953:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.808980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:20.809038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:20.810235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:20.812415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:20.812733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470956460816221:2050] [node 3] ICS01 socket: 226 reason# EndOfStream
2025-11-27T18:17:20.812785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470952165848826:2050] [node 3] ICP30 unregister session Session# [5:7577470956460816221:2050] VirtualId# [5:7577470956460816216:0]
2025-11-27T18:17:20.812813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470956460816221:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.812910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:20.813021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:20.810482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470959116877817:2050] [node 3] ICS01 socket: 178 reason# EndOfStream
2025-11-27T18:17:20.810531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470954821910436:2050] [node 3] ICP30 unregister session Session# [6:7577470959116877817:2050] VirtualId# [6:7577470959116877810:0]
2025-11-27T18:17:20.810558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470959116877817:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.810632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:20.810664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:20.828717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:20.828917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470956460816214:2108] [node 4] ICS07 socket disconnect 221 reason# EndOfStream
2025-11-27T18:17:20.828945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470956460816214:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.829001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:20.829030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470956460816214:2108] [node 4] ICS15 start handshake
2025-11-27T18:17:20.833451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:20.833743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470959116877830:2092] [node 4] ICS01 socket: 191 reason# EndOfStream
2025-11-27T18:17:20.833607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470956460816184:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21795) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:20.833963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470959116877782:2092] [node 4] ICP30 unregister session Session# [6:7577470959116877830:2092] VirtualId# [6:7577470959116877824:0]
2025-11-27T18:17:20.833921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470956460816214:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:20.834010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470959116877830:2092] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:20.837372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470956460816184:2108] [node 4] ICP30 unregister session Session# [5:7577470956460816214:2108] VirtualId# [5:7577470956460816210:0]
2025-11-27T18:17:20.837436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:20.837472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470956460816184:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21795) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:20.837471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:20.837487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21795) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:20.837528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:20.851118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470956460816184:2108] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:21.009793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470962512132596:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:21.010680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470962512132595:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:21.030396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470964007437177:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:21.030459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470964007437178:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:21.055344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470962518860778:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:21.055386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470962518860779:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:21.082485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470960893491028:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:21.082519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470960893491029:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:21.082543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470960893491030:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:21.106646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470964685139953:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:21.106699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470964685139954:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:21.106728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470964685139955:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:21.151434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470962509997620:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:21.151495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470962509997621:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:21.151523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470962509997622:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:21.250848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470962509997682:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:21.260752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470962512132656:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:21.287130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470962509997682:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(35982743ns) {Configure:104(35975573ns) {ProcessConfigured:135(30196550ns) {StartInitialHandshake:157(30192251ns) {DropHandshakes:470(3325ns) {DropIncomingHandshake:442(1026ns) DropOutgoingHandshake:457(309ns)} GenerateSessionVirtualId:988(1689ns) PrepareNewSessionHandshake:477(3930ns) {SwitchToState:179(1991ns)}}}}}
2025-11-27T18:17:21.329626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470964007437244:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:21.337462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470964007437243:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:21.337765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470962518860844:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:21.383247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470962518860845:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:21.410988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470962512132662:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:21.444739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470964007437245:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:21.451441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470960893491091:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:21.457954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470960893491092:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:21.451902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470962512132663:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:21.457573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470962518860846:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:21.466557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470962509997687:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:21.470037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470964685140016:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:21.470175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470964685140017:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:21.455287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470964007437244:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(117556285ns) {Configure:104(117550621ns) {ProcessConfigured:135(81741ns) {StartInitialHandshake:157(76953ns) {DropHandshakes:470(2244ns) {DropIncomingHandshake:442(688ns) DropOutgoingHandshake:457(275ns)} GenerateSessionVirtualId:988(798ns) PrepareNewSessionHandshake:477(1418ns) {SwitchToState:179(531ns)}}}}}
2025-11-27T18:17:21.470382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470962512132656:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(196632309ns) {Configure:104(196627956ns) {ProcessConfigured:135(5533282ns) {StartInitialHandshake:157(5531664ns) {DropHandshakes:470(2356ns) {DropIncomingHandshake:442(666ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(724ns) PrepareNewSessionHandshake:477(3046ns) {SwitchToState:179(1291ns)}}}}}
2025-11-27T18:17:21.485623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470962512132595:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(225168738ns) {Configure:104(225164541ns) {ProcessConfigured:135(7601466ns) {StartInitialHandshake:157(7598536ns) {DropHandshakes:470(2297ns) {DropIncomingHandshake:442(613ns) DropOutgoingHandshake:457(312ns)} GenerateSessionVirtualId:988(819ns) PrepareNewSessionHandshake:477(4226ns) {SwitchToState:179(2285ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.481595Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470964007437178:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(134061282ns) {Configure:104(134055757ns) {ProcessConfigured:135(7901599ns) {StartInitialHandshake:157(7899306ns) {DropHandshakes:470(1603ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(588ns)} GenerateSessionVirtualId:988(466ns) PrepareNewSessionHandshake:477(3111ns) {SwitchToState:179(1240ns)}}}}}
2025-11-27T18:17:21.507682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470962518860779:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(54653771ns) {Configure:104(54644064ns) {ProcessConfigured:135(72516ns) {StartInitialHandshake:157(70204ns) {DropHandshakes:470(3113ns) {DropIncomingHandshake:442(866ns) DropOutgoingHandshake:457(315ns)} GenerateSessionVirtualId:988(913ns) PrepareNewSessionHandshake:477(1668ns) {SwitchToState:179(453ns)}}}}}
2025-11-27T18:17:21.494878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470960893491029:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(33183967ns) {Configure:104(33179054ns) {ProcessConfigured:135(71980ns) {StartInitialHandshake:157(69216ns) {DropHandshakes:470(2315ns) {DropIncomingHandshake:442(609ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(1101ns) PrepareNewSessionHandshake:477(1835ns) {SwitchToState:179(595ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.528474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470962512132666:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51212
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.536496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470962512132671:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51212
2025-11-27T18:17:21.501898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470960893491030:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(50158374ns) {Configure:104(50154383ns) {ProcessConfigured:135(57866ns) {StartInitialHandshake:157(55977ns) {DropHandshakes:470(1829ns) {DropIncomingHandshake:442(443ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(941ns) PrepareNewSessionHandshake:477(1434ns) {SwitchToState:179(413ns)}}}}}
2025-11-27T18:17:21.513703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470964007437243:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(30172926ns) {Configure:104(30166914ns) {ProcessConfigured:135(12145785ns) {StartInitialHandshake:157(12141838ns) {DropHandshakes:470(1075705ns) {DropIncomingHandshake:442(1497ns) DropOutgoingHandshake:457(184ns)} GenerateSessionVirtualId:988(1081ns) PrepareNewSessionHandshake:477(3099ns) {SwitchToState:179(1472ns)}}}}}
2025-11-27T18:17:21.509215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470962518860840:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49692
2025-11-27T18:17:21.526553Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470962518860845:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(64799734ns) {Configure:104(64794705ns) {ProcessConfigured:135(78515ns) {StartInitialHandshake:157(76513ns) {DropHandshakes:470(2218ns) {DropIncomingHandshake:442(491ns) DropOutgoingHandshake:457(277ns)} GenerateSessionVirtualId:988(764ns) PrepareNewSessionHandshake:477(1763ns) {SwitchToState:179(624ns)}}}}}
2025-11-27T18:17:21.527296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470962518860856:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49692
2025-11-27T18:17:21.549513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470962518860844:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(126124119ns) {Configure:104(126119384ns) {ProcessConfigured:135(25259249ns) {StartInitialHandshake:157(25256599ns) {DropHandshakes:470(2809ns) {DropIncomingHandshake:442(546ns) DropOutgoingHandshake:457(420ns)} GenerateSessionVirtualId:988(1078ns) PrepareNewSessionHandshake:477(2414ns) {SwitchToState:179(876ns)}}}}}
2025-11-27T18:17:21.530441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470962509997622:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52519207ns) {Configure:104(52513517ns) {ProcessConfigured:135(53955ns) {StartInitialHandshake:157(52000ns) {DropHandshakes:470(1909ns) {DropIncomingHandshake:442(507ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(861ns) PrepareNewSessionHandshake:477(1519ns) {SwitchToState:179(558ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.564390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470962509997620:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(176825909ns) {Configure:104(176816064ns) {ProcessConfigured:135(27788520ns) {StartInitialHandshake:157(27786081ns) {DropHandshakes:470(2194ns) {DropIncomingHandshake:442(663ns) DropOutgoingHandshake:457(289ns)} GenerateSessionVirtualId:988(1688ns) PrepareNewSessionHandshake:477(2999ns) {SwitchToState:179(1416ns)}}}}}
2025-11-27T18:17:21.565560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470964685139954:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(87495049ns) {Configure:104(87489444ns) {ProcessConfigured:135(54660ns) {StartInitialHandshake:157(52374ns) {DropHandshakes:470(3142ns) {DropIncomingHandshake:442(645ns) DropOutgoingHandshake:457(615ns)} GenerateSessionVirtualId:988(911ns) PrepareNewSessionHandshake:477(1567ns) {SwitchToState:179(339ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.564516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470962509997692:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30252
2025-11-27T18:17:21.572476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470962512132666:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.566413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470962509997691:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30252
2025-11-27T18:17:21.573928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470960893491028:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(232382944ns) {Configure:104(232375178ns) {ProcessConfigured:135(8168360ns) {StartInitialHandshake:157(8166068ns) {DropHandshakes:470(2732ns) {DropIncomingHandshake:442(765ns) DropOutgoingHandshake:457(363ns)} GenerateSessionVirtualId:988(1233ns) PrepareNewSessionHandshake:477(1480ns) {SwitchToState:179(289ns)}}}}}
2025-11-27T18:17:21.573960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470964007437256:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39433
2025-11-27T18:17:21.574138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470962518860840:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.574630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470960893491106:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42869
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.591246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470962518860778:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.582378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470962509997691:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.603297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470964007437249:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.590995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470962512132671:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.586484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470962509997621:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(137514214ns) {Configure:104(137502672ns) {ProcessConfigured:135(10989553ns) {StartInitialHandshake:157(10987326ns) {DropHandshakes:470(3095ns) {DropIncomingHandshake:442(936ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(847ns) PrepareNewSessionHandshake:477(3366ns) {SwitchToState:179(1543ns)}}}}}
2025-11-27T18:17:21.593615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470962518860856:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.613965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470962518860860:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49692
2025-11-27T18:17:21.591525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470962509997621:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.599115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470962509997692:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.603461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470962509997687:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.603338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470964007437249:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.597663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470962512132658:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.597689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470962512132658:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.609744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470962512132658:2108] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:21.591586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470960893491106:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.593303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470960893491030:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.604017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470960893491108:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42869
2025-11-27T18:17:21.615340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470964007437249:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:21.615033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470962509997696:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30252
2025-11-27T18:17:21.603997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470964685139953:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(153037219ns) {Configure:104(153031718ns) {ProcessConfigured:135(8552979ns) {StartInitialHandshake:157(8022249ns) {DropHandshakes:470(2751ns) {DropIncomingHandshake:442(675ns) DropOutgoingHandshake:457(589ns)} GenerateSessionVirtualId:988(810ns) PrepareNewSessionHandshake:477(3031ns) {SwitchToState:179(1714ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.621409Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470962512132662:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(116660333ns) {Configure:104(112280825ns) {ProcessConfigured:135(14319461ns) {StartInitialHandshake:157(14316696ns) {DropHandshakes:470(1316ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(717ns) PrepareNewSessionHandshake:477(5264ns) {SwitchToState:179(2432ns)}}}}}
2025-11-27T18:17:21.614927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470962518860862:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49692
2025-11-27T18:17:21.604093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470964685139955:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(119304581ns) {Configure:104(114967889ns) {ProcessConfigured:135(52462ns) {StartInitialHandshake:157(50538ns) {DropHandshakes:470(1940ns) {DropIncomingHandshake:442(316ns) DropOutgoingHandshake:457(325ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:477(6251ns) {SwitchToState:179(392ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.639609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470962509997696:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.615091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470962518860851:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.629464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470964007437257:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39433
2025-11-27T18:17:21.613925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470960893491098:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.618943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470964685140020:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57384
2025-11-27T18:17:21.639973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470962512132596:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.657332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470964007437247:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.613951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470960893491098:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.615117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470962518860851:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.639752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470962509997703:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30252
2025-11-27T18:17:21.619120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470964685140021:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57384
2025-11-27T18:17:21.640043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:21.619248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470964685140023:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57384
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.650036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470962518860851:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:21.657386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470964007437247:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.643707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470962509997620:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.620892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470960893491098:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:21.665962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470964007437245:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.646442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470962512132596:2049] [node 3] ICP22 created new session: [1:7577470962512132679:2049]
2025-11-27T18:17:21.627239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470964685140021:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.643799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470962509997704:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30252
2025-11-27T18:17:21.653824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470962518860860:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.666045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:21.646504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470962512132679:2049] [node 3] ICS09 handshake done sender: [1:7577470962512132658:2108] self: [1:7577470962512132657:0] peer: [3:7577470962518860859:0] socket: 168 qp: -1
2025-11-27T18:17:21.672375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470962518860846:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.653907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470962509997705:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30252
2025-11-27T18:17:21.634027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470964685140023:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.654000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470960893491108:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.646538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470962512132679:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.672434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:21.666095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470964007437245:2110] [node 6] ICP22 created new session: [2:7577470964007437259:2110]
2025-11-27T18:17:21.638898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470964685140020:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.681929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470962509997691:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:21.658774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470960893491096:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:17:21.646591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
::ffff:127.1.0.0
2025-11-27T18:17:21.672488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470962518860846:2107] [node 4] ICP22 created new session: [3:7577470962518860865:2107]
2025-11-27T18:17:21.666137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470964007437259:2110] [node 6] ICS09 handshake done sender: [2:7577470964007437249:2112] self: [2:7577470964007437248:0] peer: [6:7577470962509997701:0] socket: 175 qp: -1
2025-11-27T18:17:21.639057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470964685139954:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.682097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470962509997692:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:21.687718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470962512132670:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.666173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470964007437259:2110] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.658794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470960893491096:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.672531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470962518860865:2107] [node 4] ICS09 handshake done sender: [3:7577470962518860851:2113] self: [3:7577470962518860850:0] peer: [4:7577470960893491107:0] socket: 183 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.682309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470962509997687:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.654396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470964685140016:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(68018085ns) {Configure:104(68007861ns) {ProcessConfigured:135(80649ns) {StartInitialHandshake:157(78485ns) {DropHandshakes:470(2313ns) {DropIncomingHandshake:442(738ns) DropOutgoingHandshake:457(342ns)} GenerateSessionVirtualId:988(1039ns) PrepareNewSessionHandshake:477(15726ns) {SwitchToState:179(385ns)}}}}}
2025-11-27T18:17:21.659962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470960893491096:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:21.687755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470962512132670:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.672568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470962518860865:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:21.666226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:21.654493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470964685140016:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.682360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:21.660757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470960893491091:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.689737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470962512132683:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51212
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.672599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:21.697038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470962512132682:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51212
2025-11-27T18:17:21.667474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470964007437256:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.691548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470964685140037:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57384
2025-11-27T18:17:21.682410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470962509997687:2110] [node 4] ICP22 created new session: [6:7577470962509997707:2110]
2025-11-27T18:17:21.673923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470962518860778:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470962518860840:2109] held: no
2025-11-27T18:17:21.660801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:21.697287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470962512132683:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.713705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470964685140038:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57384
2025-11-27T18:17:21.683183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470964007437257:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.660849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470960893491091:2108] [node 5] ICP22 created new session: [4:7577470960893491115:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.682452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470962509997707:2110] [node 4] ICS09 handshake done sender: [6:7577470962509997692:2113] self: [6:7577470962509997702:0] peer: [4:7577470960893491097:0] socket: 185 qp: -1
2025-11-27T18:17:21.687832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470962518860840:2109] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:21.714354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470964685140028:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.660889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470960893491115:2108] [node 5] ICS09 handshake done sender: [4:7577470960893491096:2111] self: [4:7577470960893491095:0] peer: [5:7577470964685140036:0] socket: 177 qp: -1
2025-11-27T18:17:21.693384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470964007437247:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:21.701853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470962512132670:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:21.713428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470962512132668:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.713470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470962512132668:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.715737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470962512132663:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.721349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:21.721406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470962512132663:2110] [node 6] ICP22 created new session: [1:7577470962512132686:2110]
2025-11-27T18:17:21.728104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470962512132686:2110] [node 6] ICS09 handshake done sender: [1:7577470962512132670:2114] self: [1:7577470962512132669:0] peer: [6:7577470962509997706:0] socket: 191 qp: -1
2025-11-27T18:17:21.728854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470962512132682:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.729779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470962512132686:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:21.732235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:21.735215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470962512132665:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.735250Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470962512132665:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.740469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470962512132668:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:21.741283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470962512132665:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:21.748342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470962512132663:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(28465749ns) {HandleHandshakeStatus:329(28316904ns) {DropIncomingHandshake:442(932ns) DropOutgoingHandshake:457(1073ns) SwitchToState:179(320ns) ProcessPendingSessionEvents:495(1987864ns) {ForwardSessionEventToSession:612(1932627ns) ForwardSessionEventToSession:612(9046ns) ForwardSessionEventToSession:612(8151ns) ForwardSessionEventToSession:612(2818ns) ForwardSessionEventToSession:612(6045ns) ForwardSessionEventToSession:612(4166ns) ForwardSessionEventToSession:612(2557ns) ForwardSessionEventToSession:612(5306ns) ForwardSessionEventToSession:612(4599ns) ForwardSessionEventToSession:612(2669ns)}}}
2025-11-27T18:17:21.750650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470962512132595:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.753280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470962512132656:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.753334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:21.753375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470962512132656:2107] [node 4] ICP22 created new session: [1:7577470962512132693:2107]
2025-11-27T18:17:21.753413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470962512132693:2107] [node 4] ICS09 handshake done sender: [1:7577470962512132665:2111] self: [1:7577470962512132664:0] peer: [4:7577470960893491123:0] socket: 217 qp: -1
2025-11-27T18:17:21.753441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470962512132693:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:21.753480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:21.771184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:21.772016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470962512132692:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51212
2025-11-27T18:17:21.775147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470962512132676:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.775169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470962512132676:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.776365Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470962512132676:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:21.783774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470962512132595:2048] [node 2] ICP22 created new session: [1:7577470962512132695:2048]
2025-11-27T18:17:21.785801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470962512132662:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.708523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470962518860778:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27889898ns) {HandleHandshakeStatus:395(27887376ns) {LogHandshakeFail:472(13355ns) {UpdateErrorStateLog:545(5429ns)} DropOutgoingHandshake:457(2354ns) ProcessPendingSessionEvents:495(8848315ns) {DropSessionEvent:512(10278ns) DropSessionEvent:512(5240ns) DropSessionEvent:512(8794354ns) DropSessionEvent:512(5301ns) DropSessionEvent:512(4110ns) DropSessionEvent:512(4240ns) DropSessionEvent:512(3902ns) DropSessionEvent:512(4093ns) DropSessionEvent:512(4516ns) DropSessionEvent:512(5383ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.695874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470964007437261:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39433
2025-11-27T18:17:21.660917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470960893491115:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:21.682488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470962509997707:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:21.714385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470964685140028:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.790887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.791279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470962512132695:2048] [node 2] ICS09 handshake done sender: [1:7577470962512132668:2113] self: [1:7577470962512132667:0] peer: [2:7577470964007437265:0] socket: 190 qp: -1
2025-11-27T18:17:21.791311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470962512132695:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:21.792222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470962512132662:2109] [node 5] ICP22 created new session: [1:7577470962512132697:2109]
2025-11-27T18:17:21.792753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.660946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:21.701690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470964007437177:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.715768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470962518860778:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.715016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470964685140023:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:21.682614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470962509997705:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.664350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470960893491113:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42869
2025-11-27T18:17:21.800006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470962512132697:2109] [node 5] ICS09 handshake done sender: [1:7577470962512132676:2116] self: [1:7577470962512132675:0] peer: [5:7577470964685140049:0] socket: 235 qp: -1
2025-11-27T18:17:21.728041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470962518860864:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49692
2025-11-27T18:17:21.686867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:21.715269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470964685140016:2108] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.664875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470960893491114:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42869
2025-11-27T18:17:21.707708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470964007437244:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.800059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470962512132697:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:21.715311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:21.687321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470962509997621:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.713512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:21.665134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470960893491106:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:21.728045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:21.670596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470960893491030:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.670654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:21.670705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470960893491030:2050] [node 3] ICP22 created new session: [4:7577470960893491118:2050]
2025-11-27T18:17:21.670749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470960893491118:2050] [node 3] ICS09 handshake done sender: [4:7577470960893491106:2115] self: [4:7577470960893491107:0] peer: [3:7577470962518860850:0] socket: 194 qp: -1
2025-11-27T18:17:21.670784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470960893491118:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:21.670813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:21.682850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470960893491092:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.691969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:21.693153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470960893491092:2109] [node 6] ICP22 created new session: [4:7577470960893491120:2109]
2025-11-27T18:17:21.701848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470960893491120:2109] [node 6] ICS09 handshake done sender: [4:7577470960893491098:2112] self: [4:7577470960893491097:0] peer: [6:7577470962509997702:0] socket: 178 qp: -1
2025-11-27T18:17:21.800092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:21.687384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:21.715346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470964685140016:2108] [node 4] ICP22 created new session: [5:7577470964685140040:2108]
2025-11-27T18:17:21.729800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470962518860778:2048] [node 1] ICP22 created new session: [3:7577470962518860868:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.705560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470960893491120:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:21.708528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470960893491121:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42869
2025-11-27T18:17:21.708587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470960893491114:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.708683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:21.715785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470960893491028:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.721531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470960893491121:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.721696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470960893491029:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.733672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470960893491125:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42869
2025-11-27T18:17:21.742131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470960893491121:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:21.749843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470960893491092:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(69294827ns) {HandleHandshakeStatus:329(69293573ns) {DropIncomingHandshake:442(24267ns) DropOutgoingHandshake:457(302ns) SwitchToState:179(4432ns) ProcessPendingSessionEvents:495(20923488ns) {ForwardSessionEventToSession:612(12497479ns) ForwardSessionEventToSession:612(5323ns) ForwardSessionEventToSession:612(1486818ns) ForwardSessionEventToSession:612(4864ns) ForwardSessionEventToSession:612(2336794ns) ForwardSessionEventToSession:612(3227819ns) ForwardSessionEventToSession:612(3755ns) ForwardSessionEventToSession:612(1820ns) ForwardSessionEventToSession:612(1337375ns) ForwardSessionEventToSession:612(2831ns)}}}
2025-11-27T18:17:21.749895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470960893491029:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.771228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:21.777575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470960893491029:2049] [node 2] ICP22 created new session: [4:7577470960893491127:2049]
2025-11-27T18:17:21.781399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470960893491127:2049] [node 2] ICS09 handshake done sender: [4:7577470960893491121:2119] self: [4:7577470960893491124:0] peer: [2:7577470964007437254:0] socket: 201 qp: -1
2025-11-27T18:17:21.714018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470964007437261:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.736314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470962518860868:2048] [node 1] ICS09 handshake done sender: [3:7577470962518860840:2109] self: [3:7577470962518860859:0] peer: [1:7577470962512132657:0] socket: 169 qp: -1
2025-11-27T18:17:21.687430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470962509997621:2049] [node 2] ICP22 created new session: [6:7577470962509997709:2049]
2025-11-27T18:17:21.781445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470960893491127:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:21.715391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470964685140040:2108] [node 4] ICS09 handshake done sender: [5:7577470964685140023:2113] self: [5:7577470964685140036:0] peer: [4:7577470960893491095:0] socket: 198 qp: -1
2025-11-27T18:17:21.715685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470964007437244:2109] [node 5] ICP22 created new session: [2:7577470964007437266:2109]
2025-11-27T18:17:21.687473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470962509997709:2049] [node 2] ICS09 handshake done sender: [6:7577470962509997691:2112] self: [6:7577470962509997701:0] peer: [2:7577470964007437248:0] socket: 184 qp: -1
2025-11-27T18:17:21.781494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:21.737050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470962518860868:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:21.715435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470964685140040:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:21.815744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470962512132595:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(57255904ns) {HandleHandshakeStatus:329(57253244ns) {DropIncomingHandshake:442(14968ns) DropOutgoingHandshake:457(332ns) SwitchToState:179(3593ns) ProcessPendingSessionEvents:495(8079425ns) {ForwardSessionEventToSession:612(4527361ns) ForwardSessionEventToSession:612(4130ns) ForwardSessionEventToSession:612(306183ns) ForwardSessionEventToSession:612(2714ns) ForwardSessionEventToSession:612(2241ns) ForwardSessionEventToSession:612(1274013ns) ForwardSessionEventToSession:612(2823ns) ForwardSessionEventToSession:612(1047247ns) ForwardSessionEventToSession:612(889159ns) ForwardSessionEventToSession:612(2389ns)}}}
2025-11-27T18:17:21.717692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470964007437266:2109] [node 5] ICS09 handshake done sender: [2:7577470964007437247:2111] self: [2:7577470964007437246:0] peer: [5:7577470964685140032:0] socket: 174 qp: -1
2025-11-27T18:17:21.692395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470962509997709:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:21.738722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:21.715498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:21.781892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470960893491029:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(31817542ns) {HandleHandshakeStatus:329(31814611ns) {DropIncomingHandshake:442(680ns) DropOutgoingHandshake:457(9828ns) SwitchToState:179(183ns) ProcessPendingSessionEvents:495(102812ns) {ForwardSessionEventToSession:612(51175ns) ForwardSessionEventToSession:612(6803ns) ForwardSessionEventToSession:612(4243ns) ForwardSessionEventToSession:612(3367ns) ForwardSessionEventToSession:612(3653ns) ForwardSessionEventToSession:612(5219ns) ForwardSessionEventToSession:612(3157ns) ForwardSessionEventToSession:612(7976ns) ForwardSessionEventToSession:612(2454ns) ForwardSessionEventToSession:612(6195ns)}}}
2025-11-27T18:17:21.697450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:21.728215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470964007437266:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:21.742487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470962518860864:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.729023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470964685140028:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.742410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470964685140017:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.797745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470960893491028:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470960893491114:2118] held: no
2025-11-27T18:17:21.729697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470964007437255:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.712095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470962509997682:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.744346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:21.750777Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470962518860778:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(30124247ns) {HandleHandshakeStatus:329(30122500ns) {DropIncomingHandshake:442(1255ns) DropOutgoingHandshake:457(141ns) SwitchToState:179(621ns) ProcessPendingSessionEvents:495(1399ns)}}
2025-11-27T18:17:21.798078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470960893491126:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42869
2025-11-27T18:17:21.729737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470964007437255:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.744400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470964685140017:2109] [node 6] ICP22 created new session: [5:7577470964685140042:2109]
2025-11-27T18:17:21.712613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470962509997711:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30252
2025-11-27T18:17:21.798282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470962518860872:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49692
2025-11-27T18:17:21.744447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470964685140042:2109] [node 6] ICS09 handshake done sender: [5:7577470964685140028:2116] self: [5:7577470964685140027:0] peer: [6:7577470962509997712:0] socket: 210 qp: -1
2025-11-27T18:17:21.803913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470960893491114:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:21.729776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:21.739028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470964007437255:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:21.725574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470962509997696:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:21.744476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470964685140042:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:21.799791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470962518860855:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.808732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470960893491028:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.799817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470962518860855:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.808788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:21.804661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470962518860855:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:21.808828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470960893491028:2048] [node 1] ICP22 created new session: [4:7577470960893491129:2048]
2025-11-27T18:17:21.806027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470962518860852:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.808865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470960893491129:2048] [node 1] ICS09 handshake done sender: [4:7577470960893491114:2118] self: [4:7577470960893491123:0] peer: [1:7577470962512132664:0] socket: 220 qp: -1
2025-11-27T18:17:21.806052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470962518860852:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.810855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470962518860845:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.808894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470960893491129:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:21.808917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:21.810915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:21.810968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470962518860845:2111] [node 6] ICP22 created new session: [3:7577470962518860875:2111]
2025-11-27T18:17:21.811037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470962518860875:2111] [node 6] ICS09 handshake done sender: [3:7577470962518860855:2115] self: [3:7577470962518860854:0] peer: [6:7577470962509997720:0] socket: 232 qp: -1
2025-11-27T18:17:21.811076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470962518860875:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:21.811107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:21.814434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470962518860872:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.816348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470962518860779:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.816650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470962518860852:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:21.823160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470962518860844:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.750874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470964007437243:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.825036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:21.726548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470962509997620:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.825086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470962518860844:2108] [node 5] ICP22 created new session: [3:7577470962518860879:2108]
2025-11-27T18:17:21.771186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:21.744542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:21.726589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:21.825134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470962518860879:2108] [node 5] ICS09 handshake done sender: [3:7577470962518860852:2114] self: [3:7577470962518860847:0] peer: [5:7577470964685140053:0] socket: 239 qp: -1
2025-11-27T18:17:21.825171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470962518860879:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:21.825210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:21.777378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470964007437243:2108] [node 4] ICP22 created new session: [2:7577470964007437272:2108]
2025-11-27T18:17:21.827252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470962518860881:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49692
2025-11-27T18:17:21.791010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470964007437272:2108] [node 4] ICS09 handshake done sender: [2:7577470964007437255:2114] self: [2:7577470964007437254:0] peer: [4:7577470960893491124:0] socket: 179 qp: -1
2025-11-27T18:17:21.726620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470962509997620:2048] [node 1] ICP22 created new session: [6:7577470962509997715:2048]
2025-11-27T18:17:21.829421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470962518860872:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:21.791096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470964007437244:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(43342756ns) {HandleHandshakeStatus:329(43340029ns) {DropIncomingHandshake:442(552ns) DropOutgoingHandshake:457(836ns) SwitchToState:179(229ns) ProcessPendingSessionEvents:495(11135601ns) {ForwardSessionEventToSession:612(2493880ns) ForwardSessionEventToSession:612(1264019ns) ForwardSessionEventToSession:612(4648ns) ForwardSessionEventToSession:612(3356ns) ForwardSessionEventToSession:612(1615922ns) ForwardSessionEventToSession:612(2041216ns) ForwardSessionEventToSession:612(2831ns) ForwardSessionEventToSession:612(835939ns) ForwardSessionEventToSession:612(2085ns) ForwardSessionEventToSession:612(2849392ns)}}}
2025-11-27T18:17:21.745007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470964685140021:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:21.832337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470962518860779:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.750251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470964685139954:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.726648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470962509997715:2048] [node 1] ICS09 handshake done sender: [6:7577470962509997696:2116] self: [6:7577470962509997706:0] peer: [1:7577470962512132669:0] socket: 195 qp: -1
2025-11-27T18:17:21.832413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:21.792592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470964007437272:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:21.832458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470962518860779:2049] [node 2] ICP22 created new session: [3:7577470962518860882:2049]
2025-11-27T18:17:21.792641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:21.726677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470962509997715:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:21.798892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470964007437177:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577470964007437257:2116] held: no
2025-11-27T18:17:21.726707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:21.750348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:21.832500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470962518860882:2049] [node 2] ICS09 handshake done sender: [3:7577470962518860872:2120] self: [3:7577470962518860878:0] peer: [2:7577470964007437252:0] socket: 236 qp: -1
2025-11-27T18:17:21.799261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470964007437271:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39433
2025-11-27T18:17:21.832532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470962518860882:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:21.801252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470964007437243:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(50229520ns) {HandleHandshakeStatus:329(50226427ns) {DropIncomingHandshake:442(24210ns) DropOutgoingHandshake:457(439ns) SwitchToState:179(703ns) ProcessPendingSessionEvents:495(97911ns) {ForwardSessionEventToSession:612(45353ns) ForwardSessionEventToSession:612(6643ns) ForwardSessionEventToSession:612(1816ns) ForwardSessionEventToSession:612(8074ns) ForwardSessionEventToSession:612(1827ns) ForwardSessionEventToSession:612(1632ns) ForwardSessionEventToSession:612(16686ns) ForwardSessionEventToSession:612(1663ns) ForwardSessionEventToSession:612(1566ns) ForwardSessionEventToSession:612(6100ns)}}}
2025-11-27T18:17:21.732086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470962509997714:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30252
2025-11-27T18:17:21.750404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470964685139954:2049] [node 2] ICP22 created new session: [5:7577470964685140046:2049]
2025-11-27T18:17:21.832564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:21.805098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470964007437273:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39433
2025-11-27T18:17:21.750885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470964685140046:2049] [node 2] ICS09 handshake done sender: [5:7577470964685140021:2112] self: [5:7577470964685140032:0] peer: [2:7577470964007437246:0] socket: 197 qp: -1
2025-11-27T18:17:21.739968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470962509997705:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:21.832791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577470962518860779:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:21.805804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470964007437257:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:21.750891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470964685140016:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(21703477ns) {HandleHandshakeStatus:329(21701509ns) {DropIncomingHandshake:442(658ns) DropOutgoingHandshake:457(5212ns) SwitchToState:179(232ns) ProcessPendingSessionEvents:495(20878374ns) {ForwardSessionEventToSession:612(11326199ns) ForwardSessionEventToSession:612(1576912ns) ForwardSessionEventToSession:612(7978ns) ForwardSessionEventToSession:612(800229ns) ForwardSessionEventToSession:612(2593145ns) ForwardSessionEventToSession:612(3672ns) ForwardSessionEventToSession:612(3530ns) ForwardSessionEventToSession:612(2912942ns) ForwardSessionEventToSession:612(3607ns) ForwardSessionEventToSession:612(1628079ns)}}}
2025-11-27T18:17:21.807122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470964007437177:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.740865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470962509997621:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(34963992ns) {HandleHandshakeStatus:329(34961850ns) {DropIncomingHandshake:442(409ns) DropOutgoingHandshake:457(4036ns) SwitchToState:179(478ns) ProcessPendingSessionEvents:495(14659035ns) {ForwardSessionEventToSession:612(4103379ns) ForwardSessionEventToSession:612(8876ns) ForwardSessionEventToSession:612(1947405ns) ForwardSessionEventToSession:612(2385ns) ForwardSessionEventToSession:612(3760829ns) ForwardSessionEventToSession:612(194365ns) ForwardSessionEventToSession:612(3319ns) ForwardSessionEventToSession:612(4613663ns) ForwardSessionEventToSession:612(3624ns) ForwardSessionEventToSession:612(3274ns)}}}
2025-11-27T18:17:21.807183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:21.754841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470964685140046:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:21.741704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470962509997717:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30252
2025-11-27T18:17:21.807232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470964007437177:2048] [node 1] ICP22 created new session: [2:7577470964007437275:2048]
2025-11-27T18:17:21.745386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470962509997682:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.761676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470964685140044:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57384
2025-11-27T18:17:21.807273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470964007437275:2048] [node 1] ICS09 handshake done sender: [2:7577470964007437257:2116] self: [2:7577470964007437265:0] peer: [1:7577470962512132667:0] socket: 192 qp: -1
2025-11-27T18:17:21.751051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:21.761759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470964685140047:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57384
2025-11-27T18:17:21.807301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470964007437275:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:21.754990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470962509997682:2107] [node 5] ICP22 created new session: [6:7577470962509997718:2107]
2025-11-27T18:17:21.807334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:21.766647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:21.820381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470964007437253:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.755041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470962509997718:2107] [node 5] ICS09 handshake done sender: [6:7577470962509997705:2120] self: [6:7577470962509997712:0] peer: [5:7577470964685140027:0] socket: 213 qp: -1
2025-11-27T18:17:21.820408Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470964007437253:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:21.774656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470964685140044:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.755070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470962509997718:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:21.826308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470964007437253:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:21.755123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:21.774785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470964685139953:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.830362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470964007437178:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:21.785829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470962509997717:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.776192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470964685140050:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57384
2025-11-27T18:17:21.830447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:21.791672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470962509997622:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.830496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470964007437178:2049] [node 3] ICP22 created new session: [2:7577470964007437278:2049]
2025-11-27T18:17:21.776805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470964685140044:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:21.814591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470962509997721:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30252
2025-11-27T18:17:21.830552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470964007437278:2049] [node 3] ICS09 handshake done sender: [2:7577470964007437253:2113] self: [2:7577470964007437252:0] peer: [3:7577470962518860878:0] socket: 200 qp: -1
2025-11-27T18:17:21.776991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470964685139953:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.830583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470964007437278:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:21.777069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:21.830696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:21.777108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470964685139953:2048] [node 1] ICP22 created new session: [5:7577470964685140051:2048]
2025-11-27T18:17:21.777155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470964685140051:2048] [node 1] ICS09 handshake done sender: [5:7577470964685140044:2120] self: [5:7577470964685140049:0] peer: [1:7577470962512132675:0] socket: 240 qp: -1
2025-11-27T18:17:21.777190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470964685140051:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:21.777236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:21.793780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470964685140047:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:21.793925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470964685139955:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:21.809042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470964685139954:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(48161785ns) {HandleHandshakeStatus:329(48159048ns) {DropIncomingHandshake:442(442ns) DropOutgoingHandshake:457(3626ns) SwitchToState:179(566ns) ProcessPendingSessionEvents:495(26639102ns) {ForwardSessionEventToSession:612(14231412ns) ForwardSessionEventToSession:612(4197ns) ForwardSessionEventToSession:612(5334410ns) ForwardSessionEventToSession:612(903378ns) ForwardSessionEventToSession:612(3243ns) ForwardSessionEventToSession:612(1551889ns) ForwardSessionEventToSession:612(3156ns) ForwardSessionEventToSession:612(2016ns) ForwardSessionEventToSession:612(4158572ns) ForwardSessionEventToSession:612(2687ns)}}}
2025-11-27T18:17:21.810422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470964685139953:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24556117ns) {HandleHandshakeStatus:329(24554492ns) {DropIncomingHandshake:442(606ns) DropOutgoingHandshake:457(4977ns) SwitchToState:179(531ns) ProcessPendingSessionEvents:495(10526064ns) {ForwardSessionEventToSession:612(1513389ns) ForwardSessionEventToSession:612(1800335ns) ForwardSessionEventToSession:612(4121498ns) ForwardSessionEventToSession:612(2115ns) ForwardSessionEventToSession:612(1545123ns) ForwardSessionEventToSession:612(3367ns) ForwardSessionEventToSession:612(919467ns) ForwardSessionEventToSession:612(599849ns) ForwardSessionEventToSession:612(2657ns) ForwardSessionEventToSession:612(4846ns)}}}
2025-11-27T18:17:21.822776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470964685140054:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57384
2025-11-27T18:17:21.826508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470964685140047:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:21.833627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470964685139955:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.833693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:21.833738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470964685139955:2050] [node 3] ICP22 created new session: [5:7577470964685140055:2050]
2025-11-27T18:17:21.833776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470964685140055:2050] [node 3] ICS09 handshake done sender: [5:7577470964685140047:2121] self: [5:7577470964685140053:0] peer: [3:7577470962518860847:0] socket: 241 qp: -1
2025-11-27T18:17:21.834127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470964685140055:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:21.834620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:21.836241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470964685139955:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:21.815513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470962509997717:2123] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:21.818209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470962509997622:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7362) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470962509997717:2123] held: no
2025-11-27T18:17:21.820283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470962509997622:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:21.820327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:21.820781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470962509997622:2050] [node 3] ICP22 created new session: [6:7577470962509997722:2050]
2025-11-27T18:17:21.824792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470962509997722:2050] [node 3] ICS09 handshake done sender: [6:7577470962509997717:2123] self: [6:7577470962509997720:0] peer: [3:7577470962518860854:0] socket: 238 qp: -1
2025-11-27T18:17:21.826157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470962509997722:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:21.826839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:21.883585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:21.883830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470964007437275:2048] [node 1] ICS01 socket: 192 reason# EndOfStream
2025-11-27T18:17:21.883870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470964007437177:2048] [node 1] ICP30 unregister session Session# [2:7577470964007437275:2048] VirtualId# [2:7577470964007437265:0]
2025-11-27T18:17:21.883900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470964007437275:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:21.883972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:21.884023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:21.885444Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:21.885857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470962518860868:2048] [node 1] ICS01 socket: 169 reason# EndOfStream
2025-11-27T18:17:21.885897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470962518860778:2048] [node 1] ICP30 unregister session Session# [3:7577470962518860868:2048] VirtualId# [3:7577470962518860859:0]
2025-11-27T18:17:21.885923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470962518860868:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:21.885985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:21.886021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:21.891916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:21.892149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470960893491129:2048] [node 1] ICS01 socket: 220 reason# EndOfStream
2025-11-27T18:17:21.892182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470960893491028:2048] [node 1] ICP30 unregister session Session# [4:7577470960893491129:2048] VirtualId# [4:7577470960893491123:0]
2025-11-27T18:17:21.892215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470960893491129:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:21.892284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:21.892325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:21.896603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:21.896833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470964685140051:2048] [node 1] ICS07 socket disconnect 240 reason# EndOfStream
2025-11-27T18:17:21.896862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470964685140051:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:21.896956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:21.896995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470964685140051:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:21.898136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470964685139953:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3272) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:21.898199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470964685140051:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:21.898233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470964685139953:2048] [node 1] ICP30 unregister session Session# [5:7577470964685140051:2048] VirtualId# [5:7577470964685140049:0]
2025-11-27T18:17:21.898277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:21.898305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470964685139953:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3272) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:21.898318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3272) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:21.899720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:21.899933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470962509997715:2048] [node 1] ICS01 socket: 195 reason# EndOfStream
2025-11-27T18:17:21.899986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470962509997620:2048] [node 1] ICP30 unregister session Session# [6:7577470962509997715:2048] VirtualId# [6:7577470962509997706:0]
2025-11-27T18:17:21.900018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470962509997715:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:21.900114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:21.900157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:21.908707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470964685139953:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:21.947057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:21.947309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470962518860882:2049] [node 2] ICS01 socket: 236 reason# EndOfStream
2025-11-27T18:17:21.947356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470962518860779:2049] [node 2] ICP30 unregister session Session# [3:7577470962518860882:2049] VirtualId# [3:7577470962518860878:0]
2025-11-27T18:17:21.947387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470962518860882:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:21.947465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:21.947506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:21.950545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:21.950793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470960893491127:2049] [node 2] ICS01 socket: 201 reason# EndOfStream
2025-11-27T18:17:21.950829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470960893491029:2049] [node 2] ICP30 unregister session Session# [4:7577470960893491127:2049] VirtualId# [4:7577470960893491124:0]
2025-11-27T18:17:21.950857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470960893491127:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:21.950957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:21.951002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:21.952719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:21.952938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470962509997709:2049] [node 2] ICS07 socket disconnect 184 reason# EndOfStream
2025-11-27T18:17:21.952976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470962509997709:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:21.953044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:21.953085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470962509997709:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:21.953402Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:21.953618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470964685140046:2049] [node 2] ICS07 socket disconnect 197 reason# EndOfStream
2025-11-27T18:17:21.953648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470964685140046:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:21.954025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470962509997621:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2458) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:21.954611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470962509997709:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:21.953713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:21.954654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470962509997621:2049] [node 2] ICP30 unregister session Session# [6:7577470962509997709:2049] VirtualId# [6:7577470962509997701:0]
2025-11-27T18:17:21.954764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:21.954834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470962509997621:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2458) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:21.953750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470964685140046:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:21.954881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2458) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:21.955464Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470964685139954:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2458) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:21.955527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470964685140046:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:21.955567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470964685139954:2049] [node 2] ICP30 unregister session Session# [5:7577470964685140046:2049] VirtualId# [5:7577470964685140032:0]
2025-11-27T18:17:21.955615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:21.955642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470964685139954:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2458) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:21.955655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2458) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:21.964472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470962509997621:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:21.965363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470964685139954:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:22.000799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:22.001064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470960893491118:2050] [node 3] ICS01 socket: 194 reason# EndOfStream
2025-11-27T18:17:22.001130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470960893491030:2050] [node 3] ICP30 unregister session Session# [4:7577470960893491118:2050] VirtualId# [4:7577470960893491107:0]
2025-11-27T18:17:22.001165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470960893491118:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:22.001276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:22.001340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:22.002211Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:22.002460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470964685140055:2050] [node 3] ICS07 socket disconnect 241 reason# EndOfStream
2025-11-27T18:17:22.002499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470964685140055:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:22.002565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:22.002604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470964685140055:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:22.002773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:22.002976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470962509997722:2050] [node 3] ICS01 socket: 238 reason# EndOfStream
2025-11-27T18:17:22.003017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470962509997622:2050] [node 3] ICP30 unregister session Session# [6:7577470962509997722:2050] VirtualId# [6:7577470962509997720:0]
2025-11-27T18:17:22.003071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470962509997722:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:22.003133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:22.003170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:22.003408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470964685139955:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7362) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:22.003537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470964685140055:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:22.003572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470964685139955:2050] [node 3] ICP30 unregister session Session# [5:7577470964685140055:2050] VirtualId# [5:7577470964685140053:0]
2025-11-27T18:17:22.003608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:22.003641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470964685139955:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7362) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:22.003656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7362) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:22.013501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470964685139955:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:22.056064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:22.056298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470962509997707:2110] [node 4] ICS01 socket: 185 reason# EndOfStream
2025-11-27T18:17:22.056341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470962509997687:2110] [node 4] ICP30 unregister session Session# [6:7577470962509997707:2110] VirtualId# [6:7577470962509997702:0]
2025-11-27T18:17:22.056373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470962509997707:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:22.056463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:22.056508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:22.057643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:22.057873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470964685140040:2108] [node 4] ICS07 socket disconnect 198 reason# EndOfStream
2025-11-27T18:17:22.057899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470964685140040:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:22.057971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:22.058013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470964685140040:2108] [node 4] ICS15 start handshake
2025-11-27T18:17:22.099417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:22.099617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470962509997718:2107] [node 5] ICS01 socket: 213 reason# EndOfStream
2025-11-27T18:17:22.099675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470962509997682:2107] [node 5] ICP30 unregister session Session# [6:7577470962509997718:2107] VirtualId# [6:7577470962509997712:0]
2025-11-27T18:17:22.099705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470962509997718:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:22.099802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:22.099846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:22.188178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470966462497453:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:22.188263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470966462497454:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:22.210096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470966142505722:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:22.210435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470966142505723:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:22.229774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470966290013616:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:22.229821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470966290013617:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:22.248352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470967492226411:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:22.248391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470967492226412:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:22.248415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470967492226413:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:22.268547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470968928828336:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:22.268547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470968928828335:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:22.268584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470968928828337:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:22.294168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470965860468243:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:22.294214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470965860468244:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:22.294255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470965860468245:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:22.420516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470965860468306:2103] [node 5] ICP01 ready to work
2025-11-27T18:17:22.420863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470965860468307:2106] [node 4] ICP01 ready to work
2025-11-27T18:17:22.436807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470966462497516:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:22.437154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470966462497517:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:22.521945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470967492226475:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:22.568088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470968928828398:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:22.554611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470966142505786:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:22.568222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470966462497518:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:22.577323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470968928828399:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:22.578007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470966142505787:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:22.589440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470966142505788:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:22.583305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470966290013680:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:22.585491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470966290013679:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:22.590550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470966290013681:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:22.608942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470965860468307:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(175715754ns) {Configure:104(175711032ns) {ProcessConfigured:135(3428433ns) {StartInitialHandshake:157(3424471ns) {DropHandshakes:470(5085ns) {DropIncomingHandshake:442(437ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(16547ns) PrepareNewSessionHandshake:477(5398ns) {SwitchToState:179(3483ns)}}}}}
2025-11-27T18:17:22.593665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470967492226478:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:22.616674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470966462497517:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48218167ns) {Configure:104(48201896ns) {ProcessConfigured:135(59882ns) {StartInitialHandshake:157(57411ns) {DropHandshakes:470(1750ns) {DropIncomingHandshake:442(490ns) DropOutgoingHandshake:457(403ns)} GenerateSessionVirtualId:988(829ns) PrepareNewSessionHandshake:477(826ns) {SwitchToState:179(335ns)}}}}}
2025-11-27T18:17:22.614528Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470965860468306:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(184805023ns) {Configure:104(184795605ns) {ProcessConfigured:135(9636564ns) {StartInitialHandshake:157(9633378ns) {DropHandshakes:470(1828ns) {DropIncomingHandshake:442(418ns) DropOutgoingHandshake:457(297ns)} GenerateSessionVirtualId:988(909ns) PrepareNewSessionHandshake:477(4645ns) {SwitchToState:179(2078ns)}}}}}
2025-11-27T18:17:22.609001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470967492226475:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(85046837ns) {Configure:104(85043067ns) {ProcessConfigured:135(3369791ns) {StartInitialHandshake:157(3366498ns) {DropHandshakes:470(9057ns) {DropIncomingHandshake:442(621ns) DropOutgoingHandshake:457(326ns)} GenerateSessionVirtualId:988(1004ns) PrepareNewSessionHandshake:477(4235ns) {SwitchToState:179(1827ns)}}}}}
2025-11-27T18:17:22.625356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470967492226478:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23857116ns) {Configure:104(23850328ns) {ProcessConfigured:135(10723245ns) {StartInitialHandshake:157(10719904ns) {DropHandshakes:470(4834ns) {DropIncomingHandshake:442(788ns) DropOutgoingHandshake:457(608ns)} GenerateSessionVirtualId:988(632ns) PrepareNewSessionHandshake:477(4068ns) {SwitchToState:179(1869ns)}}}}}
2025-11-27T18:17:22.638190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470967492226412:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47689519ns) {Configure:104(47685667ns) {ProcessConfigured:135(114342ns) {StartInitialHandshake:157(112766ns) {DropHandshakes:470(2221ns) {DropIncomingHandshake:442(498ns) DropOutgoingHandshake:457(523ns)} GenerateSessionVirtualId:988(601ns) PrepareNewSessionHandshake:477(1815ns) {SwitchToState:179(613ns)}}}}}
2025-11-27T18:17:22.638682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470965860468243:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29502248ns) {Configure:104(29497116ns) {ProcessConfigured:135(5953109ns) {StartInitialHandshake:157(5951127ns) {DropHandshakes:470(2345ns) {DropIncomingHandshake:442(521ns) DropOutgoingHandshake:457(314ns)} GenerateSessionVirtualId:988(1133ns) PrepareNewSessionHandshake:477(2033ns) {SwitchToState:179(779ns)}}}}}
2025-11-27T18:17:22.649317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470966462497453:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(223327136ns) {Configure:104(223322653ns) {ProcessConfigured:135(59661ns) {StartInitialHandshake:157(57724ns) {DropHandshakes:470(1745ns) {DropIncomingHandshake:442(600ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(854ns) PrepareNewSessionHandshake:477(1380ns) {SwitchToState:179(482ns)}}}}}
2025-11-27T18:17:22.658467Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470966142505722:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(188821475ns) {Configure:104(188817539ns) {ProcessConfigured:135(20444774ns) {StartInitialHandshake:157(20442409ns) {DropHandshakes:470(1945ns) {DropIncomingHandshake:442(508ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(1105ns) PrepareNewSessionHandshake:477(3358ns) {SwitchToState:179(1626ns)}}}}}
2025-11-27T18:17:22.681588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470968928828337:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(139624356ns) {Configure:104(139619051ns) {ProcessConfigured:135(19773169ns) {StartInitialHandshake:157(19769343ns) {DropHandshakes:470(1510ns) {DropIncomingHandshake:442(387ns) DropOutgoingHandshake:457(277ns)} GenerateSessionVirtualId:988(905ns) PrepareNewSessionHandshake:477(4544ns) {SwitchToState:179(2528ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.683410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470967492226489:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27704
2025-11-27T18:17:22.685410Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470966462497454:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(223051019ns) {Configure:104(223044951ns) {ProcessConfigured:135(3788098ns) {StartInitialHandshake:157(3786309ns) {DropHandshakes:470(1829ns) {DropIncomingHandshake:442(189ns) DropOutgoingHandshake:457(218ns)} GenerateSessionVirtualId:988(641ns) PrepareNewSessionHandshake:477(2724ns) {SwitchToState:179(933ns)}}}}}
2025-11-27T18:17:22.686684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470967492226489:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.686796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470965860468245:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(66235256ns) {Configure:104(66230111ns) {ProcessConfigured:135(21914950ns) {StartInitialHandshake:157(21911976ns) {DropHandshakes:470(1674ns) {DropIncomingHandshake:442(408ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(828ns) PrepareNewSessionHandshake:477(2578ns) {SwitchToState:179(971ns)}}}}}
2025-11-27T18:17:22.686835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470966142505787:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(84835424ns) {Configure:104(84828811ns) {ProcessConfigured:135(16170353ns) {StartInitialHandshake:157(16166592ns) {DropHandshakes:470(3092ns) {DropIncomingHandshake:442(1013ns) DropOutgoingHandshake:457(318ns)} GenerateSessionVirtualId:988(1026ns) PrepareNewSessionHandshake:477(4585ns) {SwitchToState:179(2354ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.685826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470966462497516:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(236096624ns) {Configure:104(236083482ns) {ProcessConfigured:135(26514955ns) {StartInitialHandshake:157(26512240ns) {DropHandshakes:470(1365ns) {DropIncomingHandshake:442(456ns) DropOutgoingHandshake:457(183ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:477(5068ns) {SwitchToState:179(2639ns)}}}}}
2025-11-27T18:17:22.687182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470966142505723:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(211755676ns) {Configure:104(211750598ns) {ProcessConfigured:135(9474721ns) {StartInitialHandshake:157(9472813ns) {DropHandshakes:470(2914ns) {DropIncomingHandshake:442(756ns) DropOutgoingHandshake:457(624ns)} GenerateSessionVirtualId:988(852ns) PrepareNewSessionHandshake:477(3499ns) {SwitchToState:179(1706ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.697070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470965860468320:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11285
2025-11-27T18:17:22.697928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470966290013616:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(132437007ns) {Configure:104(132424720ns) {ProcessConfigured:135(444064ns) {StartInitialHandshake:157(441384ns) {DropHandshakes:470(1909ns) {DropIncomingHandshake:442(415ns) DropOutgoingHandshake:457(218ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:477(2010ns) {SwitchToState:179(821ns)}}}}}
2025-11-27T18:17:22.688608Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470968928828335:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(200399969ns) {Configure:104(200387779ns) {ProcessConfigured:135(1387649ns) {StartInitialHandshake:157(1384545ns) {DropHandshakes:470(2146ns) {DropIncomingHandshake:442(574ns) DropOutgoingHandshake:457(276ns)} GenerateSessionVirtualId:988(822ns) PrepareNewSessionHandshake:477(3145ns) {SwitchToState:179(1596ns)}}}}}
2025-11-27T18:17:22.690650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470966462497532:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23328
2025-11-27T18:17:22.697876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470965860468318:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11285
2025-11-27T18:17:22.689379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470968928828405:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-11-27T18:17:22.700237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470967492226488:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27704
2025-11-27T18:17:22.700283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470965860468320:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.703233Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470967492226411:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(172940938ns) {Configure:104(172926177ns) {ProcessConfigured:135(3332161ns) {StartInitialHandshake:157(3330154ns) {DropHandshakes:470(1693ns) {DropIncomingHandshake:442(649ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(620ns) PrepareNewSessionHandshake:477(2826ns) {SwitchToState:179(1314ns)}}}}}
2025-11-27T18:17:22.703160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470966462497532:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.689517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470968928828406:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-11-27T18:17:22.694003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470968928828336:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(116108497ns) {Configure:104(116099013ns) {ProcessConfigured:135(2522899ns) {StartInitialHandshake:157(2520998ns) {DropHandshakes:470(478ns) {DropIncomingHandshake:442(103ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(308ns) PrepareNewSessionHandshake:477(2294ns) {SwitchToState:179(440ns)}}}}}
2025-11-27T18:17:22.705708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470965860468243:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.702667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470968928828405:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.706229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470965860468244:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(115275164ns) {Configure:104(115268379ns) {ProcessConfigured:135(2411172ns) {StartInitialHandshake:157(2409367ns) {DropHandshakes:470(2535ns) {DropIncomingHandshake:442(598ns) DropOutgoingHandshake:457(289ns)} GenerateSessionVirtualId:988(839ns) PrepareNewSessionHandshake:477(2582ns) {SwitchToState:179(1082ns)}}}}}
2025-11-27T18:17:22.703399Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470968928828398:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(107454048ns) {Configure:104(107445166ns) {ProcessConfigured:135(5502320ns) {StartInitialHandshake:157(5500163ns) {DropHandshakes:470(989070ns) {DropIncomingHandshake:442(1367ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(1502ns) PrepareNewSessionHandshake:477(2853ns) {SwitchToState:179(798ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.706981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470966142505786:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(117293266ns) {Configure:104(117282638ns) {ProcessConfigured:135(1008636ns) {StartInitialHandshake:157(1006887ns) {DropHandshakes:470(1985ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(566ns)} GenerateSessionVirtualId:988(630ns) PrepareNewSessionHandshake:477(2573ns) {SwitchToState:179(1362ns)}}}}}
2025-11-27T18:17:22.707094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470965860468318:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.706845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470966462497522:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.703992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470967492226411:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.704165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470968928828398:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.709642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470965860468307:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.704251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470967492226488:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.709509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470967492226484:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.706886Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470966462497522:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.709540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470967492226484:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.709356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470966290013617:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(116155331ns) {Configure:104(116147297ns) {ProcessConfigured:135(1833549ns) {StartInitialHandshake:157(1831198ns) {DropHandshakes:470(2825ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(468ns)} GenerateSessionVirtualId:988(923ns) PrepareNewSessionHandshake:477(1931ns) {SwitchToState:179(913ns)}}}}}
2025-11-27T18:17:22.721466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470968928828406:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.721248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470966142505795:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26943
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.710360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470966462497520:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.710705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470966290013680:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(112819000ns) {Configure:104(112811560ns) {ProcessConfigured:135(1856647ns) {StartInitialHandshake:157(1855359ns) {DropHandshakes:470(1854ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(362ns)} GenerateSessionVirtualId:988(621ns) PrepareNewSessionHandshake:477(1380ns) {SwitchToState:179(142ns)}}}}}
2025-11-27T18:17:22.713380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470967492226493:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27704
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.717102Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470966290013679:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(122086628ns) {Configure:104(122078753ns) {ProcessConfigured:135(7575333ns) {StartInitialHandshake:157(7572787ns) {DropHandshakes:470(2122ns) {DropIncomingHandshake:442(431ns) DropOutgoingHandshake:457(427ns)} GenerateSessionVirtualId:988(554ns) PrepareNewSessionHandshake:477(4877ns) {SwitchToState:179(1684ns)}}}}}
2025-11-27T18:17:22.710394Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470966462497520:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.726545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470968928828399:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(39955773ns) {Configure:104(39947215ns) {ProcessConfigured:135(8162488ns) {StartInitialHandshake:157(8158908ns) {DropHandshakes:470(3808ns) {DropIncomingHandshake:442(535ns) DropOutgoingHandshake:457(644ns)} GenerateSessionVirtualId:988(824ns) PrepareNewSessionHandshake:477(3111ns) {SwitchToState:179(994ns)}}}}}
2025-11-27T18:17:22.717909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470967492226484:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:22.726391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470966142505795:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.735956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470968928828418:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-11-27T18:17:22.727977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470966290013681:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24438511ns) {Configure:104(24430954ns) {ProcessConfigured:135(70002ns) {StartInitialHandshake:157(66967ns) {DropHandshakes:470(1887ns) {DropIncomingHandshake:442(404ns) DropOutgoingHandshake:457(387ns)} GenerateSessionVirtualId:988(848ns) PrepareNewSessionHandshake:477(1902ns) {SwitchToState:179(556ns)}}}}}
2025-11-27T18:17:22.734280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470966142505722:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.726519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470966462497520:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:22.725231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470967492226480:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.745088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470966462497522:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:22.725301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470967492226480:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.728587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470966290013690:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-11-27T18:17:22.737658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470966142505801:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26943
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.747339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470966462497527:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.728729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470966290013691:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-11-27T18:17:22.749304Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470966142505788:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(47247748ns) {Configure:104(45068082ns) {ProcessConfigured:135(2666603ns) {StartInitialHandshake:157(2664116ns) {DropHandshakes:470(1925ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(663ns)} GenerateSessionVirtualId:988(527ns) PrepareNewSessionHandshake:477(1654ns) {SwitchToState:179(559ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.726457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470967492226493:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.751238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470968928828416:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-11-27T18:17:22.747384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470966462497527:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.729273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470966290013691:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.726490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470967492226478:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.748012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470966462497517:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.753138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470965860468326:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11285
2025-11-27T18:17:22.737997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470966290013690:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.734205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:22.758526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470968928828405:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:22.762366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470966142505801:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.749397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:22.760595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470966290013697:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-11-27T18:17:22.735424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470967492226478:2108] [node 5] ICP22 created new session: [4:7577470967492226497:2108]
2025-11-27T18:17:22.736506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470967492226497:2108] [node 5] ICS09 handshake done sender: [4:7577470967492226484:2113] self: [4:7577470967492226483:0] peer: [5:7577470968928828413:0] socket: 176 qp: -1
2025-11-27T18:17:22.736892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470967492226497:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:22.737896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470967492226480:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:22.745708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:22.749360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470967492226499:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27704
2025-11-27T18:17:22.749371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470967492226475:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.749448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:22.749495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470967492226475:2109] [node 6] ICP22 created new session: [4:7577470967492226501:2109]
2025-11-27T18:17:22.751977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470967492226501:2109] [node 6] ICS09 handshake done sender: [4:7577470967492226480:2112] self: [4:7577470967492226479:0] peer: [6:7577470965860468324:0] socket: 175 qp: -1
2025-11-27T18:17:22.770340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470965860468325:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11285
2025-11-27T18:17:22.768991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470968928828418:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.749460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470966462497517:2109] [node 6] ICP22 created new session: [1:7577470966462497537:2109]
2025-11-27T18:17:22.762161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470966290013699:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-11-27T18:17:22.771592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470968928828419:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-11-27T18:17:22.775584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470966290013697:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.772726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470968928828336:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.782554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470966290013699:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.782534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470966142505805:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26943
2025-11-27T18:17:22.749971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470966462497537:2109] [node 6] ICS09 handshake done sender: [1:7577470966462497520:2110] self: [1:7577470966462497519:0] peer: [6:7577470965860468323:0] socket: 181 qp: -1
2025-11-27T18:17:22.750594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470966462497518:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.750627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:22.750782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470966462497518:2107] [node 4] ICP22 created new session: [1:7577470966462497538:2107]
2025-11-27T18:17:22.750794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470966462497537:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:22.750971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:22.750991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470966462497538:2107] [node 4] ICS09 handshake done sender: [1:7577470966462497522:2111] self: [1:7577470966462497521:0] peer: [4:7577470967492226492:0] socket: 182 qp: -1
2025-11-27T18:17:22.751545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470966462497538:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:22.751583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:22.768875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470966462497535:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23328
2025-11-27T18:17:22.782528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470966462497527:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:22.795604Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470966462497517:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(26321280ns) {HandleHandshakeStatus:329(26317812ns) {DropIncomingHandshake:442(864ns) DropOutgoingHandshake:457(605ns) SwitchToState:179(439ns) ProcessPendingSessionEvents:495(20544078ns) {ForwardSessionEventToSession:612(8803405ns) ForwardSessionEventToSession:612(24237ns) ForwardSessionEventToSession:612(3206ns) ForwardSessionEventToSession:612(349442ns) ForwardSessionEventToSession:612(2753ns) ForwardSessionEventToSession:612(6605866ns) ForwardSessionEventToSession:612(9780ns) ForwardSessionEventToSession:612(3376ns) ForwardSessionEventToSession:612(4708489ns) ForwardSessionEventToSession:612(9819ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.753179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470967492226501:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:22.787744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470966290013617:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.775422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470968928828398:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.775635Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470965860468307:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14444) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470965860468318:2114] held: no
2025-11-27T18:17:22.782702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470966142505794:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.796440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470966462497535:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.791171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470965860468326:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.805533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470966290013693:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.757613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.782874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:22.796493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470966462497453:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.758287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470967492226489:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:22.805332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470965860468327:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11285
2025-11-27T18:17:22.782738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470966142505794:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.791323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470968928828419:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.805583Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470966290013693:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.796855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:22.806177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470965860468307:2106] [node 4] ICP35 event processing took too much time PendingConnection:279(27843255ns) {HandleHandshakeStatus:395(27839128ns) {LogHandshakeFail:472(15991ns) {UpdateErrorStateLog:545(5809ns)} DropOutgoingHandshake:457(1377ns) ProcessPendingSessionEvents:495(9785205ns) {DropSessionEvent:512(12421ns) DropSessionEvent:512(4551ns) DropSessionEvent:512(5102ns) DropSessionEvent:512(4324ns) DropSessionEvent:512(3733ns) DropSessionEvent:512(5318ns) DropSessionEvent:512(4255ns) DropSessionEvent:512(25283ns) DropSessionEvent:512(5392ns) DropSessionEvent:512(5617ns)}}}
2025-11-27T18:17:22.762057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470967492226411:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.794840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470966142505805:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.793784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470968928828398:2108] [node 4] ICP22 created new session: [5:7577470968928828421:2108]
2025-11-27T18:17:22.811097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470966290013703:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-11-27T18:17:22.806074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470966462497453:2048] [node 2] ICP22 created new session: [1:7577470966462497545:2048]
2025-11-27T18:17:22.794976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470968928828337:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.795736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470968928828421:2108] [node 4] ICS09 handshake done sender: [5:7577470968928828405:2112] self: [5:7577470968928828413:0] peer: [4:7577470967492226483:0] socket: 191 qp: -1
2025-11-27T18:17:22.796552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470968928828421:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:22.796595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:22.801393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470968928828398:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(25707621ns) {HandleHandshakeStatus:329(25704496ns) {DropIncomingHandshake:442(1170ns) DropOutgoingHandshake:457(6362ns) SwitchToState:179(507ns) ProcessPendingSessionEvents:495(112075ns) {ForwardSessionEventToSession:612(67741ns) ForwardSessionEventToSession:612(2884ns) ForwardSessionEventToSession:612(5548ns) ForwardSessionEventToSession:612(2697ns) ForwardSessionEventToSession:612(2845ns) ForwardSessionEventToSession:612(6790ns) ForwardSessionEventToSession:612(5180ns) ForwardSessionEventToSession:612(6609ns) ForwardSessionEventToSession:612(2134ns) ForwardSessionEventToSession:612(1824ns)}}}
2025-11-27T18:17:22.806567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470968928828422:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-11-27T18:17:22.807649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470965860468320:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:22.796276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470966142505792:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.765823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470967492226475:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16225585ns) {HandleHandshakeStatus:329(16223076ns) {DropIncomingHandshake:442(13932ns) DropOutgoingHandshake:457(332ns) SwitchToState:179(417ns) ProcessPendingSessionEvents:495(127592ns) {ForwardSessionEventToSession:612(76652ns) ForwardSessionEventToSession:612(5710ns) ForwardSessionEventToSession:612(11921ns) ForwardSessionEventToSession:612(2468ns) ForwardSessionEventToSession:612(4755ns) ForwardSessionEventToSession:612(2656ns) ForwardSessionEventToSession:612(3036ns) ForwardSessionEventToSession:612(5919ns) ForwardSessionEventToSession:612(2341ns) ForwardSessionEventToSession:612(4708ns)}}}
2025-11-27T18:17:22.813630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470966290013693:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:22.822668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470968928828422:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.822818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470968928828335:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.806626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470966462497545:2048] [node 2] ICS09 handshake done sender: [1:7577470966462497527:2112] self: [1:7577470966462497526:0] peer: [2:7577470966142505802:0] socket: 186 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.807974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470965860468318:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:22.814531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470966290013687:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.773736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:22.796314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470966142505792:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.809109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470966462497545:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:22.808140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470965860468243:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.780745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470967492226411:2048] [node 1] ICP22 created new session: [4:7577470967492226504:2048]
2025-11-27T18:17:22.805728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470966142505807:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26943
2025-11-27T18:17:22.814569Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470966290013687:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.809967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:22.829576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470968928828426:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-11-27T18:17:22.808196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:22.780798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470967492226504:2048] [node 1] ICS09 handshake done sender: [4:7577470967492226489:2116] self: [4:7577470967492226492:0] peer: [1:7577470966462497521:0] socket: 185 qp: -1
2025-11-27T18:17:22.825033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470966290013703:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.806354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470966142505794:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:22.808240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470965860468243:2048] [node 1] ICP22 created new session: [6:7577470965860468328:2048]
2025-11-27T18:17:22.811210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470966462497518:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(44918080ns) {HandleHandshakeStatus:329(44915185ns) {DropIncomingHandshake:442(263ns) DropOutgoingHandshake:457(292ns) SwitchToState:179(243ns) ProcessPendingSessionEvents:495(38011301ns) {ForwardSessionEventToSession:612(16452046ns) ForwardSessionEventToSession:612(1902547ns) ForwardSessionEventToSession:612(3665ns) ForwardSessionEventToSession:612(3231ns) ForwardSessionEventToSession:612(9707789ns) ForwardSessionEventToSession:612(2913ns) ForwardSessionEventToSession:612(4917346ns) ForwardSessionEventToSession:612(7187ns) ForwardSessionEventToSession:612(449841ns) ForwardSessionEventToSession:612(4538548ns)}}}
2025-11-27T18:17:22.780835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470967492226504:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:22.816487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470966142505792:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:22.837787Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470968928828337:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9374) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470968928828419:2119] held: no
2025-11-27T18:17:22.825086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470966290013681:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.808274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470965860468328:2048] [node 1] ICS09 handshake done sender: [6:7577470965860468320:2115] self: [6:7577470965860468323:0] peer: [1:7577470966462497519:0] socket: 188 qp: -1
2025-11-27T18:17:22.808305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470965860468328:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:22.780911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:22.816865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470966462497544:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23328
2025-11-27T18:17:22.839216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470968928828418:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:22.823160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470966142505795:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.825783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470966290013705:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-11-27T18:17:22.808338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:22.824472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470966142505787:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.781397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470967492226411:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19138686ns) {HandleHandshakeStatus:329(19136269ns) {DropIncomingHandshake:442(1285ns) DropOutgoingHandshake:457(5837ns) SwitchToState:179(447ns) ProcessPendingSessionEvents:495(111054ns) {ForwardSessionEventToSession:612(57916ns) ForwardSessionEventToSession:612(3788ns) ForwardSessionEventToSession:612(7735ns) ForwardSessionEventToSession:612(1766ns) ForwardSessionEventToSession:612(2485ns) ForwardSessionEventToSession:612(4956ns) ForwardSessionEventToSession:612(3434ns) ForwardSessionEventToSession:612(16174ns) ForwardSessionEventToSession:612(2532ns) ForwardSessionEventToSession:612(2146ns)}}}
2025-11-27T18:17:22.820669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470966462497453:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(23942302ns) {HandleHandshakeStatus:329(23939812ns) {DropIncomingHandshake:442(244ns) DropOutgoingHandshake:457(1078ns) SwitchToState:179(405ns) ProcessPendingSessionEvents:495(2737931ns) {ForwardSessionEventToSession:612(2683681ns) ForwardSessionEventToSession:612(5030ns) ForwardSessionEventToSession:612(6089ns) ForwardSessionEventToSession:612(2683ns) ForwardSessionEventToSession:612(2260ns) ForwardSessionEventToSession:612(4744ns) ForwardSessionEventToSession:612(2909ns) ForwardSessionEventToSession:612(7269ns) ForwardSessionEventToSession:612(7686ns) ForwardSessionEventToSession:612(2746ns)}}}
2025-11-27T18:17:22.809193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470965860468245:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.809981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470965860468307:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.810680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:22.812705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470965860468307:2106] [node 4] ICP22 created new session: [6:7577470965860468331:2106]
2025-11-27T18:17:22.814105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470965860468331:2106] [node 4] ICS09 handshake done sender: [6:7577470965860468318:2114] self: [6:7577470965860468324:0] peer: [4:7577470967492226479:0] socket: 177 qp: -1
2025-11-27T18:17:22.829473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:22.814160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470965860468331:2106] [node 4] ICS10 traffic start
2025-11-27T18:17:22.840445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470968928828427:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-11-27T18:17:22.824513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:22.814240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:22.843068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470965860468245:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9374) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470965860468326:2118] held: no
2025-11-27T18:17:22.836662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470966290013681:2108] [node 5] ICP22 created new session: [3:7577470966290013707:2108]
2025-11-27T18:17:22.843560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470968928828429:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.820829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470966462497453:2048] [node 2] ICP18 handshake (actor [1:7577470966462497535:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.824980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470966142505723:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.793587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470967492226478:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(25829378ns) {HandleHandshakeStatus:329(25825616ns) {DropIncomingHandshake:442(545ns) DropOutgoingHandshake:457(477ns) SwitchToState:179(358ns) ProcessPendingSessionEvents:495(4326940ns) {ForwardSessionEventToSession:612(2767767ns) ForwardSessionEventToSession:612(5292ns) ForwardSessionEventToSession:612(217774ns) ForwardSessionEventToSession:612(528572ns) ForwardSessionEventToSession:612(5145ns) ForwardSessionEventToSession:612(286693ns) ForwardSessionEventToSession:612(3341ns) ForwardSessionEventToSession:612(2183ns) ForwardSessionEventToSession:612(490540ns) ForwardSessionEventToSession:612(3776ns)}}}
2025-11-27T18:17:22.845486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470968928828336:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.825815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470966462497531:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.825179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470966142505787:2109] [node 5] ICP22 created new session: [2:7577470966142505812:2109]
2025-11-27T18:17:22.795017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577470967492226478:2108] [node 5] ICP18 handshake (actor [4:7577470967492226493:2118]) is too old
2025-11-27T18:17:22.838110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470966290013616:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.846060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:22.825848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470966462497531:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.838094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470966462497531:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:22.842193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470966462497516:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.845506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470966462497528:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.845537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470966462497528:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.846083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:22.848721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470966462497516:2108] [node 5] ICP22 created new session: [1:7577470966462497548:2108]
2025-11-27T18:17:22.852302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470966462497548:2108] [node 5] ICS09 handshake done sender: [1:7577470966462497531:2114] self: [1:7577470966462497529:0] peer: [5:7577470968928828428:0] socket: 228 qp: -1
2025-11-27T18:17:22.897914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470967492226506:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27704
2025-11-27T18:17:22.855039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470966462497548:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:22.866068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:22.829480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470966142505722:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.878117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470965860468336:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11285
2025-11-27T18:17:22.829561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470966142505812:2109] [node 5] ICS09 handshake done sender: [2:7577470966142505794:2113] self: [2:7577470966142505793:0] peer: [5:7577470968928828420:0] socket: 204 qp: -1
2025-11-27T18:17:22.884700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470965860468335:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11285
2025-11-27T18:17:22.848260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470968928828336:2049] [node 2] ICP22 created new session: [5:7577470968928828431:2049]
2025-11-27T18:17:22.838125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470966290013707:2108] [node 5] ICS09 handshake done sender: [3:7577470966290013693:2116] self: [3:7577470966290013692:0] peer: [5:7577470968928828424:0] socket: 215 qp: -1
2025-11-27T18:17:22.833948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470966142505810:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26943
2025-11-27T18:17:22.877707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470966462497551:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23328
2025-11-27T18:17:22.838242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470966290013699:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:22.851617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470968928828431:2049] [node 2] ICS09 handshake done sender: [5:7577470968928828418:2118] self: [5:7577470968928828420:0] peer: [2:7577470966142505793:0] socket: 211 qp: -1
2025-11-27T18:17:22.901505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470965860468336:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.917883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470967492226506:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.833984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470966142505812:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:22.884702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470966462497528:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:22.838675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470966290013707:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:22.908820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470965860468306:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.834032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:22.925745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470967492226413:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.852254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470968928828422:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:22.909433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470965860468326:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:22.897801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470966462497551:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.838977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470966290013687:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:22.836429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470966142505723:2049] [node 3] ICP22 created new session: [2:7577470966142505814:2049]
2025-11-27T18:17:22.852407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470968928828419:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.837023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:22.837864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470966142505814:2049] [node 3] ICS09 handshake done sender: [2:7577470966142505792:2112] self: [2:7577470966142505791:0] peer: [3:7577470966290013701:0] socket: 199 qp: -1
2025-11-27T18:17:22.838370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470966142505814:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:22.838422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:22.838542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:22.839092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470966142505722:2048] [node 1] ICP22 created new session: [2:7577470966142505817:2048]
2025-11-27T18:17:22.839276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470966142505817:2048] [node 1] ICS09 handshake done sender: [2:7577470966142505795:2114] self: [2:7577470966142505802:0] peer: [1:7577470966462497526:0] socket: 190 qp: -1
2025-11-27T18:17:22.840117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470966142505817:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:22.842537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:22.846454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470966142505722:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16815139ns) {HandleHandshakeStatus:329(16812812ns) {DropIncomingHandshake:442(896ns) DropOutgoingHandshake:457(4110ns) SwitchToState:179(3442ns) ProcessPendingSessionEvents:495(89109ns) {ForwardSessionEventToSession:612(43616ns) ForwardSessionEventToSession:612(2649ns) ForwardSessionEventToSession:612(5885ns) ForwardSessionEventToSession:612(2466ns) ForwardSessionEventToSession:612(2094ns) ForwardSessionEventToSession:612(4788ns) ForwardSessionEventToSession:612(5145ns) ForwardSessionEventToSession:612(1937ns) ForwardSessionEventToSession:612(9313ns) ForwardSessionEventToSession:612(4347ns)}}}
2025-11-27T18:17:22.873413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470966142505810:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.881429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577470966142505722:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:22.901403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470966142505813:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26943
2025-11-27T18:17:22.901638Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470966142505787:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(44537907ns) {HandleHandshakeStatus:329(44535177ns) {DropIncomingHandshake:442(620ns) DropOutgoingHandshake:457(588ns) SwitchToState:179(619ns) ProcessPendingSessionEvents:495(28404759ns) {ForwardSessionEventToSession:612(8082782ns) ForwardSessionEventToSession:612(4242ns) ForwardSessionEventToSession:612(6273305ns) ForwardSessionEventToSession:612(2691ns) ForwardSessionEventToSession:612(7087221ns) ForwardSessionEventToSession:612(3451ns) ForwardSessionEventToSession:612(2308ns) ForwardSessionEventToSession:612(6913501ns) ForwardSessionEventToSession:612(7603ns) ForwardSessionEventToSession:612(3678ns)}}}
2025-11-27T18:17:22.901667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470966142505723:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(29913402ns) {HandleHandshakeStatus:329(29910808ns) {DropIncomingHandshake:442(15354ns) DropOutgoingHandshake:457(239ns) SwitchToState:179(457ns) ProcessPendingSessionEvents:495(11971893ns) {ForwardSessionEventToSession:612(2530107ns) ForwardSessionEventToSession:612(5825ns) ForwardSessionEventToSession:612(2791886ns) ForwardSessionEventToSession:612(8228ns) ForwardSessionEventToSession:612(2987ns) ForwardSessionEventToSession:612(346088ns) ForwardSessionEventToSession:612(3750ns) ForwardSessionEventToSession:612(3814161ns) ForwardSessionEventToSession:612(3446ns) ForwardSessionEventToSession:612(2978ns)}}}
2025-11-27T18:17:22.926438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470966142505813:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.935396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470966142505787:2109] [node 5] ICP18 handshake (actor [2:7577470966142505810:2120]) is too old
2025-11-27T18:17:22.922248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470965860468245:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.898122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470966462497516:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(55736261ns) {HandleHandshakeStatus:329(55733812ns) {DropIncomingHandshake:442(10287ns) DropOutgoingHandshake:457(357ns) SwitchToState:179(716ns) ProcessPendingSessionEvents:495(13890753ns) {ForwardSessionEventToSession:612(13834139ns) ForwardSessionEventToSession:612(6126ns) ForwardSessionEventToSession:612(8995ns) ForwardSessionEventToSession:612(3385ns) ForwardSessionEventToSession:612(1987ns) ForwardSessionEventToSession:612(4881ns) ForwardSessionEventToSession:612(5039ns) ForwardSessionEventToSession:612(2194ns) ForwardSessionEventToSession:612(11269ns) ForwardSessionEventToSession:612(1946ns)}}}
2025-11-27T18:17:22.839141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:22.855097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470968928828431:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.901594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470966462497454:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.868506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470968928828335:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8283) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470968928828422:2120] held: no
2025-11-27T18:17:22.840871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470966290013680:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.903375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:22.926922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:22.868605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470968928828335:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.869028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:22.869097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:22.873352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470968928828337:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.881514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470968928828335:2048] [node 1] ICP22 created new session: [5:7577470968928828433:2048]
2025-11-27T18:17:22.881571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470968928828433:2048] [node 1] ICS09 handshake done sender: [5:7577470968928828422:2120] self: [5:7577470968928828428:0] peer: [1:7577470966462497529:0] socket: 231 qp: -1
2025-11-27T18:17:22.881616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470968928828433:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:22.881662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:22.889733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:22.893934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470968928828337:2050] [node 3] ICP22 created new session: [5:7577470968928828435:2050]
2025-11-27T18:17:22.909519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470968928828435:2050] [node 3] ICS09 handshake done sender: [5:7577470968928828419:2119] self: [5:7577470968928828424:0] peer: [3:7577470966290013692:0] socket: 219 qp: -1
2025-11-27T18:17:22.914111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470968928828435:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:22.914176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:22.918483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470968928828336:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(44053851ns) {HandleHandshakeStatus:329(44047896ns) {DropIncomingHandshake:442(1186ns) DropOutgoingHandshake:457(8767ns) SwitchToState:179(766ns) ProcessPendingSessionEvents:495(131321ns) {ForwardSessionEventToSession:612(82178ns) ForwardSessionEventToSession:612(3645ns) ForwardSessionEventToSession:612(7530ns) ForwardSessionEventToSession:612(3003ns) ForwardSessionEventToSession:612(5152ns) ForwardSessionEventToSession:612(5386ns) ForwardSessionEventToSession:612(2916ns) ForwardSessionEventToSession:612(4603ns) ForwardSessionEventToSession:612(2141ns) ForwardSessionEventToSession:612(2308ns)}}}
2025-11-27T18:17:22.922176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470968928828415:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.840947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:22.927602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470965860468245:2050] [node 3] ICP22 created new session: [6:7577470965860468339:2050]
2025-11-27T18:17:22.922217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470968928828415:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.949711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470967492226507:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27704
2025-11-27T18:17:22.840995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470966290013680:2109] [node 6] ICP22 created new session: [3:7577470966290013711:2109]
2025-11-27T18:17:22.905984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470966462497454:2049] [node 3] ICP22 created new session: [1:7577470966462497553:2049]
2025-11-27T18:17:22.841039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470966290013711:2109] [node 6] ICS09 handshake done sender: [3:7577470966290013687:2112] self: [3:7577470966290013685:0] peer: [6:7577470965860468330:0] socket: 213 qp: -1
2025-11-27T18:17:22.957512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470967492226413:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9374) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470967492226506:2120] held: no
2025-11-27T18:17:22.841071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470966290013711:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:22.960085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470967492226507:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.841106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:22.841903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470966290013617:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.848166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.853347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470966290013617:2049] [node 2] ICP22 created new session: [3:7577470966290013713:2049]
2025-11-27T18:17:22.861815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470966290013713:2049] [node 2] ICS09 handshake done sender: [3:7577470966290013699:2118] self: [3:7577470966290013701:0] peer: [2:7577470966142505791:0] socket: 220 qp: -1
2025-11-27T18:17:22.961092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470967492226509:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27704
2025-11-27T18:17:22.865923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470966290013713:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:22.933708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470968928828337:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(57243259ns) {HandleHandshakeStatus:329(57241356ns) {DropIncomingHandshake:442(3709ns) DropOutgoingHandshake:457(237ns) SwitchToState:179(733ns) ProcessPendingSessionEvents:495(1075ns)}}
2025-11-27T18:17:22.918084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470966462497553:2049] [node 3] ICS09 handshake done sender: [1:7577470966462497528:2113] self: [1:7577470966462497525:0] peer: [3:7577470966290013708:0] socket: 226 qp: -1
2025-11-27T18:17:22.934926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470965860468339:2050] [node 3] ICS09 handshake done sender: [6:7577470965860468326:2118] self: [6:7577470965860468330:0] peer: [3:7577470966290013685:0] socket: 222 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.956942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470968928828415:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:22.865994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:22.963245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470967492226412:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.918168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470966462497553:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:22.935397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470965860468339:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:22.961311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470968928828399:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:22.974834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:22.942255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:22.870406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470966290013617:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(29777791ns) {HandleHandshakeStatus:329(29775331ns) {DropIncomingHandshake:442(2071ns) DropOutgoingHandshake:457(19610ns) SwitchToState:179(633ns) ProcessPendingSessionEvents:495(92720ns) {ForwardSessionEventToSession:612(52200ns) ForwardSessionEventToSession:612(2511ns) ForwardSessionEventToSession:612(4959ns) ForwardSessionEventToSession:612(2539ns) ForwardSessionEventToSession:612(4962ns) ForwardSessionEventToSession:612(5188ns) ForwardSessionEventToSession:612(2956ns) ForwardSessionEventToSession:612(4648ns) ForwardSessionEventToSession:612(2134ns) ForwardSessionEventToSession:612(2971ns)}}}
2025-11-27T18:17:22.969404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470966142505804:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.969477Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470966142505804:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.945681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470965860468338:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11285
2025-11-27T18:17:22.977766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470967492226506:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:22.922451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:22.976623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470968928828399:2109] [node 6] ICP22 created new session: [5:7577470968928828438:2109]
2025-11-27T18:17:22.871874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470966290013716:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-11-27T18:17:22.980589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470967492226413:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.980667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:22.957018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470965860468245:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(25940413ns) {HandleHandshakeStatus:329(25937803ns) {DropIncomingHandshake:442(589ns) DropOutgoingHandshake:457(464ns) SwitchToState:179(859ns) ProcessPendingSessionEvents:495(868ns)}}
2025-11-27T18:17:22.977735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470966142505797:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.981990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470968928828438:2109] [node 6] ICS09 handshake done sender: [5:7577470968928828415:2116] self: [5:7577470968928828414:0] peer: [6:7577470965860468337:0] socket: 206 qp: -1
2025-11-27T18:17:22.960053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470965860468338:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.980730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470967492226413:2050] [node 3] ICP22 created new session: [4:7577470967492226511:2050]
2025-11-27T18:17:22.960373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470965860468244:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:22.980773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470967492226511:2050] [node 3] ICS09 handshake done sender: [4:7577470967492226506:2120] self: [4:7577470967492226508:0] peer: [3:7577470966290013688:0] socket: 240 qp: -1
2025-11-27T18:17:22.973476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470965860468341:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11285
2025-11-27T18:17:22.980803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470967492226511:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:22.973747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470965860468336:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:22.980847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:22.978009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470965860468306:2103] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17060) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470965860468336:2121] held: no
2025-11-27T18:17:22.978583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470965860468306:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.977771Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470966142505797:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.903192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470966290013681:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(43834496ns) {HandleHandshakeStatus:329(43831692ns) {DropIncomingHandshake:442(13110ns) DropOutgoingHandshake:457(519ns) SwitchToState:179(404ns) ProcessPendingSessionEvents:495(23476599ns) {ForwardSessionEventToSession:612(3193891ns) ForwardSessionEventToSession:612(3983ns) ForwardSessionEventToSession:612(8716623ns) ForwardSessionEventToSession:612(6940ns) ForwardSessionEventToSession:612(2878ns) ForwardSessionEventToSession:612(4402815ns) ForwardSessionEventToSession:612(4434ns) ForwardSessionEventToSession:612(4104588ns) ForwardSessionEventToSession:612(3011558ns) ForwardSessionEventToSession:612(3295ns)}}}
2025-11-27T18:17:22.963117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470966462497454:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(48255998ns) {HandleHandshakeStatus:329(45204810ns) {DropIncomingHandshake:442(1727415ns) DropOutgoingHandshake:457(1035ns) SwitchToState:179(469ns) ProcessPendingSessionEvents:495(19885044ns) {ForwardSessionEventToSession:612(2424221ns) ForwardSessionEventToSession:612(4595333ns) ForwardSessionEventToSession:612(3157ns) ForwardSessionEventToSession:612(317880ns) ForwardSessionEventToSession:612(4484ns) ForwardSessionEventToSession:612(6666875ns) ForwardSessionEventToSession:612(2475932ns) ForwardSessionEventToSession:612(3781ns) ForwardSessionEventToSession:612(3024ns) ForwardSessionEventToSession:612(2011424ns)}}}
2025-11-27T18:17:22.905887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470966290013703:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:22.982029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470968928828438:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:22.977769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470966142505804:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:22.978649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:22.913481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470966290013616:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8283) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470966290013703:2119] held: no
2025-11-27T18:17:22.982437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: 2025-11-27T18:17:22.978691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470965860468306:2103] [node 5] ICP22 created new session: [6:7577470965860468344:2103]
::ffff:127.1.0.0
2025-11-27T18:17:22.918644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470966290013616:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.981872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470966142505788:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.988537Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470968928828399:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(26483804ns) {HandleHandshakeStatus:329(26481280ns) {DropIncomingHandshake:442(28722ns) DropOutgoingHandshake:457(521ns) SwitchToState:179(443ns) ProcessPendingSessionEvents:495(2720139ns) {ForwardSessionEventToSession:612(2666518ns) ForwardSessionEventToSession:612(10670ns) ForwardSessionEventToSession:612(2694ns) ForwardSessionEventToSession:612(5997ns) ForwardSessionEventToSession:612(5297ns) ForwardSessionEventToSession:612(3183ns) ForwardSessionEventToSession:612(5489ns) ForwardSessionEventToSession:612(2436ns) ForwardSessionEventToSession:612(3306ns) ForwardSessionEventToSession:612(5124ns)}}}
2025-11-27T18:17:22.978728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470965860468344:2103] [node 5] ICS09 handshake done sender: [6:7577470965860468336:2121] self: [6:7577470965860468337:0] peer: [5:7577470968928828414:0] socket: 202 qp: -1
2025-11-27T18:17:22.978773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470965860468344:2103] [node 5] ICS10 traffic start
2025-11-27T18:17:22.978819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:22.981847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470965860468343:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11285
2025-11-27T18:17:22.983382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470965860468338:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:22.985103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470965860468244:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.988626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:22.988915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470965860468244:2049] [node 2] ICP22 created new session: [6:7577470965860468346:2049]
2025-11-27T18:17:22.989340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470965860468346:2049] [node 2] ICS09 handshake done sender: [6:7577470965860468338:2122] self: [6:7577470965860468342:0] peer: [2:7577470966142505803:0] socket: 242 qp: -1
2025-11-27T18:17:22.989719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470965860468346:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:22.989761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:22.991057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470967492226513:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27704
2025-11-27T18:17:22.926322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:22.983686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:22.985021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470966142505788:2110] [node 6] ICP22 created new session: [2:7577470966142505823:2110]
2025-11-27T18:17:22.926584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470966290013616:2048] [node 1] ICP22 created new session: [3:7577470966290013717:2048]
2025-11-27T18:17:22.993655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470967492226412:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16233) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470967492226507:2121] held: no
2025-11-27T18:17:22.996689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470967492226507:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:22.987503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470966142505797:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:22.999030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470967492226412:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:22.928099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470966290013689:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:22.999146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:22.928123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470966290013689:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:22.999589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470967492226412:2049] [node 2] ICP22 created new session: [4:7577470967492226514:2049]
2025-11-27T18:17:22.987707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470966142505823:2110] [node 6] ICS09 handshake done sender: [2:7577470966142505804:2117] self: [2:7577470966142505803:0] peer: [6:7577470965860468342:0] socket: 214 qp: -1
2025-11-27T18:17:23.000239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470967492226514:2049] [node 2] ICS09 handshake done sender: [4:7577470967492226507:2121] self: [4:7577470967492226510:0] peer: [2:7577470966142505796:0] socket: 241 qp: -1
2025-11-27T18:17:22.929055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470966290013689:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:22.933658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470966290013717:2048] [node 1] ICS09 handshake done sender: [3:7577470966290013703:2119] self: [3:7577470966290013708:0] peer: [1:7577470966462497525:0] socket: 229 qp: -1
2025-11-27T18:17:22.933714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470966290013717:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:22.935465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:22.944724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470966290013679:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.947413Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470966290013616:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27092632ns) {HandleHandshakeStatus:329(27089039ns) {DropIncomingHandshake:442(1004ns) DropOutgoingHandshake:457(328ns) SwitchToState:179(977ns) ProcessPendingSessionEvents:495(924ns)}}
2025-11-27T18:17:22.959028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:22.959082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470966290013679:2107] [node 4] ICP22 created new session: [3:7577470966290013720:2107]
2025-11-27T18:17:22.960590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470966290013720:2107] [node 4] ICS09 handshake done sender: [3:7577470966290013689:2113] self: [3:7577470966290013688:0] peer: [4:7577470967492226508:0] socket: 193 qp: -1
2025-11-27T18:17:22.961187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470966290013720:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:22.987748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470966142505823:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:22.965476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:22.987821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:22.989453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470966142505786:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:22.987918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470966290013679:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(41460075ns) {HandleHandshakeStatus:329(41456643ns) {DropIncomingHandshake:442(1606638ns) DropOutgoingHandshake:457(933ns) SwitchToState:179(2117ns) ProcessPendingSessionEvents:495(5291369ns) {ForwardSessionEventToSession:612(3507950ns) ForwardSessionEventToSession:612(4935ns) ForwardSessionEventToSession:612(382386ns) ForwardSessionEventToSession:612(2340ns) ForwardSessionEventToSession:612(3302ns) ForwardSessionEventToSession:612(900605ns) ForwardSessionEventToSession:612(3084ns) ForwardSessionEventToSession:612(2819ns) ForwardSessionEventToSession:612(467728ns) ForwardSessionEventToSession:612(3188ns)}}}
2025-11-27T18:17:22.989505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:22.990066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470966142505786:2107] [node 4] ICP22 created new session: [2:7577470966142505826:2107]
2025-11-27T18:17:22.990868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470966142505826:2107] [node 4] ICS09 handshake done sender: [2:7577470966142505797:2115] self: [2:7577470966142505796:0] peer: [4:7577470967492226510:0] socket: 212 qp: -1
2025-11-27T18:17:22.990966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470966142505826:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:23.000640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470967492226514:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:22.992750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:23.000701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:23.030360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:23.030611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470966142505817:2048] [node 1] ICS07 socket disconnect 190 reason# EndOfStream
2025-11-27T18:17:23.030650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470966142505817:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.030724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:23.030769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470966142505817:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:23.031792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:23.032011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470966290013717:2048] [node 1] ICS01 socket: 229 reason# EndOfStream
2025-11-27T18:17:23.032065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470966290013616:2048] [node 1] ICP30 unregister session Session# [3:7577470966290013717:2048] VirtualId# [3:7577470966290013708:0]
2025-11-27T18:17:23.032096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470966290013717:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.032162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:23.032206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:23.033747Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470966142505722:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8283) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:23.033797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470966142505817:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:23.036896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:23.037844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:23.037548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470968928828433:2048] [node 1] ICS01 socket: 231 reason# EndOfStream
2025-11-27T18:17:23.037589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470968928828335:2048] [node 1] ICP30 unregister session Session# [5:7577470968928828433:2048] VirtualId# [5:7577470968928828428:0]
2025-11-27T18:17:23.037621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470968928828433:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.037695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:23.037732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:23.038032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470965860468328:2048] [node 1] ICS01 socket: 188 reason# EndOfStream
2025-11-27T18:17:23.038076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470965860468243:2048] [node 1] ICP30 unregister session Session# [6:7577470965860468328:2048] VirtualId# [6:7577470965860468323:0]
2025-11-27T18:17:23.037458Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:23.038097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470965860468328:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.038145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:23.038180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:23.037660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470967492226504:2048] [node 1] ICS01 socket: 185 reason# EndOfStream
2025-11-27T18:17:23.040270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470966142505722:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8283) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:23.037692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470967492226411:2048] [node 1] ICP30 unregister session Session# [4:7577470967492226504:2048] VirtualId# [4:7577470967492226492:0]
2025-11-27T18:17:23.037717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470967492226504:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.037781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:23.037824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:23.040348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470966142505817:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:23.040449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470966142505722:2048] [node 1] ICP30 unregister session Session# [2:7577470966142505817:2048] VirtualId# [2:7577470966142505802:0]
2025-11-27T18:17:23.040497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:23.040528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470966142505722:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8283) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:23.040540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8283) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:23.049840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577470966142505722:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:23.089400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:23.089572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470966290013713:2049] [node 2] ICS01 socket: 220 reason# EndOfStream
2025-11-27T18:17:23.089602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470966290013617:2049] [node 2] ICP30 unregister session Session# [3:7577470966290013713:2049] VirtualId# [3:7577470966290013701:0]
2025-11-27T18:17:23.089626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470966290013713:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.089691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:23.089724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:23.090016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:23.090138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470967492226514:2049] [node 2] ICS01 socket: 241 reason# EndOfStream
2025-11-27T18:17:23.090157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470967492226412:2049] [node 2] ICP30 unregister session Session# [4:7577470967492226514:2049] VirtualId# [4:7577470967492226510:0]
2025-11-27T18:17:23.090171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470967492226514:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.090205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:23.090227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:23.092239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:23.092450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470965860468346:2049] [node 2] ICS01 socket: 242 reason# EndOfStream
2025-11-27T18:17:23.092495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470965860468244:2049] [node 2] ICP30 unregister session Session# [6:7577470965860468346:2049] VirtualId# [6:7577470965860468342:0]
2025-11-27T18:17:23.092517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470965860468346:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.092565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:23.092617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:23.091736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:23.091936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470968928828431:2049] [node 2] ICS01 socket: 211 reason# EndOfStream
2025-11-27T18:17:23.091978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470968928828336:2049] [node 2] ICP30 unregister session Session# [5:7577470968928828431:2049] VirtualId# [5:7577470968928828420:0]
2025-11-27T18:17:23.092002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470968928828431:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.092080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:23.092122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:23.170431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:23.170643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470967492226511:2050] [node 3] ICS01 socket: 240 reason# EndOfStream
2025-11-27T18:17:23.170685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470967492226413:2050] [node 3] ICP30 unregister session Session# [4:7577470967492226511:2050] VirtualId# [4:7577470967492226508:0]
2025-11-27T18:17:23.170713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470967492226511:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.170782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:23.170830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:23.171480Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:23.171683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470968928828435:2050] [node 3] ICS01 socket: 219 reason# EndOfStream
2025-11-27T18:17:23.171735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470968928828337:2050] [node 3] ICP30 unregister session Session# [5:7577470968928828435:2050] VirtualId# [5:7577470968928828424:0]
2025-11-27T18:17:23.171760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470968928828435:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.171827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:23.171904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:23.172220Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:23.172456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470965860468339:2050] [node 3] ICS01 socket: 222 reason# EndOfStream
2025-11-27T18:17:23.172489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470965860468245:2050] [node 3] ICP30 unregister session Session# [6:7577470965860468339:2050] VirtualId# [6:7577470965860468330:0]
2025-11-27T18:17:23.172514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470965860468339:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.172566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:23.172600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:23.225478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:23.226705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470968928828421:2108] [node 4] ICS01 socket: 191 reason# EndOfStream
2025-11-27T18:17:23.226750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470968928828398:2108] [node 4] ICP30 unregister session Session# [5:7577470968928828421:2108] VirtualId# [5:7577470968928828413:0]
2025-11-27T18:17:23.226783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470968928828421:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.226863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:23.226914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:23.225479Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:23.225743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470965860468331:2106] [node 4] ICS01 socket: 177 reason# EndOfStream
2025-11-27T18:17:23.225800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470965860468307:2106] [node 4] ICP30 unregister session Session# [6:7577470965860468331:2106] VirtualId# [6:7577470965860468324:0]
2025-11-27T18:17:23.225836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470965860468331:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.225966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:23.226020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:23.302223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:23.302595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470965860468344:2103] [node 5] ICS01 socket: 202 reason# EndOfStream
2025-11-27T18:17:23.302643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470965860468306:2103] [node 5] ICP30 unregister session Session# [6:7577470965860468344:2103] VirtualId# [6:7577470965860468337:0]
2025-11-27T18:17:23.302695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470965860468344:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:23.302794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:23.302849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:23.417800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470971406361229:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:23.417846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470971406361230:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:23.440562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470970275723936:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:23.440669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470970275723937:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:23.475398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470973241786676:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:23.475529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470973241786677:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:23.496210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470969462271352:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:23.496251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470969462271353:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:23.496277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470969462271354:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:23.519772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470971543018843:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:23.519808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470971543018844:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:23.519838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470971543018845:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:23.543370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470971542633489:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:23.543401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470971542633490:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:23.543433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470971542633491:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:23.693224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470971542633549:2104] [node 5] ICP01 ready to work
2025-11-27T18:17:23.709708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470971406361293:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:23.706463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470971406361292:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:23.774807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470971406361294:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:23.790782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470969462271415:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:23.793612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470973241786748:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:23.797503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470973241786749:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:23.802643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470971543018906:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:23.804163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470971543018907:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:23.790906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470970275724004:2113] [node 6] ICP01 ready to work
2025-11-27T18:17:23.797638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470969462271416:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:23.797361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470970275724003:2111] [node 5] ICP01 ready to work
2025-11-27T18:17:23.802887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470970275724005:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:23.805319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470973241786750:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:23.809007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470971406361229:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(113274005ns) {Configure:104(113268820ns) {ProcessConfigured:135(509570ns) {StartInitialHandshake:157(507332ns) {DropHandshakes:470(1839ns) {DropIncomingHandshake:442(467ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(964ns) PrepareNewSessionHandshake:477(1943ns) {SwitchToState:179(741ns)}}}}}
2025-11-27T18:17:23.809158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470971542633553:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:23.813883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470971542633549:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(119992833ns) {Configure:104(119987531ns) {ProcessConfigured:135(92765ns) {StartInitialHandshake:157(89332ns) {DropHandshakes:470(3571ns) {DropIncomingHandshake:442(904ns) DropOutgoingHandshake:457(628ns)} GenerateSessionVirtualId:988(828ns) PrepareNewSessionHandshake:477(1916ns) {SwitchToState:179(743ns)}}}}}
2025-11-27T18:17:23.845673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470971406361293:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(96587825ns) {Configure:104(96580732ns) {ProcessConfigured:135(1100951ns) {StartInitialHandshake:157(1098139ns) {DropHandshakes:470(3896ns) {DropIncomingHandshake:442(1246ns) DropOutgoingHandshake:457(359ns)} GenerateSessionVirtualId:988(645ns) PrepareNewSessionHandshake:477(1880ns) {SwitchToState:179(801ns)}}}}}
2025-11-27T18:17:23.849591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470971406361230:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(132938425ns) {Configure:104(132932528ns) {ProcessConfigured:135(12071952ns) {StartInitialHandshake:157(12067356ns) {DropHandshakes:470(3763ns) {DropIncomingHandshake:442(1014ns) DropOutgoingHandshake:457(668ns)} GenerateSessionVirtualId:988(1862ns) PrepareNewSessionHandshake:477(3655ns) {SwitchToState:179(1193ns)}}}}}
2025-11-27T18:17:23.859300Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470970275724003:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(40327062ns) {Configure:104(40322776ns) {ProcessConfigured:135(4803990ns) {StartInitialHandshake:157(4801515ns) {DropHandshakes:470(938ns) {DropIncomingHandshake:442(266ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(301ns) PrepareNewSessionHandshake:477(2956ns) {SwitchToState:179(837ns)}}}}}
2025-11-27T18:17:23.859269Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470969462271415:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50669955ns) {Configure:104(50662913ns) {ProcessConfigured:135(14487924ns) {StartInitialHandshake:157(14485971ns) {DropHandshakes:470(1040ns) {DropIncomingHandshake:442(300ns) DropOutgoingHandshake:457(209ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:477(2923ns) {SwitchToState:179(1474ns)}}}}}
2025-11-27T18:17:23.859441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470973241786676:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(123498844ns) {Configure:104(123492898ns) {ProcessConfigured:135(14646758ns) {StartInitialHandshake:157(14643907ns) {DropHandshakes:470(1717ns) {DropIncomingHandshake:442(500ns) DropOutgoingHandshake:457(522ns)} GenerateSessionVirtualId:988(494ns) PrepareNewSessionHandshake:477(4169ns) {SwitchToState:179(1669ns)}}}}}
2025-11-27T18:17:23.857511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470971406361292:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(102933964ns) {Configure:104(102929742ns) {ProcessConfigured:135(6747472ns) {StartInitialHandshake:157(6745267ns) {DropHandshakes:470(297727ns) {DropIncomingHandshake:442(689ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(613ns) PrepareNewSessionHandshake:477(2184ns) {SwitchToState:179(790ns)}}}}}
2025-11-27T18:17:23.859772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470971406361294:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(61199147ns) {Configure:104(61188488ns) {ProcessConfigured:135(10691707ns) {StartInitialHandshake:157(10689781ns) {DropHandshakes:470(1017ns) {DropIncomingHandshake:442(298ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(507ns) PrepareNewSessionHandshake:477(2712ns) {SwitchToState:179(1100ns)}}}}}
2025-11-27T18:17:23.859457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470971542633491:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(62888326ns) {Configure:104(62883442ns) {ProcessConfigured:135(10042354ns) {StartInitialHandshake:157(10040546ns) {DropHandshakes:470(1152ns) {DropIncomingHandshake:442(294ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(460ns) PrepareNewSessionHandshake:477(2637ns) {SwitchToState:179(1585ns)}}}}}
2025-11-27T18:17:23.859511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470969462271352:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(70369239ns) {Configure:104(70362398ns) {ProcessConfigured:135(9090627ns) {StartInitialHandshake:157(9088765ns) {DropHandshakes:470(859ns) {DropIncomingHandshake:442(231ns) DropOutgoingHandshake:457(190ns)} GenerateSessionVirtualId:988(417ns) PrepareNewSessionHandshake:477(1705ns) {SwitchToState:179(353ns)}}}}}
2025-11-27T18:17:23.859715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470970275724004:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(51193450ns) {Configure:104(51181881ns) {ProcessConfigured:135(14446747ns) {StartInitialHandshake:157(14444312ns) {DropHandshakes:470(1124ns) {DropIncomingHandshake:442(306ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(503ns) PrepareNewSessionHandshake:477(2294ns) {SwitchToState:179(1322ns)}}}}}
2025-11-27T18:17:23.860097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470971542633490:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(62980985ns) {Configure:104(62975779ns) {ProcessConfigured:135(13499758ns) {StartInitialHandshake:157(13497813ns) {DropHandshakes:470(1104ns) {DropIncomingHandshake:442(350ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(878ns) PrepareNewSessionHandshake:477(1405ns) {SwitchToState:179(1059ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.865598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470969462271416:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(50247043ns) {Configure:104(50239799ns) {ProcessConfigured:135(8913877ns) {StartInitialHandshake:157(8912792ns) {DropHandshakes:470(744ns) {DropIncomingHandshake:442(228ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:477(859022ns) {SwitchToState:179(1188ns)}}}}}
2025-11-27T18:17:23.860180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470973241786677:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(126446021ns) {Configure:104(126440995ns) {ProcessConfigured:135(18498929ns) {StartInitialHandshake:157(18497363ns) {DropHandshakes:470(870ns) {DropIncomingHandshake:442(204ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(499ns) PrepareNewSessionHandshake:477(2352ns) {SwitchToState:179(156ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.859807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470970275723936:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(136300704ns) {Configure:104(136294484ns) {ProcessConfigured:135(14200981ns) {StartInitialHandshake:157(14197992ns) {DropHandshakes:470(1830ns) {DropIncomingHandshake:442(678ns) DropOutgoingHandshake:457(404ns)} GenerateSessionVirtualId:988(2096ns) PrepareNewSessionHandshake:477(2078ns) {SwitchToState:179(693ns)}}}}}
2025-11-27T18:17:23.865675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470969462271354:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(70692003ns) {Configure:104(70671149ns) {ProcessConfigured:135(9131011ns) {StartInitialHandshake:157(9127886ns) {DropHandshakes:470(1149ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(1949ns) PrepareNewSessionHandshake:477(1984ns) {SwitchToState:179(544ns)}}}}}
2025-11-27T18:17:23.865716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470973241786750:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48631110ns) {Configure:104(48626576ns) {ProcessConfigured:135(10730954ns) {StartInitialHandshake:157(10729040ns) {DropHandshakes:470(1243ns) {DropIncomingHandshake:442(399ns) DropOutgoingHandshake:457(296ns)} GenerateSessionVirtualId:988(522ns) PrepareNewSessionHandshake:477(1628ns) {SwitchToState:179(43ns)}}}}}
2025-11-27T18:17:23.880670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470971542633489:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(67029536ns) {Configure:104(67020234ns) {ProcessConfigured:135(7244479ns) {StartInitialHandshake:157(7242126ns) {DropHandshakes:470(1104ns) {DropIncomingHandshake:442(279ns) DropOutgoingHandshake:457(213ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:477(2436ns) {SwitchToState:179(949ns)}}}}}
2025-11-27T18:17:23.878609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470970275724005:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(53980281ns) {Configure:104(53975019ns) {ProcessConfigured:135(1760607ns) {StartInitialHandshake:157(1758542ns) {DropHandshakes:470(1541ns) {DropIncomingHandshake:442(363ns) DropOutgoingHandshake:457(417ns)} GenerateSessionVirtualId:988(934ns) PrepareNewSessionHandshake:477(1320ns) {SwitchToState:179(490ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.880108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470969462271353:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(75597141ns) {Configure:104(75591244ns) {ProcessConfigured:135(5168570ns) {StartInitialHandshake:157(5166972ns) {DropHandshakes:470(1741ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(641ns) PrepareNewSessionHandshake:477(1267ns) {SwitchToState:179(374ns)}}}}}
2025-11-27T18:17:23.901760Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470971542633553:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(67796405ns) {Configure:104(67787591ns) {ProcessConfigured:135(20126621ns) {StartInitialHandshake:157(20123263ns) {DropHandshakes:470(2539ns) {DropIncomingHandshake:442(591ns) DropOutgoingHandshake:457(480ns)} GenerateSessionVirtualId:988(836ns) PrepareNewSessionHandshake:477(4114ns) {SwitchToState:179(1746ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.911088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470971406361308:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44073
2025-11-27T18:17:23.875645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470973241786748:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(49767410ns) {Configure:104(49762316ns) {ProcessConfigured:135(9012758ns) {StartInitialHandshake:157(9010676ns) {DropHandshakes:470(1105ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(388ns) PrepareNewSessionHandshake:477(2063ns) {SwitchToState:179(612ns)}}}}}
2025-11-27T18:17:23.885296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470971543018843:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(66864433ns) {Configure:104(66854398ns) {ProcessConfigured:135(981877ns) {StartInitialHandshake:157(980023ns) {DropHandshakes:470(1220ns) {DropIncomingHandshake:442(326ns) DropOutgoingHandshake:457(283ns)} GenerateSessionVirtualId:988(849ns) PrepareNewSessionHandshake:477(1516ns) {SwitchToState:179(65ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.878690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470970275723937:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(116588280ns) {Configure:104(116580208ns) {ProcessConfigured:135(9448102ns) {StartInitialHandshake:157(9445612ns) {DropHandshakes:470(1324ns) {DropIncomingHandshake:442(292ns) DropOutgoingHandshake:457(419ns)} GenerateSessionVirtualId:988(809ns) PrepareNewSessionHandshake:477(2180ns) {SwitchToState:179(765ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.909101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470969462271429:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-11-27T18:17:23.893305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470973241786749:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(62352473ns) {Configure:104(62346963ns) {ProcessConfigured:135(7503823ns) {StartInitialHandshake:157(7501449ns) {DropHandshakes:470(587ns) {DropIncomingHandshake:442(151ns) DropOutgoingHandshake:457(101ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:477(2016ns) {SwitchToState:179(590ns)}}}}}
2025-11-27T18:17:23.885983Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470971543018907:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(55868406ns) {Configure:104(55864603ns) {ProcessConfigured:135(1324602ns) {StartInitialHandshake:157(1322979ns) {DropHandshakes:470(1160ns) {DropIncomingHandshake:442(319ns) DropOutgoingHandshake:457(231ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:477(899ns) {SwitchToState:179(109ns)}}}}}
2025-11-27T18:17:23.899114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470970275724016:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6240
2025-11-27T18:17:23.893710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470971543018906:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(60769597ns) {Configure:104(60763634ns) {ProcessConfigured:135(6266008ns) {StartInitialHandshake:157(6263191ns) {DropHandshakes:470(1670ns) {DropIncomingHandshake:442(341ns) DropOutgoingHandshake:457(739ns)} GenerateSessionVirtualId:988(392ns) PrepareNewSessionHandshake:477(3626ns) {SwitchToState:179(1994ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.931048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470971542633566:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10034
2025-11-27T18:17:23.911140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470973241786765:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46577
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.921391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470970275724020:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6240
2025-11-27T18:17:23.937957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470971542633568:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10034
2025-11-27T18:17:23.893750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470971543018844:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(92885371ns) {Configure:104(75952507ns) {ProcessConfigured:135(883730ns) {StartInitialHandshake:157(882551ns) {DropHandshakes:470(918ns) {DropIncomingHandshake:442(292ns) DropOutgoingHandshake:457(94ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(1047ns) {SwitchToState:179(98ns)}}}}}
2025-11-27T18:17:23.931010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470973241786761:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46577
2025-11-27T18:17:23.959655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470971406361308:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.898234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470971543018845:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(68665027ns) {Configure:104(68657765ns) {ProcessConfigured:135(270199ns) {StartInitialHandshake:157(268755ns) {DropHandshakes:470(772ns) {DropIncomingHandshake:442(211ns) DropOutgoingHandshake:457(165ns)} GenerateSessionVirtualId:988(404ns) PrepareNewSessionHandshake:477(934ns) {SwitchToState:179(281ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.966807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470969462271429:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.933925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470970275724016:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.957568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470973241786761:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.967481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470971542633568:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.968350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470969462271353:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:23.959777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470971406361296:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.956949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470970275723936:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.968634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470969462271431:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-11-27T18:17:23.898983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470971543018918:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57456
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.969690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470969462271432:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-11-27T18:17:23.929095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470971543018918:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.968003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470971542633566:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.961159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470973241786765:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.959807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470971406361296:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:23.959469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470970275724021:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6240
2025-11-27T18:17:23.967702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470971543018920:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57456
2025-11-27T18:17:23.972650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470969462271422:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.968757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470971542633553:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:23.966402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470973241786677:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:23.959588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470970275724020:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.973022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470971543018922:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57456
2025-11-27T18:17:23.969599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470971542633549:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:23.972683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470969462271422:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:23.966741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470971406361309:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44073
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.967831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470973241786676:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:17:23.967517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470971406361296:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:23.973896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470969462271432:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.968785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470970275724021:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.973903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470971543018923:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57456
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.971828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470973241786766:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46577
2025-11-27T18:17:23.968834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470970275724014:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.968859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470970275724014:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:23.969297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470970275724015:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.969322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470970275724015:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:23.969340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470970275724024:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6240
2025-11-27T18:17:23.970823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470970275724014:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:23.972837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470970275724016:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:23.973451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470970275723936:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:23.974050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:23.974120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470970275723936:2048] [node 1] ICP22 created new session: [2:7577470970275724027:2048]
2025-11-27T18:17:23.974879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470970275723937:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:23.976203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470970275724027:2048] [node 1] ICS09 handshake done sender: [2:7577470970275724016:2119] self: [2:7577470970275724022:0] peer: [1:7577470971406361295:0] socket: 187 qp: -1
2025-11-27T18:17:23.976529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470970275724027:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:23.976578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:23.978766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:23.980476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470970275723937:2049] [node 3] ICP22 created new session: [2:7577470970275724030:2049]
2025-11-27T18:17:23.983019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470970275724030:2049] [node 3] ICS09 handshake done sender: [2:7577470970275724014:2117] self: [2:7577470970275724010:0] peer: [3:7577470973241786767:0] socket: 192 qp: -1
2025-11-27T18:17:23.984261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470970275724030:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:23.985383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470970275724015:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:23.985527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:23.969589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470971406361310:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44073
2025-11-27T18:17:23.974898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470971543018922:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.980291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470973241786766:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.990089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470971542633571:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10034
2025-11-27T18:17:23.976090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470969462271431:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.000035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470970275724005:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:23.974858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470971406361298:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.978763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470969462271422:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.974892Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470971406361298:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:23.983982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470973241786760:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.975753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470971543018917:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.975779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470971543018917:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:23.976554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470971543018845:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:23.976145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470971406361309:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.045829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470975837600870:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10034
2025-11-27T18:17:24.022847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:23.984013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470973241786760:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:23.981163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470969462271354:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:23.976660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470971543018920:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.049630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470974570691331:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6240
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:24.089769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470971542633553:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13878) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470971542633566:2115] held: no
2025-11-27T18:17:24.089922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470971542633549:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28896) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470971542633568:2116] held: no
2025-11-27T18:17:24.090091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470971542633566:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:24.090751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470971542633572:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10034
2025-11-27T18:17:24.013802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470969462271416:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:23.984374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470973241786769:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46577
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.978618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470971406361310:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:24.053574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470970275724005:2107] [node 4] ICP22 created new session: [2:7577470974570691330:2107]
2025-11-27T18:17:24.091257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470971542633553:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:23.982398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470971543018923:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.980529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470971406361229:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.091343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:23.986354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470973241786677:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24600) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470973241786765:2125] held: no
2025-11-27T18:17:24.069528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470974570691330:2107] [node 4] ICS09 handshake done sender: [2:7577470970275724015:2118] self: [2:7577470970275724013:0] peer: [4:7577470969462271433:0] socket: 193 qp: -1
2025-11-27T18:17:23.982496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470971543018843:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:24.013876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470969462271353:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24600) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470969462271429:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:24.091377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470971542633553:2109] [node 4] ICP22 created new session: [6:7577470975837600872:2109]
2025-11-27T18:17:24.017655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470969462271421:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.982460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:24.069576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470974570691330:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.985531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470971406361229:2048] [node 2] ICP22 created new session: [1:7577470971406361315:2048]
2025-11-27T18:17:23.990764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470971406361298:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:23.987075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470973241786765:2125] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:24.091425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470975837600872:2109] [node 4] ICS09 handshake done sender: [6:7577470971542633566:2115] self: [6:7577470971542633569:0] peer: [4:7577470969462271420:0] socket: 197 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:24.017693Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470969462271421:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:23.988836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470971543018930:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57456
2025-11-27T18:17:23.987887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470973241786677:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.075098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.989110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470971543018922:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.995956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470971406361300:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.091454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470975837600872:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:24.025409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:23.987929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:24.091486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:24.111519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470975837600870:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.989240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470971543018845:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.111612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470971542633572:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:23.987967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470973241786677:2049] [node 2] ICP22 created new session: [3:7577470973241786772:2049]
2025-11-27T18:17:23.989347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:24.111947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470975837600874:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10034
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.988016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470973241786772:2049] [node 2] ICS09 handshake done sender: [3:7577470973241786765:2125] self: [3:7577470973241786767:0] peer: [2:7577470970275724010:0] socket: 199 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:23.995986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470971406361315:2048] [node 2] ICS09 handshake done sender: [1:7577470971406361296:2110] self: [1:7577470971406361295:0] peer: [2:7577470970275724022:0] socket: 182 qp: -1
2025-11-27T18:17:23.988073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470973241786772:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:23.988118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:23.995994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470971406361300:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:23.989744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470973241786756:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.001341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470971406361315:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:24.001403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:23.989783Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470973241786756:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:23.989834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470973241786760:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:24.013578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470971406361300:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:23.999305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470973241786770:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46577
2025-11-27T18:17:24.014031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470971406361230:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.014076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:24.011847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470973241786761:2124] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:24.032357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470971406361230:2049] [node 3] ICP22 created new session: [1:7577470975701328616:2049]
2025-11-27T18:17:24.012199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470973241786756:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:24.025453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470969462271416:2109] [node 6] ICP22 created new session: [4:7577470973757238735:2109]
2025-11-27T18:17:23.989399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470971543018845:2050] [node 3] ICP22 created new session: [5:7577470971543018931:2050]
2025-11-27T18:17:24.063487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470971406361293:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.088959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470970275723936:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(44063402ns) {HandleHandshakeStatus:329(44059957ns) {DropIncomingHandshake:442(1470ns) DropOutgoingHandshake:457(4537ns) SwitchToState:179(344ns) ProcessPendingSessionEvents:495(33503604ns) {ForwardSessionEventToSession:612(6741645ns) ForwardSessionEventToSession:612(5272782ns) ForwardSessionEventToSession:612(3698ns) ForwardSessionEventToSession:612(8666807ns) ForwardSessionEventToSession:612(9660ns) ForwardSessionEventToSession:612(4011ns) ForwardSessionEventToSession:612(4628904ns) ForwardSessionEventToSession:612(3985443ns) ForwardSessionEventToSession:612(8962ns) ForwardSessionEventToSession:612(4143272ns)}}}
2025-11-27T18:17:24.021528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470973241786749:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.025501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470973757238735:2109] [node 6] ICS09 handshake done sender: [4:7577470969462271422:2111] self: [4:7577470969462271420:0] peer: [6:7577470971542633569:0] socket: 196 qp: -1
2025-11-27T18:17:24.063555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:24.112094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470971542633489:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:24.021598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:24.112154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470971542633490:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:24.117720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470975837600874:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.117841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470971542633491:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:24.133444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470975837600869:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10034
2025-11-27T18:17:24.146132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470975837600880:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10034
2025-11-27T18:17:24.174076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470971542633568:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:24.174441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470975837600870:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:24.182287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470971542633489:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470975837600870:2120] held: no
2025-11-27T18:17:24.182403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470971542633489:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.182456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:24.182504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470971542633489:2048] [node 1] ICP22 created new session: [6:7577470975837600882:2048]
2025-11-27T18:17:24.182550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470975837600882:2048] [node 1] ICS09 handshake done sender: [6:7577470975837600870:2120] self: [6:7577470975837600875:0] peer: [1:7577470971406361301:0] socket: 238 qp: -1
2025-11-27T18:17:24.182588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470975837600882:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:24.182622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:24.182879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470971542633549:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.182908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:24.182953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470971542633549:2104] [node 5] ICP22 created new session: [6:7577470975837600884:2104]
2025-11-27T18:17:24.183005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470975837600884:2104] [node 5] ICS09 handshake done sender: [6:7577470971542633568:2116] self: [6:7577470971542633570:0] peer: [5:7577470971543018912:0] socket: 214 qp: -1
2025-11-27T18:17:24.183038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470975837600884:2104] [node 5] ICS10 traffic start
2025-11-27T18:17:24.183083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:24.183676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470975837600881:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10034
2025-11-27T18:17:24.186297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470975837600874:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:24.186611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470971542633491:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.186674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:24.186723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470971542633491:2050] [node 3] ICP22 created new session: [6:7577470975837600886:2050]
2025-11-27T18:17:24.186765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470975837600886:2050] [node 3] ICS09 handshake done sender: [6:7577470975837600874:2121] self: [6:7577470975837600877:0] peer: [3:7577470973241786753:0] socket: 245 qp: -1
2025-11-27T18:17:24.186794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470975837600886:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:24.186839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:24.194811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470975837600878:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10034
2025-11-27T18:17:24.098177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470970275723937:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(40576339ns) {HandleHandshakeStatus:329(40573269ns) {DropIncomingHandshake:442(17299ns) DropOutgoingHandshake:457(552ns) SwitchToState:179(236ns) ProcessPendingSessionEvents:495(4652281ns) {ForwardSessionEventToSession:612(4563916ns) ForwardSessionEventToSession:612(3949ns) ForwardSessionEventToSession:612(5548ns) ForwardSessionEventToSession:612(2722ns) ForwardSessionEventToSession:612(7280ns) ForwardSessionEventToSession:612(2992ns) ForwardSessionEventToSession:612(3669ns) ForwardSessionEventToSession:612(7556ns) ForwardSessionEventToSession:612(40925ns) ForwardSessionEventToSession:612(3647ns)}}}
2025-11-27T18:17:24.021676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470973241786749:2108] [node 5] ICP22 created new session: [3:7577470977536754072:2108]
2025-11-27T18:17:24.042791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470969462271354:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61877) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470969462271431:2116] held: no
2025-11-27T18:17:24.065040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470975701328616:2049] [node 3] ICS09 handshake done sender: [1:7577470971406361298:2111] self: [1:7577470971406361297:0] peer: [3:7577470973241786768:0] socket: 183 qp: -1
2025-11-27T18:17:24.204366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470971542633490:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24600) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470971542633572:2118] held: no
2025-11-27T18:17:24.069409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470975701328616:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:24.043046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470973757238734:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-11-27T18:17:23.989434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470971543018931:2050] [node 3] ICS09 handshake done sender: [5:7577470971543018922:2117] self: [5:7577470971543018926:0] peer: [3:7577470973241786759:0] socket: 216 qp: -1
2025-11-27T18:17:24.102836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470974570691331:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.021725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470977536754072:2108] [node 5] ICS09 handshake done sender: [3:7577470973241786760:2123] self: [3:7577470973241786759:0] peer: [5:7577470971543018926:0] socket: 206 qp: -1
2025-11-27T18:17:24.121207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470970275724011:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.076646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:23.989465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470971543018931:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:24.043930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470973757238738:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-11-27T18:17:24.021760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470977536754072:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:24.088654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470971406361293:2108] [node 5] ICP22 created new session: [1:7577470975701328617:2108]
2025-11-27T18:17:24.208599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470971542633572:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:23.989499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:24.021799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:24.209688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470971542633490:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.121244Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470970275724011:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:24.044393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470973757238739:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-11-27T18:17:23.989760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470971543018907:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:23.990260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470971543018917:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:24.121870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470970275724009:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.209741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:24.025991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470973241786748:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.010149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470971543018906:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.097693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470971406361229:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(99594836ns) {HandleHandshakeStatus:329(99592199ns) {DropIncomingHandshake:442(12111ns) DropOutgoingHandshake:457(534ns) SwitchToState:179(107ns) ProcessPendingSessionEvents:495(14085961ns) {ForwardSessionEventToSession:612(14015345ns) ForwardSessionEventToSession:612(5779ns) ForwardSessionEventToSession:612(3188ns) ForwardSessionEventToSession:612(12867ns) ForwardSessionEventToSession:612(18191ns) ForwardSessionEventToSession:612(3055ns) ForwardSessionEventToSession:612(7146ns) ForwardSessionEventToSession:612(1901ns) ForwardSessionEventToSession:612(4764ns) ForwardSessionEventToSession:612(2317ns)}}}
2025-11-27T18:17:24.044495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470969462271429:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:24.215999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470971542633490:2049] [node 2] ICP22 created new session: [6:7577470975837600888:2049]
2025-11-27T18:17:24.121889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470970275724009:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:24.045011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470969462271431:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:24.010217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:24.026030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:24.112811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470971406361306:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.010270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470971543018906:2109] [node 6] ICP22 created new session: [5:7577470975837986229:2109]
2025-11-27T18:17:24.129368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470970275724009:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:24.010335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470975837986229:2109] [node 6] ICS09 handshake done sender: [5:7577470971543018917:2114] self: [5:7577470971543018912:0] peer: [6:7577470971542633570:0] socket: 212 qp: -1
2025-11-27T18:17:24.010364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470975837986229:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:24.133340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470970275724011:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:24.010396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:24.137989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470970275724004:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.138055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:24.030394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470975837986231:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57456
2025-11-27T18:17:24.138107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470970275724004:2113] [node 6] ICP22 created new session: [2:7577470974570691336:2113]
2025-11-27T18:17:24.030531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470975837986232:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57456
2025-11-27T18:17:24.138153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470974570691336:2113] [node 6] ICS09 handshake done sender: [2:7577470970275724011:2115] self: [2:7577470970275724006:0] peer: [6:7577470975837600876:0] socket: 189 qp: -1
2025-11-27T18:17:24.052707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470975837986235:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57456
2025-11-27T18:17:24.138191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470974570691336:2113] [node 6] ICS10 traffic start
2025-11-27T18:17:24.061484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470971543018920:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:24.138227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:24.070109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470975837986231:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.141417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470970275724003:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.070155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470971543018843:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.157407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:24.157464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470970275724003:2111] [node 5] ICP22 created new session: [2:7577470974570691338:2111]
2025-11-27T18:17:24.070215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:24.157516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470974570691338:2111] [node 5] ICS09 handshake done sender: [2:7577470970275724009:2114] self: [2:7577470970275724007:0] peer: [5:7577470975837986238:0] socket: 233 qp: -1
2025-11-27T18:17:24.070268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470971543018843:2048] [node 1] ICP22 created new session: [5:7577470975837986236:2048]
2025-11-27T18:17:24.157555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470974570691338:2111] [node 5] ICS10 traffic start
2025-11-27T18:17:24.070313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470975837986236:2048] [node 1] ICS09 handshake done sender: [5:7577470971543018920:2116] self: [5:7577470971543018927:0] peer: [1:7577470971406361299:0] socket: 194 qp: -1
2025-11-27T18:17:24.157620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:24.070350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470975837986236:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:24.070403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:24.158096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470970275724003:2111] [node 5] ICP35 event processing took too much time PendingConnection:279(23702820ns) {HandleHandshakeStatus:329(23698629ns) {DropIncomingHandshake:442(32633ns) DropOutgoingHandshake:457(449ns) SwitchToState:179(715ns) ProcessPendingSessionEvents:495(110282ns) {ForwardSessionEventToSession:612(54785ns) ForwardSessionEventToSession:612(9564ns) ForwardSessionEventToSession:612(1932ns) ForwardSessionEventToSession:612(2639ns) ForwardSessionEventToSession:612(9635ns) ForwardSessionEventToSession:612(1971ns) ForwardSessionEventToSession:612(2925ns) ForwardSessionEventToSession:612(11960ns) ForwardSessionEventToSession:612(1866ns) ForwardSessionEventToSession:612(4364ns)}}}
2025-11-27T18:17:24.169355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470974570691340:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6240
2025-11-27T18:17:24.097812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470971543018845:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(64494054ns) {HandleHandshakeStatus:329(64491313ns) {DropIncomingHandshake:442(1068ns) DropOutgoingHandshake:457(32931ns) SwitchToState:179(184ns) ProcessPendingSessionEvents:495(64041239ns) {ForwardSessionEventToSession:612(16124984ns) ForwardSessionEventToSession:612(3217039ns) ForwardSessionEventToSession:612(7347ns) ForwardSessionEventToSession:612(3522ns) ForwardSessionEventToSession:612(2768069ns) ForwardSessionEventToSession:612(4095ns) ForwardSessionEventToSession:612(3648ns) ForwardSessionEventToSession:612(34670629ns) ForwardSessionEventToSession:612(14097ns) ForwardSessionEventToSession:612(7191712ns)}}}
2025-11-27T18:17:24.188721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470974570691340:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.118162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470971543018844:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:24.132347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470971543018923:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:24.188734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470970275724005:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(141484345ns) {HandleHandshakeStatus:329(141480440ns) {DropIncomingHandshake:442(13931ns) DropOutgoingHandshake:457(598ns) SwitchToState:179(567ns) ProcessPendingSessionEvents:495(32827629ns) {ForwardSessionEventToSession:612(20783641ns) ForwardSessionEventToSession:612(9929ns) ForwardSessionEventToSession:612(2706ns) ForwardSessionEventToSession:612(2510133ns) ForwardSessionEventToSession:612(4654ns) ForwardSessionEventToSession:612(3793ns) ForwardSessionEventToSession:612(1410057ns) ForwardSessionEventToSession:612(2972ns) ForwardSessionEventToSession:612(8067710ns) ForwardSessionEventToSession:612(8491ns)}}}
2025-11-27T18:17:24.193546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470970275724004:2113] [node 6] ICP18 handshake (actor [2:7577470974570691340:2124]) is too old
2025-11-27T18:17:24.134435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470971543018907:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.153379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:24.157088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470971543018907:2108] [node 4] ICP22 created new session: [5:7577470975837986240:2108]
2025-11-27T18:17:24.218564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470975837600888:2049] [node 2] ICS09 handshake done sender: [6:7577470971542633572:2118] self: [6:7577470975837600876:0] peer: [2:7577470970275724006:0] socket: 229 qp: -1
2025-11-27T18:17:24.161956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470971543018843:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(50550493ns) {HandleHandshakeStatus:329(50548147ns) {DropIncomingHandshake:442(966ns) DropOutgoingHandshake:457(4840ns) SwitchToState:179(682ns) ProcessPendingSessionEvents:495(43783003ns) {ForwardSessionEventToSession:612(69778ns) ForwardSessionEventToSession:612(11092301ns) ForwardSessionEventToSession:612(9611094ns) ForwardSessionEventToSession:612(12996ns) ForwardSessionEventToSession:612(11357321ns) ForwardSessionEventToSession:612(4079ns) ForwardSessionEventToSession:612(6790334ns) ForwardSessionEventToSession:612(4776ns) ForwardSessionEventToSession:612(11591ns) ForwardSessionEventToSession:612(4797800ns)}}}
2025-11-27T18:17:24.165771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470975837986240:2108] [node 4] ICS09 handshake done sender: [5:7577470971543018923:2118] self: [5:7577470971543018928:0] peer: [4:7577470969462271417:0] socket: 217 qp: -1
2025-11-27T18:17:24.218604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470975837600888:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:24.165830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470975837986240:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:24.220529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:24.165887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:24.166328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470971543018907:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(31719137ns) {HandleHandshakeStatus:329(31717038ns) {DropIncomingHandshake:442(1032ns) DropOutgoingHandshake:457(5765ns) SwitchToState:179(914ns) ProcessPendingSessionEvents:495(112338ns) {ForwardSessionEventToSession:612(60555ns) ForwardSessionEventToSession:612(5263ns) ForwardSessionEventToSession:612(2263ns) ForwardSessionEventToSession:612(3539ns) ForwardSessionEventToSession:612(4866ns) ForwardSessionEventToSession:612(5457ns) ForwardSessionEventToSession:612(7686ns) ForwardSessionEventToSession:612(2881ns) ForwardSessionEventToSession:612(2250ns) ForwardSessionEventToSession:612(8751ns)}}}
2025-11-27T18:17:24.170392Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470971543018844:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24600) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470975837986231:2120] held: no
2025-11-27T18:17:24.194226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470975837986239:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57456
2025-11-27T18:17:24.197482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470975837986231:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:24.204280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470971543018844:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.204327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:24.206358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470971543018844:2049] [node 2] ICP22 created new session: [5:7577470975837986242:2049]
2025-11-27T18:17:24.209296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470975837986242:2049] [node 2] ICS09 handshake done sender: [5:7577470975837986231:2120] self: [5:7577470975837986238:0] peer: [2:7577470970275724007:0] socket: 235 qp: -1
2025-11-27T18:17:24.212810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470975837986242:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:24.212876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:24.045236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470969462271353:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.045313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:24.045612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470973757238735:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:24.057563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470969462271421:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:24.061920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:24.064875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470969462271353:2049] [node 2] ICP22 created new session: [4:7577470973757238740:2049]
2025-11-27T18:17:24.073678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470969462271354:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.081408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470973757238740:2049] [node 2] ICS09 handshake done sender: [4:7577470969462271429:2115] self: [4:7577470969462271433:0] peer: [2:7577470970275724013:0] socket: 201 qp: -1
2025-11-27T18:17:24.081477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470973757238740:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:24.081525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:24.081751Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470969462271353:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(36460011ns) {HandleHandshakeStatus:329(36457460ns) {DropIncomingHandshake:442(1045ns) DropOutgoingHandshake:457(440ns) SwitchToState:179(726ns) ProcessPendingSessionEvents:495(1841ns)}}
2025-11-27T18:17:24.088570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470969462271415:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.097919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:24.108933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470969462271354:2050] [node 3] ICP22 created new session: [4:7577470973757238743:2050]
2025-11-27T18:17:24.119693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470973757238739:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.123018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470973757238743:2050] [node 3] ICS09 handshake done sender: [4:7577470969462271431:2116] self: [4:7577470969462271436:0] peer: [3:7577470973241786754:0] socket: 207 qp: -1
2025-11-27T18:17:24.123256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:24.132168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470973757238743:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:24.133411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470969462271415:2108] [node 5] ICP22 created new session: [4:7577470973757238745:2108]
2025-11-27T18:17:24.133746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:24.133952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470973757238745:2108] [node 5] ICS09 handshake done sender: [4:7577470969462271421:2110] self: [4:7577470969462271417:0] peer: [5:7577470971543018928:0] socket: 210 qp: -1
2025-11-27T18:17:24.133998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470973757238745:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:24.141367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:24.174328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470969462271354:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(92554702ns) {HandleHandshakeStatus:329(92551960ns) {DropIncomingHandshake:442(1268ns) DropOutgoingHandshake:457(384ns) SwitchToState:179(726ns) ProcessPendingSessionEvents:495(1545ns)}}
2025-11-27T18:17:24.177831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470973757238744:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-11-27T18:17:24.189829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470973757238744:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.194406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470969462271415:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(104965516ns) {HandleHandshakeStatus:329(104961283ns) {DropIncomingHandshake:442(772ns) DropOutgoingHandshake:457(606ns) SwitchToState:179(872ns) ProcessPendingSessionEvents:495(39611893ns) {ForwardSessionEventToSession:612(30297693ns) ForwardSessionEventToSession:612(6672ns) ForwardSessionEventToSession:612(2380ns) ForwardSessionEventToSession:612(4475834ns) ForwardSessionEventToSession:612(3144ns) ForwardSessionEventToSession:612(3219ns) ForwardSessionEventToSession:612(4774666ns) ForwardSessionEventToSession:612(14471ns) ForwardSessionEventToSession:612(5853ns) ForwardSessionEventToSession:612(3645ns)}}}
2025-11-27T18:17:24.194490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577470969462271415:2108] [node 5] ICP18 handshake (actor [4:7577470973757238739:2120]) is too old
2025-11-27T18:17:24.194618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470969462271416:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(127736296ns) {HandleHandshakeStatus:329(127732596ns) {DropIncomingHandshake:442(16498ns) DropOutgoingHandshake:457(1042ns) SwitchToState:179(662ns) ProcessPendingSessionEvents:495(31296412ns) {ForwardSessionEventToSession:612(20538936ns) ForwardSessionEventToSession:612(9217ns) ForwardSessionEventToSession:612(1618251ns) ForwardSessionEventToSession:612(4273ns) ForwardSessionEventToSession:612(2477ns) ForwardSessionEventToSession:612(1403644ns) ForwardSessionEventToSession:612(4256ns) ForwardSessionEventToSession:612(366439ns) ForwardSessionEventToSession:612(7320149ns) ForwardSessionEventToSession:612(7874ns)}}}
2025-11-27T18:17:24.194845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470969462271352:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:24.216888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470973757238749:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-11-27T18:17:24.220020Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470973757238744:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:24.222050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470969462271352:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.222707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:24.222756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470969462271352:2048] [node 1] ICP22 created new session: [4:7577470973757238750:2048]
2025-11-27T18:17:24.222799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470973757238750:2048] [node 1] ICS09 handshake done sender: [4:7577470973757238744:2121] self: [4:7577470973757238748:0] peer: [1:7577470971406361305:0] socket: 246 qp: -1
2025-11-27T18:17:24.222838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470973757238750:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:24.222873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:24.223123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470969462271352:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:24.112833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470971406361306:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:24.116066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470975701328617:2108] [node 5] ICS09 handshake done sender: [1:7577470971406361300:2112] self: [1:7577470971406361299:0] peer: [5:7577470971543018927:0] socket: 184 qp: -1
2025-11-27T18:17:24.116118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470975701328617:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:24.118851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470971406361230:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(104572302ns) {HandleHandshakeStatus:329(104570044ns) {DropIncomingHandshake:442(212ns) DropOutgoingHandshake:457(151ns) SwitchToState:179(353ns) ProcessPendingSessionEvents:495(9864138ns) {ForwardSessionEventToSession:612(9813813ns) ForwardSessionEventToSession:612(5527ns) ForwardSessionEventToSession:612(2747ns) ForwardSessionEventToSession:612(4799ns) ForwardSessionEventToSession:612(2439ns) ForwardSessionEventToSession:612(2611ns) ForwardSessionEventToSession:612(4841ns) ForwardSessionEventToSession:612(3438ns) ForwardSessionEventToSession:612(10542ns) ForwardSessionEventToSession:612(2114ns)}}}
2025-11-27T18:17:24.123324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:24.134301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470971406361306:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:24.145699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470971406361292:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.145768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:24.145821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470971406361292:2109] [node 6] ICP22 created new session: [1:7577470975701328623:2109]
2025-11-27T18:17:24.149488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470975701328623:2109] [node 6] ICS09 handshake done sender: [1:7577470971406361306:2113] self: [1:7577470971406361301:0] peer: [6:7577470975837600875:0] socket: 236 qp: -1
2025-11-27T18:17:24.155619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470975701328623:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:24.169332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:24.193704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470975701328619:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44073
2025-11-27T18:17:24.193749Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470971406361293:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(98453224ns) {HandleHandshakeStatus:329(98449718ns) {DropIncomingHandshake:442(16824ns) DropOutgoingHandshake:457(541ns) SwitchToState:179(802ns) ProcessPendingSessionEvents:495(20252805ns) {ForwardSessionEventToSession:612(12170465ns) ForwardSessionEventToSession:612(7785ns) ForwardSessionEventToSession:612(2349ns) ForwardSessionEventToSession:612(7560ns) ForwardSessionEventToSession:612(2931ns) ForwardSessionEventToSession:612(7938ns) ForwardSessionEventToSession:612(7475ns) ForwardSessionEventToSession:612(3500ns) ForwardSessionEventToSession:612(8012469ns) ForwardSessionEventToSession:612(10896ns)}}}
2025-11-27T18:17:24.194290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470971406361292:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(47912082ns) {HandleHandshakeStatus:329(47907863ns) {DropIncomingHandshake:442(16913ns) DropOutgoingHandshake:457(555ns) SwitchToState:179(732ns) ProcessPendingSessionEvents:495(9366292ns) {ForwardSessionEventToSession:612(9309075ns) ForwardSessionEventToSession:612(14512ns) ForwardSessionEventToSession:612(2408ns) ForwardSessionEventToSession:612(2353ns) ForwardSessionEventToSession:612(5068ns) ForwardSessionEventToSession:612(2386ns) ForwardSessionEventToSession:612(6850ns) ForwardSessionEventToSession:612(4808ns) ForwardSessionEventToSession:612(2687ns) ForwardSessionEventToSession:612(5494ns)}}}
2025-11-27T18:17:24.203408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470975701328619:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.204060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470971406361307:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.204100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470971406361307:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:24.212935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470971406361307:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:24.220578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470971406361294:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.222193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:24.222250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470971406361294:2107] [node 4] ICP22 created new session: [1:7577470975701328627:2107]
2025-11-27T18:17:24.223709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470975701328627:2107] [node 4] ICS09 handshake done sender: [1:7577470971406361307:2114] self: [1:7577470971406361305:0] peer: [4:7577470973757238748:0] socket: 202 qp: -1
2025-11-27T18:17:24.026070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470973241786748:2107] [node 4] ICP22 created new session: [3:7577470977536754075:2107]
2025-11-27T18:17:24.026108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470977536754075:2107] [node 4] ICS09 handshake done sender: [3:7577470973241786756:2120] self: [3:7577470973241786754:0] peer: [4:7577470969462271436:0] socket: 203 qp: -1
2025-11-27T18:17:24.026139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470977536754075:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:24.026170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:24.026773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470977536754074:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46577
2025-11-27T18:17:24.029398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470973241786676:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:24.031744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470977536754074:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.037924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577470973241786748:2107] [node 4] ICP18 handshake (actor [3:7577470977536754074:2129]) is too old
2025-11-27T18:17:24.069353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:24.088791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470973241786676:2048] [node 1] ICP22 created new session: [3:7577470977536754079:2048]
2025-11-27T18:17:24.117449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470977536754079:2048] [node 1] ICS09 handshake done sender: [3:7577470973241786761:2124] self: [3:7577470973241786768:0] peer: [1:7577470971406361297:0] socket: 185 qp: -1
2025-11-27T18:17:24.117492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470977536754079:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:24.118164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470973241786757:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.118189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470973241786757:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:24.128954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470977536754080:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46577
2025-11-27T18:17:24.129483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:24.158667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470977536754080:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:24.161798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470973241786757:2121] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:24.185350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470973241786750:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:24.185441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:24.185483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470973241786750:2109] [node 6] ICP22 created new session: [3:7577470977536754084:2109]
2025-11-27T18:17:24.185521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470977536754084:2109] [node 6] ICS09 handshake done sender: [3:7577470973241786757:2121] self: [3:7577470973241786753:0] peer: [6:7577470975837600877:0] socket: 243 qp: -1
2025-11-27T18:17:24.185552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470977536754084:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:24.185597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:24.193667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470973241786676:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(132943539ns) {HandleHandshakeStatus:329(132940904ns) {DropIncomingHandshake:442(1107ns) DropOutgoingHandshake:457(7407ns) SwitchToState:179(620ns) ProcessPendingSessionEvents:495(28383928ns) {ForwardSessionEventToSession:612(3936517ns) ForwardSessionEventToSession:612(5605ns) ForwardSessionEventToSession:612(3787865ns) ForwardSessionEventToSession:612(5156ns) ForwardSessionEventToSession:612(2790ns) ForwardSessionEventToSession:612(7844387ns) ForwardSessionEventToSession:612(4500ns) ForwardSessionEventToSession:612(4438848ns) ForwardSessionEventToSession:612(8330182ns) ForwardSessionEventToSession:612(2745ns)}}}
2025-11-27T18:17:24.227956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470975701328627:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:24.228034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:24.279115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:24.279356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470977536754079:2048] [node 1] ICS07 socket disconnect 185 reason# EndOfStream
2025-11-27T18:17:24.279383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470977536754079:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.279459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:24.279496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470977536754079:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:24.281407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:24.281883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470970275724027:2048] [node 1] ICS01 socket: 187 reason# EndOfStream
2025-11-27T18:17:24.282133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470970275723936:2048] [node 1] ICP30 unregister session Session# [2:7577470970275724027:2048] VirtualId# [2:7577470970275724022:0]
2025-11-27T18:17:24.282176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470970275724027:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.282267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:24.282321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:24.284629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470973241786676:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10668) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:24.284678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470977536754079:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:24.288032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470973241786676:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10668) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:24.288118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470977536754079:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:24.288176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470973241786676:2048] [node 1] ICP30 unregister session Session# [3:7577470977536754079:2048] VirtualId# [3:7577470973241786768:0]
2025-11-27T18:17:24.288243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:24.288295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470973241786676:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:24.288327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:24.291215Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:24.291453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470975837986236:2048] [node 1] ICS01 socket: 194 reason# EndOfStream
2025-11-27T18:17:24.291499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470971543018843:2048] [node 1] ICP30 unregister session Session# [5:7577470975837986236:2048] VirtualId# [5:7577470971543018927:0]
2025-11-27T18:17:24.291535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470975837986236:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.291609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:24.291691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:24.292143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:24.292341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470975837600882:2048] [node 1] ICS01 socket: 238 reason# EndOfStream
2025-11-27T18:17:24.292370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470971542633489:2048] [node 1] ICP30 unregister session Session# [6:7577470975837600882:2048] VirtualId# [6:7577470975837600875:0]
2025-11-27T18:17:24.292401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470975837600882:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.292478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:24.292514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:24.296373Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:24.296689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470973757238750:2048] [node 1] ICS07 socket disconnect 246 reason# EndOfStream
2025-11-27T18:17:24.299727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470973241786676:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:24.296718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470973757238750:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.296799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:24.296919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470973757238750:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:24.298003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470969462271352:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10668) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:24.298089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470973757238750:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:24.298117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470969462271352:2048] [node 1] ICP30 unregister session Session# [4:7577470973757238750:2048] VirtualId# [4:7577470973757238748:0]
2025-11-27T18:17:24.298164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:24.298193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470969462271352:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:24.298209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:24.309476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470969462271352:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:24.376650Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:24.376889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470973241786772:2049] [node 2] ICS01 socket: 199 reason# EndOfStream
2025-11-27T18:17:24.376942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470973241786677:2049] [node 2] ICP30 unregister session Session# [3:7577470973241786772:2049] VirtualId# [3:7577470973241786767:0]
2025-11-27T18:17:24.376978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470973241786772:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.377077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:24.377130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:24.379726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:24.379961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470973757238740:2049] [node 2] ICS01 socket: 201 reason# EndOfStream
2025-11-27T18:17:24.379995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470969462271353:2049] [node 2] ICP30 unregister session Session# [4:7577470973757238740:2049] VirtualId# [4:7577470969462271433:0]
2025-11-27T18:17:24.380027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470973757238740:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.380092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:24.380137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:24.380789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:24.380991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470975837986242:2049] [node 2] ICS01 socket: 235 reason# EndOfStream
2025-11-27T18:17:24.381037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470971543018844:2049] [node 2] ICP30 unregister session Session# [5:7577470975837986242:2049] VirtualId# [5:7577470975837986238:0]
2025-11-27T18:17:24.381068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470975837986242:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.381123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:24.381160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:24.393449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:24.397855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470975837600888:2049] [node 2] ICS01 socket: 229 reason# EndOfStream
2025-11-27T18:17:24.397913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470971542633490:2049] [node 2] ICP30 unregister session Session# [6:7577470975837600888:2049] VirtualId# [6:7577470975837600876:0]
2025-11-27T18:17:24.397955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470975837600888:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.398052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:24.398102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:24.425499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:24.425773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470971543018931:2050] [node 3] ICS07 socket disconnect 216 reason# EndOfStream
2025-11-27T18:17:24.425815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470971543018931:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.425904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:24.425947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470971543018931:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:24.425494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:24.426270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:24.426469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470975837600886:2050] [node 3] ICS07 socket disconnect 245 reason# EndOfStream
2025-11-27T18:17:24.426495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470975837600886:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.426562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:24.426599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470975837600886:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:24.429588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470973757238743:2050] [node 3] ICS01 socket: 207 reason# EndOfStream
2025-11-27T18:17:24.429643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470969462271354:2050] [node 3] ICP30 unregister session Session# [4:7577470973757238743:2050] VirtualId# [4:7577470969462271436:0]
2025-11-27T18:17:24.429828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470971543018845:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61877) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:24.429897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470971543018931:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:24.429675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470973757238743:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.429769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:24.429939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470971543018845:2050] [node 3] ICP30 unregister session Session# [5:7577470971543018931:2050] VirtualId# [5:7577470971543018926:0]
2025-11-27T18:17:24.429826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:24.429991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:24.430026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470971543018845:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61877) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:24.430040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61877) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:24.430252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470971542633491:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61877) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:24.430490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470975837600886:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:24.430533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470971542633491:2050] [node 3] ICP30 unregister session Session# [6:7577470975837600886:2050] VirtualId# [6:7577470975837600877:0]
2025-11-27T18:17:24.430572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:24.430597Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470971542633491:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61877) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:24.430626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61877) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:24.439374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470971543018845:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:24.440362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470971542633491:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:24.489846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:24.490101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470975837986240:2108] [node 4] ICS01 socket: 217 reason# EndOfStream
2025-11-27T18:17:24.490145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470971543018907:2108] [node 4] ICP30 unregister session Session# [5:7577470975837986240:2108] VirtualId# [5:7577470971543018928:0]
2025-11-27T18:17:24.490181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470975837986240:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.490255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:24.490298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:24.493412Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:24.493642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470975837600872:2109] [node 4] ICS01 socket: 197 reason# EndOfStream
2025-11-27T18:17:24.493684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470971542633553:2109] [node 4] ICP30 unregister session Session# [6:7577470975837600872:2109] VirtualId# [6:7577470971542633569:0]
2025-11-27T18:17:24.493717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470975837600872:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.493794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:24.493839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:24.554597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:24.554875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470975837600884:2104] [node 5] ICS01 socket: 214 reason# EndOfStream
2025-11-27T18:17:24.554910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470971542633549:2104] [node 5] ICP30 unregister session Session# [6:7577470975837600884:2104] VirtualId# [6:7577470971542633570:0]
2025-11-27T18:17:24.554949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470975837600884:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:24.555043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:24.555104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:24.647408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470976002658092:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:24.647445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470976002658093:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:24.670468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470974521339266:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:24.670523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470974521339267:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:24.693651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470976594747754:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:24.693714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470976594747755:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:24.721941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470975984123058:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:24.722038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470975984123059:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:24.722075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470975984123060:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:24.748227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470974172597833:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:24.748269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470974172597834:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:24.748339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470974172597835:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:24.769936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470973710201272:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:24.769993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470973710201273:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:24.770021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470973710201274:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:24.900566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470973710201334:2106] [node 5] ICP01 ready to work
2025-11-27T18:17:24.969178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470974521339335:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:25.012309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470978816306632:2110] [node 5] ICP01 ready to work
2025-11-27T18:17:25.012675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470978816306633:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:25.098712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470978005168635:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:25.102497Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470976002658092:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(106335677ns) {Configure:104(106329831ns) {ProcessConfigured:135(24851846ns) {StartInitialHandshake:157(24848567ns) {DropHandshakes:470(3292ns) {DropIncomingHandshake:442(762ns) DropOutgoingHandshake:457(380ns)} GenerateSessionVirtualId:988(826ns) PrepareNewSessionHandshake:477(5484ns) {SwitchToState:179(3095ns)}}}}}
2025-11-27T18:17:25.130831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470976594747754:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(161594367ns) {Configure:104(161579214ns) {ProcessConfigured:135(80420ns) {StartInitialHandshake:157(77910ns) {DropHandshakes:470(2993ns) {DropIncomingHandshake:442(713ns) DropOutgoingHandshake:457(559ns)} GenerateSessionVirtualId:988(990ns) PrepareNewSessionHandshake:477(1720ns) {SwitchToState:179(568ns)}}}}}
2025-11-27T18:17:25.131324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470980889715115:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:25.143486Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470976002658093:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40719113ns) {Configure:104(40712554ns) {ProcessConfigured:135(70536ns) {StartInitialHandshake:157(68780ns) {DropHandshakes:470(5991ns) {DropIncomingHandshake:442(731ns) DropOutgoingHandshake:457(393ns)} GenerateSessionVirtualId:988(999ns) PrepareNewSessionHandshake:477(1609ns) {SwitchToState:179(861ns)}}}}}
2025-11-27T18:17:25.144076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470980297625455:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:25.144442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470980297625456:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:25.145036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470980297625457:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:25.134093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470978467565193:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:25.131635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470980889715116:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:25.131905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470980889715117:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:25.149927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470975984123058:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(184108422ns) {Configure:104(184103164ns) {ProcessConfigured:135(63477ns) {StartInitialHandshake:157(61635ns) {DropHandshakes:470(2740ns) {DropIncomingHandshake:442(751ns) DropOutgoingHandshake:457(442ns)} GenerateSessionVirtualId:988(525ns) PrepareNewSessionHandshake:477(1085ns) {SwitchToState:179(585ns)}}}}}
2025-11-27T18:17:25.154673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470978816306633:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(50701947ns) {Configure:104(50693975ns) {ProcessConfigured:135(74464ns) {StartInitialHandshake:157(73031ns) {DropHandshakes:470(1247ns) {DropIncomingHandshake:442(386ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(665ns) PrepareNewSessionHandshake:477(1310ns) {SwitchToState:179(565ns)}}}}}
2025-11-27T18:17:25.134560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470978467565194:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:25.165691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470974172597834:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(30654262ns) {Configure:104(30647182ns) {ProcessConfigured:135(12000204ns) {StartInitialHandshake:157(11997272ns) {DropHandshakes:470(1364ns) {DropIncomingHandshake:442(381ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(407ns) PrepareNewSessionHandshake:477(3936ns) {SwitchToState:179(1904ns)}}}}}
2025-11-27T18:17:25.152695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470980279090424:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:25.152963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470980279090425:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.190364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470978467565192:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17762
2025-11-27T18:17:25.191659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470978005168635:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(73614376ns) {Configure:104(73608048ns) {ProcessConfigured:135(894750ns) {StartInitialHandshake:157(892481ns) {DropHandshakes:470(3018ns) {DropIncomingHandshake:442(710ns) DropOutgoingHandshake:457(371ns)} GenerateSessionVirtualId:988(1001ns) PrepareNewSessionHandshake:477(1719ns) {SwitchToState:179(709ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.194383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470980889715118:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65297
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.191041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470980279090424:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(21283776ns) {Configure:104(21280039ns) {ProcessConfigured:135(50480ns) {StartInitialHandshake:157(48122ns) {DropHandshakes:470(3352ns) {DropIncomingHandshake:442(265ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(175ns) PrepareNewSessionHandshake:477(2431ns) {SwitchToState:179(937ns)}}}}}
2025-11-27T18:17:25.200138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470980279090433:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24609
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.203430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470978816306641:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27439
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.204133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470978816306642:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27439
2025-11-27T18:17:25.206976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470978816306643:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27439
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.212630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470978467565192:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.217637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470980889715118:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.213338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470978005168644:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37953
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.230715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470974521339329:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.230752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470974521339329:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.231700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470978816306644:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27439
2025-11-27T18:17:25.218113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470980889715130:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65297
2025-11-27T18:17:25.213530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470978005168645:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37953
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.243249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470980297625458:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14347
2025-11-27T18:17:25.218201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470976594747755:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.244394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470980279090433:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.244650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470975984123060:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.234781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470974521339329:2108] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:25.229751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470978005168644:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.239166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470980889715132:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65297
2025-11-27T18:17:25.237072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470973710201273:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.247164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470978005168645:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.247301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470973710201334:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.248096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470978816306644:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.248188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470978816306642:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.248286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470978816306641:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.248591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470978816306635:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.248612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470978816306635:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.249377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470978816306643:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.250102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470978467565198:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.251294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470980297625459:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14347
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.251460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470980297625460:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14347
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.251554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470980297625461:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14347
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.251657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470980297625462:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14347
2025-11-27T18:17:25.251851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470980297625458:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.249418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470974521339267:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.252561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470980297625460:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.250131Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470978467565198:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.249918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470980889715130:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.250931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470980889715122:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.250950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470980889715122:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.249499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:25.253272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470978467565198:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:25.260350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470980889715118:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:25.258978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470980279090435:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24609
2025-11-27T18:17:25.249552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470974521339267:2049] [node 3] ICP22 created new session: [2:7577470978816306647:2049]
2025-11-27T18:17:25.249604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470978816306647:2049] [node 3] ICS09 handshake done sender: [2:7577470974521339329:2108] self: [2:7577470974521339328:0] peer: [3:7577470980889715131:0] socket: 187 qp: -1
2025-11-27T18:17:25.249632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470978816306647:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:25.265158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470976594747755:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.268587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470980297625462:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.268662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470980297625459:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.249866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:25.254913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470974521339267:2049] [node 3] ICP18 handshake (actor [2:7577470978816306641:2115]) is too old
2025-11-27T18:17:25.270259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470980297625461:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.266710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470978467565193:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.255002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470978816306646:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27439
2025-11-27T18:17:25.256136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470978816306635:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:25.256526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470978816306633:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.256602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:25.256643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470978816306633:2111] [node 6] ICP22 created new session: [2:7577470978816306654:2111]
2025-11-27T18:17:25.256696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470978816306654:2111] [node 6] ICS09 handshake done sender: [2:7577470978816306635:2112] self: [2:7577470978816306634:0] peer: [6:7577470978005168648:0] socket: 194 qp: -1
2025-11-27T18:17:25.256729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470978816306654:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:25.256759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:25.266763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:25.273413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470978816306646:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.274068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470974521339266:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.265315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:25.269563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470978467565193:2108] [node 6] ICP22 created new session: [5:7577470978467565211:2108]
2025-11-27T18:17:25.270470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470978467565211:2108] [node 6] ICS09 handshake done sender: [5:7577470978467565198:2112] self: [5:7577470978467565197:0] peer: [6:7577470978005168649:0] socket: 204 qp: -1
2025-11-27T18:17:25.270518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470978467565211:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:25.270567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:25.265360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470976594747755:2049] [node 2] ICP22 created new session: [3:7577470980889715135:2049]
2025-11-27T18:17:25.265942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470980889715135:2049] [node 2] ICS09 handshake done sender: [3:7577470980889715118:2111] self: [3:7577470980889715131:0] peer: [2:7577470974521339328:0] socket: 192 qp: -1
2025-11-27T18:17:25.269661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470980889715135:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:25.270252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:25.271762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577470976594747755:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.274740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470976002658156:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.272359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470980889715133:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65297
2025-11-27T18:17:25.274773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470976002658156:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.272603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470980889715122:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:25.275116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470980889715133:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.275161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470980889715115:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.275209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:25.275989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470980889715115:2107] [node 4] ICP22 created new session: [3:7577470980889715138:2107]
2025-11-27T18:17:25.276403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470980889715138:2107] [node 4] ICS09 handshake done sender: [3:7577470980889715122:2113] self: [3:7577470980889715121:0] peer: [4:7577470980279090434:0] socket: 197 qp: -1
2025-11-27T18:17:25.276575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470980279090436:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24609
2025-11-27T18:17:25.276614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470976002658156:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:25.276778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470980889715138:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.276840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:25.276898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470978467565210:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17762
2025-11-27T18:17:25.277873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470980279090435:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.278010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470980279090436:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.278310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470980279090437:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24609
2025-11-27T18:17:25.278365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470978816306658:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27439
2025-11-27T18:17:25.277165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470976002658092:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.277245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:25.277314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470976002658092:2048] [node 2] ICP22 created new session: [1:7577470980297625478:2048]
2025-11-27T18:17:25.277362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470980297625478:2048] [node 2] ICS09 handshake done sender: [1:7577470976002658156:2110] self: [1:7577470976002658155:0] peer: [2:7577470978816306657:0] socket: 223 qp: -1
2025-11-27T18:17:25.277396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470980297625478:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:25.277427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:25.278566Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470973710201273:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12139) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470978005168644:2115] held: no
2025-11-27T18:17:25.278933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470978467565210:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.279313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470974172597834:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.280491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470978816306646:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:25.279999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470975984123058:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.285954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470980297625464:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.285986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470980297625464:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.280059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470975984123059:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.287235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470980297625464:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:25.280105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470980279090433:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:25.288514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470980297625455:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.288572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:25.288622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470980297625455:2107] [node 4] ICP22 created new session: [1:7577470980297625482:2107]
2025-11-27T18:17:25.288663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470980297625482:2107] [node 4] ICS09 handshake done sender: [1:7577470980297625464:2117] self: [1:7577470980297625463:0] peer: [4:7577470980279090439:0] socket: 229 qp: -1
2025-11-27T18:17:25.288696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470980297625482:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:25.288723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:25.290450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470976594747754:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.280352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470980279090438:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24609
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.290671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470973710201334:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25157) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470978005168645:2116] held: no
2025-11-27T18:17:25.290949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470978005168650:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37953
2025-11-27T18:17:25.287366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470975984123060:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.288376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:25.289675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470975984123060:2050] [node 3] ICP22 created new session: [4:7577470980279090441:2050]
2025-11-27T18:17:25.291604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470980279090441:2050] [node 3] ICS09 handshake done sender: [4:7577470980279090433:2116] self: [4:7577470980279090434:0] peer: [3:7577470980889715121:0] socket: 199 qp: -1
2025-11-27T18:17:25.294898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470980279090441:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:25.295439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470980297625454:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.295462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470980297625454:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.295023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:25.295290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470978816306637:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.296936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470978005168651:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37953
2025-11-27T18:17:25.295334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470978816306637:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.299063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470978005168645:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.298121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470974521339266:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.298038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470978467565215:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17762
2025-11-27T18:17:25.299471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470978467565216:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17762
2025-11-27T18:17:25.300284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470978467565216:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.300433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470974172597835:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.299400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470980889715143:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65297
2025-11-27T18:17:25.300940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470980889715124:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.300961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470980889715124:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.300743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470980279090438:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.302823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470978005168644:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:25.301612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470980889715144:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65297
2025-11-27T18:17:25.301966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470980889715124:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:25.302656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470980297625454:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:25.300426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:25.300489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470974521339266:2048] [node 1] ICP22 created new session: [2:7577470978816306659:2048]
2025-11-27T18:17:25.300554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470978816306659:2048] [node 1] ICS09 handshake done sender: [2:7577470978816306646:2119] self: [2:7577470978816306657:0] peer: [1:7577470976002658155:0] socket: 225 qp: -1
2025-11-27T18:17:25.302525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470978467565215:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.302376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470980889715116:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.300589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470978816306659:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.305304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470974172597833:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.305398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470978467565218:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17762
2025-11-27T18:17:25.305379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470973710201334:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.305423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:25.305470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470973710201334:2106] [node 5] ICP22 created new session: [6:7577470978005168656:2106]
2025-11-27T18:17:25.305514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470978005168656:2106] [node 5] ICS09 handshake done sender: [6:7577470978005168645:2116] self: [6:7577470978005168649:0] peer: [5:7577470978467565197:0] socket: 210 qp: -1
2025-11-27T18:17:25.305549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470978005168656:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:25.305580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:25.300630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:25.302413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.302455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470980889715116:2108] [node 5] ICP22 created new session: [3:7577470980889715146:2108]
2025-11-27T18:17:25.305909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470973710201273:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.305280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577470974521339266:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:25.306160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:25.302497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470980889715146:2108] [node 5] ICS09 handshake done sender: [3:7577470980889715124:2114] self: [3:7577470980889715123:0] peer: [5:7577470978467565217:0] socket: 218 qp: -1
2025-11-27T18:17:25.306381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470973710201273:2049] [node 2] ICP22 created new session: [6:7577470978005168658:2049]
2025-11-27T18:17:25.305808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470978816306639:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.305830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470978816306639:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.317977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470978816306637:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:25.306425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470978005168658:2049] [node 2] ICS09 handshake done sender: [6:7577470978005168644:2115] self: [6:7577470978005168648:0] peer: [2:7577470978816306634:0] socket: 206 qp: -1
2025-11-27T18:17:25.303088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470980889715146:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:25.323236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470978816306639:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:25.306452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470978005168658:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:25.306499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:25.306951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470978005168654:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37953
2025-11-27T18:17:25.307039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470978005168655:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37953
2025-11-27T18:17:25.323509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470976002658093:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.323571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:25.323598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470978005168654:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.323610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470976002658093:2049] [node 3] ICP22 created new session: [1:7577470980297625486:2049]
2025-11-27T18:17:25.323649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470980297625486:2049] [node 3] ICS09 handshake done sender: [1:7577470980297625454:2111] self: [1:7577470980297625453:0] peer: [3:7577470980889715142:0] socket: 224 qp: -1
2025-11-27T18:17:25.303165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:25.323674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470980297625486:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:25.322875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470980889715133:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:25.324966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470978816306632:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.325025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:25.325071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470978816306632:2110] [node 5] ICP22 created new session: [2:7577470978816306663:2110]
2025-11-27T18:17:25.325108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470978816306663:2110] [node 5] ICS09 handshake done sender: [2:7577470978816306637:2113] self: [2:7577470978816306636:0] peer: [5:7577470978467565214:0] socket: 207 qp: -1
2025-11-27T18:17:25.325148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470978816306663:2110] [node 5] ICS10 traffic start
2025-11-27T18:17:25.325178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:25.325585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470974521339335:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.325625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:25.325656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470974521339335:2109] [node 4] ICP22 created new session: [2:7577470978816306665:2109]
2025-11-27T18:17:25.325696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470978816306665:2109] [node 4] ICS09 handshake done sender: [2:7577470978816306639:2114] self: [2:7577470978816306638:0] peer: [4:7577470980279090440:0] socket: 233 qp: -1
2025-11-27T18:17:25.325724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470978816306665:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:25.325755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:25.326809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470973710201274:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.323697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.326719Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470978467565216:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:25.330977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470980297625466:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.331006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470980297625466:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.327403Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470975984123058:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470980279090435:2117] held: no
2025-11-27T18:17:25.328014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470980279090443:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24609
2025-11-27T18:17:25.330447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470980889715144:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.336476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470980297625466:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:25.328323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470974172597835:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.328399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:25.328443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470974172597835:2050] [node 3] ICP22 created new session: [5:7577470978467565221:2050]
2025-11-27T18:17:25.328482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470978467565221:2050] [node 3] ICS09 handshake done sender: [5:7577470978467565216:2118] self: [5:7577470978467565217:0] peer: [3:7577470980889715123:0] socket: 214 qp: -1
2025-11-27T18:17:25.328512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470978467565221:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:25.328543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:25.329434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470978467565220:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17762
2025-11-27T18:17:25.329816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470980279090444:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24609
2025-11-27T18:17:25.330497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470976594747754:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.330215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470980279090445:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24609
2025-11-27T18:17:25.330531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:25.332010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470976594747754:2048] [node 1] ICP22 created new session: [3:7577470980889715148:2048]
2025-11-27T18:17:25.332064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470980889715148:2048] [node 1] ICS09 handshake done sender: [3:7577470980889715133:2118] self: [3:7577470980889715142:0] peer: [1:7577470980297625453:0] socket: 226 qp: -1
2025-11-27T18:17:25.337238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470980279090435:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:25.332095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470980889715148:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:25.332132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:25.332669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470980889715126:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.332688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470980889715126:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.337663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470980279090444:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.338773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470978467565210:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:25.339051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470980297625456:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.339117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:25.339161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470980297625456:2108] [node 5] ICP22 created new session: [1:7577470980297625489:2108]
2025-11-27T18:17:25.339207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470980297625489:2108] [node 5] ICS09 handshake done sender: [1:7577470980297625466:2118] self: [1:7577470980297625465:0] peer: [5:7577470978467565219:0] socket: 193 qp: -1
2025-11-27T18:17:25.339243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470980297625489:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:25.339275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:25.339950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470975984123058:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.340025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:25.340076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470975984123058:2048] [node 1] ICP22 created new session: [4:7577470980279090449:2048]
2025-11-27T18:17:25.340136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470980279090449:2048] [node 1] ICS09 handshake done sender: [4:7577470980279090435:2117] self: [4:7577470980279090439:0] peer: [1:7577470980297625463:0] socket: 231 qp: -1
2025-11-27T18:17:25.340172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470980279090449:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:25.340211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:25.340617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470980279090436:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.341836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470974172597834:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12139) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470978467565210:2116] held: no
2025-11-27T18:17:25.341931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470974172597834:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.341962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:25.341998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470974172597834:2049] [node 2] ICP22 created new session: [5:7577470978467565224:2049]
2025-11-27T18:17:25.342032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470978467565224:2049] [node 2] ICS09 handshake done sender: [5:7577470978467565210:2116] self: [5:7577470978467565214:0] peer: [2:7577470978816306636:0] socket: 235 qp: -1
2025-11-27T18:17:25.342057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470978467565224:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:25.342085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:25.342844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470978467565223:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17762
2025-11-27T18:17:25.342965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470978467565226:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17762
2025-11-27T18:17:25.344215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470975984123059:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12139) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470980279090436:2118] held: no
2025-11-27T18:17:25.344296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470975984123059:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.344348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:25.344383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470975984123059:2049] [node 2] ICP22 created new session: [4:7577470980279090451:2049]
2025-11-27T18:17:25.344424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470980279090451:2049] [node 2] ICS09 handshake done sender: [4:7577470980279090436:2118] self: [4:7577470980279090440:0] peer: [2:7577470978816306638:0] socket: 234 qp: -1
2025-11-27T18:17:25.344445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470980279090451:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:25.344467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:25.348265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470980889715126:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:25.348498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470980889715117:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.348557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:25.348608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470980889715117:2109] [node 6] ICP22 created new session: [3:7577470980889715152:2109]
2025-11-27T18:17:25.348657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470980889715152:2109] [node 6] ICS09 handshake done sender: [3:7577470980889715126:2115] self: [3:7577470980889715125:0] peer: [6:7577470978005168660:0] socket: 240 qp: -1
2025-11-27T18:17:25.348690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470980889715152:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:25.348737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:25.350859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470974172597833:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470978467565215:2117] held: no
2025-11-27T18:17:25.350999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470978467565215:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:25.351347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470974172597833:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.351387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:25.351428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470974172597833:2048] [node 1] ICP22 created new session: [5:7577470978467565227:2048]
2025-11-27T18:17:25.351466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470978467565227:2048] [node 1] ICS09 handshake done sender: [5:7577470978467565215:2117] self: [5:7577470978467565219:0] peer: [1:7577470980297625465:0] socket: 189 qp: -1
2025-11-27T18:17:25.351498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470978467565227:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:25.351522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:25.352718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470978467565223:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.352861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470978467565194:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.353199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470980279090427:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.353247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470980279090427:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.354381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470978467565230:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17762
2025-11-27T18:17:25.354622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470980279090427:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:25.355131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470978467565223:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:25.355344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470978467565194:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.355390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:25.355432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470978467565194:2109] [node 4] ICP22 created new session: [5:7577470978467565231:2109]
2025-11-27T18:17:25.355476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470978467565231:2109] [node 4] ICS09 handshake done sender: [5:7577470978467565223:2121] self: [5:7577470978467565229:0] peer: [4:7577470980279090426:0] socket: 209 qp: -1
2025-11-27T18:17:25.355509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470978467565231:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:25.355542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:25.357012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470980279090424:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.357063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:25.357105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470980279090424:2108] [node 5] ICP22 created new session: [4:7577470980279090455:2108]
2025-11-27T18:17:25.357150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470980279090455:2108] [node 5] ICS09 handshake done sender: [4:7577470980279090427:2114] self: [4:7577470980279090426:0] peer: [5:7577470978467565229:0] socket: 254 qp: -1
2025-11-27T18:17:25.357174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470980279090455:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:25.357223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:25.389741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470978005168655:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.391328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470973710201274:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470978005168654:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.391656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470978005168661:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37953
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.391827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470978005168662:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37953
2025-11-27T18:17:25.392517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470973710201272:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.392740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470978005168654:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:25.392954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470973710201274:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.393040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:25.393033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470980297625468:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.393092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470973710201274:2050] [node 3] ICP22 created new session: [6:7577470978005168664:2050]
2025-11-27T18:17:25.393079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470980297625468:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.393126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470978005168664:2050] [node 3] ICS09 handshake done sender: [6:7577470978005168654:2119] self: [6:7577470978005168660:0] peer: [3:7577470980889715125:0] socket: 243 qp: -1
2025-11-27T18:17:25.393152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470978005168664:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:25.393216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:25.394163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470978005168661:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.394354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470978005168635:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:25.394684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470980297625468:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.395992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470978005168667:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37953
2025-11-27T18:17:25.394714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470980279090429:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:25.395559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470980297625457:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.394746Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470980279090429:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:25.395617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:25.395661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470980297625457:2109] [node 6] ICP22 created new session: [1:7577470980297625492:2109]
2025-11-27T18:17:25.396188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470980297625492:2109] [node 6] ICS09 handshake done sender: [1:7577470980297625468:2119] self: [1:7577470980297625467:0] peer: [6:7577470978005168663:0] socket: 186 qp: -1
2025-11-27T18:17:25.396234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470980297625492:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:25.396894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470980279090429:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:25.397113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470980279090425:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:25.397143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:25.397168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470980279090425:2109] [node 6] ICP22 created new session: [4:7577470980279090458:2109]
2025-11-27T18:17:25.396273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:25.397199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470980279090458:2109] [node 6] ICS09 handshake done sender: [4:7577470980279090429:2115] self: [4:7577470980279090428:0] peer: [6:7577470978005168666:0] socket: 255 qp: -1
2025-11-27T18:17:25.397232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470980279090458:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:25.397271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:25.398694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470978005168655:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:25.400198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470973710201272:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.400253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:25.400299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470973710201272:2048] [node 1] ICP22 created new session: [6:7577470978005168669:2048]
2025-11-27T18:17:25.400341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470978005168669:2048] [node 1] ICS09 handshake done sender: [6:7577470978005168655:2120] self: [6:7577470978005168663:0] peer: [1:7577470980297625467:0] socket: 244 qp: -1
2025-11-27T18:17:25.400370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470978005168669:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:25.400389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:25.400568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577470973710201272:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:25.402201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470978005168668:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37953
2025-11-27T18:17:25.402632Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470978005168635:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8544) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470978005168661:2121] held: no
2025-11-27T18:17:25.402862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470978005168661:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:25.403057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470978005168635:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:25.403097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:25.403135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470978005168635:2110] [node 4] ICP22 created new session: [6:7577470978005168671:2110]
2025-11-27T18:17:25.403191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470978005168671:2110] [node 4] ICS09 handshake done sender: [6:7577470978005168661:2121] self: [6:7577470978005168666:0] peer: [4:7577470980279090428:0] socket: 208 qp: -1
2025-11-27T18:17:25.403225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470978005168671:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:25.403276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:25.416819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:25.417083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577470978816306659:2048] [node 1] ICS07 socket disconnect 225 reason# EndOfStream
2025-11-27T18:17:25.417116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470978816306659:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.417236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:25.417562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470978816306659:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:25.418014Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:25.418225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470980889715148:2048] [node 1] ICS01 socket: 226 reason# EndOfStream
2025-11-27T18:17:25.418267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470976594747754:2048] [node 1] ICP30 unregister session Session# [3:7577470980889715148:2048] VirtualId# [3:7577470980889715142:0]
2025-11-27T18:17:25.418302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470980889715148:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.418380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:25.418432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:25.422129Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:25.422364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470980279090449:2048] [node 1] ICS01 socket: 231 reason# EndOfStream
2025-11-27T18:17:25.422420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470975984123058:2048] [node 1] ICP30 unregister session Session# [4:7577470980279090449:2048] VirtualId# [4:7577470980279090439:0]
2025-11-27T18:17:25.422453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470980279090449:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.422534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:25.422588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:25.424419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:25.424631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470978005168669:2048] [node 1] ICS07 socket disconnect 244 reason# EndOfStream
2025-11-27T18:17:25.424672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470978005168669:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.422577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470974521339266:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2872) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:25.422618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577470978816306659:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:25.423536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470974521339266:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2872) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:25.423583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470978816306659:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:25.424747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:25.429418Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:25.448438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470978467565227:2048] [node 1] ICS01 socket: 189 reason# EndOfStream
2025-11-27T18:17:25.448507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470974172597833:2048] [node 1] ICP30 unregister session Session# [5:7577470978467565227:2048] VirtualId# [5:7577470978467565219:0]
2025-11-27T18:17:25.448566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470978467565227:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.423615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470974521339266:2048] [node 1] ICP30 unregister session Session# [2:7577470978816306659:2048] VirtualId# [2:7577470978816306657:0]
2025-11-27T18:17:25.424788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470978005168669:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:25.423657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:25.423685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577470974521339266:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2872) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:25.423728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2872) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:25.445294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577470974521339266:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:25.448674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:25.448748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:25.453482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470973710201272:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2872) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:25.453542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470978005168669:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:25.453574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470973710201272:2048] [node 1] ICP30 unregister session Session# [6:7577470978005168669:2048] VirtualId# [6:7577470978005168663:0]
2025-11-27T18:17:25.453631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:25.453659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470973710201272:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2872) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:25.453683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2872) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:25.463365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470973710201272:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:25.485505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:25.485798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470980889715135:2049] [node 2] ICS01 socket: 192 reason# EndOfStream
2025-11-27T18:17:25.485846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470976594747755:2049] [node 2] ICP30 unregister session Session# [3:7577470980889715135:2049] VirtualId# [3:7577470980889715131:0]
2025-11-27T18:17:25.485940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470980889715135:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.486045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:25.486092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:25.490263Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:25.490469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470980279090451:2049] [node 2] ICS01 socket: 234 reason# EndOfStream
2025-11-27T18:17:25.490498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470975984123059:2049] [node 2] ICP30 unregister session Session# [4:7577470980279090451:2049] VirtualId# [4:7577470980279090440:0]
2025-11-27T18:17:25.490522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470980279090451:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.490601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:25.490640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:25.491770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:25.491928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470978005168658:2049] [node 2] ICS01 socket: 206 reason# EndOfStream
2025-11-27T18:17:25.491953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470973710201273:2049] [node 2] ICP30 unregister session Session# [6:7577470978005168658:2049] VirtualId# [6:7577470978005168648:0]
2025-11-27T18:17:25.491976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470978005168658:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.492034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:25.492064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:25.489333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:25.489495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470978467565224:2049] [node 2] ICS01 socket: 235 reason# EndOfStream
2025-11-27T18:17:25.489534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470974172597834:2049] [node 2] ICP30 unregister session Session# [5:7577470978467565224:2049] VirtualId# [5:7577470978467565214:0]
2025-11-27T18:17:25.489559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470978467565224:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.489624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:25.489654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:25.528375Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:25.528998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470978005168664:2050] [node 3] ICS01 socket: 243 reason# EndOfStream
2025-11-27T18:17:25.529044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470973710201274:2050] [node 3] ICP30 unregister session Session# [6:7577470978005168664:2050] VirtualId# [6:7577470978005168660:0]
2025-11-27T18:17:25.529078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470978005168664:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.529145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:25.529181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:25.527533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:25.528512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:25.527796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470978467565221:2050] [node 3] ICS01 socket: 214 reason# EndOfStream
2025-11-27T18:17:25.529034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470980279090441:2050] [node 3] ICS01 socket: 199 reason# EndOfStream
2025-11-27T18:17:25.527840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470974172597835:2050] [node 3] ICP30 unregister session Session# [5:7577470978467565221:2050] VirtualId# [5:7577470978467565217:0]
2025-11-27T18:17:25.529075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470975984123060:2050] [node 3] ICP30 unregister session Session# [4:7577470980279090441:2050] VirtualId# [4:7577470980279090434:0]
2025-11-27T18:17:25.527878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470978467565221:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.529102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470980279090441:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.529179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:25.528003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:25.529241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:25.528049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:25.559474Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:25.559732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470978005168671:2110] [node 4] ICS01 socket: 208 reason# EndOfStream
2025-11-27T18:17:25.559773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470978005168635:2110] [node 4] ICP30 unregister session Session# [6:7577470978005168671:2110] VirtualId# [6:7577470978005168666:0]
2025-11-27T18:17:25.559812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470978005168671:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.559903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:25.559966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:25.561389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:25.561692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470978467565231:2109] [node 4] ICS01 socket: 209 reason# EndOfStream
2025-11-27T18:17:25.561806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470978467565194:2109] [node 4] ICP30 unregister session Session# [5:7577470978467565231:2109] VirtualId# [5:7577470978467565229:0]
2025-11-27T18:17:25.561871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470978467565231:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.561948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:25.562041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:25.601093Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:25.601340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470978005168656:2106] [node 5] ICS01 socket: 210 reason# EndOfStream
2025-11-27T18:17:25.601373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470973710201334:2106] [node 5] ICP30 unregister session Session# [6:7577470978005168656:2106] VirtualId# [6:7577470978005168649:0]
2025-11-27T18:17:25.601405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470978005168656:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:25.601479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:25.601519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:25.671684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470980792245459:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:25.671711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470980792245460:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:25.693177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470981568777256:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:25.693230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470981568777257:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:25.717515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470980544159926:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:25.717553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470980544159927:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:25.742273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470978713294101:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:25.742283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470978713294100:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:25.742316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470978713294102:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:25.765340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470978800737050:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:25.765384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470978800737052:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:25.771343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470978800737051:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:25.791207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470978003244116:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:25.791206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470978003244115:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:25.791241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470978003244117:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:26.052905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470982298211461:2099] [node 4] ICP01 ready to work
2025-11-27T18:17:26.091992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470985863744615:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:26.093435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470982298211469:2095] [node 5] ICP01 ready to work
2025-11-27T18:17:26.141587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470985863744616:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:26.234205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470985863744617:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:26.238231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470983095704409:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:26.255363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470984839127289:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:26.249919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470983095704410:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:26.277661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470982298211469:2095] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(168707357ns) {Configure:104(168701365ns) {ProcessConfigured:135(16848290ns) {StartInitialHandshake:157(16844484ns) {DropHandshakes:470(4234863ns) {DropIncomingHandshake:442(700ns) DropOutgoingHandshake:457(1828ns)} GenerateSessionVirtualId:988(2268ns) PrepareNewSessionHandshake:477(4227ns) {SwitchToState:179(1667ns)}}}}}
2025-11-27T18:17:26.281917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470978800737050:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28286124ns) {Configure:104(28279985ns) {ProcessConfigured:135(53592ns) {StartInitialHandshake:157(51792ns) {DropHandshakes:470(1363ns) {DropIncomingHandshake:442(361ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(493ns) PrepareNewSessionHandshake:477(1092ns) {SwitchToState:179(563ns)}}}}}
2025-11-27T18:17:26.278750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470984839127290:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:26.286435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470981568777257:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(221255969ns) {Configure:104(218193782ns) {ProcessConfigured:135(51853ns) {StartInitialHandshake:157(49150ns) {DropHandshakes:470(913ns) {DropIncomingHandshake:442(281ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(420ns) PrepareNewSessionHandshake:477(1464ns) {SwitchToState:179(684ns)}}}}}
2025-11-27T18:17:26.279497Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470982298211461:2099] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(209395387ns) {Configure:104(209387799ns) {ProcessConfigured:135(24496850ns) {StartInitialHandshake:157(24494099ns) {DropHandshakes:470(2389ns) {DropIncomingHandshake:442(510ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(1254ns) PrepareNewSessionHandshake:477(2908ns) {SwitchToState:179(1063ns)}}}}}
2025-11-27T18:17:26.279548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470984839127291:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:26.290033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470980792245459:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(227765648ns) {Configure:104(227758033ns) {ProcessConfigured:135(1215621ns) {StartInitialHandshake:157(1211680ns) {DropHandshakes:470(3036ns) {DropIncomingHandshake:442(578ns) DropOutgoingHandshake:457(623ns)} GenerateSessionVirtualId:988(774ns) PrepareNewSessionHandshake:477(4931ns) {SwitchToState:179(3863ns)}}}}}
2025-11-27T18:17:26.287522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470980544159927:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(187020459ns) {Configure:104(187015222ns) {ProcessConfigured:135(45754ns) {StartInitialHandshake:157(44640ns) {DropHandshakes:470(826ns) {DropIncomingHandshake:442(224ns) DropOutgoingHandshake:457(184ns)} GenerateSessionVirtualId:988(582ns) PrepareNewSessionHandshake:477(837ns) {SwitchToState:179(276ns)}}}}}
2025-11-27T18:17:26.295838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470980792245460:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(240296000ns) {Configure:104(240292924ns) {ProcessConfigured:135(10300724ns) {StartInitialHandshake:157(10298246ns) {DropHandshakes:470(1359ns) {DropIncomingHandshake:442(394ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(462ns) PrepareNewSessionHandshake:477(339577ns) {SwitchToState:179(2830ns)}}}}}
2025-11-27T18:17:26.296490Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470978713294100:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(170981077ns) {Configure:104(170965738ns) {ProcessConfigured:135(3995341ns) {StartInitialHandshake:157(3993674ns) {DropHandshakes:470(992703ns) {DropIncomingHandshake:442(1023ns) DropOutgoingHandshake:457(133ns)} GenerateSessionVirtualId:988(1509ns) PrepareNewSessionHandshake:477(2364ns) {SwitchToState:179(1154ns)}}}}}
2025-11-27T18:17:26.296043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470985087212822:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:26.301438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470985087212823:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:26.292444Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470985863744615:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29647528ns) {Configure:104(29641931ns) {ProcessConfigured:135(36332ns) {StartInitialHandshake:157(34613ns) {DropHandshakes:470(955ns) {DropIncomingHandshake:442(218ns) DropOutgoingHandshake:457(169ns)} GenerateSessionVirtualId:988(308ns) PrepareNewSessionHandshake:477(574ns) {SwitchToState:179(171ns)}}}}}
2025-11-27T18:17:26.326068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470983095704410:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(41419989ns) {Configure:104(41416225ns) {ProcessConfigured:135(12047022ns) {StartInitialHandshake:157(12044449ns) {DropHandshakes:470(3135ns) {DropIncomingHandshake:442(681ns) DropOutgoingHandshake:457(558ns)} GenerateSessionVirtualId:988(965ns) PrepareNewSessionHandshake:477(3999ns) {SwitchToState:179(1950ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.323543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470978713294101:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(164055644ns) {Configure:104(164051186ns) {ProcessConfigured:135(5245702ns) {StartInitialHandshake:157(5244532ns) {DropHandshakes:470(2048158ns) {DropIncomingHandshake:442(212ns) DropOutgoingHandshake:457(1246ns)} GenerateSessionVirtualId:988(791ns) PrepareNewSessionHandshake:477(2160ns) {SwitchToState:179(876ns)}}}}}
2025-11-27T18:17:26.337862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470985087212824:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:26.326395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470983008261464:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:26.350697Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470978003244117:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(60957744ns) {Configure:104(60953865ns) {ProcessConfigured:135(9421159ns) {StartInitialHandshake:157(9418811ns) {DropHandshakes:470(2552ns) {DropIncomingHandshake:442(570ns) DropOutgoingHandshake:457(314ns)} GenerateSessionVirtualId:988(1160ns) PrepareNewSessionHandshake:477(3699ns) {SwitchToState:179(1920ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.329250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470983008261465:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-27T18:17:26.354040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470978003244115:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(57284243ns) {Configure:104(57281228ns) {ProcessConfigured:135(36666ns) {StartInitialHandshake:157(34929ns) {DropHandshakes:470(527ns) {DropIncomingHandshake:442(131ns) DropOutgoingHandshake:457(48ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:477(1395ns) {SwitchToState:179(428ns)}}}}}
2025-11-27T18:17:26.351335Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470985863744616:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(63941556ns) {Configure:104(63938010ns) {ProcessConfigured:135(55761ns) {StartInitialHandshake:157(53673ns) {DropHandshakes:470(1863ns) {DropIncomingHandshake:442(349ns) DropOutgoingHandshake:457(299ns)} GenerateSessionVirtualId:988(947ns) PrepareNewSessionHandshake:477(1249ns) {SwitchToState:179(486ns)}}}}}
2025-11-27T18:17:26.353128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470985863744631:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56899
2025-11-27T18:17:26.333561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470983008261472:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.370933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470978003244116:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20048905ns) {Configure:104(20045010ns) {ProcessConfigured:135(59206ns) {StartInitialHandshake:157(57305ns) {DropHandshakes:470(2624ns) {DropIncomingHandshake:442(579ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(699ns) PrepareNewSessionHandshake:477(3823ns) {SwitchToState:179(338ns)}}}}}
2025-11-27T18:17:26.366162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470984839127291:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(74011516ns) {Configure:104(74007660ns) {ProcessConfigured:135(11966261ns) {StartInitialHandshake:157(11963091ns) {DropHandshakes:470(684ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(182ns)} GenerateSessionVirtualId:988(574ns) PrepareNewSessionHandshake:477(3667ns) {SwitchToState:179(877ns)}}}}}
2025-11-27T18:17:26.371523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470985863744631:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.366995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470984839127299:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64837
No RDMA context found for GID: 2025-11-27T18:17:26.371717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470983008261465:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:17:26.367107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470984839127300:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64837
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.380576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470983008261476:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-27T18:17:26.380708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470983095704409:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(92393728ns) {Configure:104(91896466ns) {ProcessConfigured:135(17808086ns) {StartInitialHandshake:157(17805983ns) {DropHandshakes:470(4063205ns) {DropIncomingHandshake:442(500ns) DropOutgoingHandshake:457(1745ns)} GenerateSessionVirtualId:988(1143ns) PrepareNewSessionHandshake:477(2969ns) {SwitchToState:179(1555ns)}}}}}
2025-11-27T18:17:26.380679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470985087212825:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27415
2025-11-27T18:17:26.367275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470984839127305:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64837
2025-11-27T18:17:26.381117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470978713294101:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.381660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470985087212826:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27415
2025-11-27T18:17:26.388433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470985087212823:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(37764055ns) {Configure:104(37756628ns) {ProcessConfigured:135(57137ns) {StartInitialHandshake:157(53671ns) {DropHandshakes:470(1888ns) {DropIncomingHandshake:442(426ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(882ns) PrepareNewSessionHandshake:477(1656ns) {SwitchToState:179(454ns)}}}}}
2025-11-27T18:17:26.393555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470985863744623:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.368736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470984839127305:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.393607Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470985863744623:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.394492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470983008261476:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.392397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470984839127300:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.395905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470983095704421:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62330
2025-11-27T18:17:26.396089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470983095704422:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62330
2025-11-27T18:17:26.391740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470985087212826:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.394125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470984839127299:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.397376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470983008261477:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.398408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470983008261477:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.398588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470978713294102:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.397987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470980544159927:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.398001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470985863744633:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56899
2025-11-27T18:17:26.397810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470983095704423:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62330
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.399888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470985863744619:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.398177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470983095704422:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.398314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470985087212837:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27415
2025-11-27T18:17:26.399464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470984839127306:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64837
2025-11-27T18:17:26.399183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470983095704421:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.400255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470978800737051:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.399449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470985087212825:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.400490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470983095704423:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.399656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470984839127296:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.402578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470978800737050:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.399678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470984839127296:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.399914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470985863744619:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.401810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470984839127306:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.402923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470985087212821:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.401246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470985863744633:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.402955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470985087212821:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.403914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470985087212828:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.403934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470985087212828:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.402459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470985863744634:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56899
2025-11-27T18:17:26.402026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470980544159926:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.402874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470985863744621:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.407873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470983008261481:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-27T18:17:26.402222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470984839127296:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:26.408120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470985087212821:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:26.402892Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470985863744621:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.408242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470985087212837:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.402679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470984839127308:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64837
2025-11-27T18:17:26.404210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470985863744635:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56899
2025-11-27T18:17:26.409702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470985087212828:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:26.404969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470984839127289:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.406143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:26.407501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470984839127289:2108] [node 4] ICP22 created new session: [3:7577470984839127314:2108]
2025-11-27T18:17:26.408568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470984839127314:2108] [node 4] ICS09 handshake done sender: [3:7577470984839127296:2114] self: [3:7577470984839127295:0] peer: [4:7577470983008261480:0] socket: 213 qp: -1
2025-11-27T18:17:26.408604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470984839127314:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:26.408643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.412354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470984839127316:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64837
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.414025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470983095704430:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62330
2025-11-27T18:17:26.405042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470985863744623:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:26.412847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470984839127306:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:26.407604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470985863744634:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.413272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470985087212820:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.414360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470978713294102:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17917) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470983008261477:2118] held: no
2025-11-27T18:17:26.419536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470980544159926:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.436287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470983008261482:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-27T18:17:26.408400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470981568777256:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.440474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470983008261465:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:26.413301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470985087212820:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.421434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470983095704421:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.449365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470978713294101:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.409495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470985863744619:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:26.449420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:26.419588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.434683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470983095704432:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62330
2025-11-27T18:17:26.449465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470978713294101:2049] [node 2] ICP22 created new session: [4:7577470983008261484:2049]
2025-11-27T18:17:26.416302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470980792245460:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.416381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:26.416426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470980792245460:2049] [node 3] ICP22 created new session: [1:7577470985087212842:2049]
2025-11-27T18:17:26.416466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470985087212842:2049] [node 3] ICS09 handshake done sender: [1:7577470985087212821:2111] self: [1:7577470985087212819:0] peer: [3:7577470984839127312:0] socket: 221 qp: -1
2025-11-27T18:17:26.416497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470985087212842:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:26.416528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:26.416949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470985087212824:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.417010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:26.417049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470985087212824:2108] [node 5] ICP22 created new session: [1:7577470985087212844:2108]
2025-11-27T18:17:26.417081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470985087212844:2108] [node 5] ICS09 handshake done sender: [1:7577470985087212828:2114] self: [1:7577470985087212827:0] peer: [5:7577470983095704429:0] socket: 222 qp: -1
2025-11-27T18:17:26.417114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470985087212844:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:26.417155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:26.434496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470985087212820:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:26.410873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470985863744635:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.419635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470980544159926:2048] [node 1] ICP22 created new session: [3:7577470984839127318:2048]
2025-11-27T18:17:26.438729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470978800737051:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.419681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470984839127318:2048] [node 1] ICS09 handshake done sender: [3:7577470984839127306:2120] self: [3:7577470984839127312:0] peer: [1:7577470985087212819:0] socket: 225 qp: -1
2025-11-27T18:17:26.448407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470980792245459:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.449505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470983008261484:2049] [node 2] ICS09 handshake done sender: [4:7577470983008261465:2113] self: [4:7577470983008261478:0] peer: [2:7577470985863744622:0] socket: 197 qp: -1
2025-11-27T18:17:26.413011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470985863744621:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.438802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:26.449536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470983008261484:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:26.448450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:26.448511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470980792245459:2048] [node 2] ICP22 created new session: [1:7577470985087212850:2048]
2025-11-27T18:17:26.448552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470985087212850:2048] [node 2] ICS09 handshake done sender: [1:7577470985087212820:2110] self: [1:7577470985087212818:0] peer: [2:7577470985863744640:0] socket: 220 qp: -1
2025-11-27T18:17:26.448582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470985087212850:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:26.448626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:26.449049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470985087212849:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27415
2025-11-27T18:17:26.466922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470985087212830:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.466950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470985087212830:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.419710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470984839127318:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:26.430224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470985863744617:2109] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.438841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470978800737051:2049] [node 2] ICP22 created new session: [5:7577470983095704433:2049]
2025-11-27T18:17:26.449570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:26.420163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.430301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:26.482544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470985087212849:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.426198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470984839127308:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.438884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470983095704433:2049] [node 2] ICS09 handshake done sender: [5:7577470983095704421:2115] self: [5:7577470983095704428:0] peer: [2:7577470985863744620:0] socket: 216 qp: -1
2025-11-27T18:17:26.449785Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470978713294101:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:26.487740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470980792245459:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(19865428ns) {HandleHandshakeStatus:329(19863071ns) {DropIncomingHandshake:442(659ns) DropOutgoingHandshake:457(655ns) SwitchToState:179(803ns) ProcessPendingSessionEvents:495(7759375ns) {ForwardSessionEventToSession:612(1782747ns) ForwardSessionEventToSession:612(163964ns) ForwardSessionEventToSession:612(1610ns) ForwardSessionEventToSession:612(1553304ns) ForwardSessionEventToSession:612(3664ns) ForwardSessionEventToSession:612(2740ns) ForwardSessionEventToSession:612(254052ns) ForwardSessionEventToSession:612(3955ns) ForwardSessionEventToSession:612(2877525ns) ForwardSessionEventToSession:612(1095516ns)}}}
2025-11-27T18:17:26.473742Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470980544159927:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17374) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470984839127299:2116] held: no
2025-11-27T18:17:26.430362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470985863744617:2109] [node 4] ICP22 created new session: [2:7577470985863744641:2109]
2025-11-27T18:17:26.438916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470983095704433:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:26.459075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470983008261483:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.460310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470983008261477:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:26.430426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470985863744641:2109] [node 4] ICS09 handshake done sender: [2:7577470985863744623:2112] self: [2:7577470985863744622:0] peer: [4:7577470983008261478:0] socket: 195 qp: -1
2025-11-27T18:17:26.438969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:26.488029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470985087212830:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:26.486196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470984839127313:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64837
2025-11-27T18:17:26.463891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470983008261483:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.488604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470980792245459:2048] [node 2] ICP18 handshake (actor [1:7577470985087212849:2118]) is too old
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-27T18:17:26.430466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470985863744641:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:26.493796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470982298211490:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39474
2025-11-27T18:17:26.439558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470978800737050:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470983095704423:2117] held: no
2025-11-27T18:17:26.464291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470978713294102:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.490832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470984839127299:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:26.430501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:26.496069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470982298211491:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39474
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.496700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470984839127298:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.446826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470983095704431:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62330
2025-11-27T18:17:26.464333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:26.496739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470984839127298:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.430910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470981568777257:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.466515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470978713294100:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.466608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470983008261489:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-27T18:17:26.430950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:26.467182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470978713294102:2050] [node 3] ICP22 created new session: [4:7577470983008261488:2050]
2025-11-27T18:17:26.496697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470985087212822:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.460167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470978800737051:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:26.496968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470980544159927:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.496196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470982298211489:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39474
2025-11-27T18:17:26.497376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:26.460395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470983095704423:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:26.430991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470981568777257:2049] [node 3] ICP22 created new session: [2:7577470985863744643:2049]
2025-11-27T18:17:26.476903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470983008261488:2050] [node 3] ICS09 handshake done sender: [4:7577470983008261477:2118] self: [4:7577470983008261480:0] peer: [3:7577470984839127295:0] socket: 217 qp: -1
2025-11-27T18:17:26.497036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:26.464616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470983095704432:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.431033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470985863744643:2049] [node 3] ICS09 handshake done sender: [2:7577470985863744619:2110] self: [2:7577470985863744618:0] peer: [3:7577470984839127310:0] socket: 193 qp: -1
2025-11-27T18:17:26.464646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470978800737050:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.467238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:26.431060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470985863744643:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:26.431090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:26.473577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470983095704437:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62330
2025-11-27T18:17:26.431412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470985863744615:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.476958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470978800737050:2048] [node 1] ICP22 created new session: [5:7577470983095704438:2048]
2025-11-27T18:17:26.431436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:26.481862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470983095704438:2048] [node 1] ICS09 handshake done sender: [5:7577470983095704423:2117] self: [5:7577470983095704429:0] peer: [1:7577470985087212827:0] socket: 226 qp: -1
2025-11-27T18:17:26.431470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470985863744615:2107] [node 5] ICP22 created new session: [2:7577470985863744645:2107]
2025-11-27T18:17:26.431522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470985863744645:2107] [node 5] ICS09 handshake done sender: [2:7577470985863744621:2111] self: [2:7577470985863744620:0] peer: [5:7577470983095704428:0] socket: 194 qp: -1
2025-11-27T18:17:26.485370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470983095704438:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:26.431556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470985863744645:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:26.488128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:26.431586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:26.490566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470978800737050:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25169222ns) {HandleHandshakeStatus:329(25167575ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(152ns) SwitchToState:179(2070ns) ProcessPendingSessionEvents:495(789ns)}}
2025-11-27T18:17:26.434181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470985863744615:2107] [node 5] ICP18 handshake (actor [2:7577470985863744635:2118]) is too old
2025-11-27T18:17:26.491764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470978800737052:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.438436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470985863744649:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56899
2025-11-27T18:17:26.492608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470983095704437:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.458784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470985863744634:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:26.481855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470983008261488:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:26.494961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470983095704410:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.467413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470981568777256:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.467448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:26.482801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:26.487188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470983008261492:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10565
2025-11-27T18:17:26.467488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470981568777256:2048] [node 1] ICP22 created new session: [2:7577470985863744650:2048]
2025-11-27T18:17:26.490123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470983008261483:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:26.467524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470985863744650:2048] [node 1] ICS09 handshake done sender: [2:7577470985863744634:2117] self: [2:7577470985863744640:0] peer: [1:7577470985087212818:0] socket: 224 qp: -1
2025-11-27T18:17:26.490250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470978713294102:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(24635544ns) {HandleHandshakeStatus:329(24632820ns) {DropIncomingHandshake:442(1132ns) DropOutgoingHandshake:457(360ns) SwitchToState:179(439ns) ProcessPendingSessionEvents:495(675ns)}}
2025-11-27T18:17:26.467582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470985863744650:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:26.493830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470978713294100:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.467612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:26.495044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:26.495605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470978713294100:2048] [node 1] ICP22 created new session: [4:7577470983008261493:2048]
2025-11-27T18:17:26.497079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470985087212822:2107] [node 4] ICP22 created new session: [1:7577470985087212853:2107]
2025-11-27T18:17:26.496851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470983008261493:2048] [node 1] ICS09 handshake done sender: [4:7577470983008261483:2121] self: [4:7577470983008261490:0] peer: [1:7577470985087212829:0] socket: 227 qp: -1
2025-11-27T18:17:26.498312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470985087212853:2107] [node 4] ICS09 handshake done sender: [1:7577470985087212830:2115] self: [1:7577470985087212829:0] peer: [4:7577470983008261490:0] socket: 229 qp: -1
2025-11-27T18:17:26.497084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470983008261493:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:26.497719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:26.498660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470985087212853:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:26.498847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470983008261489:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.499050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:26.500483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470983095704420:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.496745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470982298211492:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39474
2025-11-27T18:17:26.497039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470982298211490:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.500514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470983095704420:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.497588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470982298211492:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.497416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470980544159927:2049] [node 2] ICP22 created new session: [3:7577470984839127322:2049]
2025-11-27T18:17:26.501095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470985087212835:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.501128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470985087212835:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.501950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470983008261475:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.498606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470978003244116:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.505141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470985087212835:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:26.501985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470983008261475:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.499921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470982298211469:2095] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.505113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470983008261475:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:26.499103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470984839127322:2049] [node 2] ICS09 handshake done sender: [3:7577470984839127299:2116] self: [3:7577470984839127310:0] peer: [2:7577470985863744618:0] socket: 202 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.500321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470982298211489:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.505784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470985863744630:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.499637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470984839127322:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.509440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470985087212823:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.509681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470983008261472:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.500425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470978003244115:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.505824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470985863744630:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.499670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:26.510450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:26.510423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:26.505070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470982298211491:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.509954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470983095704442:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62330
2025-11-27T18:17:26.505113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470984839127298:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:26.510285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470985863744630:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:26.510744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470985087212823:2109] [node 6] ICP22 created new session: [1:7577470985087212856:2109]
2025-11-27T18:17:26.506537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470982298211461:2099] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.510665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470984839127290:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.510725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470983008261472:2108] [node 5] ICP22 created new session: [4:7577470983008261497:2108]
2025-11-27T18:17:26.512104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470983095704432:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:26.511512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470985863744616:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.511246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470983008261497:2108] [node 5] ICS09 handshake done sender: [4:7577470983008261475:2116] self: [4:7577470983008261474:0] peer: [5:7577470983095704441:0] socket: 249 qp: -1
2025-11-27T18:17:26.511238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470985087212856:2109] [node 6] ICS09 handshake done sender: [1:7577470985087212835:2116] self: [1:7577470985087212834:0] peer: [6:7577470982298211496:0] socket: 247 qp: -1
2025-11-27T18:17:26.511909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:26.506883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470982298211497:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39474
2025-11-27T18:17:26.511277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470985087212856:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:26.510669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470982298211493:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39474
2025-11-27T18:17:26.512194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470983095704420:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:26.511263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470983008261469:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.512438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:26.512250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470984839127290:2109] [node 5] ICP22 created new session: [3:7577470984839127325:2109]
2025-11-27T18:17:26.511291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470983008261469:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.511312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:26.512495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470985863744616:2108] [node 6] ICP22 created new session: [2:7577470985863744653:2108]
2025-11-27T18:17:26.513564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470978800737052:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.512292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470982298211489:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:26.514257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470984839127325:2109] [node 5] ICS09 handshake done sender: [3:7577470984839127298:2115] self: [3:7577470984839127297:0] peer: [5:7577470983095704440:0] socket: 245 qp: -1
2025-11-27T18:17:26.513599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:26.515490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470985863744653:2108] [node 6] ICS09 handshake done sender: [2:7577470985863744630:2114] self: [2:7577470985863744629:0] peer: [6:7577470982298211494:0] socket: 200 qp: -1
2025-11-27T18:17:26.514566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470984839127325:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:26.515363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:26.511553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470983008261497:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:26.512132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:26.516408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470983008261469:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:26.517013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470982298211493:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.513641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470978800737052:2050] [node 3] ICP22 created new session: [5:7577470983095704445:2050]
2025-11-27T18:17:26.515799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470985863744653:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:26.514731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470983095704445:2050] [node 3] ICS09 handshake done sender: [5:7577470983095704432:2120] self: [5:7577470983095704440:0] peer: [3:7577470984839127297:0] socket: 246 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.517365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470978003244115:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.515072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470983095704445:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:26.515845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:26.517418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:26.515116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:26.517666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470978003244115:2048] [node 1] ICP22 created new session: [6:7577470982298211501:2048]
2025-11-27T18:17:26.519970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470983008261464:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.516665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470978800737052:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:26.517056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470983095704409:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.517588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:26.517729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470983095704409:2109] [node 6] ICP22 created new session: [5:7577470983095704447:2109]
2025-11-27T18:17:26.517833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470983095704447:2109] [node 6] ICS09 handshake done sender: [5:7577470983095704420:2114] self: [5:7577470983095704419:0] peer: [6:7577470982298211495:0] socket: 253 qp: -1
2025-11-27T18:17:26.518576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470982298211501:2048] [node 1] ICS09 handshake done sender: [6:7577470982298211489:2115] self: [6:7577470982298211496:0] peer: [1:7577470985087212834:0] socket: 215 qp: -1
2025-11-27T18:17:26.520039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:26.518801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470982298211501:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:26.520658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470983008261464:2109] [node 6] ICP22 created new session: [4:7577470983008261500:2109]
2025-11-27T18:17:26.518832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:26.521073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470983008261500:2109] [node 6] ICS09 handshake done sender: [4:7577470983008261469:2115] self: [4:7577470983008261468:0] peer: [6:7577470982298211498:0] socket: 228 qp: -1
2025-11-27T18:17:26.521275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470983008261500:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:26.521331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:26.518045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470983095704444:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62330
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.518627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470983095704447:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:26.518660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:26.522632Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470983095704410:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17705) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470983095704437:2121] held: no
2025-11-27T18:17:26.522919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470982298211499:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39474
2025-11-27T18:17:26.523261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470983095704437:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.523478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470983095704410:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.523554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:26.523653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470982298211500:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39474
2025-11-27T18:17:26.523671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470983095704410:2108] [node 4] ICP22 created new session: [5:7577470983095704449:2108]
2025-11-27T18:17:26.523724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470983095704449:2108] [node 4] ICS09 handshake done sender: [5:7577470983095704437:2121] self: [5:7577470983095704441:0] peer: [4:7577470983008261474:0] socket: 252 qp: -1
2025-11-27T18:17:26.523774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470983095704449:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:26.523914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:26.525836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470978003244117:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:26.525957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470982298211469:2095] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31475) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470982298211492:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.526077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470982298211502:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39474
2025-11-27T18:17:26.526080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470982298211461:2099] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17705) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470982298211491:2117] held: no
2025-11-27T18:17:26.526170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470982298211490:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:26.526987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470982298211492:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:26.527675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470982298211469:2095] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.527722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:26.527763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470982298211469:2095] [node 5] ICP22 created new session: [6:7577470982298211506:2095]
2025-11-27T18:17:26.527807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470982298211506:2095] [node 5] ICS09 handshake done sender: [6:7577470982298211492:2118] self: [6:7577470982298211495:0] peer: [5:7577470983095704419:0] socket: 254 qp: -1
2025-11-27T18:17:26.527848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470982298211506:2095] [node 5] ICS10 traffic start
2025-11-27T18:17:26.528054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:26.527421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470984839127302:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:26.527458Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470984839127302:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:26.528341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470984839127302:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:26.528525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470984839127291:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:26.528575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:26.528614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470984839127291:2110] [node 6] ICP22 created new session: [3:7577470984839127328:2110]
2025-11-27T18:17:26.528676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470984839127328:2110] [node 6] ICS09 handshake done sender: [3:7577470984839127302:2118] self: [3:7577470984839127301:0] peer: [6:7577470982298211504:0] socket: 192 qp: -1
2025-11-27T18:17:26.528706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470984839127328:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:26.528734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:26.528535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470982298211491:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:26.531626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470978003244116:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.531668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:26.531704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470978003244116:2049] [node 2] ICP22 created new session: [6:7577470982298211508:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.532264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470985863744655:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56899
2025-11-27T18:17:26.531969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470982298211508:2049] [node 2] ICS09 handshake done sender: [6:7577470982298211490:2116] self: [6:7577470982298211494:0] peer: [2:7577470985863744629:0] socket: 199 qp: -1
2025-11-27T18:17:26.532013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470982298211508:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:26.532045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:26.532607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470982298211461:2099] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:26.532641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:26.532683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470982298211461:2099] [node 4] ICP22 created new session: [6:7577470982298211511:2099]
2025-11-27T18:17:26.532726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470982298211511:2099] [node 4] ICS09 handshake done sender: [6:7577470982298211491:2117] self: [6:7577470982298211498:0] peer: [4:7577470983008261468:0] socket: 250 qp: -1
2025-11-27T18:17:26.532750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470982298211511:2099] [node 4] ICS10 traffic start
2025-11-27T18:17:26.532791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:26.533871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470978003244117:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17917) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470982298211493:2119] held: no
2025-11-27T18:17:26.535668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470982298211510:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39474
2025-11-27T18:17:26.536686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470982298211493:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:26.536933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470978003244117:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:26.537005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:26.537054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470978003244117:2050] [node 3] ICP22 created new session: [6:7577470982298211513:2050]
2025-11-27T18:17:26.537111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470982298211513:2050] [node 3] ICS09 handshake done sender: [6:7577470982298211493:2119] self: [6:7577470982298211504:0] peer: [3:7577470984839127301:0] socket: 257 qp: -1
2025-11-27T18:17:26.537146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470982298211513:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:26.537190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:26.567039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:26.567273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470985863744650:2048] [node 1] ICS01 socket: 224 reason# EndOfStream
2025-11-27T18:17:26.567315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470981568777256:2048] [node 1] ICP30 unregister session Session# [2:7577470985863744650:2048] VirtualId# [2:7577470985863744640:0]
2025-11-27T18:17:26.567348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470985863744650:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.567420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:26.567459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:26.568445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:26.568625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470984839127318:2048] [node 1] ICS01 socket: 225 reason# EndOfStream
2025-11-27T18:17:26.568671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470980544159926:2048] [node 1] ICP30 unregister session Session# [3:7577470984839127318:2048] VirtualId# [3:7577470984839127312:0]
2025-11-27T18:17:26.568712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470984839127318:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.568769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:26.568815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:26.574619Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:26.574827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470983008261493:2048] [node 1] ICS01 socket: 227 reason# EndOfStream
2025-11-27T18:17:26.574863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470978713294100:2048] [node 1] ICP30 unregister session Session# [4:7577470983008261493:2048] VirtualId# [4:7577470983008261490:0]
2025-11-27T18:17:26.574899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470983008261493:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.574984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:26.575038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:26.576753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:26.576945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470983095704438:2048] [node 1] ICS01 socket: 226 reason# EndOfStream
2025-11-27T18:17:26.576982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470978800737050:2048] [node 1] ICP30 unregister session Session# [5:7577470983095704438:2048] VirtualId# [5:7577470983095704429:0]
2025-11-27T18:17:26.577008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470983095704438:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.577072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:26.577108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:26.578347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:26.578598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470982298211501:2048] [node 1] ICS07 socket disconnect 215 reason# EndOfStream
2025-11-27T18:17:26.578625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470982298211501:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.578691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:26.578728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470982298211501:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:26.579660Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470978003244115:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5995) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:26.579779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470982298211501:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:26.579823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470978003244115:2048] [node 1] ICP30 unregister session Session# [6:7577470982298211501:2048] VirtualId# [6:7577470982298211496:0]
2025-11-27T18:17:26.579873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:26.579927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470978003244115:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:26.579946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:26.591332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470978003244115:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:26.616169Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:26.616401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470984839127322:2049] [node 2] ICS01 socket: 202 reason# EndOfStream
2025-11-27T18:17:26.616440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470980544159927:2049] [node 2] ICP30 unregister session Session# [3:7577470984839127322:2049] VirtualId# [3:7577470984839127310:0]
2025-11-27T18:17:26.616470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470984839127322:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.616539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:26.616575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:26.618751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:26.618974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470983095704433:2049] [node 2] ICS01 socket: 216 reason# EndOfStream
2025-11-27T18:17:26.619031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470978800737051:2049] [node 2] ICP30 unregister session Session# [5:7577470983095704433:2049] VirtualId# [5:7577470983095704428:0]
2025-11-27T18:17:26.619064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470983095704433:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.619142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:26.619183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:26.624488Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:26.624740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470983008261484:2049] [node 2] ICS01 socket: 197 reason# EndOfStream
2025-11-27T18:17:26.624778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470978713294101:2049] [node 2] ICP30 unregister session Session# [4:7577470983008261484:2049] VirtualId# [4:7577470983008261478:0]
2025-11-27T18:17:26.624836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470983008261484:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.624903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:26.624957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:26.625492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:26.625728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470982298211508:2049] [node 2] ICS01 socket: 199 reason# EndOfStream
2025-11-27T18:17:26.625764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470978003244116:2049] [node 2] ICP30 unregister session Session# [6:7577470982298211508:2049] VirtualId# [6:7577470982298211494:0]
2025-11-27T18:17:26.625794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470982298211508:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.625870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:26.625918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:26.660334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:26.660574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470983008261488:2050] [node 3] ICS01 socket: 217 reason# EndOfStream
2025-11-27T18:17:26.660620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470978713294102:2050] [node 3] ICP30 unregister session Session# [4:7577470983008261488:2050] VirtualId# [4:7577470983008261480:0]
2025-11-27T18:17:26.660656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470983008261488:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.660730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:26.660792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:26.661248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:26.661464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470983095704445:2050] [node 3] ICS01 socket: 246 reason# EndOfStream
2025-11-27T18:17:26.661495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470978800737052:2050] [node 3] ICP30 unregister session Session# [5:7577470983095704445:2050] VirtualId# [5:7577470983095704440:0]
2025-11-27T18:17:26.661522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470983095704445:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.661584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:26.661620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:26.665577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:26.665996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470982298211513:2050] [node 3] ICS01 socket: 257 reason# EndOfStream
2025-11-27T18:17:26.666120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470978003244117:2050] [node 3] ICP30 unregister session Session# [6:7577470982298211513:2050] VirtualId# [6:7577470982298211504:0]
2025-11-27T18:17:26.666193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470982298211513:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.666301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:26.666344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:26.696496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:26.696729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470983095704449:2108] [node 4] ICS01 socket: 252 reason# EndOfStream
2025-11-27T18:17:26.696764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470983095704410:2108] [node 4] ICP30 unregister session Session# [5:7577470983095704449:2108] VirtualId# [5:7577470983095704441:0]
2025-11-27T18:17:26.696795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470983095704449:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.696882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:26.696928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:26.699712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:26.700113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470982298211511:2099] [node 4] ICS01 socket: 250 reason# EndOfStream
2025-11-27T18:17:26.700212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470982298211461:2099] [node 4] ICP30 unregister session Session# [6:7577470982298211511:2099] VirtualId# [6:7577470982298211498:0]
2025-11-27T18:17:26.700282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470982298211511:2099] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.700455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:26.700555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:26.745094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:26.745402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470982298211506:2095] [node 5] ICS01 socket: 254 reason# EndOfStream
2025-11-27T18:17:26.745457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470982298211469:2095] [node 5] ICP30 unregister session Session# [6:7577470982298211506:2095] VirtualId# [6:7577470982298211495:0]
2025-11-27T18:17:26.745499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470982298211506:2095] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:26.745580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:26.745643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:26.828384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470983727687388:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:26.831854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470983727687387:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:26.852883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470983774417441:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:26.852926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470983774417442:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:26.877782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470985049937163:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:26.877820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470985049937164:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:26.901651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470985255871557:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:26.901692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470985255871558:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:26.902044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470985255871556:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:26.925916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470983923043796:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:26.925974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470983923043798:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:26.927004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470983923043797:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:26.950751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470984234694499:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:26.950772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470984234694500:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:26.950793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470984234694501:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:27.100987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470988529661856:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:27.089763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470988069384802:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:27.173726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470988022654746:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:27.089969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470988069384803:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:27.189716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470983774417441:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(99511567ns) {Configure:104(99506226ns) {ProcessConfigured:135(15471558ns) {StartInitialHandshake:157(15468642ns) {DropHandshakes:470(2936ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(370ns)} GenerateSessionVirtualId:988(1184ns) PrepareNewSessionHandshake:477(3292ns) {SwitchToState:179(1560ns)}}}}}
2025-11-27T18:17:27.190656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470989550838916:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:27.174199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470989344904549:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:27.201360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470989344904550:2116] [node 5] ICP01 ready to work
2025-11-27T18:17:27.198566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470988218011154:2106] [node 4] ICP01 ready to work
2025-11-27T18:17:27.193948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470989550838917:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:27.197699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470988529661858:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:27.209538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470988069384807:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:27.210884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470988022654748:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:27.208725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470988218011155:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:27.213883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470989344904555:2121] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.213490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470988022654747:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:27.225220Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470988069384803:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22093460ns) {Configure:104(22089181ns) {ProcessConfigured:135(20731887ns) {StartInitialHandshake:157(20730506ns) {DropHandshakes:470(2714ns) {DropIncomingHandshake:442(482ns) DropOutgoingHandshake:457(542ns)} GenerateSessionVirtualId:988(766ns) PrepareNewSessionHandshake:477(2020ns) {SwitchToState:179(948ns)}}}}}
2025-11-27T18:17:27.236824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470988069384811:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6210
2025-11-27T18:17:27.242839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470988069384802:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(54306051ns) {Configure:104(54298482ns) {ProcessConfigured:135(63534ns) {StartInitialHandshake:157(61069ns) {DropHandshakes:470(971ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(155ns)} GenerateSessionVirtualId:988(649ns) PrepareNewSessionHandshake:477(2064ns) {SwitchToState:179(824ns)}}}}}
2025-11-27T18:17:27.275897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470988529661856:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(174602958ns) {Configure:104(174598061ns) {ProcessConfigured:135(62441ns) {StartInitialHandshake:157(59691ns) {DropHandshakes:470(1716ns) {DropIncomingHandshake:442(340ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(742ns) PrepareNewSessionHandshake:477(2111ns) {SwitchToState:179(865ns)}}}}}
2025-11-27T18:17:27.262656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470983923043798:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(55250616ns) {Configure:104(55243547ns) {ProcessConfigured:135(59441ns) {StartInitialHandshake:157(57378ns) {DropHandshakes:470(1177ns) {DropIncomingHandshake:442(332ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(549ns) PrepareNewSessionHandshake:477(1506ns) {SwitchToState:179(444ns)}}}}}
2025-11-27T18:17:27.281461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470985255871556:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(163814663ns) {Configure:104(163810144ns) {ProcessConfigured:135(14935164ns) {StartInitialHandshake:157(14932120ns) {DropHandshakes:470(3394ns) {DropIncomingHandshake:442(975ns) DropOutgoingHandshake:457(722ns)} GenerateSessionVirtualId:988(576ns) PrepareNewSessionHandshake:477(3469ns) {SwitchToState:179(1644ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.290193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470983727687388:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(67999794ns) {Configure:104(67994855ns) {ProcessConfigured:135(55863ns) {StartInitialHandshake:157(53345ns) {DropHandshakes:470(3438ns) {DropIncomingHandshake:442(586ns) DropOutgoingHandshake:457(620ns)} GenerateSessionVirtualId:988(838ns) PrepareNewSessionHandshake:477(1165ns) {SwitchToState:179(459ns)}}}}}
2025-11-27T18:17:27.294285Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470989344904550:2116] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(71552222ns) {Configure:104(71549305ns) {ProcessConfigured:135(16469105ns) {StartInitialHandshake:157(16466502ns) {DropHandshakes:470(2809ns) {DropIncomingHandshake:442(519ns) DropOutgoingHandshake:457(445ns)} GenerateSessionVirtualId:988(847ns) PrepareNewSessionHandshake:477(3142ns) {SwitchToState:179(1842ns)}}}}}
2025-11-27T18:17:27.288316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470985255871557:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(90109444ns) {Configure:104(90105766ns) {ProcessConfigured:135(24032068ns) {StartInitialHandshake:157(24029206ns) {DropHandshakes:470(2697ns) {DropIncomingHandshake:442(572ns) DropOutgoingHandshake:457(373ns)} GenerateSessionVirtualId:988(716ns) PrepareNewSessionHandshake:477(3768ns) {SwitchToState:179(1664ns)}}}}}
2025-11-27T18:17:27.304086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470984234694500:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(192765117ns) {Configure:104(192760365ns) {ProcessConfigured:135(2324829ns) {StartInitialHandshake:157(2322051ns) {DropHandshakes:470(1487ns) {DropIncomingHandshake:442(508ns) DropOutgoingHandshake:457(299ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(2155ns) {SwitchToState:179(1065ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.305093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470988529661867:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63551
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.305307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470988529661868:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63551
2025-11-27T18:17:27.303538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470983774417442:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(76959701ns) {Configure:104(76953408ns) {ProcessConfigured:135(56323ns) {StartInitialHandshake:157(54174ns) {DropHandshakes:470(1261ns) {DropIncomingHandshake:442(332ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(604ns) PrepareNewSessionHandshake:477(899ns) {SwitchToState:179(301ns)}}}}}
2025-11-27T18:17:27.297620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470983727687387:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(228199154ns) {Configure:104(228194305ns) {ProcessConfigured:135(12167750ns) {StartInitialHandshake:157(12164476ns) {DropHandshakes:470(2074ns) {DropIncomingHandshake:442(504ns) DropOutgoingHandshake:457(291ns)} GenerateSessionVirtualId:988(1052ns) PrepareNewSessionHandshake:477(3277ns) {SwitchToState:179(2057ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.298203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470988022654757:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34348
2025-11-27T18:17:27.309965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470989550838916:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(95387910ns) {Configure:104(95382269ns) {ProcessConfigured:135(562920ns) {StartInitialHandshake:157(559692ns) {DropHandshakes:470(1602ns) {DropIncomingHandshake:442(412ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(365ns) PrepareNewSessionHandshake:477(4067ns) {SwitchToState:179(2126ns)}}}}}
2025-11-27T18:17:27.313684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470988069384819:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6210
2025-11-27T18:17:27.309419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470988022654757:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.314612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470983923043797:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(118378574ns) {Configure:104(118370086ns) {ProcessConfigured:135(1790477ns) {StartInitialHandshake:157(1787952ns) {DropHandshakes:470(2675ns) {DropIncomingHandshake:442(543ns) DropOutgoingHandshake:457(391ns)} GenerateSessionVirtualId:988(457ns) PrepareNewSessionHandshake:477(1594ns) {SwitchToState:179(741ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.313927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470984234694499:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(108338085ns) {Configure:104(108332551ns) {ProcessConfigured:135(8454199ns) {StartInitialHandshake:157(8451793ns) {DropHandshakes:470(1238ns) {DropIncomingHandshake:442(306ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(630ns) PrepareNewSessionHandshake:477(3246ns) {SwitchToState:179(1233ns)}}}}}
2025-11-27T18:17:27.312031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470989550838922:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36652
2025-11-27T18:17:27.337228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470988218011155:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(34428522ns) {Configure:104(33927112ns) {ProcessConfigured:135(46687ns) {StartInitialHandshake:157(44801ns) {DropHandshakes:470(4849ns) {DropIncomingHandshake:442(588ns) DropOutgoingHandshake:457(325ns)} GenerateSessionVirtualId:988(635ns) PrepareNewSessionHandshake:477(689ns) {SwitchToState:179(229ns)}}}}}
2025-11-27T18:17:27.321583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470988529661867:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.337427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470985049937163:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(127674804ns) {Configure:104(127667977ns) {ProcessConfigured:135(19406947ns) {StartInitialHandshake:157(19403625ns) {DropHandshakes:470(955ns) {DropIncomingHandshake:442(233ns) DropOutgoingHandshake:457(208ns)} GenerateSessionVirtualId:988(349ns) PrepareNewSessionHandshake:477(6836500ns) {SwitchToState:179(5005ns)}}}}}
2025-11-27T18:17:27.338022Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470988022654746:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(96161192ns) {Configure:104(96154399ns) {ProcessConfigured:135(5106606ns) {StartInitialHandshake:157(5103494ns) {DropHandshakes:470(1000ns) {DropIncomingHandshake:442(345ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(341ns) PrepareNewSessionHandshake:477(3069ns) {SwitchToState:179(1354ns)}}}}}
2025-11-27T18:17:27.337423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470988069384820:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6210
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.339545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470988529661858:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(55422353ns) {Configure:104(55401683ns) {ProcessConfigured:135(16083211ns) {StartInitialHandshake:157(16081930ns) {DropHandshakes:470(1285ns) {DropIncomingHandshake:442(386ns) DropOutgoingHandshake:457(296ns)} GenerateSessionVirtualId:988(729ns) PrepareNewSessionHandshake:477(1670ns) {SwitchToState:179(484ns)}}}}}
2025-11-27T18:17:27.312192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470989550838923:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36652
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.341104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470989344904566:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-27T18:17:27.340484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470983923043796:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(107771369ns) {Configure:104(107760464ns) {ProcessConfigured:135(15139946ns) {StartInitialHandshake:157(15137421ns) {DropHandshakes:470(1415ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(175ns)} GenerateSessionVirtualId:988(395ns) PrepareNewSessionHandshake:477(2566ns) {SwitchToState:179(1305ns)}}}}}
2025-11-27T18:17:27.339903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470988529661868:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.337662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470988069384811:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.346234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470984234694501:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.340095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470988069384819:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.316547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470989550838923:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.341846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470988218011166:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20341
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.346792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470989344904554:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.340230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470988069384810:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.340258Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470988069384810:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.340714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470988069384820:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.332135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470989550838922:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.354309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470988022654750:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.354357Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470988022654750:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.341972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470988218011167:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20341
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.346794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470989344904563:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-27T18:17:27.346295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470983774417441:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.347005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470988218011166:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.332232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470985255871557:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.346827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470989344904554:2138] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.361575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470989344904563:2141] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.358599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470988069384810:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.339884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470985255871558:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(44481966ns) {Configure:104(44476706ns) {ProcessConfigured:135(38594ns) {StartInitialHandshake:157(37129ns) {DropHandshakes:470(1383ns) {DropIncomingHandshake:442(214ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:477(399ns) {SwitchToState:179(119ns)}}}}}
2025-11-27T18:17:27.358527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470988529661875:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63551
2025-11-27T18:17:27.369516Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470988218011154:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36326740ns) {Configure:104(36322049ns) {ProcessConfigured:135(5582498ns) {StartInitialHandshake:157(5579603ns) {DropHandshakes:470(2153ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(451ns)} GenerateSessionVirtualId:988(662ns) PrepareNewSessionHandshake:477(1842ns) {SwitchToState:179(626ns)}}}}}
2025-11-27T18:17:27.361416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470984234694500:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.369100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470988529661876:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63551
2025-11-27T18:17:27.377462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470988529661876:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.378684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470988529661875:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.370970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470988069384821:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6210
2025-11-27T18:17:27.369942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470988218011167:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.339989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470985255871558:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.389156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470988069384807:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.389268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:27.389313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470988069384807:2107] [node 4] ICP22 created new session: [2:7577470988069384827:2107]
2025-11-27T18:17:27.389357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470988069384827:2107] [node 4] ICS09 handshake done sender: [2:7577470988069384810:2114] self: [2:7577470988069384809:0] peer: [4:7577470989550838931:0] socket: 204 qp: -1
2025-11-27T18:17:27.389386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470988069384827:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:27.389430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:27.390111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470988069384821:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.371297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470989344904557:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.374843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470989550838935:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36652
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.391604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470988069384808:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.370563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470988218011154:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.371326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470989344904557:2139] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.377997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470989344904566:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.378542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470989344904554:2138] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:27.378650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470985049937163:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.381715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470989344904557:2139] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:27.392768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470989344904555:2121] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.392818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:27.392860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470989344904555:2121] [node 6] ICP22 created new session: [3:7577470989344904572:2121]
2025-11-27T18:17:27.392907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470989344904572:2121] [node 6] ICS09 handshake done sender: [3:7577470989344904557:2139] self: [3:7577470989344904556:0] peer: [6:7577470988529661877:0] socket: 200 qp: -1
2025-11-27T18:17:27.392949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470989344904572:2121] [node 6] ICS10 traffic start
2025-11-27T18:17:27.392975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:27.391665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470988022654750:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:27.392469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470988529661858:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.401798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470988022654754:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.398192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470988218011175:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20341
2025-11-27T18:17:27.391651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470988069384808:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.401832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470988022654754:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.381560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470989550838927:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.392439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470984234694499:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.403283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470988529661878:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63551
2025-11-27T18:17:27.403300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470988022654752:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.381600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470989550838927:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.403098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470989344904549:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.400049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470988069384826:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6210
2025-11-27T18:17:27.420433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470988069384808:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:27.420510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470988069384826:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.403326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470988022654752:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.392808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470989550838923:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:27.403374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.414771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470984234694501:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470988529661867:2113] held: no
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.435010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470983774417441:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11398) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577470988069384819:2118] held: no
::ffff:127.1.0.0
2025-11-27T18:17:27.422941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470988218011177:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20341
2025-11-27T18:17:27.393293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470985255871557:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.405241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470989344904549:2107] [node 4] ICP22 created new session: [3:7577470989344904574:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.442160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470988069384803:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.442231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:27.403904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470983727687387:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.442277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470988069384803:2111] [node 6] ICP22 created new session: [2:7577470988069384833:2111]
2025-11-27T18:17:27.414990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470989344904574:2107] [node 4] ICS09 handshake done sender: [3:7577470989344904554:2138] self: [3:7577470989344904553:0] peer: [4:7577470989550838934:0] socket: 199 qp: -1
2025-11-27T18:17:27.433510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470984234694501:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17500516ns) {HandleHandshakeStatus:395(17498140ns) {LogHandshakeFail:472(12265ns) {UpdateErrorStateLog:545(4598ns)} DropOutgoingHandshake:457(1739ns) ProcessPendingSessionEvents:495(8655383ns) {DropSessionEvent:512(2679143ns) DropSessionEvent:512(6007ns) DropSessionEvent:512(3782ns) DropSessionEvent:512(4209ns) DropSessionEvent:512(3929ns) DropSessionEvent:512(4601ns) DropSessionEvent:512(2741358ns) DropSessionEvent:512(13232ns) DropSessionEvent:512(3182559ns) DropSessionEvent:512(4854ns)}}}
2025-11-27T18:17:27.437357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470988529661880:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63551
2025-11-27T18:17:27.441588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470988529661878:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.453871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470988529661883:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63551
2025-11-27T18:17:27.454161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470988529661884:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63551
2025-11-27T18:17:27.423093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470988218011178:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20341
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.393358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:27.403961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:27.417666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470989344904574:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:27.421757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470989344904567:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-27T18:17:27.422431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470989344904575:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-27T18:17:27.423629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:27.437897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470989344904567:2144] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.453016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470989344904555:2121] [node 6] ICP35 event processing took too much time PendingConnection:279(24813191ns) {HandleHandshakeStatus:329(24811176ns) {DropIncomingHandshake:442(11258ns) DropOutgoingHandshake:457(460ns) SwitchToState:179(100ns) ProcessPendingSessionEvents:495(14964701ns) {ForwardSessionEventToSession:612(670061ns) ForwardSessionEventToSession:612(4989ns) ForwardSessionEventToSession:612(225268ns) ForwardSessionEventToSession:612(3292ns) ForwardSessionEventToSession:612(9911693ns) ForwardSessionEventToSession:612(6979ns) ForwardSessionEventToSession:612(14256ns) ForwardSessionEventToSession:612(2650876ns) ForwardSessionEventToSession:612(3649ns) ForwardSessionEventToSession:612(2142ns)}}}
2025-11-27T18:17:27.457364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470989344904563:2141] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.404003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470983727687387:2048] [node 2] ICP22 created new session: [1:7577470988022654765:2048]
2025-11-27T18:17:27.430068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470988218011176:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20341
2025-11-27T18:17:27.397597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470985255871557:2049] [node 2] ICP22 created new session: [4:7577470989550838938:2049]
2025-11-27T18:17:27.468502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470985049937163:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.450742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470988069384829:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6210
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.468570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:27.449814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470988218011177:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.458360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470988529661885:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63551
2025-11-27T18:17:27.449937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470983923043796:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.457990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470988218011166:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:27.466518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470988218011154:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.471867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470988218011176:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.476502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:27.404045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470988022654765:2048] [node 2] ICS09 handshake done sender: [1:7577470988022654750:2110] self: [1:7577470988022654749:0] peer: [2:7577470988069384823:0] socket: 210 qp: -1
2025-11-27T18:17:27.450868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470988069384833:2111] [node 6] ICS09 handshake done sender: [2:7577470988069384808:2113] self: [2:7577470988069384806:0] peer: [6:7577470988529661879:0] socket: 203 qp: -1
2025-11-27T18:17:27.398557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470989550838938:2049] [node 2] ICS09 handshake done sender: [4:7577470989550838923:2114] self: [4:7577470989550838931:0] peer: [2:7577470988069384809:0] socket: 209 qp: -1
2025-11-27T18:17:27.485840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470988218011154:2106] [node 4] ICP22 created new session: [5:7577470988218011180:2106]
2025-11-27T18:17:27.487702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470983923043797:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.487775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470988218011180:2106] [node 4] ICS09 handshake done sender: [5:7577470988218011166:2115] self: [5:7577470988218011174:0] peer: [4:7577470989550838926:0] socket: 230 qp: -1
2025-11-27T18:17:27.458715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470988529661867:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:27.468602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470985049937163:2048] [node 1] ICP22 created new session: [3:7577470989344904579:2048]
2025-11-27T18:17:27.488230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470988218011180:2106] [node 4] ICS10 traffic start
2025-11-27T18:17:27.404074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470988022654765:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:27.398596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470989550838938:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:27.458959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470988529661856:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.490225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470988218011165:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.490252Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470988218011165:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.490004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.457936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470988069384833:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:27.494684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470988218011178:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.468633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470989344904579:2048] [node 1] ICS09 handshake done sender: [3:7577470989344904563:2141] self: [3:7577470989344904571:0] peer: [1:7577470988022654751:0] socket: 216 qp: -1
2025-11-27T18:17:27.471808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470984234694501:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.404102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:27.398632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:27.460373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:27.468652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470989344904579:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.499813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470983923043798:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.499905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470988218011183:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20341
2025-11-27T18:17:27.500290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470988218011154:2106] [node 4] ICP35 event processing took too much time PendingConnection:279(33599150ns) {HandleHandshakeStatus:329(33596935ns) {DropIncomingHandshake:442(2808ns) DropOutgoingHandshake:457(8748ns) SwitchToState:179(281ns) ProcessPendingSessionEvents:495(1138221ns) {ForwardSessionEventToSession:612(1093485ns) ForwardSessionEventToSession:612(6077ns) ForwardSessionEventToSession:612(2780ns) ForwardSessionEventToSession:612(5665ns) ForwardSessionEventToSession:612(2896ns) ForwardSessionEventToSession:612(3026ns) ForwardSessionEventToSession:612(5375ns) ForwardSessionEventToSession:612(5517ns) ForwardSessionEventToSession:612(2317ns) ForwardSessionEventToSession:612(4045ns)}}}
2025-11-27T18:17:27.500439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470988218011165:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:27.501639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470988218011155:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.409742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470988022654754:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:27.471843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:27.420157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470989550838925:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.463438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470988069384819:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:27.466051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470988069384803:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(23710523ns) {HandleHandshakeStatus:329(23707837ns) {DropIncomingHandshake:442(15219ns) DropOutgoingHandshake:457(403ns) SwitchToState:179(604ns) ProcessPendingSessionEvents:495(113245ns) {ForwardSessionEventToSession:612(65240ns) ForwardSessionEventToSession:612(5766ns) ForwardSessionEventToSession:612(2436ns) ForwardSessionEventToSession:612(3096ns) ForwardSessionEventToSession:612(6096ns) ForwardSessionEventToSession:612(2632ns) ForwardSessionEventToSession:612(8968ns) ForwardSessionEventToSession:612(2399ns) ForwardSessionEventToSession:612(2585ns) ForwardSessionEventToSession:612(5194ns)}}}
2025-11-27T18:17:27.476908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470983774417441:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.476944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:27.476993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470983774417441:2048] [node 1] ICP22 created new session: [2:7577470988069384837:2048]
2025-11-27T18:17:27.477039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470988069384837:2048] [node 1] ICS09 handshake done sender: [2:7577470988069384819:2118] self: [2:7577470988069384823:0] peer: [1:7577470988022654749:0] socket: 213 qp: -1
2025-11-27T18:17:27.477069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470988069384837:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:27.477100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:27.489630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470988069384818:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.489663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470988069384818:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.505684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470988069384816:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.505716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470988069384816:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.505920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470988069384818:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:27.468676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:27.410773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470988022654752:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:27.501696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:27.420213Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470989550838925:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.472165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470988529661868:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:27.481518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470989344904581:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-27T18:17:27.477757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470988529661876:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:27.423880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470985255871557:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:27.501734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470988218011155:2108] [node 6] ICP22 created new session: [5:7577470988218011186:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.410800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470988022654748:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.484844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470989344904581:2146] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.518195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470983774417442:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.518254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:27.518298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470983774417442:2049] [node 3] ICP22 created new session: [2:7577470988069384841:2049]
2025-11-27T18:17:27.518346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470988069384841:2049] [node 3] ICS09 handshake done sender: [2:7577470988069384818:2117] self: [2:7577470988069384817:0] peer: [3:7577470989344904583:0] socket: 266 qp: -1
2025-11-27T18:17:27.518379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470988069384841:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:27.477957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470988529661875:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:27.484961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470985049937164:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.501793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470988218011186:2108] [node 6] ICS09 handshake done sender: [5:7577470988218011165:2114] self: [5:7577470988218011164:0] peer: [6:7577470988529661887:0] socket: 233 qp: -1
2025-11-27T18:17:27.518421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:27.410839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.479116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470984234694500:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.490511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470989344904549:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(73595610ns) {HandleHandshakeStatus:329(73592187ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(613ns) SwitchToState:179(798ns) ProcessPendingSessionEvents:495(42591982ns) {ForwardSessionEventToSession:612(29570794ns) ForwardSessionEventToSession:612(7135ns) ForwardSessionEventToSession:612(3317ns) ForwardSessionEventToSession:612(6584946ns) ForwardSessionEventToSession:612(2854ns) ForwardSessionEventToSession:612(1634847ns) ForwardSessionEventToSession:612(3794ns) ForwardSessionEventToSession:612(3318ns) ForwardSessionEventToSession:612(4755712ns) ForwardSessionEventToSession:612(4982ns)}}}
2025-11-27T18:17:27.424715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470989550838936:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36652
2025-11-27T18:17:27.501819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470988218011186:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:27.519190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470988069384816:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:27.507732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470989344904562:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.479154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:27.410885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470988022654748:2109] [node 6] ICP22 created new session: [1:7577470988022654770:2109]
2025-11-27T18:17:27.425795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470989550838940:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36652
2025-11-27T18:17:27.501848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:27.507758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470989344904562:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.526630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470988069384802:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.526702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:27.526746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470988069384802:2109] [node 5] ICP22 created new session: [2:7577470988069384843:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:27.526788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470988069384843:2109] [node 5] ICS09 handshake done sender: [2:7577470988069384816:2116] self: [2:7577470988069384815:0] peer: [5:7577470988218011181:0] socket: 249 qp: -1
2025-11-27T18:17:27.479198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470984234694500:2049] [node 2] ICP22 created new session: [6:7577470988529661889:2049]
2025-11-27T18:17:27.508853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470988218011177:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:27.425906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470989550838922:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:27.520629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470989344904562:2140] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:27.426160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470989550838927:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:27.526813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470988069384843:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:27.443162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470989550838925:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:27.454483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470985255871558:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.454569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:27.454614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470985255871558:2050] [node 3] ICP22 created new session: [4:7577470989550838943:2050]
2025-11-27T18:17:27.454658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470989550838943:2050] [node 3] ICS09 handshake done sender: [4:7577470989550838922:2113] self: [4:7577470989550838934:0] peer: [3:7577470989344904553:0] socket: 208 qp: -1
2025-11-27T18:17:27.454687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470989550838943:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:27.454722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:27.455406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470989550838917:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.455468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:27.455514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470989550838917:2108] [node 5] ICP22 created new session: [4:7577470989550838945:2108]
2025-11-27T18:17:27.455549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470989550838945:2108] [node 5] ICS09 handshake done sender: [4:7577470989550838927:2116] self: [4:7577470989550838926:0] peer: [5:7577470988218011174:0] socket: 219 qp: -1
2025-11-27T18:17:27.455595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470989550838945:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:27.455623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:27.463578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470989550838940:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.470293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470989550838916:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.470331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:27.471949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470989550838916:2110] [node 6] ICP22 created new session: [4:7577470989550838947:2110]
2025-11-27T18:17:27.473251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470985255871556:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:27.481915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470989550838947:2110] [node 6] ICS09 handshake done sender: [4:7577470989550838925:2115] self: [4:7577470989550838924:0] peer: [6:7577470988529661882:0] socket: 218 qp: -1
2025-11-27T18:17:27.485353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470989550838947:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:27.487858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:27.492427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470989550838916:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(28645824ns) {HandleHandshakeStatus:329(28644180ns) {DropIncomingHandshake:442(1191ns) DropOutgoingHandshake:457(708ns) SwitchToState:179(476ns) ProcessPendingSessionEvents:495(2065015ns) {ForwardSessionEventToSession:612(2019086ns) ForwardSessionEventToSession:612(6888ns) ForwardSessionEventToSession:612(2349ns) ForwardSessionEventToSession:612(2032ns) ForwardSessionEventToSession:612(9443ns) ForwardSessionEventToSession:612(2039ns) ForwardSessionEventToSession:612(1987ns) ForwardSessionEventToSession:612(4288ns) ForwardSessionEventToSession:612(1966ns) ForwardSessionEventToSession:612(4409ns)}}}
2025-11-27T18:17:27.523844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470989550838950:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36652
2025-11-27T18:17:27.538623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470985255871556:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11398) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470989550838940:2120] held: no
2025-11-27T18:17:27.540941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470989550838940:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:27.542606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470985255871556:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.545620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:27.545673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470985255871556:2048] [node 1] ICP22 created new session: [4:7577470989550838951:2048]
2025-11-27T18:17:27.545722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470989550838951:2048] [node 1] ICS09 handshake done sender: [4:7577470989550838940:2120] self: [4:7577470989550838948:0] peer: [1:7577470988022654762:0] socket: 258 qp: -1
2025-11-27T18:17:27.545763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470989550838951:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:27.549384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:27.412303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470983727687388:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.479233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470988529661889:2049] [node 2] ICS09 handshake done sender: [6:7577470988529661868:2114] self: [6:7577470988529661879:0] peer: [2:7577470988069384806:0] socket: 212 qp: -1
2025-11-27T18:17:27.514832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470983923043796:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.479265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470988529661889:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:27.522866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470989344904585:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53870
2025-11-27T18:17:27.526843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:27.412351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:27.525730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470989344904550:2116] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.525802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:27.514896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:27.525845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470989344904550:2116] [node 5] ICP22 created new session: [3:7577470989344904586:2116]
2025-11-27T18:17:27.479295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:27.525877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470989344904586:2116] [node 5] ICS09 handshake done sender: [3:7577470989344904562:2140] self: [3:7577470989344904561:0] peer: [5:7577470988218011185:0] socket: 255 qp: -1
2025-11-27T18:17:27.525902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470989344904586:2116] [node 5] ICS10 traffic start
2025-11-27T18:17:27.525936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:27.538061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470985049937164:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577470989344904581:2146] held: no
2025-11-27T18:17:27.538329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470989344904581:2146] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:27.412400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470983727687388:2049] [node 3] ICP22 created new session: [1:7577470988022654771:2049]
2025-11-27T18:17:27.514953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470983923043796:2048] [node 1] ICP22 created new session: [5:7577470988218011189:2048]
2025-11-27T18:17:27.412439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470988022654771:2049] [node 3] ICS09 handshake done sender: [1:7577470988022654752:2111] self: [1:7577470988022654751:0] peer: [3:7577470989344904571:0] socket: 214 qp: -1
2025-11-27T18:17:27.412471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470988022654771:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:27.541808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470985049937164:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.412505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:27.420344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470988022654770:2109] [node 6] ICS09 handshake done sender: [1:7577470988022654754:2112] self: [1:7577470988022654753:0] peer: [6:7577470988529661881:0] socket: 215 qp: -1
2025-11-27T18:17:27.423590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470988022654770:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:27.432920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:27.479519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577470984234694500:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:27.479723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470988529661858:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.434598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470988022654748:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(23600531ns) {HandleHandshakeStatus:329(23598175ns) {DropIncomingHandshake:442(424ns) DropOutgoingHandshake:457(432ns) SwitchToState:179(797ns) ProcessPendingSessionEvents:495(127844ns) {ForwardSessionEventToSession:612(73272ns) ForwardSessionEventToSession:612(13096ns) ForwardSessionEventToSession:612(2377ns) ForwardSessionEventToSession:612(2799ns) ForwardSessionEventToSession:612(6420ns) ForwardSessionEventToSession:612(2120ns) ForwardSessionEventToSession:612(2652ns) ForwardSessionEventToSession:612(4668ns) ForwardSessionEventToSession:612(2222ns) ForwardSessionEventToSession:612(7293ns)}}}
2025-11-27T18:17:27.479754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:27.521139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470988218011189:2048] [node 1] ICS09 handshake done sender: [5:7577470988218011177:2120] self: [5:7577470988218011179:0] peer: [1:7577470988022654755:0] socket: 254 qp: -1
2025-11-27T18:17:27.479786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470988529661858:2108] [node 4] ICP22 created new session: [6:7577470988529661891:2108]
2025-11-27T18:17:27.439779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470988022654775:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34348
2025-11-27T18:17:27.541868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:27.541922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470985049937164:2049] [node 2] ICP22 created new session: [3:7577470989344904588:2049]
2025-11-27T18:17:27.541982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470989344904588:2049] [node 2] ICS09 handshake done sender: [3:7577470989344904581:2146] self: [3:7577470989344904583:0] peer: [2:7577470988069384817:0] socket: 267 qp: -1
2025-11-27T18:17:27.542012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470989344904588:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:27.542043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:27.521189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470988218011189:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:27.479823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470988529661891:2108] [node 4] ICS09 handshake done sender: [6:7577470988529661876:2118] self: [6:7577470988529661882:0] peer: [4:7577470989550838924:0] socket: 228 qp: -1
2025-11-27T18:17:27.479851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470988529661891:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:27.479874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:27.480136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470984234694499:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.480167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:27.480218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470984234694499:2048] [node 1] ICP22 created new session: [6:7577470988529661893:2048]
2025-11-27T18:17:27.480254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470988529661893:2048] [node 1] ICS09 handshake done sender: [6:7577470988529661875:2117] self: [6:7577470988529661881:0] peer: [1:7577470988022654753:0] socket: 224 qp: -1
2025-11-27T18:17:27.480281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470988529661893:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:27.480312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:27.481823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470984234694501:2050] [node 3] ICP22 created new session: [6:7577470988529661888:2050]
2025-11-27T18:17:27.485748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470988529661888:2050] [node 3] ICS09 handshake done sender: [6:7577470988529661867:2113] self: [6:7577470988529661877:0] peer: [3:7577470989344904556:0] socket: 211 qp: -1
2025-11-27T18:17:27.488198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470988529661888:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:27.488249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:27.499091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470984234694501:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(23658626ns) {HandleHandshakeStatus:329(23657187ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(207ns) SwitchToState:179(134ns) ProcessPendingSessionEvents:495(905ns)}}
2025-11-27T18:17:27.500776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470988529661896:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63551
2025-11-27T18:17:27.516104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470988529661856:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30031) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577470988529661878:2119] held: no
2025-11-27T18:17:27.516869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470988529661878:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:27.524347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470988529661856:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.524422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:27.524478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470988529661856:2107] [node 5] ICP22 created new session: [6:7577470988529661897:2107]
2025-11-27T18:17:27.524532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470988529661897:2107] [node 5] ICS09 handshake done sender: [6:7577470988529661878:2119] self: [6:7577470988529661887:0] peer: [5:7577470988218011164:0] socket: 236 qp: -1
2025-11-27T18:17:27.524560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470988529661897:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:27.524586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:27.455466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470988022654775:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.521235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:27.522180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470988218011191:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20341
2025-11-27T18:17:27.532332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470988218011178:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:27.532600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470988218011190:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20341
2025-11-27T18:17:27.533128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470983923043798:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.533178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:27.533230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470983923043798:2050] [node 3] ICP22 created new session: [5:7577470988218011193:2050]
2025-11-27T18:17:27.533288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470988218011193:2050] [node 3] ICS09 handshake done sender: [5:7577470988218011178:2121] self: [5:7577470988218011185:0] peer: [3:7577470989344904561:0] socket: 257 qp: -1
2025-11-27T18:17:27.538526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470988218011193:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:27.538587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:27.550564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470988218011176:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:27.559792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470983923043797:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:27.559840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:27.559880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470983923043797:2049] [node 2] ICP22 created new session: [5:7577470988218011195:2049]
2025-11-27T18:17:27.559929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470988218011195:2049] [node 2] ICS09 handshake done sender: [5:7577470988218011176:2119] self: [5:7577470988218011181:0] peer: [2:7577470988069384815:0] socket: 227 qp: -1
2025-11-27T18:17:27.559964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470988218011195:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:27.560008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:27.560214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577470983923043797:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:27.463281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470988022654778:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34348
2025-11-27T18:17:27.469983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470988022654756:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.470016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470988022654756:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.473056Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470988022654756:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:27.490016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470988022654763:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:27.490042Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470988022654763:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:27.491264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470988022654747:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.491300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:27.491337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470988022654747:2108] [node 5] ICP22 created new session: [1:7577470988022654780:2108]
2025-11-27T18:17:27.491379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470988022654780:2108] [node 5] ICS09 handshake done sender: [1:7577470988022654756:2113] self: [1:7577470988022654755:0] peer: [5:7577470988218011179:0] socket: 250 qp: -1
2025-11-27T18:17:27.491405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470988022654780:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:27.491428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:27.491880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470988022654774:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34348
2025-11-27T18:17:27.505530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470988022654763:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:27.510052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577470983727687388:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:27.515003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470988022654746:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:27.516945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:27.520425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470988022654746:2107] [node 4] ICP22 created new session: [1:7577470988022654784:2107]
2025-11-27T18:17:27.520795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470988022654784:2107] [node 4] ICS09 handshake done sender: [1:7577470988022654763:2115] self: [1:7577470988022654762:0] peer: [4:7577470989550838948:0] socket: 253 qp: -1
2025-11-27T18:17:27.523272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470988022654784:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:27.523973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:27.541445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470988022654746:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(25471817ns) {HandleHandshakeStatus:329(25469060ns) {DropIncomingHandshake:442(1894614ns) DropOutgoingHandshake:457(637ns) SwitchToState:179(619ns) ProcessPendingSessionEvents:495(1647897ns) {ForwardSessionEventToSession:612(1348714ns) ForwardSessionEventToSession:612(3281ns) ForwardSessionEventToSession:612(2285ns) ForwardSessionEventToSession:612(4621ns) ForwardSessionEventToSession:612(3566ns) ForwardSessionEventToSession:612(8503ns) ForwardSessionEventToSession:612(5216ns) ForwardSessionEventToSession:612(2472ns) ForwardSessionEventToSession:612(256833ns) ForwardSessionEventToSession:612(3634ns)}}}
2025-11-27T18:17:27.557065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470988022654783:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34348
2025-11-27T18:17:27.595283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:27.595523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470988069384837:2048] [node 1] ICS01 socket: 213 reason# EndOfStream
2025-11-27T18:17:27.595562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470983774417441:2048] [node 1] ICP30 unregister session Session# [2:7577470988069384837:2048] VirtualId# [2:7577470988069384823:0]
2025-11-27T18:17:27.595597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470988069384837:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.595669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:27.595707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:27.597326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:27.597559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470989344904579:2048] [node 1] ICS07 socket disconnect 216 reason# EndOfStream
2025-11-27T18:17:27.597587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470989344904579:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.597681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:27.597734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470989344904579:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:27.598525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470985049937163:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11398) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:27.598623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470989344904579:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:27.598658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470985049937163:2048] [node 1] ICP30 unregister session Session# [3:7577470989344904579:2048] VirtualId# [3:7577470989344904571:0]
2025-11-27T18:17:27.598709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:27.598740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470985049937163:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11398) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:27.598754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11398) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:27.600926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:27.601137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470989550838951:2048] [node 1] ICS01 socket: 258 reason# EndOfStream
2025-11-27T18:17:27.601174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470985255871556:2048] [node 1] ICP30 unregister session Session# [4:7577470989550838951:2048] VirtualId# [4:7577470989550838948:0]
2025-11-27T18:17:27.601218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470989550838951:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.601337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:27.601403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:27.602178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:27.602466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470988218011189:2048] [node 1] ICS01 socket: 254 reason# EndOfStream
2025-11-27T18:17:27.602510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470983923043796:2048] [node 1] ICP30 unregister session Session# [5:7577470988218011189:2048] VirtualId# [5:7577470988218011179:0]
2025-11-27T18:17:27.602576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470988218011189:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.602668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:27.602721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:27.606647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:27.607033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470988529661893:2048] [node 1] ICS01 socket: 224 reason# EndOfStream
2025-11-27T18:17:27.607080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470984234694499:2048] [node 1] ICP30 unregister session Session# [6:7577470988529661893:2048] VirtualId# [6:7577470988529661881:0]
2025-11-27T18:17:27.607111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470988529661893:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.607194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:27.607279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:27.609794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470985049937163:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:27.638864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:27.639090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470989344904588:2049] [node 2] ICS01 socket: 267 reason# EndOfStream
2025-11-27T18:17:27.639132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470985049937164:2049] [node 2] ICP30 unregister session Session# [3:7577470989344904588:2049] VirtualId# [3:7577470989344904583:0]
2025-11-27T18:17:27.639164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470989344904588:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.639237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:27.639275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:27.645448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:27.645921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:27.646089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470988529661889:2049] [node 2] ICS01 socket: 212 reason# EndOfStream
2025-11-27T18:17:27.646122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470984234694500:2049] [node 2] ICP30 unregister session Session# [6:7577470988529661889:2049] VirtualId# [6:7577470988529661879:0]
2025-11-27T18:17:27.646142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470988529661889:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.646186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:27.646217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:27.646513Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:27.645665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470988218011195:2049] [node 2] ICS01 socket: 227 reason# EndOfStream
2025-11-27T18:17:27.645703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470983923043797:2049] [node 2] ICP30 unregister session Session# [5:7577470988218011195:2049] VirtualId# [5:7577470988218011181:0]
2025-11-27T18:17:27.645739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470988218011195:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.645812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:27.645852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:27.646731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470989550838938:2049] [node 2] ICS07 socket disconnect 209 reason# EndOfStream
2025-11-27T18:17:27.646762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470989550838938:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.646838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:27.646897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470989550838938:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:27.657896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470985255871557:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:27.658155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470989550838938:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:27.658356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470985255871557:2049] [node 2] ICP30 unregister session Session# [4:7577470989550838938:2049] VirtualId# [4:7577470989550838931:0]
2025-11-27T18:17:27.658489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:27.658529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470985255871557:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:27.658549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:27.668413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470985255871557:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:27.709500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:27.709743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470988218011193:2050] [node 3] ICS07 socket disconnect 257 reason# EndOfStream
2025-11-27T18:17:27.709768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470988218011193:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.709842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:27.709878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470988218011193:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:27.710917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470983923043798:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28370) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:27.710972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470988218011193:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:27.711002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470983923043798:2050] [node 3] ICP30 unregister session Session# [5:7577470988218011193:2050] VirtualId# [5:7577470988218011185:0]
2025-11-27T18:17:27.711045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:27.711073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470983923043798:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28370) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:27.711086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28370) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:27.710813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:27.711017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470989550838943:2050] [node 3] ICS01 socket: 208 reason# EndOfStream
2025-11-27T18:17:27.711050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470985255871558:2050] [node 3] ICP30 unregister session Session# [4:7577470989550838943:2050] VirtualId# [4:7577470989550838934:0]
2025-11-27T18:17:27.711079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470989550838943:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.711152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:27.711196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:27.711341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:27.711536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470988529661888:2050] [node 3] ICS01 socket: 211 reason# EndOfStream
2025-11-27T18:17:27.711564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470984234694501:2050] [node 3] ICP30 unregister session Session# [6:7577470988529661888:2050] VirtualId# [6:7577470988529661877:0]
2025-11-27T18:17:27.711586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470988529661888:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.711638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:27.711672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:27.721326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470983923043798:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:27.761367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:27.761617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470988218011180:2106] [node 4] ICS01 socket: 230 reason# EndOfStream
2025-11-27T18:17:27.761662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470988218011154:2106] [node 4] ICP30 unregister session Session# [5:7577470988218011180:2106] VirtualId# [5:7577470988218011174:0]
2025-11-27T18:17:27.761692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470988218011180:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.761770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:27.761839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:27.762260Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:27.762461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470988529661891:2108] [node 4] ICS01 socket: 228 reason# EndOfStream
2025-11-27T18:17:27.762495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470988529661858:2108] [node 4] ICP30 unregister session Session# [6:7577470988529661891:2108] VirtualId# [6:7577470988529661882:0]
2025-11-27T18:17:27.762521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470988529661891:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.762577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:27.762613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:27.827575Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:27.828024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470988529661897:2107] [node 5] ICS01 socket: 236 reason# EndOfStream
2025-11-27T18:17:27.828103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470988529661856:2107] [node 5] ICP30 unregister session Session# [6:7577470988529661897:2107] VirtualId# [6:7577470988529661887:0]
2025-11-27T18:17:27.828169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470988529661897:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:27.828360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:27.828452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:27.913618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470989013364916:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:27.913659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470989013364917:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:27.943106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470987384150617:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:27.943148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470987384150618:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:27.970922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470986915142420:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:27.971000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470986915142421:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:27.996951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470988658858431:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:27.997002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470988658858433:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:27.997064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470988658858432:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:28.029072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470993380446853:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:28.029112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470993380446854:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:28.029138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470993380446855:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:28.055271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470990743734812:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:28.055309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470990743734813:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:28.055337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470990743734814:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:28.142982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470990743734867:2097] [node 4] ICP01 ready to work
2025-11-27T18:17:28.142892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470990743734866:2099] [node 5] ICP01 ready to work
2025-11-27T18:17:28.153717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470993308332275:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:28.154105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470993308332276:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:28.154718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470993308332277:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:28.183509Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470989013364916:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46049525ns) {Configure:104(46042497ns) {ProcessConfigured:135(472858ns) {StartInitialHandshake:157(469958ns) {DropHandshakes:470(3165ns) {DropIncomingHandshake:442(669ns) DropOutgoingHandshake:457(373ns)} GenerateSessionVirtualId:988(600ns) PrepareNewSessionHandshake:477(406030ns) {SwitchToState:179(2244ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.195558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470992953825793:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:28.210967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470991679117977:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:28.196983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470992953825790:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10570
2025-11-27T18:17:28.206156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470991210109777:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:28.218007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470991210109780:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:28.212738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470991679117978:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:28.237827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470993380446916:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:28.232391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470992953825798:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.231301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470989013364917:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(59547699ns) {Configure:104(59543582ns) {ProcessConfigured:135(24321602ns) {StartInitialHandshake:157(24318637ns) {DropHandshakes:470(2002ns) {DropIncomingHandshake:442(478ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(637ns) PrepareNewSessionHandshake:477(4224ns) {SwitchToState:179(1968ns)}}}}}
2025-11-27T18:17:28.243115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470993380446920:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:28.231136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470991210109781:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:28.225856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470991679117979:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.282839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470992953825800:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10570
2025-11-27T18:17:28.282911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470992953825801:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10570
2025-11-27T18:17:28.282960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470992953825802:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10570
2025-11-27T18:17:28.260651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470993308332276:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(94977792ns) {Configure:104(94972450ns) {ProcessConfigured:135(61528ns) {StartInitialHandshake:157(58512ns) {DropHandshakes:470(1509ns) {DropIncomingHandshake:442(341ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(733ns) PrepareNewSessionHandshake:477(1118ns) {SwitchToState:179(522ns)}}}}}
2025-11-27T18:17:28.251698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470991679117978:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18797881ns) {Configure:104(18793368ns) {ProcessConfigured:135(6360982ns) {StartInitialHandshake:157(6359629ns) {DropHandshakes:470(15801ns) {DropIncomingHandshake:442(14219ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(565ns) PrepareNewSessionHandshake:477(2079ns) {SwitchToState:179(146ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.255418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470990743734867:2097] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(88744378ns) {Configure:104(88739549ns) {ProcessConfigured:135(2864036ns) {StartInitialHandshake:157(2860391ns) {DropHandshakes:470(3309ns) {DropIncomingHandshake:442(789ns) DropOutgoingHandshake:457(569ns)} GenerateSessionVirtualId:988(909ns) PrepareNewSessionHandshake:477(2124ns) {SwitchToState:179(880ns)}}}}}
2025-11-27T18:17:28.257019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470993380446853:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24481295ns) {Configure:104(24476259ns) {ProcessConfigured:135(2626864ns) {StartInitialHandshake:157(2624936ns) {DropHandshakes:470(564ns) {DropIncomingHandshake:442(201ns) DropOutgoingHandshake:457(119ns)} GenerateSessionVirtualId:988(512ns) PrepareNewSessionHandshake:477(2069ns) {SwitchToState:179(882ns)}}}}}
2025-11-27T18:17:28.270726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470991210109777:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(38079703ns) {Configure:104(38073340ns) {ProcessConfigured:135(56941ns) {StartInitialHandshake:157(55486ns) {DropHandshakes:470(1602ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(683ns) PrepareNewSessionHandshake:477(1548ns) {SwitchToState:179(457ns)}}}}}
2025-11-27T18:17:28.302084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470992953825790:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.253427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470991679117980:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22867
2025-11-27T18:17:28.279543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470993380446919:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52496
2025-11-27T18:17:28.271477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470991210109782:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
2025-11-27T18:17:28.313786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470988658858432:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(131012275ns) {Configure:104(131008358ns) {ProcessConfigured:135(10867539ns) {StartInitialHandshake:157(8091802ns) {DropHandshakes:470(3139ns) {DropIncomingHandshake:442(762ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(1881ns) PrepareNewSessionHandshake:477(3723ns) {SwitchToState:179(1674ns)}}}}}
2025-11-27T18:17:28.264020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470990743734882:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21543
2025-11-27T18:17:28.257581Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470987384150618:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(89161086ns) {Configure:104(88914692ns) {ProcessConfigured:135(8822293ns) {StartInitialHandshake:157(8820968ns) {DropHandshakes:470(979ns) {DropIncomingHandshake:442(262ns) DropOutgoingHandshake:457(137ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:477(2025ns) {SwitchToState:179(155ns)}}}}}
2025-11-27T18:17:28.291041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470993380446930:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52496
2025-11-27T18:17:28.271763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470991210109787:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
2025-11-27T18:17:28.313959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470992953825795:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.266057Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470990743734866:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(83936184ns) {Configure:104(83930102ns) {ProcessConfigured:135(2823055ns) {StartInitialHandshake:157(2821455ns) {DropHandshakes:470(1850ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(421ns)} GenerateSessionVirtualId:988(397ns) PrepareNewSessionHandshake:477(852ns) {SwitchToState:179(596ns)}}}}}
2025-11-27T18:17:28.324216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470993308332277:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(136341316ns) {Configure:104(136334036ns) {ProcessConfigured:135(16268833ns) {StartInitialHandshake:157(16266264ns) {DropHandshakes:470(2092ns) {DropIncomingHandshake:442(460ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(768ns) PrepareNewSessionHandshake:477(3225ns) {SwitchToState:179(1832ns)}}}}}
2025-11-27T18:17:28.285849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470987384150617:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(102326244ns) {Configure:104(102320797ns) {ProcessConfigured:135(7490862ns) {StartInitialHandshake:157(7488308ns) {DropHandshakes:470(1934ns) {DropIncomingHandshake:442(429ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(961ns) PrepareNewSessionHandshake:477(1718ns) {SwitchToState:179(656ns)}}}}}
2025-11-27T18:17:28.328140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470993308332281:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.271815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470991210109788:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.291169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470993380446931:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52496
2025-11-27T18:17:28.328170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470993308332281:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:28.326123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470991679117980:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.293846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470993380446919:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.328895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470993308332281:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:28.330668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470987384150617:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.302645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470993380446930:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.310000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470993380446916:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.292852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470991210109782:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.325973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470990743734882:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.313983Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470992953825795:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.315209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470992953825795:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:28.331167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470990743734890:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21543
2025-11-27T18:17:28.331037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470993308332275:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.330949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470986915142421:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(144381394ns) {Configure:104(144376027ns) {ProcessConfigured:135(2168885ns) {StartInitialHandshake:157(2157339ns) {DropHandshakes:470(1118ns) {DropIncomingHandshake:442(270ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(586ns) PrepareNewSessionHandshake:477(1307ns) {SwitchToState:179(680ns)}}}}}
2025-11-27T18:17:28.331014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470993380446854:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(115442209ns) {Configure:104(115437573ns) {ProcessConfigured:135(19140662ns) {StartInitialHandshake:157(19138204ns) {DropHandshakes:470(1398ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(2630ns) {SwitchToState:179(1397ns)}}}}}
2025-11-27T18:17:28.335059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470990743734890:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.331575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.318851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470988658858431:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(125116483ns) {Configure:104(125107454ns) {ProcessConfigured:135(44988804ns) {StartInitialHandshake:157(36167518ns) {DropHandshakes:470(2352ns) {DropIncomingHandshake:442(230ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(1219ns) PrepareNewSessionHandshake:477(2850ns) {SwitchToState:179(1459ns)}}}}}
2025-11-27T18:17:28.331484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470986915142420:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(142984810ns) {Configure:104(142979739ns) {ProcessConfigured:135(15472354ns) {StartInitialHandshake:157(15469343ns) {DropHandshakes:470(2197ns) {DropIncomingHandshake:442(418ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(2117ns) PrepareNewSessionHandshake:477(2008ns) {SwitchToState:179(604ns)}}}}}
2025-11-27T18:17:28.331482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470993380446854:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.336133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470991679117985:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.324630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470992953825802:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.338077Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470990743734813:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(98991350ns) {Configure:104(98987728ns) {ProcessConfigured:135(1488890ns) {StartInitialHandshake:157(1487276ns) {DropHandshakes:470(1613ns) {DropIncomingHandshake:442(260ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(357ns) PrepareNewSessionHandshake:477(1454ns) {SwitchToState:179(683ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.332718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470993308332279:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.335372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470991210109781:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(90061768ns) {Configure:104(90058196ns) {ProcessConfigured:135(6842455ns) {StartInitialHandshake:157(6841142ns) {DropHandshakes:470(1178ns) {DropIncomingHandshake:442(181ns) DropOutgoingHandshake:457(153ns)} GenerateSessionVirtualId:988(412ns) PrepareNewSessionHandshake:477(1653ns) {SwitchToState:179(114ns)}}}}}
2025-11-27T18:17:28.336174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470991679117985:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.324681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470992953825793:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.332747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470993308332279:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.324723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:28.340246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470990743734866:2099] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.340703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470991210109788:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.340993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470991210109787:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.342938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470991210109791:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
2025-11-27T18:17:28.338394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470993380446916:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18985) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470993380446919:2111] held: no
2025-11-27T18:17:28.339404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470993380446935:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52496
2025-11-27T18:17:28.343016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470993380446937:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52496
2025-11-27T18:17:28.337888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470991679117995:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22867
2025-11-27T18:17:28.324765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470992953825793:2109] [node 5] ICP22 created new session: [4:7577470992953825808:2109]
2025-11-27T18:17:28.333487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470993308332275:2107] [node 4] ICP22 created new session: [1:7577470993308332293:2107]
2025-11-27T18:17:28.341872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470990743734813:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.344540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470991210109798:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.343223Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470991679117977:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(106989001ns) {Configure:104(106983637ns) {ProcessConfigured:135(3202310ns) {StartInitialHandshake:157(3198860ns) {DropHandshakes:470(695079ns) {DropIncomingHandshake:442(1438ns) DropOutgoingHandshake:457(317ns)} GenerateSessionVirtualId:988(669ns) PrepareNewSessionHandshake:477(1361ns) {SwitchToState:179(692ns)}}}}}
2025-11-27T18:17:28.324815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470992953825808:2109] [node 5] ICS09 handshake done sender: [4:7577470992953825795:2112] self: [4:7577470992953825794:0] peer: [5:7577470993380446932:0] socket: 208 qp: -1
2025-11-27T18:17:28.334988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470993308332293:2107] [node 4] ICS09 handshake done sender: [1:7577470993308332281:2111] self: [1:7577470993308332280:0] peer: [4:7577470992953825811:0] socket: 205 qp: -1
2025-11-27T18:17:28.344642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470993380446931:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.343356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470991679117992:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22867
2025-11-27T18:17:28.346904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470986915142421:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.345466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470993380446922:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.335019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470993308332291:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29220
2025-11-27T18:17:28.325524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470992953825808:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.347017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470991210109791:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.345500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470993380446922:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:28.345723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470991679117980:2111] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:28.325560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:28.335701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470993308332293:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:28.348674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470991210109798:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.346605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470991679117992:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.346264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470993380446937:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.335760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:28.327711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470988658858431:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.348619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470991679117985:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:28.346354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470993380446919:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:28.350378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470991210109784:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.338163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470993308332291:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.327895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470992953825800:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.349922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470991679117987:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.384715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470990743734897:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21543
2025-11-27T18:17:28.350407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470991210109784:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.386408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470990743734896:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21543
2025-11-27T18:17:28.333517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577470992953825793:2109] [node 5] ICP18 handshake (actor [4:7577470992953825802:2115]) is too old
2025-11-27T18:17:28.347302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470993380446916:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.349945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470991679117987:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.341892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470993308332279:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:28.343846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470989013364916:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.351484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470986915142420:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.347331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:28.335588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470992953825798:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(49761844ns) {Configure:104(49757816ns) {ProcessConfigured:135(5966044ns) {StartInitialHandshake:157(5963386ns) {DropHandshakes:470(703106ns) {DropIncomingHandshake:442(999ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(945ns) PrepareNewSessionHandshake:477(2291ns) {SwitchToState:179(795ns)}}}}}
2025-11-27T18:17:28.357990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470991210109784:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:28.353372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470991679117982:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.343913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:28.392074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470990743734895:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21543
2025-11-27T18:17:28.353392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470991679117982:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:28.347352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470993380446916:2108] [node 4] ICP22 created new session: [5:7577470993380446939:2108]
2025-11-27T18:17:28.354072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470987384150617:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.344304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470992953825814:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10570
2025-11-27T18:17:28.362968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470991210109786:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.394093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470990743734895:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.347384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470993380446939:2108] [node 4] ICS09 handshake done sender: [5:7577470993380446919:2111] self: [5:7577470993380446932:0] peer: [4:7577470992953825794:0] socket: 212 qp: -1
2025-11-27T18:17:28.343964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470989013364916:2048] [node 2] ICP22 created new session: [1:7577470993308332299:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.354116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:28.348375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470992953825790:2110] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:28.363003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470991210109786:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:28.344012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470993308332299:2048] [node 2] ICS09 handshake done sender: [1:7577470993308332279:2110] self: [1:7577470993308332278:0] peer: [2:7577470991679117993:0] socket: 204 qp: -1
2025-11-27T18:17:28.347404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470993380446939:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:28.394147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470990743734890:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.354164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470987384150617:2048] [node 1] ICP22 created new session: [2:7577470991679118000:2048]
2025-11-27T18:17:28.348629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470992953825801:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.394422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470990743734882:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:28.347425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:28.385217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470991210109786:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:28.344043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470993308332299:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.344116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:28.344361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470993308332294:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29220
2025-11-27T18:17:28.351718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470993308332294:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.351772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470993308332287:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.351792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470993308332287:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:28.356568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470993308332287:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:28.361766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470993308332275:2107] [node 4] ICP18 handshake (actor [1:7577470993308332294:2116]) is too old
2025-11-27T18:17:28.381578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470993308332285:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.381633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470993308332285:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:28.383674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470993308332276:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.383738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:28.383810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470993308332276:2108] [node 5] ICP22 created new session: [1:7577470993308332304:2108]
2025-11-27T18:17:28.383854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470993308332304:2108] [node 5] ICS09 handshake done sender: [1:7577470993308332287:2113] self: [1:7577470993308332286:0] peer: [5:7577470993380446943:0] socket: 251 qp: -1
2025-11-27T18:17:28.383883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470993308332304:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:28.383921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:28.388652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470993308332285:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:28.351563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470988658858433:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.354204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470991679118000:2048] [node 1] ICS09 handshake done sender: [2:7577470991679117980:2111] self: [2:7577470991679117993:0] peer: [1:7577470993308332278:0] socket: 210 qp: -1
2025-11-27T18:17:28.388485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470991210109777:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.394672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470990743734866:2099] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.347777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470993380446855:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.391972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470993308332301:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29220
2025-11-27T18:17:28.392785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470989013364917:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.347983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470993380446853:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.348153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470993380446938:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52496
2025-11-27T18:17:28.357766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470993380446922:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:28.357853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470993380446930:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:28.360100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470993380446946:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52496
2025-11-27T18:17:28.354231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470991679118000:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:28.394705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:28.358286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470988658858431:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.388526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:28.354275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:28.358100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470991679117987:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:28.359847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470991679117982:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:28.361480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470991679117978:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.362987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:28.367863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470991679117978:2109] [node 5] ICP22 created new session: [2:7577470991679118005:2109]
2025-11-27T18:17:28.374121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470991679118005:2109] [node 5] ICS09 handshake done sender: [2:7577470991679117985:2113] self: [2:7577470991679117983:0] peer: [5:7577470993380446936:0] socket: 215 qp: -1
2025-11-27T18:17:28.381826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470991679118005:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:28.384877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:28.386617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470991679117979:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.388405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:28.388883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470991679117979:2110] [node 6] ICP22 created new session: [2:7577470991679118007:2110]
2025-11-27T18:17:28.390873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470991679118007:2110] [node 6] ICS09 handshake done sender: [2:7577470991679117987:2114] self: [2:7577470991679117986:0] peer: [6:7577470990743734894:0] socket: 216 qp: -1
2025-11-27T18:17:28.392641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470991679118007:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:28.392674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:28.392809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470987384150618:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.396367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:28.398722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470987384150618:2049] [node 3] ICP22 created new session: [2:7577470991679118009:2049]
2025-11-27T18:17:28.406801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577470991679118004:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22867
2025-11-27T18:17:28.410806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470991679118009:2049] [node 3] ICS09 handshake done sender: [2:7577470991679117982:2112] self: [2:7577470991679117981:0] peer: [3:7577470991210109802:0] socket: 214 qp: -1
2025-11-27T18:17:28.410903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470991679117979:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(19509422ns) {HandleHandshakeStatus:329(19506068ns) {DropIncomingHandshake:442(150ns) DropOutgoingHandshake:457(255ns) SwitchToState:179(263ns) ProcessPendingSessionEvents:495(7392112ns) {ForwardSessionEventToSession:612(7323754ns) ForwardSessionEventToSession:612(8628ns) ForwardSessionEventToSession:612(11615ns) ForwardSessionEventToSession:612(6127ns) ForwardSessionEventToSession:612(9358ns) ForwardSessionEventToSession:612(2002ns) ForwardSessionEventToSession:612(5497ns) ForwardSessionEventToSession:612(2757ns) ForwardSessionEventToSession:612(3637ns) ForwardSessionEventToSession:612(5915ns)}}}
2025-11-27T18:17:28.416189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470991679118009:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:28.416249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:28.429673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470991679117996:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.430441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470991679117978:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(35005712ns) {HandleHandshakeStatus:329(34997473ns) {DropIncomingHandshake:442(402ns) DropOutgoingHandshake:457(1755ns) SwitchToState:179(675ns) ProcessPendingSessionEvents:495(5594809ns) {ForwardSessionEventToSession:612(1811142ns) ForwardSessionEventToSession:612(3681ns) ForwardSessionEventToSession:612(721414ns) ForwardSessionEventToSession:612(3555ns) ForwardSessionEventToSession:612(2697ns) ForwardSessionEventToSession:612(935869ns) ForwardSessionEventToSession:612(2936ns) ForwardSessionEventToSession:612(2087246ns) ForwardSessionEventToSession:612(5397ns) ForwardSessionEventToSession:612(2386ns)}}}
2025-11-27T18:17:28.430483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577470991679117996:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:28.453423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577470991679117996:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:28.458894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470987384150618:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(40438705ns) {HandleHandshakeStatus:329(40436893ns) {DropIncomingHandshake:442(794ns) DropOutgoingHandshake:457(332ns) SwitchToState:179(280ns) ProcessPendingSessionEvents:495(1629566ns) {ForwardSessionEventToSession:612(1364852ns) ForwardSessionEventToSession:612(3936ns) ForwardSessionEventToSession:612(2504ns) ForwardSessionEventToSession:612(5951ns) ForwardSessionEventToSession:612(6585ns) ForwardSessionEventToSession:612(3149ns) ForwardSessionEventToSession:612(4839ns) ForwardSessionEventToSession:612(2765ns) ForwardSessionEventToSession:612(220250ns) ForwardSessionEventToSession:612(2572ns)}}}
2025-11-27T18:17:28.465604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577470991679118004:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.370725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470993380446854:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.390808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470991210109780:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.393451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:28.358330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:28.358379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470988658858431:2048] [node 1] ICP22 created new session: [4:7577470992953825817:2048]
2025-11-27T18:17:28.358416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470992953825817:2048] [node 1] ICS09 handshake done sender: [4:7577470992953825790:2110] self: [4:7577470992953825811:0] peer: [1:7577470993308332280:0] socket: 206 qp: -1
2025-11-27T18:17:28.358442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470992953825817:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:28.358473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:28.373734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470992953825820:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10570
2025-11-27T18:17:28.387984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470992953825801:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:28.391989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470988658858433:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.392035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:28.392832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470988658858433:2050] [node 3] ICP22 created new session: [4:7577470992953825822:2050]
2025-11-27T18:17:28.393473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470992953825819:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10570
2025-11-27T18:17:28.394415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470992953825822:2050] [node 3] ICS09 handshake done sender: [4:7577470992953825801:2114] self: [4:7577470992953825816:0] peer: [3:7577470991210109785:0] socket: 233 qp: -1
2025-11-27T18:17:28.396795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470992953825822:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:28.396845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:28.414238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470992953825819:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.421437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470988658858432:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.431883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470992953825812:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.431906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470992953825812:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:28.439726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470988658858433:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(42231246ns) {HandleHandshakeStatus:329(42228918ns) {DropIncomingHandshake:442(1137ns) DropOutgoingHandshake:457(3927ns) SwitchToState:179(301ns) ProcessPendingSessionEvents:495(11447799ns) {ForwardSessionEventToSession:612(8821996ns) ForwardSessionEventToSession:612(813235ns) ForwardSessionEventToSession:612(3096ns) ForwardSessionEventToSession:612(2306ns) ForwardSessionEventToSession:612(22438ns) ForwardSessionEventToSession:612(1900ns) ForwardSessionEventToSession:612(2763ns) ForwardSessionEventToSession:612(7595ns) ForwardSessionEventToSession:612(1988ns) ForwardSessionEventToSession:612(1049683ns)}}}
2025-11-27T18:17:28.439810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577470988658858433:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:28.444452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470992953825812:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:28.481959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470992953825798:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.482026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:28.482083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470992953825798:2108] [node 6] ICP22 created new session: [4:7577470992953825827:2108]
2025-11-27T18:17:28.482122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470992953825827:2108] [node 6] ICS09 handshake done sender: [4:7577470992953825812:2118] self: [4:7577470992953825809:0] peer: [6:7577470990743734906:0] socket: 272 qp: -1
2025-11-27T18:17:28.482165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470992953825827:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:28.482199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:28.370800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:28.394749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470990743734866:2099] [node 5] ICP22 created new session: [6:7577470990743734899:2099]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.495310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577470987384150618:2049] [node 3] ICP18 handshake (actor [2:7577470991679118004:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.395679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470989013364917:2049] [node 3] ICP22 created new session: [1:7577470993308332307:2049]
2025-11-27T18:17:28.401511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470993308332307:2049] [node 3] ICS09 handshake done sender: [1:7577470993308332285:2112] self: [1:7577470993308332284:0] peer: [3:7577470991210109804:0] socket: 239 qp: -1
2025-11-27T18:17:28.401549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470993308332307:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:28.410745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:28.410939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470993308332290:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.410970Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470993308332290:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.394793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470990743734899:2099] [node 5] ICS09 handshake done sender: [6:7577470990743734890:2116] self: [6:7577470990743734893:0] peer: [5:7577470993380446921:0] socket: 232 qp: -1
2025-11-27T18:17:28.394817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470990743734899:2099] [node 5] ICS10 traffic start
2025-11-27T18:17:28.394842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:28.395496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470990743734812:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.398908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470990743734813:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.398948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:28.409455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470990743734813:2049] [node 2] ICP22 created new session: [6:7577470990743734902:2049]
2025-11-27T18:17:28.414345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470990743734902:2049] [node 2] ICS09 handshake done sender: [6:7577470990743734882:2113] self: [6:7577470990743734894:0] peer: [2:7577470991679117986:0] socket: 218 qp: -1
2025-11-27T18:17:28.416128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470990743734898:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21543
2025-11-27T18:17:28.417364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470990743734902:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:28.425553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:28.426033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470990743734813:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(28991794ns) {HandleHandshakeStatus:329(28989347ns) {DropIncomingHandshake:442(1874ns) DropOutgoingHandshake:457(3760ns) SwitchToState:179(369ns) ProcessPendingSessionEvents:495(93300ns) {ForwardSessionEventToSession:612(43414ns) ForwardSessionEventToSession:612(6863ns) ForwardSessionEventToSession:612(3268ns) ForwardSessionEventToSession:612(2843ns) ForwardSessionEventToSession:612(8489ns) ForwardSessionEventToSession:612(2417ns) ForwardSessionEventToSession:612(5470ns) ForwardSessionEventToSession:612(5299ns) ForwardSessionEventToSession:612(2032ns) ForwardSessionEventToSession:612(5612ns)}}}
2025-11-27T18:17:28.429811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470990743734898:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.431463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470990743734867:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.438952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470990743734905:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21543
2025-11-27T18:17:28.453947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470990743734895:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:28.454247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470990743734812:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.454311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:28.454351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470990743734812:2048] [node 1] ICP22 created new session: [6:7577470990743734908:2048]
2025-11-27T18:17:28.454400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470990743734908:2048] [node 1] ICS09 handshake done sender: [6:7577470990743734895:2118] self: [6:7577470990743734901:0] peer: [1:7577470993308332288:0] socket: 260 qp: -1
2025-11-27T18:17:28.454434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470990743734908:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:28.454468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:28.495967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470990743734910:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21543
2025-11-27T18:17:28.498157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470990743734898:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:28.509851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470990743734867:2097] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.509934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:28.509995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470990743734867:2097] [node 4] ICP22 created new session: [6:7577470990743734911:2097]
2025-11-27T18:17:28.417613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470993308332301:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.371231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470993380446854:2049] [node 2] ICP22 created new session: [5:7577470993380446947:2049]
2025-11-27T18:17:28.510268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470991679117977:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.391608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470991210109777:2107] [node 4] ICP22 created new session: [3:7577470991210109808:2107]
2025-11-27T18:17:28.384763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470993380446920:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.527917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470992953825826:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10570
2025-11-27T18:17:28.392141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470991210109808:2107] [node 4] ICS09 handshake done sender: [3:7577470991210109786:2112] self: [3:7577470991210109785:0] peer: [4:7577470992953825816:0] socket: 227 qp: -1
2025-11-27T18:17:28.432050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470993308332290:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:28.533884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470990743734911:2097] [node 4] ICS09 handshake done sender: [6:7577470990743734898:2121] self: [6:7577470990743734906:0] peer: [4:7577470992953825809:0] socket: 273 qp: -1
2025-11-27T18:17:28.533889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:28.533804Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470988658858432:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470992953825819:2120] held: no
2025-11-27T18:17:28.535625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470990743734911:2097] [node 4] ICS10 traffic start
2025-11-27T18:17:28.392235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.449556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470993308332277:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.384841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:28.539535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470992953825819:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:28.533937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470991679117977:2107] [node 4] ICP22 created new session: [2:7577470991679118012:2107]
2025-11-27T18:17:28.392847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470991210109808:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:28.449626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:28.385002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470993380446947:2049] [node 2] ICS09 handshake done sender: [5:7577470993380446930:2115] self: [5:7577470993380446936:0] peer: [2:7577470991679117983:0] socket: 228 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:28.538423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577470991679118012:2107] [node 4] ICS09 handshake done sender: [2:7577470991679117996:2118] self: [2:7577470991679117994:0] peer: [4:7577470992953825824:0] socket: 263 qp: -1
2025-11-27T18:17:28.535683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:28.545491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470988658858432:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.393375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470991210109780:2108] [node 5] ICP22 created new session: [3:7577470991210109810:2108]
2025-11-27T18:17:28.385039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470993380446947:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:28.449678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470993308332277:2109] [node 6] ICP22 created new session: [1:7577470993308332310:2109]
2025-11-27T18:17:28.538956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577470991679118012:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:28.540984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470990743734907:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21543
2025-11-27T18:17:28.545537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:28.394362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:28.385128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470993380446920:2109] [node 6] ICP22 created new session: [5:7577470993380446948:2109]
2025-11-27T18:17:28.539588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:28.545580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470988658858432:2049] [node 2] ICP22 created new session: [4:7577470992953825829:2049]
2025-11-27T18:17:28.449729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470993308332310:2109] [node 6] ICS09 handshake done sender: [1:7577470993308332290:2114] self: [1:7577470993308332288:0] peer: [6:7577470990743734901:0] socket: 258 qp: -1
2025-11-27T18:17:28.541850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470990743734867:2097] [node 4] ICP35 event processing took too much time PendingConnection:279(31456509ns) {HandleHandshakeStatus:329(31453415ns) {DropIncomingHandshake:442(1204ns) DropOutgoingHandshake:457(7762ns) SwitchToState:179(536ns) ProcessPendingSessionEvents:495(2452750ns) {ForwardSessionEventToSession:612(2369914ns) ForwardSessionEventToSession:612(5365ns) ForwardSessionEventToSession:612(23085ns) ForwardSessionEventToSession:612(5673ns) ForwardSessionEventToSession:612(2647ns) ForwardSessionEventToSession:612(19905ns) ForwardSessionEventToSession:612(2526ns) ForwardSessionEventToSession:612(2075ns) ForwardSessionEventToSession:612(4796ns) ForwardSessionEventToSession:612(2330ns)}}}
2025-11-27T18:17:28.386305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:28.545047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470991679117977:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(51148230ns) {HandleHandshakeStatus:329(51143660ns) {DropIncomingHandshake:442(28177ns) DropOutgoingHandshake:457(366ns) SwitchToState:179(379ns) ProcessPendingSessionEvents:495(88304ns) {ForwardSessionEventToSession:612(40720ns) ForwardSessionEventToSession:612(2527ns) ForwardSessionEventToSession:612(8187ns) ForwardSessionEventToSession:612(4628ns) ForwardSessionEventToSession:612(5085ns) ForwardSessionEventToSession:612(2447ns) ForwardSessionEventToSession:612(2616ns) ForwardSessionEventToSession:612(10249ns) ForwardSessionEventToSession:612(2575ns) ForwardSessionEventToSession:612(1850ns)}}}
2025-11-27T18:17:28.545625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470992953825829:2049] [node 2] ICS09 handshake done sender: [4:7577470992953825819:2120] self: [4:7577470992953825824:0] peer: [2:7577470991679117994:0] socket: 268 qp: -1
2025-11-27T18:17:28.545656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470992953825829:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:28.545690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:28.394569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470991210109810:2108] [node 5] ICS09 handshake done sender: [3:7577470991210109784:2111] self: [3:7577470991210109783:0] peer: [5:7577470993380446941:0] socket: 226 qp: -1
2025-11-27T18:17:28.449762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470993308332310:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:28.546042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577470990743734907:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.396828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470991210109810:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:28.546537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470990743734814:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:28.548776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577470990743734914:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21543
2025-11-27T18:17:28.550430Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577470990743734907:2123] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:28.567003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470990743734814:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.597344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:28.597419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470990743734814:2050] [node 3] ICP22 created new session: [6:7577470990743734915:2050]
2025-11-27T18:17:28.597481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577470990743734915:2050] [node 3] ICS09 handshake done sender: [6:7577470990743734907:2123] self: [6:7577470990743734913:0] peer: [3:7577470991210109795:0] socket: 236 qp: -1
2025-11-27T18:17:28.597522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577470990743734915:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:28.597587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:28.387297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470993380446854:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16420308ns) {HandleHandshakeStatus:329(16418554ns) {DropIncomingHandshake:442(816ns) DropOutgoingHandshake:457(4930ns) SwitchToState:179(672ns) ProcessPendingSessionEvents:495(128189ns) {ForwardSessionEventToSession:612(83218ns) ForwardSessionEventToSession:612(8431ns) ForwardSessionEventToSession:612(4980ns) ForwardSessionEventToSession:612(2174ns) ForwardSessionEventToSession:612(2087ns) ForwardSessionEventToSession:612(4693ns) ForwardSessionEventToSession:612(3318ns) ForwardSessionEventToSession:612(5139ns) ForwardSessionEventToSession:612(2187ns) ForwardSessionEventToSession:612(4270ns)}}}
2025-11-27T18:17:28.387732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470993380446945:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52496
2025-11-27T18:17:28.387979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470993380446931:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:28.388667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470993380446948:2109] [node 6] ICS09 handshake done sender: [5:7577470993380446922:2112] self: [5:7577470993380446921:0] peer: [6:7577470990743734893:0] socket: 229 qp: -1
2025-11-27T18:17:28.390424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470993380446948:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:28.390504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470993380446937:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:28.390887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:28.394396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470993380446855:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.394437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:28.396669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470993380446855:2050] [node 3] ICP22 created new session: [5:7577470993380446951:2050]
2025-11-27T18:17:28.396962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470993380446853:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.396997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:28.397034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470993380446853:2048] [node 1] ICP22 created new session: [5:7577470993380446952:2048]
2025-11-27T18:17:28.397080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470993380446952:2048] [node 1] ICS09 handshake done sender: [5:7577470993380446937:2119] self: [5:7577470993380446943:0] peer: [1:7577470993308332286:0] socket: 252 qp: -1
2025-11-27T18:17:28.397106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470993380446952:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:28.397150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:28.399125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470993380446951:2050] [node 3] ICS09 handshake done sender: [5:7577470993380446931:2116] self: [5:7577470993380446941:0] peer: [3:7577470991210109783:0] socket: 237 qp: -1
2025-11-27T18:17:28.402331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470993380446951:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:28.409373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:28.459054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470993380446920:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(36251001ns) {HandleHandshakeStatus:329(36246614ns) {DropIncomingHandshake:442(1176ns) DropOutgoingHandshake:457(1065ns) SwitchToState:179(268ns) ProcessPendingSessionEvents:495(21968212ns) {ForwardSessionEventToSession:612(3155608ns) ForwardSessionEventToSession:612(6605809ns) ForwardSessionEventToSession:612(2709ns) ForwardSessionEventToSession:612(5364822ns) ForwardSessionEventToSession:612(13354ns) ForwardSessionEventToSession:612(3880ns) ForwardSessionEventToSession:612(5424312ns) ForwardSessionEventToSession:612(4349ns) ForwardSessionEventToSession:612(2496ns) ForwardSessionEventToSession:612(1372114ns)}}}
2025-11-27T18:17:28.505584Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470993380446855:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(43440039ns) {HandleHandshakeStatus:329(43437386ns) {DropIncomingHandshake:442(414ns) DropOutgoingHandshake:457(4867ns) SwitchToState:179(296ns) ProcessPendingSessionEvents:495(20269403ns) {ForwardSessionEventToSession:612(12373691ns) ForwardSessionEventToSession:612(655445ns) ForwardSessionEventToSession:612(2696ns) ForwardSessionEventToSession:612(651392ns) ForwardSessionEventToSession:612(2704ns) ForwardSessionEventToSession:612(2819ns) ForwardSessionEventToSession:612(721675ns) ForwardSessionEventToSession:612(3290ns) ForwardSessionEventToSession:612(1076508ns) ForwardSessionEventToSession:612(4763323ns)}}}
2025-11-27T18:17:28.449798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:28.396857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:28.416309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470991210109806:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
2025-11-27T18:17:28.430679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577470991210109809:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
2025-11-27T18:17:28.437842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470991210109777:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(34415616ns) {HandleHandshakeStatus:329(34413801ns) {DropIncomingHandshake:442(12697ns) DropOutgoingHandshake:457(283ns) SwitchToState:179(223ns) ProcessPendingSessionEvents:495(15722628ns) {ForwardSessionEventToSession:612(6932229ns) ForwardSessionEventToSession:612(4509ns) ForwardSessionEventToSession:612(3181ns) ForwardSessionEventToSession:612(3273312ns) ForwardSessionEventToSession:612(4608ns) ForwardSessionEventToSession:612(3279ns) ForwardSessionEventToSession:612(4284523ns) ForwardSessionEventToSession:612(6468ns) ForwardSessionEventToSession:612(1188160ns) ForwardSessionEventToSession:612(4764ns)}}}
2025-11-27T18:17:28.438191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470991210109780:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(40602081ns) {HandleHandshakeStatus:329(40599897ns) {DropIncomingHandshake:442(775490ns) DropOutgoingHandshake:457(1060ns) SwitchToState:179(863ns) ProcessPendingSessionEvents:495(11359466ns) {ForwardSessionEventToSession:612(10144082ns) ForwardSessionEventToSession:612(9455ns) ForwardSessionEventToSession:612(2216ns) ForwardSessionEventToSession:612(6585ns) ForwardSessionEventToSession:612(683540ns) ForwardSessionEventToSession:612(7482ns) ForwardSessionEventToSession:612(3310ns) ForwardSessionEventToSession:612(2970ns) ForwardSessionEventToSession:612(475784ns) ForwardSessionEventToSession:612(2779ns)}}}
2025-11-27T18:17:28.438839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470991210109787:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:28.453540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470991210109798:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:28.473861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470986915142421:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.473922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:28.473967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470986915142421:2049] [node 2] ICP22 created new session: [3:7577470991210109815:2049]
2025-11-27T18:17:28.474016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470991210109815:2049] [node 2] ICS09 handshake done sender: [3:7577470991210109787:2113] self: [3:7577470991210109802:0] peer: [2:7577470991679117981:0] socket: 224 qp: -1
2025-11-27T18:17:28.474053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470991210109815:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:28.474090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:28.496343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470986915142420:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:28.505674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:28.513449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470986915142420:2048] [node 1] ICP22 created new session: [3:7577470991210109817:2048]
2025-11-27T18:17:28.529369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470991210109817:2048] [node 1] ICS09 handshake done sender: [3:7577470991210109798:2119] self: [3:7577470991210109804:0] peer: [1:7577470993308332284:0] socket: 246 qp: -1
2025-11-27T18:17:28.533994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470991210109817:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:28.536541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:28.546844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577470991210109797:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:28.546863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577470991210109797:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:28.548988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577470991210109797:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:28.563554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470991210109781:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:28.563615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:28.594852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470991210109781:2109] [node 6] ICP22 created new session: [3:7577470991210109820:2109]
2025-11-27T18:17:28.633820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577470991210109820:2109] [node 6] ICS09 handshake done sender: [3:7577470991210109797:2118] self: [3:7577470991210109795:0] peer: [6:7577470990743734913:0] socket: 230 qp: -1
2025-11-27T18:17:28.633906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577470991210109820:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:28.633977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470986915142420:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(53052563ns) {HandleHandshakeStatus:329(53050230ns) {DropIncomingHandshake:442(1078ns) DropOutgoingHandshake:457(3783ns) SwitchToState:179(368ns) ProcessPendingSessionEvents:495(10318295ns) {ForwardSessionEventToSession:612(5158598ns) ForwardSessionEventToSession:612(506728ns) ForwardSessionEventToSession:612(2900ns) ForwardSessionEventToSession:612(1179423ns) ForwardSessionEventToSession:612(3673ns) ForwardSessionEventToSession:612(2370ns) ForwardSessionEventToSession:612(2095880ns) ForwardSessionEventToSession:612(4057ns) ForwardSessionEventToSession:612(3245ns) ForwardSessionEventToSession:612(1347025ns)}}}
2025-11-27T18:17:28.641524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:28.682839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470991210109781:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(118129378ns) {HandleHandshakeStatus:329(118126497ns) {DropIncomingHandshake:442(15371ns) DropOutgoingHandshake:457(386ns) SwitchToState:179(450ns) ProcessPendingSessionEvents:495(154956ns) {ForwardSessionEventToSession:612(94624ns) ForwardSessionEventToSession:612(7503ns) ForwardSessionEventToSession:612(3034ns) ForwardSessionEventToSession:612(14500ns) ForwardSessionEventToSession:612(4859ns) ForwardSessionEventToSession:612(3187ns) ForwardSessionEventToSession:612(7723ns) ForwardSessionEventToSession:612(2536ns) ForwardSessionEventToSession:612(2026ns) ForwardSessionEventToSession:612(5424ns)}}}
2025-11-27T18:17:28.483729Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470989013364917:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(48660693ns) {HandleHandshakeStatus:329(48658710ns) {DropIncomingHandshake:442(1146ns) DropOutgoingHandshake:457(1264ns) SwitchToState:179(204ns) ProcessPendingSessionEvents:495(11844549ns) {ForwardSessionEventToSession:612(2302075ns) ForwardSessionEventToSession:612(4767ns) ForwardSessionEventToSession:612(1058390ns) ForwardSessionEventToSession:612(3754ns) ForwardSessionEventToSession:612(3083ns) ForwardSessionEventToSession:612(4717993ns) ForwardSessionEventToSession:612(3106ns) ForwardSessionEventToSession:612(3726980ns) ForwardSessionEventToSession:612(3632ns) ForwardSessionEventToSession:612(2429ns)}}}
2025-11-27T18:17:28.490349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470993308332276:2108] [node 5] ICP18 handshake (actor [1:7577470993308332301:2117]) is too old
2025-11-27T18:17:28.521607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470993308332312:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29220
2025-11-27T18:17:28.537298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577470989013364917:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:28.684755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470990743734814:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(115049008ns) {HandleHandshakeStatus:329(115044917ns) {DropIncomingHandshake:442(1179ns) DropOutgoingHandshake:457(5540ns) SwitchToState:179(490ns) ProcessPendingSessionEvents:495(44214594ns) {ForwardSessionEventToSession:612(15718797ns) ForwardSessionEventToSession:612(9618502ns) ForwardSessionEventToSession:612(5279ns) ForwardSessionEventToSession:612(2233ns) ForwardSessionEventToSession:612(6733108ns) ForwardSessionEventToSession:612(3713101ns) ForwardSessionEventToSession:612(3873343ns) ForwardSessionEventToSession:612(4175ns) ForwardSessionEventToSession:612(2996ns) ForwardSessionEventToSession:612(4513872ns)}}}
2025-11-27T18:17:28.725586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:28.725806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577470991679118000:2048] [node 1] ICS01 socket: 210 reason# EndOfStream
2025-11-27T18:17:28.725850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470987384150617:2048] [node 1] ICP30 unregister session Session# [2:7577470991679118000:2048] VirtualId# [2:7577470991679117993:0]
2025-11-27T18:17:28.725883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577470991679118000:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.725964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:28.726008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:28.730809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:28.731020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577470991210109817:2048] [node 1] ICS07 socket disconnect 246 reason# EndOfStream
2025-11-27T18:17:28.731046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470991210109817:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.731122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:28.731162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470991210109817:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:28.738152Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470986915142420:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9330) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:28.738204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577470991210109817:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:28.739120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470986915142420:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9330) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:28.739174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470991210109817:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:28.739201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470986915142420:2048] [node 1] ICP30 unregister session Session# [3:7577470991210109817:2048] VirtualId# [3:7577470991210109804:0]
2025-11-27T18:17:28.739245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:28.739309Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470986915142420:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:28.739330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:28.743074Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:28.743287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470993380446952:2048] [node 1] ICS01 socket: 252 reason# EndOfStream
2025-11-27T18:17:28.743327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470993380446853:2048] [node 1] ICP30 unregister session Session# [5:7577470993380446952:2048] VirtualId# [5:7577470993380446943:0]
2025-11-27T18:17:28.743356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470993380446952:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.743430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:28.743494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:28.744267Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:28.744453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470990743734908:2048] [node 1] ICS07 socket disconnect 260 reason# EndOfStream
2025-11-27T18:17:28.744491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470990743734908:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.744554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:28.744590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470990743734908:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:28.749373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470986915142420:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:28.745432Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470990743734812:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9330) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:28.753722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:28.751274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470990743734908:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:28.751312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470990743734812:2048] [node 1] ICP30 unregister session Session# [6:7577470990743734908:2048] VirtualId# [6:7577470990743734901:0]
2025-11-27T18:17:28.751373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:28.751407Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470990743734812:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:28.751429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:28.761571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470990743734812:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:28.754507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470992953825817:2048] [node 1] ICS01 socket: 206 reason# EndOfStream
2025-11-27T18:17:28.754554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470988658858431:2048] [node 1] ICP30 unregister session Session# [4:7577470992953825817:2048] VirtualId# [4:7577470992953825811:0]
2025-11-27T18:17:28.754589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470992953825817:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.754689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:28.754744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:28.835000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:28.835234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577470991210109815:2049] [node 2] ICS01 socket: 224 reason# EndOfStream
2025-11-27T18:17:28.835277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470986915142421:2049] [node 2] ICP30 unregister session Session# [3:7577470991210109815:2049] VirtualId# [3:7577470991210109802:0]
2025-11-27T18:17:28.835307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577470991210109815:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.835384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:28.835448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:28.838736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:28.838964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470992953825829:2049] [node 2] ICS01 socket: 268 reason# EndOfStream
2025-11-27T18:17:28.839002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470988658858432:2049] [node 2] ICP30 unregister session Session# [4:7577470992953825829:2049] VirtualId# [4:7577470992953825824:0]
2025-11-27T18:17:28.839039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470992953825829:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.839120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:28.839160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:28.840612Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:28.840837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470990743734902:2049] [node 2] ICS01 socket: 218 reason# EndOfStream
2025-11-27T18:17:28.840880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470990743734813:2049] [node 2] ICP30 unregister session Session# [6:7577470990743734902:2049] VirtualId# [6:7577470990743734894:0]
2025-11-27T18:17:28.840904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470990743734902:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.840965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:28.841000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:28.841391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:28.841568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470993380446947:2049] [node 2] ICS01 socket: 228 reason# EndOfStream
2025-11-27T18:17:28.841615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470993380446854:2049] [node 2] ICP30 unregister session Session# [5:7577470993380446947:2049] VirtualId# [5:7577470993380446936:0]
2025-11-27T18:17:28.841659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470993380446947:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.841739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:28.841793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:28.925439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:28.925439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:28.925657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577470990743734915:2050] [node 3] ICS07 socket disconnect 236 reason# EndOfStream
2025-11-27T18:17:28.925439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:28.925663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470992953825822:2050] [node 3] ICS07 socket disconnect 233 reason# EndOfStream
2025-11-27T18:17:28.925695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470992953825822:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.925777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:28.925820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470992953825822:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:28.925688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470990743734915:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.925776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:28.925815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577470990743734915:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:28.927315Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470990743734814:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25912) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:28.926936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577470993380446951:2050] [node 3] ICS07 socket disconnect 237 reason# EndOfStream
2025-11-27T18:17:28.945508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470988658858433:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25912) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:28.945567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470992953825822:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:28.945606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470988658858433:2050] [node 3] ICP30 unregister session Session# [4:7577470992953825822:2050] VirtualId# [4:7577470992953825816:0]
2025-11-27T18:17:28.945655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:28.945682Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470988658858433:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:28.945699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:28.926974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470993380446951:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:28.933625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470990743734915:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:28.941712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470990743734814:2050] [node 3] ICP30 unregister session Session# [6:7577470990743734915:2050] VirtualId# [6:7577470990743734913:0]
2025-11-27T18:17:28.941793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:28.941836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577470990743734814:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:28.941855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:28.953103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577470990743734814:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:28.927044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:28.955707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470988658858433:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:28.927078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577470993380446951:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:28.937859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470993380446855:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25912) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:28.937913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470993380446951:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:28.937949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470993380446855:2050] [node 3] ICP30 unregister session Session# [5:7577470993380446951:2050] VirtualId# [5:7577470993380446941:0]
2025-11-27T18:17:28.937996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:28.938024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577470993380446855:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:28.938039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:28.947010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577470993380446855:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:29.009450Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:29.009730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470990743734911:2097] [node 4] ICS01 socket: 273 reason# EndOfStream
2025-11-27T18:17:29.009791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470990743734867:2097] [node 4] ICP30 unregister session Session# [6:7577470990743734911:2097] VirtualId# [6:7577470990743734906:0]
2025-11-27T18:17:29.009841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470990743734911:2097] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:29.009939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:29.009992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:29.049424Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:29.049675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577470990743734899:2099] [node 5] ICS01 socket: 232 reason# EndOfStream
2025-11-27T18:17:29.049718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470990743734866:2099] [node 5] ICP30 unregister session Session# [6:7577470990743734899:2099] VirtualId# [6:7577470990743734893:0]
2025-11-27T18:17:29.049753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577470990743734899:2099] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:29.049858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:29.049903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:29.148373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470995420920632:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:29.148411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470995420920633:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:29.182550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470997058733630:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:29.182594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470997058733631:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:29.207248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470997091331424:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:29.207290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470997091331425:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:29.234233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470995041949356:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:29.234270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470995041949357:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:29.234293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470995041949358:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:29.257330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470995072600838:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:29.257369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470995072600839:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:29.257394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470995072600840:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:29.378101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470999102183315:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:29.378190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470999102183317:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:29.383035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470999102183316:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:29.845838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470995420920695:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:29.884415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470997058733692:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:29.910082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470995041949418:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:29.867469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470999102183352:2084] [node 5] ICP01 ready to work
2025-11-27T18:17:29.962310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470997058733694:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:29.893533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577470999102183360:2086] [node 4] ICP01 ready to work
2025-11-27T18:17:30.015101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470999102183317:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(190093440ns) {Configure:104(190087519ns) {ProcessConfigured:135(76763ns) {StartInitialHandshake:157(73921ns) {DropHandshakes:470(2670ns) {DropIncomingHandshake:442(1006ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(971ns) PrepareNewSessionHandshake:477(1721ns) {SwitchToState:179(597ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:29.934088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470995420920699:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:30.010479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470995072600901:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:30.025386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003397150676:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25360
2025-11-27T18:17:30.042066Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470999102183360:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(58159552ns) {Configure:104(58147790ns) {ProcessConfigured:135(70774ns) {StartInitialHandshake:157(68379ns) {DropHandshakes:470(2616ns) {DropIncomingHandshake:442(661ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(1222ns) PrepareNewSessionHandshake:477(2433ns) {SwitchToState:179(812ns)}}}}}
2025-11-27T18:17:30.016198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470995041949418:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(105794264ns) {Configure:104(105789353ns) {ProcessConfigured:135(15029758ns) {StartInitialHandshake:157(15025690ns) {DropHandshakes:470(2901ns) {DropIncomingHandshake:442(736ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(1016ns) PrepareNewSessionHandshake:477(5247ns) {SwitchToState:179(2609ns)}}}}}
2025-11-27T18:17:30.023604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577470997058733695:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:30.019504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577470997091331487:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:30.001597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470995420920700:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:30.010405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577470995072600902:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:30.042992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577470999336916720:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:30.051170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470997058733692:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(153617858ns) {Configure:104(153611850ns) {ProcessConfigured:135(74101ns) {StartInitialHandshake:157(70882ns) {DropHandshakes:470(3036ns) {DropIncomingHandshake:442(525ns) DropOutgoingHandshake:457(722ns)} GenerateSessionVirtualId:988(1099ns) PrepareNewSessionHandshake:477(2241ns) {SwitchToState:179(1089ns)}}}}}
2025-11-27T18:17:30.029951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471001386298784:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:30.043192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471001386298785:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.098869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471003397150676:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.105518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470999102183352:2084] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(211716603ns) {Configure:104(190988638ns) {ProcessConfigured:135(115569ns) {StartInitialHandshake:157(100133ns) {DropHandshakes:470(3897ns) {DropIncomingHandshake:442(994ns) DropOutgoingHandshake:457(515ns)} GenerateSessionVirtualId:988(843ns) PrepareNewSessionHandshake:477(1910ns) {SwitchToState:179(628ns)}}}}}
2025-11-27T18:17:30.106341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471001386298789:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42245
2025-11-27T18:17:30.099376Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470995041949357:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(77380492ns) {Configure:104(77371110ns) {ProcessConfigured:135(77867ns) {StartInitialHandshake:157(74255ns) {DropHandshakes:470(2873ns) {DropIncomingHandshake:442(758ns) DropOutgoingHandshake:457(524ns)} GenerateSessionVirtualId:988(1081ns) PrepareNewSessionHandshake:477(2257ns) {SwitchToState:179(665ns)}}}}}
2025-11-27T18:17:30.117982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470999102183316:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(162817811ns) {Configure:104(162812649ns) {ProcessConfigured:135(88524ns) {StartInitialHandshake:157(85373ns) {DropHandshakes:470(3415ns) {DropIncomingHandshake:442(832ns) DropOutgoingHandshake:457(641ns)} GenerateSessionVirtualId:988(543ns) PrepareNewSessionHandshake:477(1870ns) {SwitchToState:179(752ns)}}}}}
2025-11-27T18:17:30.111365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470995420920632:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(252910525ns) {Configure:104(252900967ns) {ProcessConfigured:135(90721ns) {StartInitialHandshake:157(88623ns) {DropHandshakes:470(3923ns) {DropIncomingHandshake:442(916ns) DropOutgoingHandshake:457(446ns)} GenerateSessionVirtualId:988(1158ns) PrepareNewSessionHandshake:477(1880ns) {SwitchToState:179(679ns)}}}}}
2025-11-27T18:17:30.107614Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470995041949358:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(142738681ns) {Configure:104(142734403ns) {ProcessConfigured:135(37406700ns) {StartInitialHandshake:157(37403390ns) {DropHandshakes:470(3369ns) {DropIncomingHandshake:442(752ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(942ns) PrepareNewSessionHandshake:477(7766ns) {SwitchToState:179(4075ns)}}}}}
2025-11-27T18:17:30.123718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471001386298789:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.139501Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577470999102183315:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(205111667ns) {Configure:104(205103451ns) {ProcessConfigured:135(5621247ns) {StartInitialHandshake:157(5618160ns) {DropHandshakes:470(3592ns) {DropIncomingHandshake:442(798ns) DropOutgoingHandshake:457(377ns)} GenerateSessionVirtualId:988(1572ns) PrepareNewSessionHandshake:477(3315ns) {SwitchToState:179(1626ns)}}}}}
2025-11-27T18:17:30.139754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470999102183315:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.146017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470999336916726:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.140737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471001386298784:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(88122766ns) {Configure:104(88114690ns) {ProcessConfigured:135(103489ns) {StartInitialHandshake:157(95340ns) {DropHandshakes:470(3925ns) {DropIncomingHandshake:442(934ns) DropOutgoingHandshake:457(522ns)} GenerateSessionVirtualId:988(1013ns) PrepareNewSessionHandshake:477(2644ns) {SwitchToState:179(913ns)}}}}}
2025-11-27T18:17:30.139401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470997058733630:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(281316456ns) {Configure:104(281310817ns) {ProcessConfigured:135(19009780ns) {StartInitialHandshake:157(13947726ns) {DropHandshakes:470(5533ns) {DropIncomingHandshake:442(1233ns) DropOutgoingHandshake:457(474ns)} GenerateSessionVirtualId:988(2856ns) PrepareNewSessionHandshake:477(4531ns) {SwitchToState:179(2515ns)}}}}}
2025-11-27T18:17:30.146206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470999336916722:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.159133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471001386298795:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42245
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.157623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470995072600838:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(219522452ns) {Configure:104(219517244ns) {ProcessConfigured:135(72131ns) {StartInitialHandshake:157(70131ns) {DropHandshakes:470(2637ns) {DropIncomingHandshake:442(582ns) DropOutgoingHandshake:457(579ns)} GenerateSessionVirtualId:988(990ns) PrepareNewSessionHandshake:477(2281ns) {SwitchToState:179(895ns)}}}}}
2025-11-27T18:17:30.151190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470995420920697:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.151259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470995420920697:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:30.146292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470999336916725:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-11-27T18:17:30.158199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470999336916725:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.171580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470999336916726:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.166697Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470995420920633:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(272789696ns) {Configure:104(272784618ns) {ProcessConfigured:135(8474328ns) {StartInitialHandshake:157(8471581ns) {DropHandshakes:470(723ns) {DropIncomingHandshake:442(205ns) DropOutgoingHandshake:457(192ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:477(3839ns) {SwitchToState:179(1467ns)}}}}}
2025-11-27T18:17:30.170542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470995420920697:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.160295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471001353700999:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24095
2025-11-27T18:17:30.165949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470997091331425:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(164362097ns) {Configure:104(152396044ns) {ProcessConfigured:135(11742771ns) {StartInitialHandshake:157(7814987ns) {DropHandshakes:470(2180ns) {DropIncomingHandshake:442(251ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(1326ns) PrepareNewSessionHandshake:477(4615ns) {SwitchToState:179(2243ns)}}}}}
2025-11-27T18:17:30.174619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577470997091331424:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(172615885ns) {Configure:104(172608565ns) {ProcessConfigured:135(5942364ns) {StartInitialHandshake:157(5938897ns) {DropHandshakes:470(2583ns) {DropIncomingHandshake:442(660ns) DropOutgoingHandshake:457(396ns)} GenerateSessionVirtualId:988(1414ns) PrepareNewSessionHandshake:477(3856ns) {SwitchToState:179(1926ns)}}}}}
2025-11-27T18:17:30.175282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471001386298798:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42245
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.177809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470995420920699:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(136214436ns) {Configure:104(136207748ns) {ProcessConfigured:135(3202578ns) {StartInitialHandshake:157(3200262ns) {DropHandshakes:470(2489ns) {DropIncomingHandshake:442(596ns) DropOutgoingHandshake:457(404ns)} GenerateSessionVirtualId:988(890ns) PrepareNewSessionHandshake:477(3780ns) {SwitchToState:179(1123ns)}}}}}
2025-11-27T18:17:30.178710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470995041949357:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.160446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471001353701000:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24095
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.204826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471001386298798:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.205072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470997091331425:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.190033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003397150692:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25360
2025-11-27T18:17:30.175208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470997058733694:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(156299399ns) {Configure:104(156292055ns) {ProcessConfigured:135(14110292ns) {StartInitialHandshake:157(14108760ns) {DropHandshakes:470(3793ns) {DropIncomingHandshake:442(502ns) DropOutgoingHandshake:457(757ns)} GenerateSessionVirtualId:988(785ns) PrepareNewSessionHandshake:477(2210ns) {SwitchToState:179(1179ns)}}}}}
2025-11-27T18:17:30.193711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470995072600839:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(236743840ns) {Configure:104(236740545ns) {ProcessConfigured:135(4809084ns) {StartInitialHandshake:157(4807117ns) {DropHandshakes:470(2930ns) {DropIncomingHandshake:442(694ns) DropOutgoingHandshake:457(415ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(3135ns) {SwitchToState:179(1280ns)}}}}}
2025-11-27T18:17:30.210543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471001386298795:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.197886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470995420920695:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.199084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470995072600840:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(224371730ns) {Configure:104(224358525ns) {ProcessConfigured:135(12745054ns) {StartInitialHandshake:157(12744165ns) {DropHandshakes:470(2720ns) {DropIncomingHandshake:442(656ns) DropOutgoingHandshake:457(380ns)} GenerateSessionVirtualId:988(982ns) PrepareNewSessionHandshake:477(2588ns) {SwitchToState:179(472ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.206435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470995041949356:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(241244353ns) {Configure:104(241240203ns) {ProcessConfigured:135(23290220ns) {StartInitialHandshake:157(23288189ns) {DropHandshakes:470(2741ns) {DropIncomingHandshake:442(579ns) DropOutgoingHandshake:457(553ns)} GenerateSessionVirtualId:988(547ns) PrepareNewSessionHandshake:477(1820ns) {SwitchToState:179(915ns)}}}}}
2025-11-27T18:17:30.197946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.197997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470995420920695:2108] [node 6] ICP22 created new session: [1:7577470999715888008:2108]
2025-11-27T18:17:30.198045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470999715888008:2108] [node 6] ICS09 handshake done sender: [1:7577470995420920697:2110] self: [1:7577470995420920696:0] peer: [6:7577471003397150690:0] socket: 210 qp: -1
2025-11-27T18:17:30.198077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470999715888008:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:30.198114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.177557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471001353700999:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.235331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470999715888007:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29041
2025-11-27T18:17:30.205506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471001353700998:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.205532Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471001353700998:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:30.214798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470997058733631:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(242456376ns) {Configure:104(242450274ns) {ProcessConfigured:135(15993458ns) {StartInitialHandshake:157(15991097ns) {DropHandshakes:470(1283ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(1303ns) PrepareNewSessionHandshake:477(2542ns) {SwitchToState:179(1381ns)}}}}}
2025-11-27T18:17:30.220008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471001353701000:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.209817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470999336916727:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-11-27T18:17:30.231161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471001353700998:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:30.223179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470999336916727:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.234810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470999336916734:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.240774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471001353700993:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.213945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003397150691:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25360
2025-11-27T18:17:30.229735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471001386298805:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42245
2025-11-27T18:17:30.234693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470999367568202:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25070
2025-11-27T18:17:30.217829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471003397150676:2109] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:30.243323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577470995420920700:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(167990808ns) {Configure:104(167979446ns) {ProcessConfigured:135(35938894ns) {StartInitialHandshake:157(35935286ns) {DropHandshakes:470(4006ns) {DropIncomingHandshake:442(950ns) DropOutgoingHandshake:457(919ns)} GenerateSessionVirtualId:988(728ns) PrepareNewSessionHandshake:477(5748ns) {SwitchToState:179(2918ns)}}}}}
2025-11-27T18:17:30.237883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470995041949358:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.240829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471001353700993:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.235093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471001386298803:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42245
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.250079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470999336916734:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.218192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470999102183315:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.234886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470999367568203:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25070
2025-11-27T18:17:30.246741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470999715888007:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.241881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577470997058733630:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.235270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471001386298785:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(167460723ns) {Configure:104(167454216ns) {ProcessConfigured:135(32260168ns) {StartInitialHandshake:157(32257904ns) {DropHandshakes:470(1692ns) {DropIncomingHandshake:442(278ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(574ns) PrepareNewSessionHandshake:477(2704ns) {SwitchToState:179(724ns)}}}}}
2025-11-27T18:17:30.218264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:30.242739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471001353701012:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24095
2025-11-27T18:17:30.251000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577470995072600902:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(199441660ns) {Configure:104(199431719ns) {ProcessConfigured:135(30868486ns) {StartInitialHandshake:157(30866057ns) {DropHandshakes:470(2855ns) {DropIncomingHandshake:442(573ns) DropOutgoingHandshake:457(462ns)} GenerateSessionVirtualId:988(868ns) PrepareNewSessionHandshake:477(3536ns) {SwitchToState:179(1372ns)}}}}}
2025-11-27T18:17:30.269588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577470995041949356:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.253553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470999715887998:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.272248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577470999336916720:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(173792616ns) {Configure:104(167766093ns) {ProcessConfigured:135(23505057ns) {StartInitialHandshake:157(23501851ns) {DropHandshakes:470(5881264ns) {DropIncomingHandshake:442(2330ns) DropOutgoingHandshake:457(1271ns)} GenerateSessionVirtualId:988(1621ns) PrepareNewSessionHandshake:477(2572ns) {SwitchToState:179(1442ns)}}}}}
2025-11-27T18:17:30.237639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471001386298803:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.255963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470999367568203:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.218317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470999102183315:2048] [node 1] ICP22 created new session: [6:7577471003397150693:2048]
2025-11-27T18:17:30.253596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470999715887998:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.237760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577470997091331424:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.242871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471001353700993:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:30.261388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470999367568202:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.263243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470995072600901:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.244517Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577470997058733695:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(173804844ns) {Configure:104(173799815ns) {ProcessConfigured:135(28099765ns) {StartInitialHandshake:157(28086624ns) {DropHandshakes:470(2329ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(875ns) PrepareNewSessionHandshake:477(2877ns) {SwitchToState:179(1209ns)}}}}}
2025-11-27T18:17:30.309663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470999336916717:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.218364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471003397150693:2048] [node 1] ICS09 handshake done sender: [6:7577471003397150676:2109] self: [6:7577471003397150690:0] peer: [1:7577470995420920696:0] socket: 211 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.261777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470999715888004:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.309715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470999336916717:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.277818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470999367568216:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25070
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.245762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471001386298798:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:30.261813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470999715888004:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.218400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471003397150693:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:30.244705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470997058733631:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.286443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470999715888004:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:30.318864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470995041949357:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470999336916726:2115] held: no
2025-11-27T18:17:30.313360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470999367568216:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.218456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:30.244751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:30.256737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471001386298787:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.249484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470997058733631:2049] [node 3] ICP22 created new session: [2:7577471001353701014:2049]
2025-11-27T18:17:30.249545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471001353701014:2049] [node 3] ICS09 handshake done sender: [2:7577471001353700998:2112] self: [2:7577471001353700997:0] peer: [3:7577471001386298802:0] socket: 239 qp: -1
2025-11-27T18:17:30.249591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471001353701014:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:30.249639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:30.257586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470997058733692:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.257650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:30.257696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470997058733692:2108] [node 4] ICP22 created new session: [2:7577471001353701016:2108]
2025-11-27T18:17:30.257747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471001353701016:2108] [node 4] ICS09 handshake done sender: [2:7577471001353700993:2110] self: [2:7577471001353700992:0] peer: [4:7577470999336916731:0] socket: 218 qp: -1
2025-11-27T18:17:30.257786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471001353701016:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:30.257818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:30.275025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471001353701006:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24095
2025-11-27T18:17:30.295266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471001353701008:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24095
2025-11-27T18:17:30.309974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471001353701012:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.330355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471001353701008:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.352442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577470997058733631:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:30.366704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471001353701003:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.366760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471001353701003:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:30.382655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471001353701003:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:30.419686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470997058733694:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.419746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:30.419800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470997058733694:2107] [node 5] ICP22 created new session: [2:7577471001353701024:2107]
2025-11-27T18:17:30.419849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471001353701024:2107] [node 5] ICS09 handshake done sender: [2:7577471001353701003:2115] self: [2:7577471001353701002:0] peer: [5:7577470999367568217:0] socket: 235 qp: -1
2025-11-27T18:17:30.419878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471001353701024:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:30.419906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:30.420519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577470997058733630:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471001353701000:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.248734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471003397150691:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.325580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470995072600839:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.309716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470999715888001:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.256788Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471001386298787:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:30.319321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470999336916717:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:30.335674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470999367568218:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25070
2025-11-27T18:17:30.261783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470999102183317:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.269439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471001386298787:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:30.440981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471001353701021:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24095
2025-11-27T18:17:30.309765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470999715888001:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:30.321691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470995041949356:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19450949ns) {IncomingHandshake:261(19448922ns) {DropIncomingHandshake:442(508ns) PrepareNewSessionHandshake:477(3762ns) {SwitchToState:179(2326ns)} GenerateSessionVirtualId:988(875ns) IssueIncomingHandshakeReply:193(12702466ns) {SwitchToState:179(2023ns)}}}
2025-11-27T18:17:30.353563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470999367568219:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25070
2025-11-27T18:17:30.265663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470999102183315:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(47263641ns) {HandleHandshakeStatus:329(47260078ns) {DropIncomingHandshake:442(1245ns) DropOutgoingHandshake:457(12841ns) SwitchToState:179(1025ns) ProcessPendingSessionEvents:495(11449601ns) {ForwardSessionEventToSession:612(11389861ns) ForwardSessionEventToSession:612(7466ns) ForwardSessionEventToSession:612(3153ns) ForwardSessionEventToSession:612(5828ns) ForwardSessionEventToSession:612(5565ns) ForwardSessionEventToSession:612(2065ns) ForwardSessionEventToSession:612(8376ns) ForwardSessionEventToSession:612(2403ns) ForwardSessionEventToSession:612(9908ns) ForwardSessionEventToSession:612(3443ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.269618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470997091331425:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.347020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470995041949418:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.347073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:30.347131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470995041949418:2108] [node 5] ICP22 created new session: [4:7577470999336916742:2108]
2025-11-27T18:17:30.347177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470999336916742:2108] [node 5] ICS09 handshake done sender: [4:7577470999336916717:2110] self: [4:7577470995041949420:0] peer: [5:7577470999367568215:0] socket: 212 qp: -1
2025-11-27T18:17:30.347214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470999336916742:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:30.347265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:30.347889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470999336916722:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.348185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470999336916741:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-11-27T18:17:30.351292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577470995041949418:2108] [node 5] ICP18 handshake (actor [4:7577470999336916722:2112]) is too old
2025-11-27T18:17:30.377489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470995041949358:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577470999336916727:2116] held: no
2025-11-27T18:17:30.381487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470999336916737:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-11-27T18:17:30.388161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470999336916727:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:30.395013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470999336916726:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:30.395706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470995041949358:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.395752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:30.395799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470995041949358:2050] [node 3] ICP22 created new session: [4:7577470999336916747:2050]
2025-11-27T18:17:30.395836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470999336916747:2050] [node 3] ICS09 handshake done sender: [4:7577470999336916727:2116] self: [4:7577470999336916735:0] peer: [3:7577471001386298786:0] socket: 229 qp: -1
2025-11-27T18:17:30.395875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470999336916747:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:30.395913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:30.401878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470995041949357:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.401953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:30.402003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470995041949357:2049] [node 2] ICP22 created new session: [4:7577470999336916749:2049]
2025-11-27T18:17:30.402045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470999336916749:2049] [node 2] ICS09 handshake done sender: [4:7577470999336916726:2115] self: [4:7577470999336916731:0] peer: [2:7577471001353700992:0] socket: 228 qp: -1
2025-11-27T18:17:30.402080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470999336916749:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:30.402115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:30.402748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577470999336916746:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-11-27T18:17:30.403727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470999336916734:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:30.404531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577470995041949356:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.422431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:30.427369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470995041949356:2048] [node 1] ICP22 created new session: [4:7577470999336916751:2048]
2025-11-27T18:17:30.434335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470999336916751:2048] [node 1] ICS09 handshake done sender: [4:7577470999336916734:2118] self: [4:7577470999336916738:0] peer: [1:7577470999715888000:0] socket: 247 qp: -1
2025-11-27T18:17:30.437892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470999336916751:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:30.455499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:30.457987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577470999336916736:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.458022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577470999336916736:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:30.461732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577470999336916736:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:30.462047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577470999336916720:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.469564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:30.474187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577470999336916720:2109] [node 6] ICP22 created new session: [4:7577470999336916754:2109]
2025-11-27T18:17:30.478933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470995041949356:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(74670659ns) {HandleHandshakeStatus:329(74667826ns) {DropIncomingHandshake:442(1833ns) DropOutgoingHandshake:457(15905ns) SwitchToState:179(8408ns) ProcessPendingSessionEvents:495(16528809ns) {ForwardSessionEventToSession:612(16481328ns) ForwardSessionEventToSession:612(4197ns) ForwardSessionEventToSession:612(7435ns) ForwardSessionEventToSession:612(1934ns) ForwardSessionEventToSession:612(1915ns) ForwardSessionEventToSession:612(5431ns) ForwardSessionEventToSession:612(4813ns) ForwardSessionEventToSession:612(3183ns) ForwardSessionEventToSession:612(5344ns) ForwardSessionEventToSession:612(2345ns)}}}
2025-11-27T18:17:30.310151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470999715887998:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:30.362853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470999367568202:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.393483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003397150697:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25360
2025-11-27T18:17:30.455854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471001353701010:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.281934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471001386298808:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42245
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.494180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577470999336916754:2109] [node 6] ICS09 handshake done sender: [4:7577470999336916736:2119] self: [4:7577470999336916733:0] peer: [6:7577471003397150701:0] socket: 279 qp: -1
2025-11-27T18:17:30.455887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471001353701010:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:30.313786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470995420920633:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.394690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003397150699:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25360
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.362985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470999367568220:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25070
2025-11-27T18:17:30.313839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:30.327208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470995420920633:2049] [node 3] ICP22 created new session: [1:7577470999715888015:2049]
2025-11-27T18:17:30.341949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470995420920632:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.342013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:30.346638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470999715888001:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:30.346738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470995420920632:2048] [node 2] ICP22 created new session: [1:7577470999715888018:2048]
2025-11-27T18:17:30.353574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470999715888015:2049] [node 3] ICS09 handshake done sender: [1:7577470999715888004:2113] self: [1:7577470999715888003:0] peer: [3:7577471001386298807:0] socket: 249 qp: -1
2025-11-27T18:17:30.365438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470999715888015:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:30.369449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470999715888018:2048] [node 2] ICS09 handshake done sender: [1:7577470999715887998:2111] self: [1:7577470999715887997:0] peer: [2:7577471001353701011:0] socket: 232 qp: -1
2025-11-27T18:17:30.376854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:30.377338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470999715888018:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:30.377384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:30.385696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470995420920699:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.385737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:30.385791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470995420920699:2107] [node 4] ICP22 created new session: [1:7577470999715888024:2107]
2025-11-27T18:17:30.385854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470999715888024:2107] [node 4] ICS09 handshake done sender: [1:7577470999715888001:2112] self: [1:7577470999715888000:0] peer: [4:7577470999336916738:0] socket: 243 qp: -1
2025-11-27T18:17:30.385888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470999715888024:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:30.385921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:30.389848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470999715888021:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29041
2025-11-27T18:17:30.395338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470999715888019:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29041
2025-11-27T18:17:30.412290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470999715888021:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.415929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577470999715888022:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29041
2025-11-27T18:17:30.423795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470995420920632:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(51497651ns) {HandleHandshakeStatus:329(51493216ns) {DropIncomingHandshake:442(15196ns) DropOutgoingHandshake:457(459ns) SwitchToState:179(494ns) ProcessPendingSessionEvents:495(7860523ns) {ForwardSessionEventToSession:612(70378ns) ForwardSessionEventToSession:612(11679ns) ForwardSessionEventToSession:612(3029ns) ForwardSessionEventToSession:612(8862ns) ForwardSessionEventToSession:612(2540ns) ForwardSessionEventToSession:612(3090ns) ForwardSessionEventToSession:612(2835018ns) ForwardSessionEventToSession:612(4815ns) ForwardSessionEventToSession:612(2988ns) ForwardSessionEventToSession:612(4906356ns)}}}
2025-11-27T18:17:30.427402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470999715888019:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.432743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470995420920699:2107] [node 4] ICP18 handshake (actor [1:7577470999715888021:2117]) is too old
2025-11-27T18:17:30.436236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470999715888022:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.440714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577470999715888011:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.440750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577470999715888011:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:30.458408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577470999715888011:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:30.462321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470995420920700:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.470046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:30.485691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470995420920700:2109] [node 5] ICP22 created new session: [1:7577470999715888028:2109]
2025-11-27T18:17:30.494291Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470995420920633:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(126943569ns) {HandleHandshakeStatus:329(126938871ns) {DropIncomingHandshake:442(143ns) DropOutgoingHandshake:457(809ns) SwitchToState:179(359ns) ProcessPendingSessionEvents:495(39013165ns) {ForwardSessionEventToSession:612(21968228ns) ForwardSessionEventToSession:612(5718447ns) ForwardSessionEventToSession:612(4166ns) ForwardSessionEventToSession:612(2916ns) ForwardSessionEventToSession:612(2430214ns) ForwardSessionEventToSession:612(840122ns) ForwardSessionEventToSession:612(1704655ns) ForwardSessionEventToSession:612(5461ns) ForwardSessionEventToSession:612(2947ns) ForwardSessionEventToSession:612(6308594ns)}}}
2025-11-27T18:17:30.497945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470995420920633:2049] [node 3] ICP18 handshake (actor [1:7577470999715888019:2116]) is too old
2025-11-27T18:17:30.508838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577470999715888028:2109] [node 5] ICS09 handshake done sender: [1:7577470999715888011:2115] self: [1:7577470999715888006:0] peer: [5:7577470999367568227:0] socket: 273 qp: -1
2025-11-27T18:17:30.508894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577470999715888028:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:30.508952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:30.520541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577470995420920700:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(58100294ns) {HandleHandshakeStatus:329(58097635ns) {DropIncomingHandshake:442(8453ns) DropOutgoingHandshake:457(177ns) SwitchToState:179(239ns) ProcessPendingSessionEvents:495(128692ns) {ForwardSessionEventToSession:612(80021ns) ForwardSessionEventToSession:612(1995ns) ForwardSessionEventToSession:612(19747ns) ForwardSessionEventToSession:612(1418ns) ForwardSessionEventToSession:612(1890ns) ForwardSessionEventToSession:612(3423ns) ForwardSessionEventToSession:612(1362ns) ForwardSessionEventToSession:612(2191ns) ForwardSessionEventToSession:612(6446ns) ForwardSessionEventToSession:612(1822ns)}}}
2025-11-27T18:17:30.427421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470999102183317:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471003397150691:2116] held: no
2025-11-27T18:17:30.305537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:30.501407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577470999336916754:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:30.457319Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471001353701000:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:30.309855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470997091331425:2049] [node 2] ICP22 created new session: [3:7577471001386298810:2049]
2025-11-27T18:17:30.319586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471001386298791:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.319629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471001386298791:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:30.321849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471001386298810:2049] [node 2] ICS09 handshake done sender: [3:7577471001386298798:2115] self: [3:7577471001386298802:0] peer: [2:7577471001353700997:0] socket: 240 qp: -1
2025-11-27T18:17:30.323189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577470997091331487:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.323254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:30.323303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470997091331487:2107] [node 4] ICP22 created new session: [3:7577471001386298812:2107]
2025-11-27T18:17:30.323340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471001386298812:2107] [node 4] ICS09 handshake done sender: [3:7577471001386298787:2110] self: [3:7577471001386298786:0] peer: [4:7577470999336916735:0] socket: 220 qp: -1
2025-11-27T18:17:30.323373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471001386298812:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:30.323407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:30.323912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471001386298808:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.353995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471001386298810:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:30.354063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:30.353735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471001386298791:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:30.385435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471001386298784:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.385501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:30.394044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471001386298784:2109] [node 6] ICP22 created new session: [3:7577471001386298819:2109]
2025-11-27T18:17:30.405396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471001386298819:2109] [node 6] ICS09 handshake done sender: [3:7577471001386298791:2112] self: [3:7577471001386298790:0] peer: [6:7577471003397150696:0] socket: 233 qp: -1
2025-11-27T18:17:30.417348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471001386298819:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:30.417404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:30.417819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471001386298784:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(32222271ns) {HandleHandshakeStatus:329(32218195ns) {DropIncomingHandshake:442(16823ns) DropOutgoingHandshake:457(450ns) SwitchToState:179(680ns) ProcessPendingSessionEvents:495(97089ns) {ForwardSessionEventToSession:612(51413ns) ForwardSessionEventToSession:612(2800ns) ForwardSessionEventToSession:612(6869ns) ForwardSessionEventToSession:612(4905ns) ForwardSessionEventToSession:612(3566ns) ForwardSessionEventToSession:612(2648ns) ForwardSessionEventToSession:612(5519ns) ForwardSessionEventToSession:612(2841ns) ForwardSessionEventToSession:612(6389ns) ForwardSessionEventToSession:612(2619ns)}}}
2025-11-27T18:17:30.418123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471001386298806:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.418146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471001386298806:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:30.425379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471001386298806:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:30.432051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471001386298811:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42245
2025-11-27T18:17:30.446382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471001386298785:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.454045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:30.458184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471001386298803:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:30.458254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471001386298785:2108] [node 5] ICP22 created new session: [3:7577471001386298822:2108]
2025-11-27T18:17:30.461526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577470997091331424:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.461583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:30.461624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577470997091331424:2048] [node 1] ICP22 created new session: [3:7577471001386298823:2048]
2025-11-27T18:17:30.461667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471001386298823:2048] [node 1] ICS09 handshake done sender: [3:7577471001386298803:2117] self: [3:7577471001386298807:0] peer: [1:7577470999715888003:0] socket: 250 qp: -1
2025-11-27T18:17:30.461700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471001386298823:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:30.461730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:30.462260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470997091331425:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(171098800ns) {HandleHandshakeStatus:329(171093403ns) {DropIncomingHandshake:442(3276ns) DropOutgoingHandshake:457(19488ns) SwitchToState:179(5777ns) ProcessPendingSessionEvents:495(44390918ns) {ForwardSessionEventToSession:612(16247450ns) ForwardSessionEventToSession:612(31351ns) ForwardSessionEventToSession:612(4449077ns) ForwardSessionEventToSession:612(4252ns) ForwardSessionEventToSession:612(5627893ns) ForwardSessionEventToSession:612(9347009ns) ForwardSessionEventToSession:612(3968ns) ForwardSessionEventToSession:612(2431ns) ForwardSessionEventToSession:612(915680ns) ForwardSessionEventToSession:612(2713ns)}}}
2025-11-27T18:17:30.469496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471001386298822:2108] [node 5] ICS09 handshake done sender: [3:7577471001386298806:2119] self: [3:7577471001386298801:0] peer: [5:7577470999367568224:0] socket: 242 qp: -1
2025-11-27T18:17:30.479794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577470997091331424:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18075631ns) {HandleHandshakeStatus:329(18072633ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(5575ns) SwitchToState:179(521ns) ProcessPendingSessionEvents:495(9939413ns) {ForwardSessionEventToSession:612(9889272ns) ForwardSessionEventToSession:612(8868ns) ForwardSessionEventToSession:612(2936ns) ForwardSessionEventToSession:612(2162ns) ForwardSessionEventToSession:612(5215ns) ForwardSessionEventToSession:612(1973ns) ForwardSessionEventToSession:612(5779ns) ForwardSessionEventToSession:612(5004ns) ForwardSessionEventToSession:612(3055ns) ForwardSessionEventToSession:612(5639ns)}}}
2025-11-27T18:17:30.485642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471001386298822:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:30.489395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:30.531949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471001386298785:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(90990905ns) {HandleHandshakeStatus:329(90986789ns) {DropIncomingHandshake:442(23369ns) DropOutgoingHandshake:457(558ns) SwitchToState:179(466ns) ProcessPendingSessionEvents:495(21803934ns) {ForwardSessionEventToSession:612(21748066ns) ForwardSessionEventToSession:612(7007ns) ForwardSessionEventToSession:612(3104ns) ForwardSessionEventToSession:612(7205ns) ForwardSessionEventToSession:612(5852ns) ForwardSessionEventToSession:612(4904ns) ForwardSessionEventToSession:612(2878ns) ForwardSessionEventToSession:612(2591ns) ForwardSessionEventToSession:612(5801ns) ForwardSessionEventToSession:612(3855ns)}}}
2025-11-27T18:17:30.375921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470995072600901:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13183) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470999367568202:2111] held: no
2025-11-27T18:17:30.457481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577470997058733630:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.501467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:30.440289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471003397150697:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:30.380049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470995072600901:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.457981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471001353701010:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:30.440625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003397150698:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25360
2025-11-27T18:17:30.440629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471003397150699:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.445072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470999102183316:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.445155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470999102183360:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.465424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:30.573435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577470999336916720:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(104358658ns) {HandleHandshakeStatus:329(104355475ns) {DropIncomingHandshake:442(17195ns) DropOutgoingHandshake:457(367ns) SwitchToState:179(359ns) ProcessPendingSessionEvents:495(29950809ns) {ForwardSessionEventToSession:612(14178071ns) ForwardSessionEventToSession:612(4898ns) ForwardSessionEventToSession:612(2676ns) ForwardSessionEventToSession:612(4195920ns) ForwardSessionEventToSession:612(3230664ns) ForwardSessionEventToSession:612(3379ns) ForwardSessionEventToSession:612(3515775ns) ForwardSessionEventToSession:612(3710ns) ForwardSessionEventToSession:612(4791177ns) ForwardSessionEventToSession:612(4019ns)}}}
2025-11-27T18:17:30.388313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:30.393464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470999367568220:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.455782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470999102183317:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(19625231ns) {HandleHandshakeStatus:395(19621728ns) {LogHandshakeFail:472(16107ns) {UpdateErrorStateLog:545(5318ns)} DropOutgoingHandshake:457(1541ns) ProcessPendingSessionEvents:495(1244722ns) {DropSessionEvent:512(41074ns) DropSessionEvent:512(14963ns) DropSessionEvent:512(5627ns) DropSessionEvent:512(1145209ns) DropSessionEvent:512(4867ns) DropSessionEvent:512(5223ns) DropSessionEvent:512(5131ns) DropSessionEvent:512(4826ns) DropSessionEvent:512(5591ns) DropSessionEvent:512(5029ns)}}}
2025-11-27T18:17:30.465467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470997058733630:2048] [node 1] ICP22 created new session: [2:7577471001353701027:2048]
2025-11-27T18:17:30.393811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470995072600901:2109] [node 4] ICP22 created new session: [5:7577470999367568222:2109]
2025-11-27T18:17:30.459251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471003397150691:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:30.465516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471001353701027:2048] [node 1] ICS09 handshake done sender: [2:7577471001353701000:2114] self: [2:7577471001353701011:0] peer: [1:7577470999715887997:0] socket: 238 qp: -1
2025-11-27T18:17:30.404622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470995072600840:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.459592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470999102183317:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.459636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:30.459680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470999102183317:2050] [node 3] ICP22 created new session: [6:7577471003397150702:2050]
2025-11-27T18:17:30.459717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471003397150702:2050] [node 3] ICS09 handshake done sender: [6:7577471003397150691:2116] self: [6:7577471003397150696:0] peer: [3:7577471001386298790:0] socket: 234 qp: -1
2025-11-27T18:17:30.459758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471003397150702:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:30.459790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:30.471248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003397150704:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25360
2025-11-27T18:17:30.471931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471003397150697:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:30.472422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470999102183316:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.472468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:30.472518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470999102183316:2049] [node 2] ICP22 created new session: [6:7577471003397150706:2049]
2025-11-27T18:17:30.472562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471003397150706:2049] [node 2] ICS09 handshake done sender: [6:7577471003397150697:2118] self: [6:7577471003397150700:0] peer: [2:7577471001353701007:0] socket: 263 qp: -1
2025-11-27T18:17:30.472603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471003397150706:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:30.472636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:30.489821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003397150708:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25360
2025-11-27T18:17:30.510204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470999102183360:2086] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13183) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471003397150699:2120] held: no
2025-11-27T18:17:30.554290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471003397150708:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.554434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577470999102183352:2084] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.564113Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577470999102183360:2086] [node 4] ICP35 event processing took too much time PendingConnection:279(18539041ns) {HandleHandshakeStatus:395(18535827ns) {LogHandshakeFail:472(23576ns) {UpdateErrorStateLog:545(14397ns)} DropOutgoingHandshake:457(1188ns) ProcessPendingSessionEvents:495(6619562ns) {DropSessionEvent:512(12288ns) DropSessionEvent:512(4257ns) DropSessionEvent:512(5830ns) DropSessionEvent:512(4236ns) DropSessionEvent:512(25797ns) DropSessionEvent:512(4796ns) DropSessionEvent:512(4202ns) DropSessionEvent:512(3491ns) DropSessionEvent:512(4918ns) DropSessionEvent:512(7284ns)}}}
2025-11-27T18:17:30.566128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003397150710:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25360
2025-11-27T18:17:30.566442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003397150705:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25360
2025-11-27T18:17:30.572198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471003397150699:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:30.572930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470999102183360:2086] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.572976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:30.574885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471003397150708:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:30.576372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470999102183360:2086] [node 4] ICP22 created new session: [6:7577471003397150711:2086]
2025-11-27T18:17:30.576416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471003397150711:2086] [node 4] ICS09 handshake done sender: [6:7577471003397150699:2120] self: [6:7577471003397150701:0] peer: [4:7577470999336916733:0] socket: 244 qp: -1
2025-11-27T18:17:30.576447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471003397150711:2086] [node 4] ICS10 traffic start
2025-11-27T18:17:30.412668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470999367568222:2109] [node 4] ICS09 handshake done sender: [5:7577470999367568202:2111] self: [5:7577470999367568215:0] peer: [4:7577470995041949420:0] socket: 222 qp: -1
2025-11-27T18:17:30.576492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:30.465548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471001353701027:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:30.577367Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577470999102183352:2084] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61025) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471003397150708:2123] held: no
2025-11-27T18:17:30.465597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:30.422859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470999367568222:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:30.577459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577470999102183352:2084] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.577525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:30.577568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577470999102183352:2084] [node 5] ICP22 created new session: [6:7577471003397150713:2084]
2025-11-27T18:17:30.577603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471003397150713:2084] [node 5] ICS09 handshake done sender: [6:7577471003397150708:2123] self: [6:7577471003397150709:0] peer: [5:7577470999367568210:0] socket: 255 qp: -1
2025-11-27T18:17:30.577646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471003397150713:2084] [node 5] ICS10 traffic start
2025-11-27T18:17:30.577688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:30.485877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577470997058733695:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.422989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470999367568219:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.498272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:30.504466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577470997058733695:2109] [node 6] ICP22 created new session: [2:7577471001353701029:2109]
2025-11-27T18:17:30.427585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:30.516706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471001353701029:2109] [node 6] ICS09 handshake done sender: [2:7577471001353701010:2118] self: [2:7577471001353701007:0] peer: [6:7577471003397150700:0] socket: 262 qp: -1
2025-11-27T18:17:30.521715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471001353701029:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:30.532452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:30.561715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577470997058733695:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(71914759ns) {HandleHandshakeStatus:329(71911758ns) {DropIncomingHandshake:442(4155334ns) DropOutgoingHandshake:457(1616ns) SwitchToState:179(576ns) ProcessPendingSessionEvents:495(6891144ns) {ForwardSessionEventToSession:612(6835348ns) ForwardSessionEventToSession:612(5511ns) ForwardSessionEventToSession:612(7251ns) ForwardSessionEventToSession:612(6719ns) ForwardSessionEventToSession:612(2861ns) ForwardSessionEventToSession:612(5791ns) ForwardSessionEventToSession:612(2585ns) ForwardSessionEventToSession:612(9749ns) ForwardSessionEventToSession:612(3099ns) ForwardSessionEventToSession:612(2685ns)}}}
2025-11-27T18:17:30.427629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577470995072600838:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:30.453274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470995072600839:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8030) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470999367568216:2117] held: no
2025-11-27T18:17:30.477388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470995072600901:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(75453710ns) {HandleHandshakeStatus:329(75450433ns) {DropIncomingHandshake:442(5385ns) DropOutgoingHandshake:457(295ns) SwitchToState:179(593ns) ProcessPendingSessionEvents:495(576ns)}}
2025-11-27T18:17:30.485879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470995072600840:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470999367568220:2120] held: no
2025-11-27T18:17:30.504566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470995072600839:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23851375ns) {HandleHandshakeStatus:395(19756875ns) {LogHandshakeFail:472(15242ns) {UpdateErrorStateLog:545(5037ns)} DropOutgoingHandshake:457(3215ns) ProcessPendingSessionEvents:495(7142893ns) {DropSessionEvent:512(13844ns) DropSessionEvent:512(4876ns) DropSessionEvent:512(4407ns) DropSessionEvent:512(4702ns) DropSessionEvent:512(2392921ns) DropSessionEvent:512(4337ns) DropSessionEvent:512(3993ns) DropSessionEvent:512(3929ns) DropSessionEvent:512(4104ns) DropSessionEvent:512(4066ns)}}}
2025-11-27T18:17:30.525368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470999367568228:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25070
2025-11-27T18:17:30.527753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470999367568223:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25070
2025-11-27T18:17:30.527994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577470995072600840:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(20945311ns) {HandleHandshakeStatus:395(12894772ns) {LogHandshakeFail:472(16029ns) {UpdateErrorStateLog:545(4729ns)} DropOutgoingHandshake:457(803ns) ProcessPendingSessionEvents:495(12477654ns) {DropSessionEvent:512(16145ns) DropSessionEvent:512(9269594ns) DropSessionEvent:512(6382ns) DropSessionEvent:512(5652ns) DropSessionEvent:512(3584ns) DropSessionEvent:512(18761ns) DropSessionEvent:512(4183ns) DropSessionEvent:512(6812ns) DropSessionEvent:512(3507ns) DropSessionEvent:512(3605ns)}}}
2025-11-27T18:17:30.528203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577470995072600838:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577470999367568219:2119] held: no
2025-11-27T18:17:30.536402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577470999367568225:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25070
2025-11-27T18:17:30.550466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470999367568219:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:30.553221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470995072600838:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.553292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:30.553353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470995072600838:2048] [node 1] ICP22 created new session: [5:7577470999367568230:2048]
2025-11-27T18:17:30.553401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470999367568230:2048] [node 1] ICS09 handshake done sender: [5:7577470999367568219:2119] self: [5:7577470999367568227:0] peer: [1:7577470999715888006:0] socket: 276 qp: -1
2025-11-27T18:17:30.553443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470999367568230:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:30.553475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:30.553724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470999367568220:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:30.554201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470995072600840:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.554235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:30.554288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470995072600840:2050] [node 3] ICP22 created new session: [5:7577470999367568232:2050]
2025-11-27T18:17:30.554330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470999367568232:2050] [node 3] ICS09 handshake done sender: [5:7577470999367568220:2120] self: [5:7577470999367568224:0] peer: [3:7577471001386298801:0] socket: 217 qp: -1
2025-11-27T18:17:30.554357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470999367568232:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:30.554384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:30.555081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470999367568216:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:30.562733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577470999367568211:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:30.562761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577470999367568211:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:30.567785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577470999367568211:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:30.568358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577470995072600839:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:30.568405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:30.568461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470995072600839:2049] [node 2] ICP22 created new session: [5:7577470999367568235:2049]
2025-11-27T18:17:30.568510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470999367568235:2049] [node 2] ICS09 handshake done sender: [5:7577470999367568216:2117] self: [5:7577470999367568217:0] peer: [2:7577471001353701002:0] socket: 264 qp: -1
2025-11-27T18:17:30.568545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470999367568235:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:30.568577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:30.573026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577470995072600902:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:30.573069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:30.575083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577470995072600902:2108] [node 6] ICP22 created new session: [5:7577470999367568237:2108]
2025-11-27T18:17:30.575160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577470999367568237:2108] [node 6] ICS09 handshake done sender: [5:7577470999367568211:2116] self: [5:7577470999367568210:0] peer: [6:7577471003397150709:0] socket: 223 qp: -1
2025-11-27T18:17:30.575207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577470999367568237:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:30.575255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:30.642660Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:30.642922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471001353701027:2048] [node 1] ICS01 socket: 238 reason# EndOfStream
2025-11-27T18:17:30.642964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577470997058733630:2048] [node 1] ICP30 unregister session Session# [2:7577471001353701027:2048] VirtualId# [2:7577471001353701011:0]
2025-11-27T18:17:30.642997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471001353701027:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.643082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:30.643129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:30.649431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:30.649734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471001386298823:2048] [node 1] ICS01 socket: 250 reason# EndOfStream
2025-11-27T18:17:30.649858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470997091331424:2048] [node 1] ICP30 unregister session Session# [3:7577471001386298823:2048] VirtualId# [3:7577471001386298807:0]
2025-11-27T18:17:30.649888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471001386298823:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.649958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:30.650008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:30.657474Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:30.657716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577470999336916751:2048] [node 1] ICS07 socket disconnect 247 reason# EndOfStream
2025-11-27T18:17:30.657740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470999336916751:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.657819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:30.657858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577470999336916751:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:30.658510Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:30.658711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471003397150693:2048] [node 1] ICS01 socket: 211 reason# EndOfStream
2025-11-27T18:17:30.658749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470999102183315:2048] [node 1] ICP30 unregister session Session# [6:7577471003397150693:2048] VirtualId# [6:7577471003397150690:0]
2025-11-27T18:17:30.658780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471003397150693:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.658845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:30.658887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:30.660252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577470995041949356:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29041) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:30.660307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470999336916751:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:30.660339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470995041949356:2048] [node 1] ICP30 unregister session Session# [4:7577470999336916751:2048] VirtualId# [4:7577470999336916738:0]
2025-11-27T18:17:30.660373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:30.660399Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577470995041949356:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29041) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:30.660411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29041) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:30.658966Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:30.659161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470999367568230:2048] [node 1] ICS01 socket: 276 reason# EndOfStream
2025-11-27T18:17:30.659195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470995072600838:2048] [node 1] ICP30 unregister session Session# [5:7577470999367568230:2048] VirtualId# [5:7577470999367568227:0]
2025-11-27T18:17:30.659218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470999367568230:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.659263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:30.659304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:30.673358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577470995041949356:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:30.697990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:30.698247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471001386298810:2049] [node 2] ICS07 socket disconnect 240 reason# EndOfStream
2025-11-27T18:17:30.698277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471001386298810:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.698357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:30.698401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471001386298810:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:30.700705Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:30.700922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470999367568235:2049] [node 2] ICS01 socket: 264 reason# EndOfStream
2025-11-27T18:17:30.700838Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470997091331425:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8030) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:30.700887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471001386298810:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:30.700980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470995072600839:2049] [node 2] ICP30 unregister session Session# [5:7577470999367568235:2049] VirtualId# [5:7577470999367568217:0]
2025-11-27T18:17:30.701172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470999367568235:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.701278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:30.701323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:30.702477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:30.702108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577470997091331425:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8030) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:30.702691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470999336916749:2049] [node 2] ICS01 socket: 228 reason# EndOfStream
2025-11-27T18:17:30.702726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470995041949357:2049] [node 2] ICP30 unregister session Session# [4:7577470999336916749:2049] VirtualId# [4:7577470999336916731:0]
2025-11-27T18:17:30.702189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471001386298810:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:30.702231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577470997091331425:2049] [node 2] ICP30 unregister session Session# [3:7577471001386298810:2049] VirtualId# [3:7577471001386298802:0]
2025-11-27T18:17:30.702754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470999336916749:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.702886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:30.702277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:30.702930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:30.702308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577470997091331425:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8030) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:30.702322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8030) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:30.703436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:30.703622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471003397150706:2049] [node 2] ICS01 socket: 263 reason# EndOfStream
2025-11-27T18:17:30.703652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470999102183316:2049] [node 2] ICP30 unregister session Session# [6:7577471003397150706:2049] VirtualId# [6:7577471003397150700:0]
2025-11-27T18:17:30.703676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471003397150706:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.703731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:30.703768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:30.712397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577470997091331425:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:30.765738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:30.765975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577470999336916747:2050] [node 3] ICS01 socket: 229 reason# EndOfStream
2025-11-27T18:17:30.766020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577470995041949358:2050] [node 3] ICP30 unregister session Session# [4:7577470999336916747:2050] VirtualId# [4:7577470999336916735:0]
2025-11-27T18:17:30.766051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577470999336916747:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.766126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:30.766185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:30.766847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:30.767048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470999367568232:2050] [node 3] ICS01 socket: 217 reason# EndOfStream
2025-11-27T18:17:30.767092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470995072600840:2050] [node 3] ICP30 unregister session Session# [5:7577470999367568232:2050] VirtualId# [5:7577470999367568224:0]
2025-11-27T18:17:30.767120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470999367568232:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.767180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:30.767232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:30.767672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:30.767860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471003397150702:2050] [node 3] ICS01 socket: 234 reason# EndOfStream
2025-11-27T18:17:30.767892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470999102183317:2050] [node 3] ICP30 unregister session Session# [6:7577471003397150702:2050] VirtualId# [6:7577471003397150696:0]
2025-11-27T18:17:30.767915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471003397150702:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.767970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:30.768007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:30.814895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:30.815148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471003397150711:2086] [node 4] ICS01 socket: 244 reason# EndOfStream
2025-11-27T18:17:30.815194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470999102183360:2086] [node 4] ICP30 unregister session Session# [6:7577471003397150711:2086] VirtualId# [6:7577471003397150701:0]
2025-11-27T18:17:30.815249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471003397150711:2086] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.815368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:30.815417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:30.817452Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:30.817855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577470999367568222:2109] [node 4] ICS01 socket: 222 reason# EndOfStream
2025-11-27T18:17:30.817940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577470995072600901:2109] [node 4] ICP30 unregister session Session# [5:7577470999367568222:2109] VirtualId# [5:7577470999367568215:0]
2025-11-27T18:17:30.818022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577470999367568222:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.818163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:30.818252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:30.899083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:30.899318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471003397150713:2084] [node 5] ICS01 socket: 255 reason# EndOfStream
2025-11-27T18:17:30.899357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577470999102183352:2084] [node 5] ICP30 unregister session Session# [6:7577471003397150713:2084] VirtualId# [6:7577471003397150709:0]
2025-11-27T18:17:30.899388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471003397150713:2084] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:30.899478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:30.899537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:30.993709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470999510612849:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:30.993748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577470999510612850:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:31.016615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471003664501203:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:31.016659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471003664501204:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:31.039343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471003838867224:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:31.039411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471003838867225:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:31.062409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471007008292514:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:31.062459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471007008292515:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:31.062671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471007008292516:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:31.085368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471003997861881:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:31.085422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471003997861882:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:31.085476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471003997861883:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:31.107904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471003604307342:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:31.107953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471003604307343:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:31.107989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471003604307344:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:31.180305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471003604307404:2106] [node 5] ICP01 ready to work
2025-11-27T18:17:31.298566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471007008292577:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:31.300207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471003664501267:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:31.309205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471003805580227:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:31.314297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471003664501268:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:31.317756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471007008292578:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:31.309234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471003604307409:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:31.321044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471003805580228:2117] [node 5] ICP01 ready to work
2025-11-27T18:17:31.320893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471003838867288:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:31.334800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471003838867289:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:31.333726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471003805580229:2124] [node 6] ICP01 ready to work
2025-11-27T18:17:31.336386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471003664501269:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:31.337308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471003838867290:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:31.341517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471003997861954:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.346334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471003997861955:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:31.349296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471003997861942:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24700
2025-11-27T18:17:31.353160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471007008292514:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31898720ns) {Configure:104(31893076ns) {ProcessConfigured:135(54600ns) {StartInitialHandshake:157(52489ns) {DropHandshakes:470(1277ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(1522ns) {SwitchToState:179(600ns)}}}}}
2025-11-27T18:17:31.358322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471003664501203:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20794878ns) {Configure:104(20788721ns) {ProcessConfigured:135(358684ns) {StartInitialHandshake:157(356957ns) {DropHandshakes:470(1382ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(1719ns) {SwitchToState:179(879ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.364574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471003838867304:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35321
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.383396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471003805580237:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30249
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.382359Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471003604307343:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23790305ns) {Configure:104(23786448ns) {ProcessConfigured:135(11958216ns) {StartInitialHandshake:157(11955041ns) {DropHandshakes:470(7770702ns) {DropIncomingHandshake:442(475ns) DropOutgoingHandshake:457(1773ns)} GenerateSessionVirtualId:988(1724ns) PrepareNewSessionHandshake:477(5768ns) {SwitchToState:179(2730ns)}}}}}
2025-11-27T18:17:31.386360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471003838867304:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.385524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471007008292589:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29463
2025-11-27T18:17:31.386763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471003838867224:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.386917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471003664501267:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25075660ns) {Configure:104(25072189ns) {ProcessConfigured:135(294853ns) {StartInitialHandshake:157(293290ns) {DropHandshakes:470(1488ns) {DropIncomingHandshake:442(250ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(1965ns) {SwitchToState:179(169ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.386375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471003805580237:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.387170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471003997861942:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.387504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471007008292590:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29463
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.397849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471003664501280:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23131
2025-11-27T18:17:31.405912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471007008292589:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.398094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471003664501281:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23131
2025-11-27T18:17:31.398212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471003664501282:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23131
2025-11-27T18:17:31.413403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471003838867306:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35321
2025-11-27T18:17:31.398278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471003664501283:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23131
2025-11-27T18:17:31.398827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471003664501281:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.419339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471003805580208:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.419173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471003664501288:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23131
2025-11-27T18:17:31.419411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471003838867307:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35321
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.421508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471007008292594:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29463
2025-11-27T18:17:31.419395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471003805580208:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.421230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471003664501283:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.421822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471007008292595:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29463
2025-11-27T18:17:31.421965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471007008292516:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.420166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471003838867306:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.422218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471007008292595:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.421927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471003838867225:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.422519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471003838867296:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.422461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471003664501273:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.422546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471003838867296:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.422490Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471003664501273:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.423805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471003664501280:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.424078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471007008292590:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.424341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471003664501203:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.424715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471003838867296:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:31.424834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471003664501273:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:31.425095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471003805580206:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.425138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471003805580206:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.426395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471003805580239:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30249
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.426483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471003664501204:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.426540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:31.426561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471003805580240:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30249
2025-11-27T18:17:31.426579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471003664501204:2049] [node 3] ICP22 created new session: [2:7577471003664501292:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.426624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471003664501292:2049] [node 3] ICS09 handshake done sender: [2:7577471003664501273:2112] self: [2:7577471003664501272:0] peer: [3:7577471003838867308:0] socket: 241 qp: -1
2025-11-27T18:17:31.426667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471003664501292:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:31.426693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.426933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471003805580241:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30249
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.427056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471003805580242:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30249
2025-11-27T18:17:31.427225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471003805580208:2108] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:31.426337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471003838867288:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.426419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:31.426463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471003838867288:2107] [node 4] ICP22 created new session: [3:7577471003838867312:2107]
2025-11-27T18:17:31.426492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471003838867312:2107] [node 4] ICS09 handshake done sender: [3:7577471003838867296:2113] self: [3:7577471003838867295:0] peer: [4:7577471007008292596:0] socket: 223 qp: -1
2025-11-27T18:17:31.426520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471003838867312:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:31.426545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:31.426978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471003838867310:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35321
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.429055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471003805580239:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.429304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471003805580240:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.429632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470999510612850:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.429714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:31.429767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470999510612850:2049] [node 3] ICP22 created new session: [1:7577471003805580246:2049]
2025-11-27T18:17:31.429782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471003664501282:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.431238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471003838867311:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35321
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.429819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471003805580246:2049] [node 3] ICS09 handshake done sender: [1:7577471003805580208:2108] self: [1:7577471003805580207:0] peer: [3:7577471003838867305:0] socket: 220 qp: -1
2025-11-27T18:17:31.429845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471003805580246:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:31.429880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:31.430588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471003805580241:2132] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.431757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471003805580206:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:31.432240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471003997861970:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24700
2025-11-27T18:17:31.431796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471007008292594:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.432429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471007008292515:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.439336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471003664501296:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23131
2025-11-27T18:17:31.437570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471003805580242:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.437623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577470999510612849:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.440754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003604307420:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35581
2025-11-27T18:17:31.444133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471003604307420:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.444268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471003604307342:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.437659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:31.439902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471003838867306:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.437708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577470999510612849:2048] [node 2] ICP22 created new session: [1:7577471003805580251:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.446800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471003664501288:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.446968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471003997861971:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24700
2025-11-27T18:17:31.446161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471007008292514:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.447441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471003997861970:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.447573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471003997861882:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.437748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471003805580251:2048] [node 2] ICS09 handshake done sender: [1:7577471003805580206:2107] self: [1:7577471003805580205:0] peer: [2:7577471003664501291:0] socket: 219 qp: -1
2025-11-27T18:17:31.446396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471007008292597:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29463
2025-11-27T18:17:31.450307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471003997861971:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.437793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471003805580251:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:31.437844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:31.438100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577470999510612849:2048] [node 2] ICP18 handshake (actor [1:7577471003805580241:2132]) is too old
2025-11-27T18:17:31.446018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471003805580235:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.446048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471003805580235:2128] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.449495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471003805580231:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.449516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471003805580231:2126] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.451029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471003805580235:2128] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:31.451813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471007008292589:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:31.452191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471007008292516:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.452255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:31.452290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471007008292516:2050] [node 3] ICP22 created new session: [4:7577471007008292602:2050]
2025-11-27T18:17:31.452334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471007008292602:2050] [node 3] ICS09 handshake done sender: [4:7577471007008292589:2115] self: [4:7577471007008292596:0] peer: [3:7577471003838867295:0] socket: 229 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.450424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471003997861881:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.452364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471007008292602:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:31.451121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471003664501275:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.454567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003604307422:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35581
2025-11-27T18:17:31.451149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471003664501275:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.452396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:31.455924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471003664501280:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:31.459019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471003664501203:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10614) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471003664501280:2116] held: no
2025-11-27T18:17:31.459121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471003664501203:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.459162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:31.459213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471003664501203:2048] [node 1] ICP22 created new session: [2:7577471003664501298:2048]
2025-11-27T18:17:31.459257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471003664501298:2048] [node 1] ICS09 handshake done sender: [2:7577471003664501280:2116] self: [2:7577471003664501291:0] peer: [1:7577471003805580205:0] socket: 236 qp: -1
2025-11-27T18:17:31.459309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471003664501298:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:31.459541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:31.459982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471003604307420:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:31.457413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471003805580233:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.457457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471003805580233:2127] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.459077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471003805580231:2126] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:31.454399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471007008292601:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29463
2025-11-27T18:17:31.459429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471003805580229:2124] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.459480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:31.462345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471003604307342:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.462395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.459517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471003805580229:2124] [node 6] ICP22 created new session: [1:7577471003805580257:2124]
2025-11-27T18:17:31.460405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471003805580257:2124] [node 6] ICS09 handshake done sender: [1:7577471003805580235:2128] self: [1:7577471003805580234:0] peer: [6:7577471003604307421:0] socket: 255 qp: -1
2025-11-27T18:17:31.460442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471003805580257:2124] [node 6] ICS10 traffic start
2025-11-27T18:17:31.460466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:31.460742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471003805580227:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.460799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:31.460832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471003805580227:2109] [node 4] ICP22 created new session: [1:7577471003805580260:2109]
2025-11-27T18:17:31.462431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471003604307342:2048] [node 1] ICP22 created new session: [6:7577471003604307423:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.463249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471003838867224:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10614) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471003838867304:2116] held: no
2025-11-27T18:17:31.460855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471003805580260:2109] [node 4] ICS09 handshake done sender: [1:7577471003805580231:2126] self: [1:7577471003805580230:0] peer: [4:7577471007008292599:0] socket: 228 qp: -1
2025-11-27T18:17:31.463005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471003997861974:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24700
2025-11-27T18:17:31.462925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471007008292590:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:31.460875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471003805580260:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:31.460893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:31.463031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471003664501277:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.462622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471003604307423:2048] [node 1] ICS09 handshake done sender: [6:7577471003604307420:2115] self: [6:7577471003604307421:0] peer: [1:7577471003805580234:0] socket: 267 qp: -1
2025-11-27T18:17:31.463742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471003997861975:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24700
2025-11-27T18:17:31.467293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471007008292514:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.465253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471003838867225:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23386) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471003838867306:2117] held: no
2025-11-27T18:17:31.463063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471003664501277:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.468582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471003997861976:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24700
2025-11-27T18:17:31.462656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471003604307423:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:31.466272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471003838867225:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.468620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471003664501275:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:31.467253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:31.462685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:31.467337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:31.465657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471003604307342:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:31.468809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471003838867225:2049] [node 2] ICP22 created new session: [3:7577471003838867315:2049]
2025-11-27T18:17:31.467383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471007008292514:2048] [node 1] ICP22 created new session: [4:7577471007008292605:2048]
2025-11-27T18:17:31.469086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471003838867315:2049] [node 2] ICS09 handshake done sender: [3:7577471003838867306:2117] self: [3:7577471003838867308:0] peer: [2:7577471003664501272:0] socket: 243 qp: -1
2025-11-27T18:17:31.469355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471003838867315:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:31.469601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:31.467426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471007008292605:2048] [node 1] ICS09 handshake done sender: [4:7577471007008292590:2116] self: [4:7577471007008292599:0] peer: [1:7577471003805580230:0] socket: 230 qp: -1
2025-11-27T18:17:31.468847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471007008292605:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:31.469119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:31.472515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471003805580233:2127] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:31.471576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471003664501277:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:31.472969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471003664501267:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.472999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471003805580228:2117] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.473019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.473074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:31.473682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471003805580228:2117] [node 5] ICP22 created new session: [1:7577471003805580262:2117]
2025-11-27T18:17:31.473335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471003664501267:2107] [node 4] ICP22 created new session: [2:7577471003664501303:2107]
2025-11-27T18:17:31.473898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471003805580262:2117] [node 5] ICS09 handshake done sender: [1:7577471003805580233:2127] self: [1:7577471003805580232:0] peer: [5:7577471003997861973:0] socket: 222 qp: -1
2025-11-27T18:17:31.473958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003604307426:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35581
2025-11-27T18:17:31.473927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471003805580262:2117] [node 5] ICS10 traffic start
2025-11-27T18:17:31.473963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:31.473403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471003664501303:2107] [node 4] ICS09 handshake done sender: [2:7577471003664501275:2113] self: [2:7577471003664501274:0] peer: [4:7577471007008292598:0] socket: 242 qp: -1
2025-11-27T18:17:31.473446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471003664501303:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:31.474500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471003838867288:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:31.473491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:31.474494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471003997861970:2125] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:31.475328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471003838867304:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:31.475637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471003997861974:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.475958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471003664501268:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.475725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471003997861882:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.475859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:31.475905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471003997861882:2049] [node 2] ICP22 created new session: [5:7577471003997861978:2049]
2025-11-27T18:17:31.476046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471003997861978:2049] [node 2] ICS09 handshake done sender: [5:7577471003997861970:2125] self: [5:7577471003997861972:0] peer: [2:7577471003664501276:0] socket: 264 qp: -1
2025-11-27T18:17:31.476077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:31.476126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471003664501268:2108] [node 5] ICP22 created new session: [2:7577471003664501305:2108]
2025-11-27T18:17:31.476486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471007008292514:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:31.476296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471003664501305:2108] [node 5] ICS09 handshake done sender: [2:7577471003664501277:2114] self: [2:7577471003664501276:0] peer: [5:7577471003997861972:0] socket: 237 qp: -1
2025-11-27T18:17:31.476245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471003997861978:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:31.476470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471003664501305:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:31.476279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:31.476541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:31.476993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471003838867224:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.477034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:31.477075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471003838867224:2048] [node 1] ICP22 created new session: [3:7577471003838867318:2048]
2025-11-27T18:17:31.477120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471003838867318:2048] [node 1] ICS09 handshake done sender: [3:7577471003838867304:2116] self: [3:7577471003838867305:0] peer: [1:7577471003805580207:0] socket: 224 qp: -1
2025-11-27T18:17:31.477143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471003838867318:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:31.477170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:31.477208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471003997861971:2126] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.477690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471007008292607:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29463
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.479231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471003838867317:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35321
2025-11-27T18:17:31.481185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471007008292515:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23386) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471007008292595:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.481358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471007008292595:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:31.483680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471003838867317:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.481609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003604307427:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35581
2025-11-27T18:17:31.481671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003604307428:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35581
2025-11-27T18:17:31.483661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471003604307426:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.484022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471007008292515:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.484069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:31.484112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471007008292515:2049] [node 2] ICP22 created new session: [4:7577471007008292609:2049]
2025-11-27T18:17:31.483951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471003604307343:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.484150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471007008292609:2049] [node 2] ICS09 handshake done sender: [4:7577471007008292595:2118] self: [4:7577471007008292598:0] peer: [2:7577471003664501274:0] socket: 247 qp: -1
2025-11-27T18:17:31.484178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471007008292609:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:31.484216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:31.485409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471003997861881:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.485458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:31.485500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471003997861881:2048] [node 1] ICP22 created new session: [5:7577471003997861980:2048]
2025-11-27T18:17:31.485532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471003997861980:2048] [node 1] ICS09 handshake done sender: [5:7577471003997861971:2126] self: [5:7577471003997861973:0] peer: [1:7577471003805580232:0] socket: 266 qp: -1
2025-11-27T18:17:31.485565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471003997861980:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:31.485590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: 2025-11-27T18:17:31.485796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471003997861881:2048] [node 1] ICP27 obsolete handshake fail ignored
::ffff:127.1.0.0
2025-11-27T18:17:31.486118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471003604307428:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.486238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003604307430:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35581
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.487002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471003997861977:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24700
2025-11-27T18:17:31.484746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471003664501279:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.484783Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471003664501279:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.486424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471003664501279:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:31.486914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471003664501269:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.486962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:31.487252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471003664501269:2110] [node 6] ICP22 created new session: [2:7577471003664501308:2110]
2025-11-27T18:17:31.493765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471003664501308:2110] [node 6] ICS09 handshake done sender: [2:7577471003664501279:2115] self: [2:7577471003664501278:0] peer: [6:7577471003604307429:0] socket: 273 qp: -1
2025-11-27T18:17:31.493908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471003664501308:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:31.493951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:31.494241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471003604307344:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.494540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471003604307426:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:31.493897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471003997861954:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.494342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471003997861977:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.495047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471003997861883:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.495127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471003604307343:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.495187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:31.495176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471007008292586:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.495241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471003604307343:2049] [node 2] ICP22 created new session: [6:7577471003604307432:2049]
2025-11-27T18:17:31.495215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471007008292586:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.495290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471003604307432:2049] [node 2] ICS09 handshake done sender: [6:7577471003604307426:2117] self: [6:7577471003604307429:0] peer: [2:7577471003664501278:0] socket: 274 qp: -1
2025-11-27T18:17:31.495325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471003604307432:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:31.495354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:31.495991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471003838867300:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.496029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471003838867300:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.497246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471007008292586:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:31.497831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471007008292577:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.497888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:31.497925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471007008292577:2108] [node 5] ICP22 created new session: [4:7577471007008292612:2108]
2025-11-27T18:17:31.497960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471007008292612:2108] [node 5] ICS09 handshake done sender: [4:7577471007008292586:2113] self: [4:7577471007008292585:0] peer: [5:7577471003997861982:0] socket: 250 qp: -1
2025-11-27T18:17:31.497987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471007008292612:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:31.498029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:31.498083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471003604307427:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.497890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471003838867298:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.497917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471003838867298:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.500425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471003997861986:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24700
2025-11-27T18:17:31.500282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471003604307343:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:31.500598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003604307434:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35581
2025-11-27T18:17:31.500735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471003604307409:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.500500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471003838867322:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35321
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.501304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003604307435:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35581
2025-11-27T18:17:31.501733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471003838867300:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:31.501800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471007008292588:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.501828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471007008292588:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.503854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471007008292614:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29463
2025-11-27T18:17:31.503985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471007008292588:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:31.504366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471003838867298:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:31.504571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471007008292578:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.504655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:31.504707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471007008292578:2109] [node 6] ICP22 created new session: [4:7577471007008292616:2109]
2025-11-27T18:17:31.506289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471003604307434:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.504752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471003838867290:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.504770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471007008292616:2109] [node 6] ICS09 handshake done sender: [4:7577471007008292588:2114] self: [4:7577471007008292587:0] peer: [6:7577471003604307436:0] socket: 238 qp: -1
2025-11-27T18:17:31.504813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:31.506491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471003604307428:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:31.505031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471007008292616:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:31.506646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:31.506083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471003838867290:2110] [node 6] ICP22 created new session: [3:7577471003838867325:2110]
2025-11-27T18:17:31.506142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471003838867325:2110] [node 6] ICS09 handshake done sender: [3:7577471003838867300:2115] self: [3:7577471003838867299:0] peer: [6:7577471003604307431:0] socket: 252 qp: -1
2025-11-27T18:17:31.507207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471003838867325:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:31.507247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:31.508030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471003604307344:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.508081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:31.508123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471003604307344:2050] [node 3] ICP22 created new session: [6:7577471003604307439:2050]
2025-11-27T18:17:31.508162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471003604307439:2050] [node 3] ICS09 handshake done sender: [6:7577471003604307428:2119] self: [6:7577471003604307431:0] peer: [3:7577471003838867299:0] socket: 278 qp: -1
2025-11-27T18:17:31.508199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471003604307439:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:31.508227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:31.508176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471003997861974:2127] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.508979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003604307438:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35581
2025-11-27T18:17:31.508538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471003997861954:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6003) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471003997861974:2127] held: no
2025-11-27T18:17:31.508643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471003997861954:2109] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.508653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471003838867322:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.508688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:31.508702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471003838867289:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.508724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471003997861954:2109] [node 4] ICP22 created new session: [5:7577471003997861988:2109]
2025-11-27T18:17:31.510050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471003604307404:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:31.508734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:31.508769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471003838867289:2109] [node 5] ICP22 created new session: [3:7577471003838867327:2109]
2025-11-27T18:17:31.508761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471003997861988:2109] [node 4] ICS09 handshake done sender: [5:7577471003997861974:2127] self: [5:7577471003997861982:0] peer: [4:7577471007008292585:0] socket: 271 qp: -1
2025-11-27T18:17:31.508804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471003838867327:2109] [node 5] ICS09 handshake done sender: [3:7577471003838867298:2114] self: [3:7577471003838867297:0] peer: [5:7577471003997861985:0] socket: 275 qp: -1
2025-11-27T18:17:31.508829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471003838867327:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:31.508791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471003997861988:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:31.508846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:31.508825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:31.509671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471003997861987:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24700
2025-11-27T18:17:31.510698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471003997861977:2130] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:31.511172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471003997861883:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.511231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:31.511286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471003997861883:2050] [node 3] ICP22 created new session: [5:7577471003997861990:2050]
2025-11-27T18:17:31.511345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471003997861990:2050] [node 3] ICS09 handshake done sender: [5:7577471003997861977:2130] self: [5:7577471003997861985:0] peer: [3:7577471003838867297:0] socket: 276 qp: -1
2025-11-27T18:17:31.511383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471003997861990:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:31.511435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:31.511262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471003838867289:2109] [node 5] ICP18 handshake (actor [3:7577471003838867322:2122]) is too old
2025-11-27T18:17:31.511784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471003604307427:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:31.511980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471007008292614:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.512068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471007008292578:2109] [node 6] ICP18 handshake (actor [4:7577471007008292614:2122]) is too old
2025-11-27T18:17:31.513607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471003997861965:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:31.513639Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471003997861965:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:31.515867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471003997861965:2124] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:31.516120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471003604307409:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.516193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:31.516240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471003604307409:2110] [node 4] ICP22 created new session: [6:7577471003604307442:2110]
2025-11-27T18:17:31.516278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471003604307442:2110] [node 4] ICS09 handshake done sender: [6:7577471003604307427:2118] self: [6:7577471003604307436:0] peer: [4:7577471007008292587:0] socket: 277 qp: -1
2025-11-27T18:17:31.516324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471003604307442:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:31.516361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:31.516231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471003997861955:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:31.516291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:31.516332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471003997861955:2110] [node 6] ICP22 created new session: [5:7577471003997861993:2110]
2025-11-27T18:17:31.516366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471003997861993:2110] [node 6] ICS09 handshake done sender: [5:7577471003997861965:2124] self: [5:7577471003997861964:0] peer: [6:7577471003604307441:0] socket: 279 qp: -1
2025-11-27T18:17:31.516575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471003604307409:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:31.516393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471003997861993:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:31.516422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:31.519670Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471003604307404:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471003604307434:2121] held: no
2025-11-27T18:17:31.519827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471003604307444:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35581
2025-11-27T18:17:31.520315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471003604307434:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:31.520547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471003604307404:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:31.520586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:31.520626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471003604307404:2106] [node 5] ICP22 created new session: [6:7577471003604307445:2106]
2025-11-27T18:17:31.520667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471003604307445:2106] [node 5] ICS09 handshake done sender: [6:7577471003604307434:2121] self: [6:7577471003604307441:0] peer: [5:7577471003997861964:0] socket: 281 qp: -1
2025-11-27T18:17:31.520699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471003604307445:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:31.520727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:31.537155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:31.537410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471003664501298:2048] [node 1] ICS01 socket: 236 reason# EndOfStream
2025-11-27T18:17:31.537457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471003664501203:2048] [node 1] ICP30 unregister session Session# [2:7577471003664501298:2048] VirtualId# [2:7577471003664501291:0]
2025-11-27T18:17:31.537488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471003664501298:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.537564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:31.537604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:31.538420Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:31.538565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471003838867318:2048] [node 1] ICS01 socket: 224 reason# EndOfStream
2025-11-27T18:17:31.538616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471003838867224:2048] [node 1] ICP30 unregister session Session# [3:7577471003838867318:2048] VirtualId# [3:7577471003838867305:0]
2025-11-27T18:17:31.538647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471003838867318:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.538696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:31.538728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:31.540562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:31.540747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471007008292605:2048] [node 1] ICS07 socket disconnect 230 reason# EndOfStream
2025-11-27T18:17:31.540774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471007008292605:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.540847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:31.540993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471007008292605:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:31.547331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471007008292514:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10614) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:31.547402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471007008292605:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:31.547437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471007008292514:2048] [node 1] ICP30 unregister session Session# [4:7577471007008292605:2048] VirtualId# [4:7577471007008292599:0]
2025-11-27T18:17:31.547496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:31.547528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471007008292514:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.547547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.549387Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:31.547138Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:31.547408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471003997861980:2048] [node 1] ICS07 socket disconnect 266 reason# EndOfStream
2025-11-27T18:17:31.547438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471003997861980:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.547531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:31.547573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471003997861980:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:31.558007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471003604307423:2048] [node 1] ICS01 socket: 267 reason# EndOfStream
2025-11-27T18:17:31.558126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471007008292514:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:31.557654Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471003997861881:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10614) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:31.557741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471003997861980:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:31.557774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471003997861881:2048] [node 1] ICP30 unregister session Session# [5:7577471003997861980:2048] VirtualId# [5:7577471003997861973:0]
2025-11-27T18:17:31.561348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471003604307342:2048] [node 1] ICP30 unregister session Session# [6:7577471003604307423:2048] VirtualId# [6:7577471003604307421:0]
2025-11-27T18:17:31.557822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:31.557844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471003997861881:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.557855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.566199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471003604307423:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.566328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:31.566390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:31.573838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471003997861881:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:31.582482Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:31.582728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471003838867315:2049] [node 2] ICS01 socket: 243 reason# EndOfStream
2025-11-27T18:17:31.582773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471003838867225:2049] [node 2] ICP30 unregister session Session# [3:7577471003838867315:2049] VirtualId# [3:7577471003838867308:0]
2025-11-27T18:17:31.582810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471003838867315:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.582888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:31.582932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:31.592257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:31.592503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471007008292609:2049] [node 2] ICS01 socket: 247 reason# EndOfStream
2025-11-27T18:17:31.592554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471007008292515:2049] [node 2] ICP30 unregister session Session# [4:7577471007008292609:2049] VirtualId# [4:7577471007008292598:0]
2025-11-27T18:17:31.592594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471007008292609:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.592673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:31.592726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:31.594207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:31.594370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471003604307432:2049] [node 2] ICS07 socket disconnect 274 reason# EndOfStream
2025-11-27T18:17:31.594791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:31.594405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471003604307432:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.595022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471003997861978:2049] [node 2] ICS01 socket: 264 reason# EndOfStream
2025-11-27T18:17:31.594490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:31.595079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471003997861882:2049] [node 2] ICP30 unregister session Session# [5:7577471003997861978:2049] VirtualId# [5:7577471003997861972:0]
2025-11-27T18:17:31.594536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471003604307432:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:31.595111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471003997861978:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.595211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:31.595259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:31.596249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471003604307343:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23386) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:31.596305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471003604307432:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:31.596335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471003604307343:2049] [node 2] ICP30 unregister session Session# [6:7577471003604307432:2049] VirtualId# [6:7577471003604307429:0]
2025-11-27T18:17:31.596368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:31.596408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471003604307343:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23386) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.596423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23386) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.613469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471003604307343:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:31.646598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:31.646816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471003997861990:2050] [node 3] ICS07 socket disconnect 276 reason# EndOfStream
2025-11-27T18:17:31.646843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471003997861990:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.646913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:31.646948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471003997861990:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:31.647890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471003997861883:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63881) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:31.647952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471003997861990:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:31.647986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471003997861883:2050] [node 3] ICP30 unregister session Session# [5:7577471003997861990:2050] VirtualId# [5:7577471003997861985:0]
2025-11-27T18:17:31.648034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:31.648061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471003997861883:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.648073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.649344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:31.649525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471007008292602:2050] [node 3] ICS01 socket: 229 reason# EndOfStream
2025-11-27T18:17:31.649568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471007008292516:2050] [node 3] ICP30 unregister session Session# [4:7577471007008292602:2050] VirtualId# [4:7577471007008292596:0]
2025-11-27T18:17:31.648675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:31.649610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471007008292602:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.649675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:31.648932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471003604307439:2050] [node 3] ICS07 socket disconnect 278 reason# EndOfStream
2025-11-27T18:17:31.649730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:31.648962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471003604307439:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.649060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:31.649113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471003604307439:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:31.660046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471003997861883:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:31.663843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471003604307344:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63881) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:31.663933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471003604307439:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:31.663967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471003604307344:2050] [node 3] ICP30 unregister session Session# [6:7577471003604307439:2050] VirtualId# [6:7577471003604307431:0]
2025-11-27T18:17:31.664029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:31.664072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471003604307344:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.664091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.674019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471003604307344:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:31.681566Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:31.681802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471003997861988:2109] [node 4] ICS01 socket: 271 reason# EndOfStream
2025-11-27T18:17:31.681843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471003997861954:2109] [node 4] ICP30 unregister session Session# [5:7577471003997861988:2109] VirtualId# [5:7577471003997861982:0]
2025-11-27T18:17:31.681876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471003997861988:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.681952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:31.682188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:31.681508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:31.681767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471003604307442:2110] [node 4] ICS07 socket disconnect 277 reason# EndOfStream
2025-11-27T18:17:31.681798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471003604307442:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.681878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:31.681922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471003604307442:2110] [node 4] ICS15 start handshake
2025-11-27T18:17:31.686605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471003604307409:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6003) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:31.686665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471003604307442:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:31.686707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471003604307409:2110] [node 4] ICP30 unregister session Session# [6:7577471003604307442:2110] VirtualId# [6:7577471003604307436:0]
2025-11-27T18:17:31.686756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:31.686790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471003604307409:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6003) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.686804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6003) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:31.694670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471003604307409:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:17:31.733444Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:31.733680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471003604307445:2106] [node 5] ICS01 socket: 281 reason# EndOfStream
2025-11-27T18:17:31.733730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471003604307404:2106] [node 5] ICP30 unregister session Session# [6:7577471003604307445:2106] VirtualId# [6:7577471003604307441:0]
2025-11-27T18:17:31.733765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471003604307445:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:31.733856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:31.733909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:31.821467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471006903464943:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:31.821512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471006903464944:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:31.842382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471003613983612:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:31.842428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471003613983613:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:31.867287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471005743197570:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:31.867325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471005743197571:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:31.890879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471007054796956:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:31.890926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471007054796957:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:31.890967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471007054796958:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:31.914024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471005271092433:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:31.914061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471005271092434:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:31.914089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471005271092435:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:31.936947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471003577761062:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:31.936990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471003577761063:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:31.937017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471003577761064:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:32.204169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471007872728418:2104] [node 4] ICP01 ready to work
2025-11-27T18:17:32.205140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471007872728422:2103] [node 5] ICP01 ready to work
2025-11-27T18:17:32.290652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471011198432303:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:32.327749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471010038164930:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:32.327671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471007908950971:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:32.360293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471010038164931:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:32.359962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471009566059792:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:32.361369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471006903464943:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(108521033ns) {Configure:104(108515218ns) {ProcessConfigured:135(90001000ns) {StartInitialHandshake:157(89996315ns) {DropHandshakes:470(11262502ns) {DropIncomingHandshake:442(1109ns) DropOutgoingHandshake:457(1433ns)} GenerateSessionVirtualId:988(2497ns) PrepareNewSessionHandshake:477(3766ns) {SwitchToState:179(1652ns)}}}}}
2025-11-27T18:17:32.361627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471010038164932:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:32.358702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471007908950973:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:32.359749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471011349764316:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:32.359899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471007908950972:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:32.365209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471009566059793:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:32.367148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471011198432305:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:32.367548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471011198432306:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:32.364839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471011349764315:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:32.394396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471005743197570:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33821250ns) {Configure:104(33248987ns) {ProcessConfigured:135(70584ns) {StartInitialHandshake:157(68584ns) {DropHandshakes:470(3289ns) {DropIncomingHandshake:442(842ns) DropOutgoingHandshake:457(413ns)} GenerateSessionVirtualId:988(882ns) PrepareNewSessionHandshake:477(1383ns) {SwitchToState:179(522ns)}}}}}
2025-11-27T18:17:32.417844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471007872728422:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(200048211ns) {Configure:104(200042477ns) {ProcessConfigured:135(54957ns) {StartInitialHandshake:157(52829ns) {DropHandshakes:470(3123ns) {DropIncomingHandshake:442(756ns) DropOutgoingHandshake:457(667ns)} GenerateSessionVirtualId:988(846ns) PrepareNewSessionHandshake:477(1765ns) {SwitchToState:179(846ns)}}}}}
2025-11-27T18:17:32.388870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471006903464944:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21462570ns) {Configure:104(20944085ns) {ProcessConfigured:135(11321620ns) {StartInitialHandshake:157(11319002ns) {DropHandshakes:470(3223ns) {DropIncomingHandshake:442(752ns) DropOutgoingHandshake:457(522ns)} GenerateSessionVirtualId:988(806ns) PrepareNewSessionHandshake:477(3786ns) {SwitchToState:179(1425ns)}}}}}
2025-11-27T18:17:32.422324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471007054796958:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(57222963ns) {Configure:104(57218003ns) {ProcessConfigured:135(53518ns) {StartInitialHandshake:157(51932ns) {DropHandshakes:470(1525ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(457ns)} GenerateSessionVirtualId:988(674ns) PrepareNewSessionHandshake:477(1239ns) {SwitchToState:179(511ns)}}}}}
2025-11-27T18:17:32.413463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471005271092434:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51874582ns) {Configure:104(51868258ns) {ProcessConfigured:135(37054520ns) {StartInitialHandshake:157(37051877ns) {DropHandshakes:470(1310ns) {DropIncomingHandshake:442(435ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:477(5427ns) {SwitchToState:179(2481ns)}}}}}
2025-11-27T18:17:32.429621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471010038164932:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49131776ns) {Configure:104(49128526ns) {ProcessConfigured:135(4017188ns) {StartInitialHandshake:157(4015029ns) {DropHandshakes:470(856ns) {DropIncomingHandshake:442(255ns) DropOutgoingHandshake:457(170ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(2632ns) {SwitchToState:179(642ns)}}}}}
2025-11-27T18:17:32.430053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471007054796957:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55948862ns) {Configure:104(55943351ns) {ProcessConfigured:135(18214274ns) {StartInitialHandshake:157(18211438ns) {DropHandshakes:470(2767ns) {DropIncomingHandshake:442(942ns) DropOutgoingHandshake:457(308ns)} GenerateSessionVirtualId:988(1027ns) PrepareNewSessionHandshake:477(6616ns) {SwitchToState:179(3273ns)}}}}}
2025-11-27T18:17:32.419964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471007872728418:2104] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(215497776ns) {Configure:104(215492490ns) {ProcessConfigured:135(30065818ns) {StartInitialHandshake:157(30062097ns) {DropHandshakes:470(2875ns) {DropIncomingHandshake:442(509ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(1047ns) PrepareNewSessionHandshake:477(2794ns) {SwitchToState:179(824ns)}}}}}
2025-11-27T18:17:32.430796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471003613983612:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(69440200ns) {Configure:104(69432722ns) {ProcessConfigured:135(61149ns) {StartInitialHandshake:157(58571ns) {DropHandshakes:470(5023ns) {DropIncomingHandshake:442(406ns) DropOutgoingHandshake:457(749ns)} GenerateSessionVirtualId:988(622ns) PrepareNewSessionHandshake:477(771ns) {SwitchToState:179(348ns)}}}}}
2025-11-27T18:17:32.437933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471003577761062:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17034759ns) {Configure:104(17029869ns) {ProcessConfigured:135(2122969ns) {StartInitialHandshake:157(2121030ns) {DropHandshakes:470(1285ns) {DropIncomingHandshake:442(256ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(415ns) PrepareNewSessionHandshake:477(2827ns) {SwitchToState:179(1645ns)}}}}}
2025-11-27T18:17:32.438406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471011198432303:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(42106175ns) {Configure:104(42100261ns) {ProcessConfigured:135(12382792ns) {StartInitialHandshake:157(12381264ns) {DropHandshakes:470(1815ns) {DropIncomingHandshake:442(436ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(576ns) PrepareNewSessionHandshake:477(2499ns) {SwitchToState:179(1417ns)}}}}}
2025-11-27T18:17:32.439945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471007908950971:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(78442964ns) {Configure:104(78437372ns) {ProcessConfigured:135(2304825ns) {StartInitialHandshake:157(1463820ns) {DropHandshakes:470(2184ns) {DropIncomingHandshake:442(371ns) DropOutgoingHandshake:457(525ns)} GenerateSessionVirtualId:988(899ns) PrepareNewSessionHandshake:477(2513ns) {SwitchToState:179(1297ns)}}}}}
2025-11-27T18:17:32.446181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471010038164931:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(51522151ns) {Configure:104(51517312ns) {ProcessConfigured:135(74761ns) {StartInitialHandshake:157(72286ns) {DropHandshakes:470(2994ns) {DropIncomingHandshake:442(617ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(875ns) PrepareNewSessionHandshake:477(9044ns) {SwitchToState:179(597ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.450484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471010038164943:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47638
2025-11-27T18:17:32.450696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471010038164944:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47638
2025-11-27T18:17:32.452981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471010038164944:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.453055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471010038164943:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.453308Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471003577761063:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17064151ns) {Configure:104(15700313ns) {ProcessConfigured:135(252366ns) {StartInitialHandshake:157(249923ns) {DropHandshakes:470(1307ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(790ns) PrepareNewSessionHandshake:477(1932ns) {SwitchToState:179(544ns)}}}}}
2025-11-27T18:17:32.453295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471011198432305:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17635438ns) {Configure:104(16946393ns) {ProcessConfigured:135(500090ns) {StartInitialHandshake:157(496836ns) {DropHandshakes:470(2419ns) {DropIncomingHandshake:442(574ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(759ns) PrepareNewSessionHandshake:477(4508ns) {SwitchToState:179(2074ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.455236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471007908950984:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
2025-11-27T18:17:32.458850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471009566059807:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37215
2025-11-27T18:17:32.458000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471007872728427:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30313
2025-11-27T18:17:32.455248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471011198432310:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14387
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.462017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471007872728427:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.461463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471011198432310:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.471057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471007872728422:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.475068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471011349764330:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27418
2025-11-27T18:17:32.474248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471010038164950:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47638
2025-11-27T18:17:32.461552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471011198432316:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14387
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.483643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471009566059797:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.475475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471007908950984:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.461667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471011198432317:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14387
2025-11-27T18:17:32.485516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471011349764331:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27418
2025-11-27T18:17:32.493325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471010038164936:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.470700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471011198432317:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.483013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471007908950989:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
2025-11-27T18:17:32.485786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471011349764330:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.483697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471009566059797:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.493359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471010038164936:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.483068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471007908950990:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.494761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471010038164936:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:32.488134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471011349764332:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27418
2025-11-27T18:17:32.492200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471009566059797:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.470840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471011198432316:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.497381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471007908950990:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.489777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471011349764331:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.496950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471010038164930:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.495838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471009566059807:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.490136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471011198432311:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.495890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471009566059792:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.499020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471007908950989:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.489893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471007054796956:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.490158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471011198432311:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.497001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:32.500416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471007872728422:2103] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24465) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471007872728427:2112] held: no
2025-11-27T18:17:32.497044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471010038164930:2107] [node 4] ICP22 created new session: [3:7577471010038164953:2107]
2025-11-27T18:17:32.500407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471011198432311:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:32.501007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471007872728438:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30313
2025-11-27T18:17:32.500994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471007908950986:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
2025-11-27T18:17:32.492675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471007054796958:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.495929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:32.497083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471010038164953:2107] [node 4] ICS09 handshake done sender: [3:7577471010038164936:2112] self: [3:7577471010038164935:0] peer: [4:7577471011349764334:0] socket: 236 qp: -1
2025-11-27T18:17:32.505248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471007908950986:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.503929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471011198432322:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14387
2025-11-27T18:17:32.505536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471003613983612:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.505188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471011198432303:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.497120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471010038164953:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:32.506032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:32.494568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471011349764335:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27418
2025-11-27T18:17:32.506495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471011198432303:2107] [node 4] ICP22 created new session: [1:7577471011198432326:2107]
2025-11-27T18:17:32.496944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471011349764332:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.497554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471011349764330:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:32.507268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471011198432326:2107] [node 4] ICS09 handshake done sender: [1:7577471011198432311:2113] self: [1:7577471011198432309:0] peer: [4:7577471011349764333:0] socket: 248 qp: -1
2025-11-27T18:17:32.495964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471009566059792:2108] [node 6] ICP22 created new session: [5:7577471009566059810:2108]
2025-11-27T18:17:32.497154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:32.505221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471007054796958:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.507300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471011198432326:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:32.508023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471007908950980:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.496024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471009566059810:2108] [node 6] ICS09 handshake done sender: [5:7577471009566059797:2111] self: [5:7577471009566059796:0] peer: [6:7577471007872728436:0] socket: 223 qp: -1
2025-11-27T18:17:32.508253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471007872728427:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:32.498050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471010038164951:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47638
2025-11-27T18:17:32.505268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:32.508050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:32.501338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471010038164951:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.496065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471009566059810:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:32.508055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471007908950980:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.505973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471007054796958:2050] [node 3] ICP22 created new session: [4:7577471011349764339:2050]
2025-11-27T18:17:32.510244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471007872728422:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.508616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471011198432308:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.505189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471005743197570:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.496096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:32.514587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:32.507112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471011349764339:2050] [node 3] ICS09 handshake done sender: [4:7577471011349764330:2115] self: [4:7577471011349764334:0] peer: [3:7577471010038164935:0] socket: 237 qp: -1
2025-11-27T18:17:32.496626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577471009566059792:2108] [node 6] ICP18 handshake (actor [5:7577471009566059807:2115]) is too old
2025-11-27T18:17:32.515070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471007872728422:2103] [node 5] ICP22 created new session: [6:7577471007872728441:2103]
2025-11-27T18:17:32.499741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471009566059808:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37215
2025-11-27T18:17:32.507359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471011349764339:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:32.515580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471007872728441:2103] [node 5] ICS09 handshake done sender: [6:7577471007872728427:2112] self: [6:7577471007872728436:0] peer: [5:7577471009566059796:0] socket: 225 qp: -1
2025-11-27T18:17:32.509478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471007908950977:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.508639Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471011198432308:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.507962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:32.506440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471009566059808:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.509500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471007908950977:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.514825Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471007054796958:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:32.507283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471005271092434:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.513393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471007908950980:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:32.515921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471007908950977:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:32.507112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471010038164950:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.508008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471005743197571:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.514812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471011198432322:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.515638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471007872728441:2103] [node 5] ICS10 traffic start
2025-11-27T18:17:32.518651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471009566059814:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37215
2025-11-27T18:17:32.519777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471010038164958:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47638
2025-11-27T18:17:32.515687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:32.515656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471011198432304:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.518626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471007872728439:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30313
2025-11-27T18:17:32.521410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471009566059808:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.515689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471011198432304:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.520751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471007908950973:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.522009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471011198432308:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:32.525035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471005271092434:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.525591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471011198432304:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:32.521830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:32.525673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471011349764337:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27418
2025-11-27T18:17:32.525744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:32.527209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471006903464944:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.521869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471003613983613:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.528828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471007054796956:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13112) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471011349764331:2116] held: no
2025-11-27T18:17:32.526054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471005271092434:2049] [node 2] ICP22 created new session: [5:7577471009566059815:2049]
2025-11-27T18:17:32.527263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:32.528948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471011349764338:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27418
2025-11-27T18:17:32.521916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:32.527316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471006903464944:2049] [node 3] ICP22 created new session: [1:7577471011198432332:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.526363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471009566059815:2049] [node 2] ICS09 handshake done sender: [5:7577471009566059808:2116] self: [5:7577471009566059813:0] peer: [2:7577471007908950979:0] socket: 252 qp: -1
2025-11-27T18:17:32.527347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471011198432332:2049] [node 3] ICS09 handshake done sender: [1:7577471011198432308:2111] self: [1:7577471011198432307:0] peer: [3:7577471010038164956:0] socket: 253 qp: -1
2025-11-27T18:17:32.529627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471007872728439:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.527370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471011198432332:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:32.524746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471007908950973:2108] [node 5] ICP22 created new session: [2:7577471007908950997:2108]
2025-11-27T18:17:32.528346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471009566059815:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.527403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:32.530977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471007872728443:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30313
2025-11-27T18:17:32.524830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471003613983613:2049] [node 3] ICP22 created new session: [2:7577471007908950998:2049]
2025-11-27T18:17:32.532007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471011349764337:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.528642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:32.531402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471003577761064:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.533536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471010038164959:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47638
2025-11-27T18:17:32.525163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471007908950998:2049] [node 3] ICS09 handshake done sender: [2:7577471007908950977:2111] self: [2:7577471007908950976:0] peer: [3:7577471010038164957:0] socket: 244 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.527670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471006903464943:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.533742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471011349764331:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:32.525307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471007908950997:2108] [node 5] ICS09 handshake done sender: [2:7577471007908950980:2112] self: [2:7577471007908950979:0] peer: [5:7577471009566059813:0] socket: 249 qp: -1
2025-11-27T18:17:32.533975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471007872728443:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.527696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:32.534311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471003577761062:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.527715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471006903464943:2048] [node 2] ICP22 created new session: [1:7577471011198432334:2048]
2025-11-27T18:17:32.527741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471011198432334:2048] [node 2] ICS09 handshake done sender: [1:7577471011198432304:2110] self: [1:7577471011198432302:0] peer: [2:7577471007908950993:0] socket: 243 qp: -1
2025-11-27T18:17:32.535359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471009566059817:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37215
2025-11-27T18:17:32.525850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471007908950997:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:32.525905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:32.526053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471007908950998:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:32.526304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:32.527757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471011198432334:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:32.535457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471007054796956:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.527793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:32.536941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.539996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471011198432315:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.540385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471005743197570:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13112) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471010038164951:2119] held: no
2025-11-27T18:17:32.540028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471011198432315:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.540160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471009566059817:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.541107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471010038164942:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.537971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471007054796956:2048] [node 1] ICP22 created new session: [4:7577471011349764344:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.541157Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471010038164942:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.541050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471003613983612:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13112) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471007908950989:2117] held: no
2025-11-27T18:17:32.540464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471011349764344:2048] [node 1] ICS09 handshake done sender: [4:7577471011349764331:2116] self: [4:7577471011349764333:0] peer: [1:7577471011198432309:0] socket: 257 qp: -1
2025-11-27T18:17:32.541578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471007872728445:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30313
2025-11-27T18:17:32.542121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471009566059818:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37215
2025-11-27T18:17:32.542186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471007908951001:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.540880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471011349764344:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:32.543729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471007872728447:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30313
2025-11-27T18:17:32.542682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471011198432315:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.544520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471007872728445:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.543346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471005271092433:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.541631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.544722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471011349764346:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27418
2025-11-27T18:17:32.543869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471005743197571:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9679) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471010038164950:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.544933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471009566059818:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.545936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471007872728418:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.547908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471007872728448:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30313
2025-11-27T18:17:32.544605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471010038164958:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.545809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471009566059793:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.546654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471011198432313:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.546312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471011349764324:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.557900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471007908950989:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:32.545833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471010038164950:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:32.546707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471011198432313:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.546340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471011349764324:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.546196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471010038164960:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47638
2025-11-27T18:17:32.547116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471011198432306:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.555170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471007872728443:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:32.549059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471011349764326:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.547154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:32.549088Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471011349764326:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.561062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471005743197571:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.547198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471011198432306:2109] [node 6] ICP22 created new session: [1:7577471011198432338:2109]
2025-11-27T18:17:32.562110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471003577761062:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.561221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471011349764326:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:32.567803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471003613983612:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.561470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:32.561450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471011349764324:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:32.567860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:32.566589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.547243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471011198432338:2109] [node 6] ICS09 handshake done sender: [1:7577471011198432315:2115] self: [1:7577471011198432314:0] peer: [6:7577471007872728446:0] socket: 275 qp: -1
2025-11-27T18:17:32.569881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471003613983612:2048] [node 1] ICP22 created new session: [2:7577471007908951004:2048]
2025-11-27T18:17:32.566676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471003577761062:2048] [node 1] ICP22 created new session: [6:7577471007872728451:2048]
2025-11-27T18:17:32.561545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471005743197571:2049] [node 2] ICP22 created new session: [3:7577471010038164962:2049]
2025-11-27T18:17:32.547292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471011198432338:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:32.581557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471009566059822:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37215
2025-11-27T18:17:32.570278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471011349764316:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.574172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471007908951004:2048] [node 1] ICS09 handshake done sender: [2:7577471007908950989:2117] self: [2:7577471007908950993:0] peer: [1:7577471011198432302:0] socket: 258 qp: -1
2025-11-27T18:17:32.561847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471010038164962:2049] [node 2] ICS09 handshake done sender: [3:7577471010038164950:2118] self: [3:7577471010038164957:0] peer: [2:7577471007908950976:0] socket: 247 qp: -1
2025-11-27T18:17:32.566724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471007872728451:2048] [node 1] ICS09 handshake done sender: [6:7577471007872728443:2118] self: [6:7577471007872728446:0] peer: [1:7577471011198432314:0] socket: 276 qp: -1
2025-11-27T18:17:32.566765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471007872728451:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:32.566823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:32.567542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471007872728450:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30313
2025-11-27T18:17:32.568379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471007872728445:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:32.568816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471007872728453:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30313
2025-11-27T18:17:32.570325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:32.568860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471007872728418:2104] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.610711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471009566059821:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37215
2025-11-27T18:17:32.548099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:32.562051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471010038164962:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:32.568910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:32.575350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471007908951004:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:32.570371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471011349764316:2108] [node 5] ICP22 created new session: [4:7577471011349764349:2108]
2025-11-27T18:17:32.554138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471011198432313:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:32.562097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:32.612372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471005271092433:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13112) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471009566059817:2118] held: no
2025-11-27T18:17:32.568961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471007872728418:2104] [node 4] ICP22 created new session: [6:7577471007872728454:2104]
2025-11-27T18:17:32.577834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:32.570408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471011349764349:2108] [node 5] ICS09 handshake done sender: [4:7577471011349764324:2113] self: [4:7577471011349764323:0] peer: [5:7577471009566059820:0] socket: 281 qp: -1
2025-11-27T18:17:32.561735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471011198432305:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.579094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471007908950983:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.569018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471007872728454:2104] [node 4] ICS09 handshake done sender: [6:7577471007872728445:2119] self: [6:7577471007872728449:0] peer: [4:7577471011349764325:0] socket: 280 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.562120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471010038164951:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:17:32.570432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471011349764349:2108] [node 5] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:17:32.562010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:32.567943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471011198432305:2108] [node 5] ICP22 created new session: [1:7577471011198432341:2108]
2025-11-27T18:17:32.568011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471011198432341:2108] [node 5] ICS09 handshake done sender: [1:7577471011198432313:2114] self: [1:7577471011198432312:0] peer: [5:7577471009566059819:0] socket: 242 qp: -1
2025-11-27T18:17:32.568062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471011198432341:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:32.568110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:32.612526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471009566059817:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:32.571325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471005743197570:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.579129Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471007908950983:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.569050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471007872728454:2104] [node 4] ICS10 traffic start
2025-11-27T18:17:32.580262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471007908950983:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:32.570456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:32.585666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471011349764346:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.589550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471011349764315:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.589609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:32.597572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471011349764315:2109] [node 6] ICP22 created new session: [4:7577471011349764351:2109]
2025-11-27T18:17:32.601478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471011349764351:2109] [node 6] ICS09 handshake done sender: [4:7577471011349764326:2114] self: [4:7577471011349764325:0] peer: [6:7577471007872728449:0] socket: 241 qp: -1
2025-11-27T18:17:32.609006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471011349764351:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:32.611538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:32.613404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471011349764316:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(27365521ns) {HandleHandshakeStatus:329(27363290ns) {DropIncomingHandshake:442(12388ns) DropOutgoingHandshake:457(237ns) SwitchToState:179(373ns) ProcessPendingSessionEvents:495(27069584ns) {ForwardSessionEventToSession:612(6469042ns) ForwardSessionEventToSession:612(7575442ns) ForwardSessionEventToSession:612(12088ns) ForwardSessionEventToSession:612(145237ns) ForwardSessionEventToSession:612(2408ns) ForwardSessionEventToSession:612(2493ns) ForwardSessionEventToSession:612(7096566ns) ForwardSessionEventToSession:612(4043ns) ForwardSessionEventToSession:612(4525571ns) ForwardSessionEventToSession:612(341850ns)}}}
2025-11-27T18:17:32.623958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471011349764315:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(34226953ns) {HandleHandshakeStatus:329(34223134ns) {DropIncomingHandshake:442(13987ns) DropOutgoingHandshake:457(350ns) SwitchToState:179(259ns) ProcessPendingSessionEvents:495(10331112ns) {ForwardSessionEventToSession:612(5785584ns) ForwardSessionEventToSession:612(6324ns) ForwardSessionEventToSession:612(3777561ns) ForwardSessionEventToSession:612(3143ns) ForwardSessionEventToSession:612(271303ns) ForwardSessionEventToSession:612(2954ns) ForwardSessionEventToSession:612(3185ns) ForwardSessionEventToSession:612(461572ns) ForwardSessionEventToSession:612(3327ns) ForwardSessionEventToSession:612(2422ns)}}}
2025-11-27T18:17:32.625699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471007054796957:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.618958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471009566059818:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:32.573373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:32.569077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:32.606755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471007908950972:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.571770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471007872728447:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.574063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471010038164942:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:32.624540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471009566059823:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37215
2025-11-27T18:17:32.606823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:32.571982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471003577761063:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.577375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471005743197570:2048] [node 1] ICP22 created new session: [3:7577471010038164966:2048]
2025-11-27T18:17:32.624592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471005271092433:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:32.624632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:32.649300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471011349764354:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27418
2025-11-27T18:17:32.606882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471007908950972:2109] [node 6] ICP22 created new session: [2:7577471007908951007:2109]
2025-11-27T18:17:32.627328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471005271092433:2048] [node 1] ICP22 created new session: [5:7577471009566059824:2048]
2025-11-27T18:17:32.606937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471007908951007:2109] [node 6] ICS09 handshake done sender: [2:7577471007908950983:2114] self: [2:7577471007908950982:0] peer: [6:7577471007872728456:0] socket: 283 qp: -1
2025-11-27T18:17:32.606976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471007908951007:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:32.630225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471009566059824:2048] [node 1] ICS09 handshake done sender: [5:7577471009566059817:2118] self: [5:7577471009566059819:0] peer: [1:7577471011198432312:0] socket: 256 qp: -1
2025-11-27T18:17:32.607007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:32.593632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471007872728439:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:32.630265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471009566059824:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:32.577429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471010038164966:2048] [node 1] ICS09 handshake done sender: [3:7577471010038164951:2119] self: [3:7577471010038164956:0] peer: [1:7577471011198432307:0] socket: 255 qp: -1
2025-11-27T18:17:32.577468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471010038164966:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:32.609048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471003613983612:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(26175808ns) {HandleHandshakeStatus:329(26172205ns) {DropIncomingHandshake:442(743ns) DropOutgoingHandshake:457(380ns) SwitchToState:179(447ns) ProcessPendingSessionEvents:495(1112ns)}}
2025-11-27T18:17:32.626114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471007908950981:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.626138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471007908950981:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.637472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471007908950981:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:32.645360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471007908950971:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.645437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:32.645489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471007908950971:2107] [node 4] ICP22 created new session: [2:7577471007908951010:2107]
2025-11-27T18:17:32.645563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471007908951010:2107] [node 4] ICS09 handshake done sender: [2:7577471007908950981:2113] self: [2:7577471007908950978:0] peer: [4:7577471011349764353:0] socket: 238 qp: -1
2025-11-27T18:17:32.645626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471007908951010:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:32.645665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:32.613818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471003577761064:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.577530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:32.630308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:32.613860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:32.597557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471010038164931:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.630693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471009566059793:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.597629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:32.614342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471003577761064:2050] [node 3] ICP22 created new session: [6:7577471007872728458:2050]
2025-11-27T18:17:32.630731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:32.597671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471010038164931:2110] [node 6] ICP22 created new session: [3:7577471010038164968:2110]
2025-11-27T18:17:32.653365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471007054796957:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9679) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471011349764346:2121] held: no
2025-11-27T18:17:32.621379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471007872728458:2050] [node 3] ICS09 handshake done sender: [6:7577471007872728439:2117] self: [6:7577471007872728444:0] peer: [3:7577471010038164941:0] socket: 245 qp: -1
2025-11-27T18:17:32.630766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471009566059793:2109] [node 4] ICP22 created new session: [5:7577471009566059826:2109]
2025-11-27T18:17:32.612032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471010038164968:2110] [node 6] ICS09 handshake done sender: [3:7577471010038164942:2115] self: [3:7577471010038164941:0] peer: [6:7577471007872728444:0] socket: 234 qp: -1
2025-11-27T18:17:32.621432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471007872728458:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:32.630808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471009566059826:2109] [node 4] ICS09 handshake done sender: [5:7577471009566059818:2119] self: [5:7577471009566059820:0] peer: [4:7577471011349764323:0] socket: 282 qp: -1
2025-11-27T18:17:32.621479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:32.612630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471010038164968:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:32.630857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471009566059826:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:32.627198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471003577761063:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9679) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471007872728447:2120] held: no
2025-11-27T18:17:32.612667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:32.630886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:32.632847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471007872728457:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30313
2025-11-27T18:17:32.624715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471010038164931:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(27100581ns) {HandleHandshakeStatus:329(27003629ns) {DropIncomingHandshake:442(25332ns) DropOutgoingHandshake:457(428ns) SwitchToState:179(531ns) ProcessPendingSessionEvents:495(1189964ns) {ForwardSessionEventToSession:612(1141494ns) ForwardSessionEventToSession:612(7427ns) ForwardSessionEventToSession:612(2661ns) ForwardSessionEventToSession:612(7221ns) ForwardSessionEventToSession:612(2250ns) ForwardSessionEventToSession:612(2409ns) ForwardSessionEventToSession:612(5309ns) ForwardSessionEventToSession:612(2618ns) ForwardSessionEventToSession:612(2430ns) ForwardSessionEventToSession:612(7856ns)}}}
2025-11-27T18:17:32.631081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471009566059793:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:32.649125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471007872728447:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:32.653631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471009566059823:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.650048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471003577761063:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.653792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471005271092435:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:32.650097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:32.650149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471003577761063:2049] [node 2] ICP22 created new session: [6:7577471007872728460:2049]
2025-11-27T18:17:32.650199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471007872728460:2049] [node 2] ICS09 handshake done sender: [6:7577471007872728447:2120] self: [6:7577471007872728456:0] peer: [2:7577471007908950982:0] socket: 235 qp: -1
2025-11-27T18:17:32.650236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471007872728460:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:32.650272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:32.654305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471011349764346:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:32.654461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471010038164940:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:32.654504Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471010038164940:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:32.655846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471007054796957:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.657271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471010038164940:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:32.655913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:32.655966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471007054796957:2049] [node 2] ICP22 created new session: [4:7577471011349764355:2049]
2025-11-27T18:17:32.656014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471011349764355:2049] [node 2] ICS09 handshake done sender: [4:7577471011349764346:2121] self: [4:7577471011349764353:0] peer: [2:7577471007908950978:0] socket: 254 qp: -1
No RDMA context found for GID: 2025-11-27T18:17:32.656047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471011349764355:2049] [node 2] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:17:32.656080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:32.657644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471010038164932:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:32.657723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471009566059829:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37215
2025-11-27T18:17:32.658073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471009566059823:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:32.657997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:32.658073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471010038164932:2109] [node 5] ICP22 created new session: [3:7577471010038164971:2109]
2025-11-27T18:17:32.658125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471010038164971:2109] [node 5] ICS09 handshake done sender: [3:7577471010038164940:2114] self: [3:7577471010038164939:0] peer: [5:7577471009566059828:0] socket: 294 qp: -1
2025-11-27T18:17:32.659083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471005271092435:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:32.658162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471010038164971:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:32.659134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:32.659706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471005271092435:2050] [node 3] ICP22 created new session: [5:7577471009566059830:2050]
2025-11-27T18:17:32.662024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471009566059830:2050] [node 3] ICS09 handshake done sender: [5:7577471009566059823:2122] self: [5:7577471009566059828:0] peer: [3:7577471010038164939:0] socket: 295 qp: -1
2025-11-27T18:17:32.662071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471009566059830:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:32.662119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:32.658333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:32.695071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:32.695313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471007908951004:2048] [node 1] ICS01 socket: 258 reason# EndOfStream
2025-11-27T18:17:32.695356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471003613983612:2048] [node 1] ICP30 unregister session Session# [2:7577471007908951004:2048] VirtualId# [2:7577471007908950993:0]
2025-11-27T18:17:32.695388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471007908951004:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.695461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:32.695501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:32.697352Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:32.697576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471010038164966:2048] [node 1] ICS01 socket: 255 reason# EndOfStream
2025-11-27T18:17:32.697621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471005743197570:2048] [node 1] ICP30 unregister session Session# [3:7577471010038164966:2048] VirtualId# [3:7577471010038164956:0]
2025-11-27T18:17:32.697647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471010038164966:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.697722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:32.697760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:32.698755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:32.699042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471011349764344:2048] [node 1] ICS01 socket: 257 reason# EndOfStream
2025-11-27T18:17:32.699117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471007054796956:2048] [node 1] ICP30 unregister session Session# [4:7577471011349764344:2048] VirtualId# [4:7577471011349764333:0]
2025-11-27T18:17:32.699157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471011349764344:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.699252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:32.699321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:32.699741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:32.699944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471009566059824:2048] [node 1] ICS01 socket: 256 reason# EndOfStream
2025-11-27T18:17:32.699984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471005271092433:2048] [node 1] ICP30 unregister session Session# [5:7577471009566059824:2048] VirtualId# [5:7577471009566059819:0]
2025-11-27T18:17:32.700013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471009566059824:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.700078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:32.700151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:32.701465Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:32.701684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471007872728451:2048] [node 1] ICS01 socket: 276 reason# EndOfStream
2025-11-27T18:17:32.701726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471003577761062:2048] [node 1] ICP30 unregister session Session# [6:7577471007872728451:2048] VirtualId# [6:7577471007872728446:0]
2025-11-27T18:17:32.701752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471007872728451:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.701827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:32.701871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:32.737601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:32.737846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471010038164962:2049] [node 2] ICS01 socket: 247 reason# EndOfStream
2025-11-27T18:17:32.737884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471005743197571:2049] [node 2] ICP30 unregister session Session# [3:7577471010038164962:2049] VirtualId# [3:7577471010038164957:0]
2025-11-27T18:17:32.737928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471010038164962:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.738012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:32.738059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:32.743376Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:32.744212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:32.743581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471009566059815:2049] [node 2] ICS07 socket disconnect 252 reason# EndOfStream
2025-11-27T18:17:32.744385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471007872728460:2049] [node 2] ICS01 socket: 235 reason# EndOfStream
2025-11-27T18:17:32.743608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471009566059815:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.744420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471003577761063:2049] [node 2] ICP30 unregister session Session# [6:7577471007872728460:2049] VirtualId# [6:7577471007872728456:0]
2025-11-27T18:17:32.743690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:32.744448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471007872728460:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.743734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471009566059815:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:32.744510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:32.744548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:32.745436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:32.745327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471005271092434:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9679) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:32.745379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471009566059815:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:32.745414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471005271092434:2049] [node 2] ICP30 unregister session Session# [5:7577471009566059815:2049] VirtualId# [5:7577471009566059813:0]
2025-11-27T18:17:32.745454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:32.745484Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471005271092434:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9679) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:32.745498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9679) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:32.745701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471011349764355:2049] [node 2] ICS01 socket: 254 reason# EndOfStream
2025-11-27T18:17:32.745748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471007054796957:2049] [node 2] ICP30 unregister session Session# [4:7577471011349764355:2049] VirtualId# [4:7577471011349764353:0]
2025-11-27T18:17:32.745786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471011349764355:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.745894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:32.745962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:32.756726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471005271092434:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:32.819657Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:32.821037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:32.820557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:32.820751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471009566059830:2050] [node 3] ICS01 socket: 295 reason# EndOfStream
2025-11-27T18:17:32.820791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471005271092435:2050] [node 3] ICP30 unregister session Session# [5:7577471009566059830:2050] VirtualId# [5:7577471009566059828:0]
2025-11-27T18:17:32.820827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471009566059830:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.820895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:32.820943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:32.819889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471011349764339:2050] [node 3] ICS07 socket disconnect 237 reason# EndOfStream
2025-11-27T18:17:32.819924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471011349764339:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.820004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:32.820048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471011349764339:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:32.821230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471007872728458:2050] [node 3] ICS01 socket: 245 reason# EndOfStream
2025-11-27T18:17:32.821296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471003577761064:2050] [node 3] ICP30 unregister session Session# [6:7577471007872728458:2050] VirtualId# [6:7577471007872728444:0]
2025-11-27T18:17:32.821324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471007872728458:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.821395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:32.821434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:32.822728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471007054796958:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5818) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:32.822780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471011349764339:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:32.822808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471007054796958:2050] [node 3] ICP30 unregister session Session# [4:7577471011349764339:2050] VirtualId# [4:7577471011349764334:0]
2025-11-27T18:17:32.822850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:32.822875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471007054796958:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5818) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:32.822891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5818) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:32.833130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471007054796958:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:32.854751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:32.855031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471009566059826:2109] [node 4] ICS01 socket: 282 reason# EndOfStream
2025-11-27T18:17:32.855087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471009566059793:2109] [node 4] ICP30 unregister session Session# [5:7577471009566059826:2109] VirtualId# [5:7577471009566059820:0]
2025-11-27T18:17:32.855126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471009566059826:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.855260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:32.855327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:32.854957Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:32.855210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471007872728454:2104] [node 4] ICS01 socket: 280 reason# EndOfStream
2025-11-27T18:17:32.855248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471007872728418:2104] [node 4] ICP30 unregister session Session# [6:7577471007872728454:2104] VirtualId# [6:7577471007872728449:0]
2025-11-27T18:17:32.855278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471007872728454:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.855361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:32.855429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:32.895086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:32.895298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471007872728441:2103] [node 5] ICS01 socket: 225 reason# EndOfStream
2025-11-27T18:17:32.895334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471007872728422:2103] [node 5] ICP30 unregister session Session# [6:7577471007872728441:2103] VirtualId# [6:7577471007872728436:0]
2025-11-27T18:17:32.895371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471007872728441:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:32.895461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:32.895506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:32.998807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471011062421425:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:32.998849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471011062421426:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:33.027024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471014463898364:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:33.027060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471014463898363:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:33.050402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471014504786080:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:33.050438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471014504786079:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:33.073317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471016154161927:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:33.073359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471016154161928:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:33.073982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471016154161926:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:33.097068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471012158109094:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:33.097138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471012158109096:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:33.097850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471012158109095:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:33.124803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471015877290798:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:33.124853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471015877290799:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:33.124876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471015877290800:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:33.258361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471015877290848:2096] [node 5] ICP01 ready to work
2025-11-27T18:17:33.305786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471015357388784:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:33.297419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471015877290856:2101] [node 4] ICP01 ready to work
2025-11-27T18:17:33.483425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471016154161989:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:33.477563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471012158109157:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:33.481178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471015357388785:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:33.474610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471014504786144:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:33.512293Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471015877290848:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(253616869ns) {Configure:104(253611468ns) {ProcessConfigured:135(75923ns) {StartInitialHandshake:157(72702ns) {DropHandshakes:470(3379ns) {DropIncomingHandshake:442(766ns) DropOutgoingHandshake:457(607ns)} GenerateSessionVirtualId:988(1173ns) PrepareNewSessionHandshake:477(2475ns) {SwitchToState:179(763ns)}}}}}
2025-11-27T18:17:33.509438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471014504786145:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:33.484407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471012158109158:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:33.512577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471014504786146:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:33.491712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471015357388786:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:33.491721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471014463898426:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:33.510497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471014463898427:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:33.510457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471016154161990:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:33.518366Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471015877290856:2101] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(200811928ns) {Configure:104(200801849ns) {ProcessConfigured:135(75069ns) {StartInitialHandshake:157(72019ns) {DropHandshakes:470(2569ns) {DropIncomingHandshake:442(493ns) DropOutgoingHandshake:457(333ns)} GenerateSessionVirtualId:988(1414ns) PrepareNewSessionHandshake:477(1473ns) {SwitchToState:179(452ns)}}}}}
2025-11-27T18:17:33.526113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471014463898428:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:33.541454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471014463898363:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(252366186ns) {Configure:104(252360374ns) {ProcessConfigured:135(86967ns) {StartInitialHandshake:157(84428ns) {DropHandshakes:470(1828ns) {DropIncomingHandshake:442(368ns) DropOutgoingHandshake:457(208ns)} GenerateSessionVirtualId:988(732ns) PrepareNewSessionHandshake:477(1633ns) {SwitchToState:179(587ns)}}}}}
2025-11-27T18:17:33.548742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471011062421426:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(234150741ns) {Configure:104(234146021ns) {ProcessConfigured:135(20955114ns) {StartInitialHandshake:157(20952353ns) {DropHandshakes:470(2767ns) {DropIncomingHandshake:442(851ns) DropOutgoingHandshake:457(547ns)} GenerateSessionVirtualId:988(918ns) PrepareNewSessionHandshake:477(4112ns) {SwitchToState:179(1845ns)}}}}}
2025-11-27T18:17:33.555888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471016154161926:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(95745104ns) {Configure:104(95738743ns) {ProcessConfigured:135(74313ns) {StartInitialHandshake:157(72257ns) {DropHandshakes:470(5532ns) {DropIncomingHandshake:442(869ns) DropOutgoingHandshake:457(396ns)} GenerateSessionVirtualId:988(1116ns) PrepareNewSessionHandshake:477(2273ns) {SwitchToState:179(712ns)}}}}}
2025-11-27T18:17:33.557380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471015877290800:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(286456880ns) {Configure:104(286451364ns) {ProcessConfigured:135(1183513ns) {StartInitialHandshake:157(1180901ns) {DropHandshakes:470(2680ns) {DropIncomingHandshake:442(724ns) DropOutgoingHandshake:457(697ns)} GenerateSessionVirtualId:988(775ns) PrepareNewSessionHandshake:477(2842ns) {SwitchToState:179(1758ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.557472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471014463898364:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(243050670ns) {Configure:104(243034372ns) {ProcessConfigured:135(467393ns) {StartInitialHandshake:157(465377ns) {DropHandshakes:470(2144ns) {DropIncomingHandshake:442(826ns) DropOutgoingHandshake:457(391ns)} GenerateSessionVirtualId:988(772ns) PrepareNewSessionHandshake:477(2259ns) {SwitchToState:179(1277ns)}}}}}
2025-11-27T18:17:33.569872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471012158109094:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(92095944ns) {Configure:104(92089752ns) {ProcessConfigured:135(49651ns) {StartInitialHandshake:157(47952ns) {DropHandshakes:470(1977ns) {DropIncomingHandshake:442(585ns) DropOutgoingHandshake:457(359ns)} GenerateSessionVirtualId:988(588ns) PrepareNewSessionHandshake:477(1159ns) {SwitchToState:179(466ns)}}}}}
2025-11-27T18:17:33.575198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471014504786079:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(261363164ns) {Configure:104(261351502ns) {ProcessConfigured:135(4413961ns) {StartInitialHandshake:157(4411721ns) {DropHandshakes:470(1307ns) {DropIncomingHandshake:442(439ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:477(4248ns) {SwitchToState:179(1998ns)}}}}}
2025-11-27T18:17:33.561339Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471011062421425:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(274672989ns) {Configure:104(274660427ns) {ProcessConfigured:135(10157523ns) {StartInitialHandshake:157(10154692ns) {DropHandshakes:470(3563ns) {DropIncomingHandshake:442(981ns) DropOutgoingHandshake:457(385ns)} GenerateSessionVirtualId:988(833ns) PrepareNewSessionHandshake:477(846975ns) {SwitchToState:179(1727ns)}}}}}
2025-11-27T18:17:33.567413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471016154161997:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37153
2025-11-27T18:17:33.563158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471015357388786:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(32451766ns) {Configure:104(32443403ns) {ProcessConfigured:135(8839219ns) {StartInitialHandshake:157(8837738ns) {DropHandshakes:470(1529ns) {DropIncomingHandshake:442(283ns) DropOutgoingHandshake:457(479ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:477(3174ns) {SwitchToState:179(2061ns)}}}}}
2025-11-27T18:17:33.581211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471014463898426:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(41402738ns) {Configure:104(41395008ns) {ProcessConfigured:135(53659ns) {StartInitialHandshake:157(49106ns) {DropHandshakes:470(1712ns) {DropIncomingHandshake:442(563ns) DropOutgoingHandshake:457(348ns)} GenerateSessionVirtualId:988(480ns) PrepareNewSessionHandshake:477(2394ns) {SwitchToState:179(1200ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.594234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471012158109158:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(65437775ns) {Configure:104(65420857ns) {ProcessConfigured:135(56302ns) {StartInitialHandshake:157(53902ns) {DropHandshakes:470(3379ns) {DropIncomingHandshake:442(179ns) DropOutgoingHandshake:457(168ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(1973ns) {SwitchToState:179(759ns)}}}}}
2025-11-27T18:17:33.596395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471012158109166:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4612
2025-11-27T18:17:33.596526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471012158109167:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4612
2025-11-27T18:17:33.596573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471012158109168:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4612
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.601726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471016154162004:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37153
2025-11-27T18:17:33.613412Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471015357388785:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(51794516ns) {Configure:104(51789939ns) {ProcessConfigured:135(97820ns) {StartInitialHandshake:157(94198ns) {DropHandshakes:470(2456ns) {DropIncomingHandshake:442(572ns) DropOutgoingHandshake:457(458ns)} GenerateSessionVirtualId:988(982ns) PrepareNewSessionHandshake:477(1687ns) {SwitchToState:179(570ns)}}}}}
2025-11-27T18:17:33.604113Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471015877290799:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(69147483ns) {Configure:104(69139782ns) {ProcessConfigured:135(15399737ns) {StartInitialHandshake:157(15397130ns) {DropHandshakes:470(2666ns) {DropIncomingHandshake:442(629ns) DropOutgoingHandshake:457(347ns)} GenerateSessionVirtualId:988(1314ns) PrepareNewSessionHandshake:477(5439ns) {SwitchToState:179(1906ns)}}}}}
2025-11-27T18:17:33.608035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471016154161927:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(63071382ns) {Configure:104(63060793ns) {ProcessConfigured:135(4770669ns) {StartInitialHandshake:157(4768532ns) {DropHandshakes:470(1982533ns) {DropIncomingHandshake:442(647ns) DropOutgoingHandshake:457(1864ns)} GenerateSessionVirtualId:988(1660ns) PrepareNewSessionHandshake:477(3351ns) {SwitchToState:179(1966ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.604110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471014504786144:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(84260589ns) {Configure:104(84247720ns) {ProcessConfigured:135(23305061ns) {StartInitialHandshake:157(23302599ns) {DropHandshakes:470(2233018ns) {DropIncomingHandshake:442(2618ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(1544ns) PrepareNewSessionHandshake:477(2464ns) {SwitchToState:179(1024ns)}}}}}
2025-11-27T18:17:33.614376Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471012158109095:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(121554434ns) {Configure:104(117590453ns) {ProcessConfigured:135(13556570ns) {StartInitialHandshake:157(13554896ns) {DropHandshakes:470(1809ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(451ns) PrepareNewSessionHandshake:477(2573ns) {SwitchToState:179(1558ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.619683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471012158109167:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.604123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471014504786080:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(91581108ns) {Configure:104(91572589ns) {ProcessConfigured:135(14466020ns) {StartInitialHandshake:157(14463303ns) {DropHandshakes:470(2569ns) {DropIncomingHandshake:442(527ns) DropOutgoingHandshake:457(417ns)} GenerateSessionVirtualId:988(836ns) PrepareNewSessionHandshake:477(4025ns) {SwitchToState:179(2004ns)}}}}}
2025-11-27T18:17:33.614469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471016154161997:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.623456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471015357388793:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61956
2025-11-27T18:17:33.626561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471014463898442:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40712
2025-11-27T18:17:33.632493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471015877290874:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21093
2025-11-27T18:17:33.629933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471014463898432:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.629972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471014463898432:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.625299Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471014504786145:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(45467921ns) {Configure:104(45459852ns) {ProcessConfigured:135(56191ns) {StartInitialHandshake:157(53847ns) {DropHandshakes:470(1845ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(751ns) PrepareNewSessionHandshake:477(3805ns) {SwitchToState:179(342ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.621065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471012158109166:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.624187Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471016154161928:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(125332985ns) {Configure:104(125324575ns) {ProcessConfigured:135(29594629ns) {StartInitialHandshake:157(29590793ns) {DropHandshakes:470(1332ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(714ns) PrepareNewSessionHandshake:477(2190ns) {SwitchToState:179(684ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.629650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471015357388794:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61956
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.622651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471012158109096:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52561856ns) {Configure:104(52555754ns) {ProcessConfigured:135(57939ns) {StartInitialHandshake:157(54926ns) {DropHandshakes:470(1873ns) {DropIncomingHandshake:442(293ns) DropOutgoingHandshake:457(555ns)} GenerateSessionVirtualId:988(622ns) PrepareNewSessionHandshake:477(992ns) {SwitchToState:179(321ns)}}}}}
2025-11-27T18:17:33.632576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471016154162004:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.636494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471015877290879:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21093
2025-11-27T18:17:33.635559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471015357388793:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.638496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471014463898432:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:33.634249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471016154161996:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.623634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471012158109095:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.636271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471015357388797:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61956
2025-11-27T18:17:33.638671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471015877290876:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21093
2025-11-27T18:17:33.634275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471016154161996:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.656861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471014463898434:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.636470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471015357388798:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61956
2025-11-27T18:17:33.629822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471012158109168:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.643901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471015877290874:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.656896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471014463898434:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.630017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471012158109158:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.650567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471015877290799:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.639262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471016154161996:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:33.642437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471015357388784:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(79794823ns) {Configure:104(75818843ns) {ProcessConfigured:135(3204359ns) {StartInitialHandshake:157(3199654ns) {DropHandshakes:470(2747ns) {DropIncomingHandshake:442(682ns) DropOutgoingHandshake:457(406ns)} GenerateSessionVirtualId:988(1311ns) PrepareNewSessionHandshake:477(2296ns) {SwitchToState:179(1084ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.639849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471016154161990:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.642959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471012158109179:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4612
2025-11-27T18:17:33.661044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471014504786154:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.657511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471014463898427:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.643443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471012158109178:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4612
2025-11-27T18:17:33.647504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471015357388794:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.651918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471015877290879:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.651017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471012158109158:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8593) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471012158109168:2115] held: no
2025-11-27T18:17:33.639907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:33.657557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:33.661090Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471014504786154:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.647823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471015357388797:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.655210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471015877290800:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.651137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471012158109167:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:33.654224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471012158109168:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:33.656582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471012158109095:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.658815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471012158109158:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.659853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:33.659504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:33.660099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471012158109158:2108] [node 4] ICP22 created new session: [5:7577471012158109180:2108]
2025-11-27T18:17:33.660137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471012158109180:2108] [node 4] ICS09 handshake done sender: [5:7577471012158109168:2115] self: [5:7577471012158109177:0] peer: [4:7577471016154161995:0] socket: 247 qp: -1
2025-11-27T18:17:33.660167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471012158109180:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:33.660201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:33.661905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471012158109095:2049] [node 2] ICP22 created new session: [5:7577471012158109181:2049]
2025-11-27T18:17:33.662830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471012158109181:2049] [node 2] ICS09 handshake done sender: [5:7577471012158109167:2114] self: [5:7577471012158109173:0] peer: [2:7577471014463898431:0] socket: 246 qp: -1
2025-11-27T18:17:33.663306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471012158109181:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:33.663350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:33.657600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471014463898427:2108] [node 5] ICP22 created new session: [2:7577471014463898444:2108]
2025-11-27T18:17:33.639967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471016154161990:2108] [node 5] ICP22 created new session: [4:7577471016154162010:2108]
2025-11-27T18:17:33.650106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471015357388798:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.662715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471014504786158:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-27T18:17:33.656453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471015877290876:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.713417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471015357388789:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.669885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471014504786146:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50753300ns) {Configure:104(50747364ns) {ProcessConfigured:135(7179071ns) {StartInitialHandshake:157(6925933ns) {DropHandshakes:470(2070ns) {DropIncomingHandshake:442(378ns) DropOutgoingHandshake:457(309ns)} GenerateSessionVirtualId:988(632ns) PrepareNewSessionHandshake:477(3240ns) {SwitchToState:179(1964ns)}}}}}
2025-11-27T18:17:33.640017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471016154162010:2108] [node 5] ICS09 handshake done sender: [4:7577471016154161996:2112] self: [4:7577471016154161995:0] peer: [5:7577471012158109177:0] socket: 242 qp: -1
2025-11-27T18:17:33.657644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471014463898444:2108] [node 5] ICS09 handshake done sender: [2:7577471014463898432:2111] self: [2:7577471014463898431:0] peer: [5:7577471012158109173:0] socket: 232 qp: -1
2025-11-27T18:17:33.658684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471015877290882:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21093
2025-11-27T18:17:33.700490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471012158109165:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.713460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471015357388789:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.700538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471012158109165:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.670235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471014504786152:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.640049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471016154162010:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:33.661016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471015877290878:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21093
2025-11-27T18:17:33.657676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471014463898444:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.714786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471015357388789:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.670259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471014504786152:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.661121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471015877290856:2101] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.657716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:33.715199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471015357388796:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.715238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471015357388796:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.701659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471012158109095:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(30725913ns) {HandleHandshakeStatus:329(30723674ns) {DropIncomingHandshake:442(1146ns) DropOutgoingHandshake:457(3215336ns) SwitchToState:179(1409ns) ProcessPendingSessionEvents:495(22396275ns) {ForwardSessionEventToSession:612(55274ns) ForwardSessionEventToSession:612(6179ns) ForwardSessionEventToSession:612(3352ns) ForwardSessionEventToSession:612(2620ns) ForwardSessionEventToSession:612(11604520ns) ForwardSessionEventToSession:612(6780ns) ForwardSessionEventToSession:612(3101ns) ForwardSessionEventToSession:612(6975077ns) ForwardSessionEventToSession:612(2502ns) ForwardSessionEventToSession:612(3720655ns)}}}
2025-11-27T18:17:33.640084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:33.682074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471014504786152:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.682264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471015877290874:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:33.711024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471012158109165:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:33.652308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471016154162007:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37153
2025-11-27T18:17:33.725038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471015357388796:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.659534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471014463898434:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.693989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471014504786158:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.700495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471014504786154:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:33.684931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471015877290878:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.703003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471014504786144:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.725157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471015357388809:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61956
2025-11-27T18:17:33.652467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471016154162012:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37153
2025-11-27T18:17:33.667846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471014463898428:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.718007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471012158109157:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.703039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:33.718879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:33.667919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:33.685704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471015877290848:2096] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.732052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471011062421426:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.654181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471016154162007:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.696977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471015877290799:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.667969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471014463898428:2109] [node 6] ICP22 created new session: [2:7577471014463898449:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.703071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471014504786144:2107] [node 4] ICP22 created new session: [3:7577471014504786165:2107]
2025-11-27T18:17:33.655872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471016154161928:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.721044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471012158109157:2109] [node 6] ICP22 created new session: [5:7577471012158109187:2109]
2025-11-27T18:17:33.668018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471014463898449:2109] [node 6] ICS09 handshake done sender: [2:7577471014463898434:2112] self: [2:7577471014463898433:0] peer: [6:7577471015877290880:0] socket: 233 qp: -1
2025-11-27T18:17:33.732107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:33.706091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:33.737630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471011062421426:2049] [node 3] ICP22 created new session: [1:7577471015357388811:2049]
2025-11-27T18:17:33.671809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471016154162015:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37153
2025-11-27T18:17:33.703104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471014504786165:2107] [node 4] ICS09 handshake done sender: [3:7577471014504786152:2114] self: [3:7577471014504786149:0] peer: [4:7577471016154162014:0] socket: 257 qp: -1
2025-11-27T18:17:33.668056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471014463898449:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:33.724994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471012158109187:2109] [node 6] ICS09 handshake done sender: [5:7577471012158109165:2112] self: [5:7577471012158109164:0] peer: [6:7577471015877290884:0] socket: 250 qp: -1
2025-11-27T18:17:33.710128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471015877290799:2049] [node 2] ICP22 created new session: [6:7577471015877290885:2049]
2025-11-27T18:17:33.672936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471016154162015:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.727379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471012158109187:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:33.747423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471015357388811:2049] [node 3] ICS09 handshake done sender: [1:7577471015357388789:2110] self: [1:7577471015357388787:0] peer: [3:7577471014504786167:0] socket: 265 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.668092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:33.703125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471014504786165:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:33.673047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471016154161927:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.748093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471015357388785:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.668891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471014463898448:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40712
2025-11-27T18:17:33.727425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:33.716853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471015877290885:2049] [node 2] ICS09 handshake done sender: [6:7577471015877290874:2114] self: [6:7577471015877290880:0] peer: [2:7577471014463898433:0] socket: 235 qp: -1
2025-11-27T18:17:33.703150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.717368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471015877290885:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:33.703747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471014504786161:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-27T18:17:33.672614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471014463898442:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.718922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:33.764085Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471015877290799:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(66074083ns) {HandleHandshakeStatus:329(66071611ns) {DropIncomingHandshake:442(2593ns) DropOutgoingHandshake:457(8453ns) SwitchToState:179(379ns) ProcessPendingSessionEvents:495(13440514ns) {ForwardSessionEventToSession:612(6516923ns) ForwardSessionEventToSession:612(4447ns) ForwardSessionEventToSession:612(586577ns) ForwardSessionEventToSession:612(2358ns) ForwardSessionEventToSession:612(786347ns) ForwardSessionEventToSession:612(3182ns) ForwardSessionEventToSession:612(2715ns) ForwardSessionEventToSession:612(1865155ns) ForwardSessionEventToSession:612(3655115ns) ForwardSessionEventToSession:612(2757ns)}}}
2025-11-27T18:17:33.766736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471015877290886:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21093
2025-11-27T18:17:33.767462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471015877290848:2096] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1042) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471015877290878:2117] held: no
2025-11-27T18:17:33.767702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471015877290856:2101] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8593) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471015877290876:2115] held: no
2025-11-27T18:17:33.768919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471015877290888:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21093
2025-11-27T18:17:33.769247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471015877290879:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:33.769362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471015877290887:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21093
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.748185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:33.673603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471016154161994:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.747844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471012158109189:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4612
2025-11-27T18:17:33.673637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471016154161994:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.753598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471012158109189:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.748231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471015357388785:2108] [node 4] ICP22 created new session: [1:7577471015357388812:2108]
2025-11-27T18:17:33.672704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471014463898448:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.703905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471014504786164:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-27T18:17:33.771950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471015877290891:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21093
2025-11-27T18:17:33.748271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471015357388812:2108] [node 4] ICS09 handshake done sender: [1:7577471015357388796:2115] self: [1:7577471015357388795:0] peer: [4:7577471016154162017:0] socket: 270 qp: -1
2025-11-27T18:17:33.772076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471015877290800:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.675684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471016154162012:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.748304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471015357388812:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:33.756627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471012158109096:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.703962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471014504786079:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.689786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471014463898441:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.693494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471016154161926:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.689827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471014463898441:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.717600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471014504786161:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.748336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:33.763183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471012158109190:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4612
2025-11-27T18:17:33.772104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:33.700434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471014463898447:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40712
2025-11-27T18:17:33.772132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471015877290800:2050] [node 3] ICP22 created new session: [6:7577471015877290892:2050]
2025-11-27T18:17:33.705864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471014463898441:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:33.772164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471015877290892:2050] [node 3] ICS09 handshake done sender: [6:7577471015877290879:2118] self: [6:7577471015877290881:0] peer: [3:7577471014504786153:0] socket: 261 qp: -1
2025-11-27T18:17:33.717448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471014463898447:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.772184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471015877290892:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:33.717504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471014463898426:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.772201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:33.717568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:33.772636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471015877290876:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:33.718289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471014463898426:2107] [node 4] ICP22 created new session: [2:7577471014463898456:2107]
2025-11-27T18:17:33.776497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471015877290856:2101] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.718328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471014463898456:2107] [node 4] ICS09 handshake done sender: [2:7577471014463898441:2114] self: [2:7577471014463898440:0] peer: [4:7577471016154162016:0] socket: 277 qp: -1
2025-11-27T18:17:33.776757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:33.718179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471014504786164:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.718362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471014463898456:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:33.719439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471014504786145:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.776795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471015877290856:2101] [node 4] ICP22 created new session: [6:7577471015877290894:2101]
2025-11-27T18:17:33.718436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:33.777586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471015877290894:2101] [node 4] ICS09 handshake done sender: [6:7577471015877290876:2115] self: [6:7577471015877290883:0] peer: [4:7577471016154161993:0] socket: 243 qp: -1
2025-11-27T18:17:33.721091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:33.777757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471015877290891:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.742416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471014463898363:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.722034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471014504786080:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.777808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471015877290878:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:33.722117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471014504786169:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-27T18:17:33.742568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471014463898439:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.710147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471016154161994:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:33.742603Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471014463898439:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.777977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471015877290798:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.722708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471014504786158:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:33.763637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471014463898439:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:33.722884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471014504786079:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.714882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471016154162020:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37153
2025-11-27T18:17:33.763687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471014463898459:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40712
2025-11-27T18:17:33.722911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:33.717820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471016154162015:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:33.765819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471014463898364:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.724858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471016154161989:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.749549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471015357388811:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:33.765897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:33.725016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471016154161927:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.766984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471014463898364:2049] [node 3] ICP22 created new session: [2:7577471014463898461:2049]
2025-11-27T18:17:33.763244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471012158109192:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4612
2025-11-27T18:17:33.725058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:33.767542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471014463898447:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:33.726048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471016154161927:2049] [node 2] ICP22 created new session: [4:7577471016154162021:2049]
2025-11-27T18:17:33.768243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471014463898461:2049] [node 3] ICS09 handshake done sender: [2:7577471014463898439:2113] self: [2:7577471014463898438:0] peer: [3:7577471014504786172:0] socket: 276 qp: -1
2025-11-27T18:17:33.732176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:33.769396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471014463898461:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:33.778188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471015877290894:2101] [node 4] ICS10 traffic start
2025-11-27T18:17:33.732239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471016154161989:2109] [node 6] ICP22 created new session: [4:7577471016154162022:2109]
2025-11-27T18:17:33.770647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:33.750918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:33.732293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471016154162022:2109] [node 6] ICS09 handshake done sender: [4:7577471016154161994:2111] self: [4:7577471016154161993:0] peer: [6:7577471015877290883:0] socket: 241 qp: -1
2025-11-27T18:17:33.722937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471014504786079:2048] [node 1] ICP22 created new session: [3:7577471014504786174:2048]
2025-11-27T18:17:33.764436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471012158109189:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:33.751364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471015357388790:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.765902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471012158109096:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.751388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471015357388790:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.776380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471014463898363:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1266) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471014463898447:2116] held: no
2025-11-27T18:17:33.778230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:33.722967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471014504786174:2048] [node 1] ICS09 handshake done sender: [3:7577471014504786158:2116] self: [3:7577471014504786167:0] peer: [1:7577471015357388787:0] socket: 267 qp: -1
2025-11-27T18:17:33.732333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471016154162022:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:33.766497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:33.753940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471015357388790:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:33.778327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471015877290848:2096] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.777235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471014463898363:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.732371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:33.763207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471011062421425:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.766641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471012158109190:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.722985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471014504786174:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:33.778105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:33.779008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:33.763265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:33.779131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471015877290848:2096] [node 5] ICP22 created new session: [6:7577471015877290897:2096]
2025-11-27T18:17:33.734821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471016154162018:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37153
2025-11-27T18:17:33.723004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:33.767034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471012158109096:2050] [node 3] ICP22 created new session: [5:7577471012158109193:2050]
2025-11-27T18:17:33.779986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471015877290897:2096] [node 5] ICS09 handshake done sender: [6:7577471015877290878:2117] self: [6:7577471015877290884:0] peer: [5:7577471012158109164:0] socket: 255 qp: -1
2025-11-27T18:17:33.735519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471016154162007:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:33.725022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471014504786145:2110] [node 6] ICP22 created new session: [3:7577471014504786170:2110]
2025-11-27T18:17:33.763316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471011062421425:2048] [node 2] ICP22 created new session: [1:7577471015357388818:2048]
2025-11-27T18:17:33.767965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471012158109094:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:33.725961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471014504786170:2110] [node 6] ICS09 handshake done sender: [3:7577471014504786154:2115] self: [3:7577471014504786153:0] peer: [6:7577471015877290881:0] socket: 258 qp: -1
2025-11-27T18:17:33.763743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471015357388818:2048] [node 2] ICS09 handshake done sender: [1:7577471015357388790:2111] self: [1:7577471015357388788:0] peer: [2:7577471014463898458:0] socket: 269 qp: -1
2025-11-27T18:17:33.764243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471015357388818:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:33.768533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471012158109193:2050] [node 3] ICS09 handshake done sender: [5:7577471012158109189:2120] self: [5:7577471012158109191:0] peer: [3:7577471014504786159:0] socket: 297 qp: -1
2025-11-27T18:17:33.764285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:33.741907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471014504786176:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-27T18:17:33.772778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471012158109193:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:33.765057Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471011062421426:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(34319168ns) {HandleHandshakeStatus:329(34316951ns) {DropIncomingHandshake:442(2599ns) DropOutgoingHandshake:457(1586ns) SwitchToState:179(380ns) ProcessPendingSessionEvents:495(7592211ns) {ForwardSessionEventToSession:612(42033ns) ForwardSessionEventToSession:612(849711ns) ForwardSessionEventToSession:612(3765ns) ForwardSessionEventToSession:612(2335ns) ForwardSessionEventToSession:612(4314365ns) ForwardSessionEventToSession:612(7568ns) ForwardSessionEventToSession:612(3453ns) ForwardSessionEventToSession:612(214940ns) ForwardSessionEventToSession:612(2574ns) ForwardSessionEventToSession:612(2135239ns)}}}
2025-11-27T18:17:33.773528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:33.743102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471014504786170:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:33.766072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471011062421426:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:33.778143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471012158109196:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4612
2025-11-27T18:17:33.768185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471015357388800:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.743172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:33.781123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471012158109094:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1266) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471012158109190:2121] held: no
2025-11-27T18:17:33.778517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471014463898363:2048] [node 1] ICP22 created new session: [2:7577471014463898463:2048]
2025-11-27T18:17:33.778557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471014463898463:2048] [node 1] ICS09 handshake done sender: [2:7577471014463898447:2116] self: [2:7577471014463898458:0] peer: [1:7577471015357388788:0] socket: 272 qp: -1
2025-11-27T18:17:33.778764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471014463898463:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:33.778911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:33.757202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471014504786160:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.781291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471012158109190:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:33.735778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471016154161928:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.757218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471014504786160:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.768199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471015357388800:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.759902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471014504786160:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:33.735826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:33.764040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471014504786146:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.735857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471016154161928:2050] [node 3] ICP22 created new session: [4:7577471016154162026:2050]
2025-11-27T18:17:33.780469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471015877290897:2096] [node 5] ICS10 traffic start
2025-11-27T18:17:33.764479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:33.735885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471016154162026:2050] [node 3] ICS09 handshake done sender: [4:7577471016154162007:2117] self: [4:7577471016154162014:0] peer: [3:7577471014504786149:0] socket: 260 qp: -1
2025-11-27T18:17:33.764719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471014504786146:2108] [node 5] ICP22 created new session: [3:7577471014504786179:2108]
2025-11-27T18:17:33.735907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471016154162026:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:33.781006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:33.764954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471014504786179:2108] [node 5] ICS09 handshake done sender: [3:7577471014504786160:2117] self: [3:7577471014504786159:0] peer: [5:7577471012158109191:0] socket: 296 qp: -1
2025-11-27T18:17:33.735925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:33.784915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471012158109094:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.765471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471014504786179:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:33.741327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471016154162021:2049] [node 2] ICS09 handshake done sender: [4:7577471016154162015:2119] self: [4:7577471016154162016:0] peer: [2:7577471014463898440:0] socket: 279 qp: -1
2025-11-27T18:17:33.765519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:33.784951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:33.745139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471016154162021:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:33.785002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471012158109094:2048] [node 1] ICP22 created new session: [5:7577471012158109197:2048]
2025-11-27T18:17:33.745212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:33.766943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471014504786145:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(42285204ns) {HandleHandshakeStatus:329(42283092ns) {DropIncomingHandshake:442(665ns) DropOutgoingHandshake:457(703ns) SwitchToState:179(224ns) ProcessPendingSessionEvents:495(10499275ns) {ForwardSessionEventToSession:612(5733212ns) ForwardSessionEventToSession:612(3028ns) ForwardSessionEventToSession:612(2437ns) ForwardSessionEventToSession:612(1000678ns) ForwardSessionEventToSession:612(2621ns) ForwardSessionEventToSession:612(431366ns) ForwardSessionEventToSession:612(1209778ns) ForwardSessionEventToSession:612(1906ns) ForwardSessionEventToSession:612(2096046ns) ForwardSessionEventToSession:612(3236ns)}}}
2025-11-27T18:17:33.786132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471012158109197:2048] [node 1] ICS09 handshake done sender: [5:7577471012158109190:2121] self: [5:7577471012158109194:0] peer: [1:7577471015357388799:0] socket: 286 qp: -1
2025-11-27T18:17:33.745648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471016154161927:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20827649ns) {HandleHandshakeStatus:329(20815274ns) {DropIncomingHandshake:442(823ns) DropOutgoingHandshake:457(3529ns) SwitchToState:179(137ns) ProcessPendingSessionEvents:495(102432ns) {ForwardSessionEventToSession:612(53502ns) ForwardSessionEventToSession:612(3517ns) ForwardSessionEventToSession:612(5399ns) ForwardSessionEventToSession:612(3361ns) ForwardSessionEventToSession:612(6085ns) ForwardSessionEventToSession:612(8179ns) ForwardSessionEventToSession:612(3471ns) ForwardSessionEventToSession:612(7933ns) ForwardSessionEventToSession:612(2004ns) ForwardSessionEventToSession:612(1713ns)}}}
2025-11-27T18:17:33.770090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471014504786181:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-27T18:17:33.786287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471012158109197:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:33.774808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471014504786182:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-27T18:17:33.746079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471016154162024:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37153
2025-11-27T18:17:33.746544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471016154161927:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:33.775292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471014504786161:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:33.750403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471016154162012:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:33.780895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471014504786080:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.782398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:33.757839Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471016154161926:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1266) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471016154162012:2118] held: no
2025-11-27T18:17:33.783018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471014504786080:2049] [node 2] ICP22 created new session: [3:7577471014504786183:2049]
2025-11-27T18:17:33.757935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471016154161926:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.783827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471014504786183:2049] [node 2] ICS09 handshake done sender: [3:7577471014504786161:2118] self: [3:7577471014504786172:0] peer: [2:7577471014463898438:0] socket: 278 qp: -1
2025-11-27T18:17:33.758064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:33.784232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471014504786183:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:33.784607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:33.771359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471015357388800:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:33.773515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471015357388784:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.786865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:33.763012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471016154161926:2048] [node 1] ICP22 created new session: [4:7577471016154162029:2048]
2025-11-27T18:17:33.773567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:33.774524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471015357388784:2107] [node 5] ICP22 created new session: [1:7577471015357388821:2107]
2025-11-27T18:17:33.787200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471015877290899:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21093
2025-11-27T18:17:33.775608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471015357388821:2107] [node 5] ICS09 handshake done sender: [1:7577471015357388800:2118] self: [1:7577471015357388799:0] peer: [5:7577471012158109194:0] socket: 253 qp: -1
2025-11-27T18:17:33.787786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471015877290891:2123] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:33.776216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471015357388821:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:33.776401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:33.763075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471016154162029:2048] [node 1] ICS09 handshake done sender: [4:7577471016154162012:2118] self: [4:7577471016154162017:0] peer: [1:7577471015357388795:0] socket: 252 qp: -1
2025-11-27T18:17:33.779768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471015357388792:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:33.763565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471016154162029:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:33.779798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471015357388792:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:33.763624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:33.788343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471014504786080:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:33.784249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471015357388792:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:33.788015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471015877290798:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:33.785005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471015357388786:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:33.785944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:33.786643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471015357388786:2109] [node 6] ICP22 created new session: [1:7577471015357388824:2109]
2025-11-27T18:17:33.787159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471015357388824:2109] [node 6] ICS09 handshake done sender: [1:7577471015357388792:2112] self: [1:7577471015357388791:0] peer: [6:7577471015877290895:0] socket: 237 qp: -1
2025-11-27T18:17:33.787191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471015357388824:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:33.788605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:33.787255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:33.788651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471015877290798:2048] [node 1] ICP22 created new session: [6:7577471015877290900:2048]
2025-11-27T18:17:33.788693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471015877290900:2048] [node 1] ICS09 handshake done sender: [6:7577471015877290891:2123] self: [6:7577471015877290895:0] peer: [1:7577471015357388791:0] socket: 229 qp: -1
2025-11-27T18:17:33.788718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471015877290900:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:33.788749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:33.789582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471015877290798:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:33.829514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:33.829766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471014463898463:2048] [node 1] ICS01 socket: 272 reason# EndOfStream
2025-11-27T18:17:33.829817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471014463898363:2048] [node 1] ICP30 unregister session Session# [2:7577471014463898463:2048] VirtualId# [2:7577471014463898458:0]
2025-11-27T18:17:33.829858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471014463898463:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.829936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:33.829983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:33.833741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:33.833991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471014504786174:2048] [node 1] ICS01 socket: 267 reason# EndOfStream
2025-11-27T18:17:33.834030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471014504786079:2048] [node 1] ICP30 unregister session Session# [3:7577471014504786174:2048] VirtualId# [3:7577471014504786167:0]
2025-11-27T18:17:33.834060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471014504786174:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.834133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:33.834169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:33.838441Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:33.838677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471012158109197:2048] [node 1] ICS01 socket: 286 reason# EndOfStream
2025-11-27T18:17:33.838725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471012158109094:2048] [node 1] ICP30 unregister session Session# [5:7577471012158109197:2048] VirtualId# [5:7577471012158109194:0]
2025-11-27T18:17:33.838757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471012158109197:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.838838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:33.838884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:33.839384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:33.839601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471016154162029:2048] [node 1] ICS01 socket: 252 reason# EndOfStream
2025-11-27T18:17:33.839648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471016154161926:2048] [node 1] ICP30 unregister session Session# [4:7577471016154162029:2048] VirtualId# [4:7577471016154162017:0]
2025-11-27T18:17:33.839673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471016154162029:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.839736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:33.839779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:33.840277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:33.840472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471015877290900:2048] [node 1] ICS07 socket disconnect 229 reason# EndOfStream
2025-11-27T18:17:33.840497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471015877290900:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.840560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:33.840600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471015877290900:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:33.846641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471015877290798:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1266) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:33.846781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471015877290900:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:33.846818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471015877290798:2048] [node 1] ICP30 unregister session Session# [6:7577471015877290900:2048] VirtualId# [6:7577471015877290895:0]
2025-11-27T18:17:33.846864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:33.846896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471015877290798:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1266) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:33.846912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1266) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:33.857103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471015877290798:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:33.922100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:33.922352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471014504786183:2049] [node 2] ICS07 socket disconnect 278 reason# EndOfStream
2025-11-27T18:17:33.922385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471014504786183:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.922466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:33.922507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471014504786183:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:33.923241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471014504786080:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2207) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:33.923303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471014504786183:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:33.923335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471014504786080:2049] [node 2] ICP30 unregister session Session# [3:7577471014504786183:2049] VirtualId# [3:7577471014504786172:0]
2025-11-27T18:17:33.923374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:33.923400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471014504786080:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2207) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:33.923413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2207) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:33.929445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:33.929648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471016154162021:2049] [node 2] ICS01 socket: 279 reason# EndOfStream
2025-11-27T18:17:33.929704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471016154161927:2049] [node 2] ICP30 unregister session Session# [4:7577471016154162021:2049] VirtualId# [4:7577471016154162016:0]
2025-11-27T18:17:33.929737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471016154162021:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.929812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:33.929855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:33.929401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:33.929633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471012158109181:2049] [node 2] ICS07 socket disconnect 246 reason# EndOfStream
2025-11-27T18:17:33.929666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471012158109181:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.929744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:33.929787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471012158109181:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:33.929976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:33.930712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471012158109095:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2207) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:33.930826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471012158109181:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:33.930861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471012158109095:2049] [node 2] ICP30 unregister session Session# [5:7577471012158109181:2049] VirtualId# [5:7577471012158109173:0]
2025-11-27T18:17:33.930898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:33.930928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471012158109095:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2207) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:33.930947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2207) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:33.930149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471015877290885:2049] [node 2] ICS07 socket disconnect 235 reason# EndOfStream
2025-11-27T18:17:33.930170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471015877290885:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.930233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:33.930269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471015877290885:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:33.933235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471014504786080:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:33.937642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471015877290799:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2207) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:33.937701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471015877290885:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:33.937739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471015877290799:2049] [node 2] ICP30 unregister session Session# [6:7577471015877290885:2049] VirtualId# [6:7577471015877290880:0]
2025-11-27T18:17:33.937786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:33.937814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471015877290799:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2207) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:33.937828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2207) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:33.943294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471012158109095:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:33.948234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471015877290799:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:33.996915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:33.997075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471015877290892:2050] [node 3] ICS07 socket disconnect 261 reason# EndOfStream
2025-11-27T18:17:33.997093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471015877290892:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.997174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:33.997200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471015877290892:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:33.997448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:33.997648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471016154162026:2050] [node 3] ICS01 socket: 260 reason# EndOfStream
2025-11-27T18:17:33.997682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471016154161928:2050] [node 3] ICP30 unregister session Session# [4:7577471016154162026:2050] VirtualId# [4:7577471016154162014:0]
2025-11-27T18:17:33.997711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471016154162026:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.997781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:33.997821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:33.997756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:33.997925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471012158109193:2050] [node 3] ICS01 socket: 297 reason# EndOfStream
2025-11-27T18:17:33.997957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471012158109096:2050] [node 3] ICP30 unregister session Session# [5:7577471012158109193:2050] VirtualId# [5:7577471012158109191:0]
2025-11-27T18:17:33.997992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471012158109193:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:33.998074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:33.998116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:34.003034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471015877290800:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13704) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:34.009602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471015877290892:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:34.009701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471015877290800:2050] [node 3] ICP30 unregister session Session# [6:7577471015877290892:2050] VirtualId# [6:7577471015877290881:0]
2025-11-27T18:17:34.009761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:34.009797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471015877290800:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13704) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:34.009811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13704) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:34.024589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471015877290800:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:34.058108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:34.058341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471012158109180:2108] [node 4] ICS01 socket: 247 reason# EndOfStream
2025-11-27T18:17:34.058379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471012158109158:2108] [node 4] ICP30 unregister session Session# [5:7577471012158109180:2108] VirtualId# [5:7577471012158109177:0]
2025-11-27T18:17:34.058413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471012158109180:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.058492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:34.058528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:34.059113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:34.059319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471015877290894:2101] [node 4] ICS01 socket: 243 reason# EndOfStream
2025-11-27T18:17:34.059352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471015877290856:2101] [node 4] ICP30 unregister session Session# [6:7577471015877290894:2101] VirtualId# [6:7577471015877290883:0]
2025-11-27T18:17:34.059375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471015877290894:2101] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.059434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:34.059513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:34.126767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:34.126986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471015877290897:2096] [node 5] ICS01 socket: 255 reason# EndOfStream
2025-11-27T18:17:34.127026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471015877290848:2096] [node 5] ICP30 unregister session Session# [6:7577471015877290897:2096] VirtualId# [6:7577471015877290884:0]
2025-11-27T18:17:34.127065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471015877290897:2096] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.127146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:34.127213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:34.211926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471020360730262:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:34.211983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471020360730263:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:34.235949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471018712876920:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:34.235987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471018712876921:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:34.265455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471020341921936:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:34.265499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471020341921937:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:34.287316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471018424870313:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:34.287354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471018424870314:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:34.287379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471018424870315:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:34.310768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471016995235169:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:34.310813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471016995235171:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:34.311089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471016995235170:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:34.333465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471017069244229:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:34.333503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471017069244230:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:34.333530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471017069244231:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:34.469505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471017069244288:2103] [node 5] ICP01 ready to work
2025-11-27T18:17:34.485809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471020360730325:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.485939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471020341922000:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:34.538980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471018712876986:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:34.486080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471020360730326:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:34.574112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471020341922001:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:34.550936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471017069244291:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:34.574021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471018712876987:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:34.581050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471018424870376:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:34.539984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471020360730327:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:34.585520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471016995235232:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:34.581060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471020341922002:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:34.581059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471018712876988:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:34.568069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471020360730328:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26404
2025-11-27T18:17:34.598746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471018712876921:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(152997240ns) {Configure:104(152992552ns) {ProcessConfigured:135(56245ns) {StartInitialHandshake:157(53625ns) {DropHandshakes:470(2242ns) {DropIncomingHandshake:442(500ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(1679ns) PrepareNewSessionHandshake:477(1490ns) {SwitchToState:179(508ns)}}}}}
2025-11-27T18:17:34.581955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471018424870377:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:34.585421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471016995235233:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:34.599327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471020360730328:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.608947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471020341922000:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(79117143ns) {Configure:104(79112667ns) {ProcessConfigured:135(73047ns) {StartInitialHandshake:157(70246ns) {DropHandshakes:470(3233ns) {DropIncomingHandshake:442(786ns) DropOutgoingHandshake:457(656ns)} GenerateSessionVirtualId:988(959ns) PrepareNewSessionHandshake:477(1659ns) {SwitchToState:179(522ns)}}}}}
2025-11-27T18:17:34.618968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471020360730327:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(45171881ns) {Configure:104(45167713ns) {ProcessConfigured:135(4229863ns) {StartInitialHandshake:157(4227253ns) {DropHandshakes:470(1515ns) {DropIncomingHandshake:442(354ns) DropOutgoingHandshake:457(172ns)} GenerateSessionVirtualId:988(453ns) PrepareNewSessionHandshake:477(2460ns) {SwitchToState:179(1153ns)}}}}}
2025-11-27T18:17:34.618980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471020360730263:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(89216616ns) {Configure:104(89212215ns) {ProcessConfigured:135(83686ns) {StartInitialHandshake:157(80967ns) {DropHandshakes:470(2094ns) {DropIncomingHandshake:442(466ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(924ns) PrepareNewSessionHandshake:477(2200ns) {SwitchToState:179(874ns)}}}}}
2025-11-27T18:17:34.628948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471016995235170:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55216223ns) {Configure:104(54561633ns) {ProcessConfigured:135(91453ns) {StartInitialHandshake:157(87742ns) {DropHandshakes:470(2520ns) {DropIncomingHandshake:442(653ns) DropOutgoingHandshake:457(391ns)} GenerateSessionVirtualId:988(1542ns) PrepareNewSessionHandshake:477(1894ns) {SwitchToState:179(702ns)}}}}}
2025-11-27T18:17:34.635891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471017069244229:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(196400862ns) {Configure:104(196396164ns) {ProcessConfigured:135(5516545ns) {StartInitialHandshake:157(5513999ns) {DropHandshakes:470(2394ns) {DropIncomingHandshake:442(671ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(1177ns) PrepareNewSessionHandshake:477(4210ns) {SwitchToState:179(2241ns)}}}}}
2025-11-27T18:17:34.633146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471020341921937:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(162327673ns) {Configure:104(162323180ns) {ProcessConfigured:135(8650360ns) {StartInitialHandshake:157(8648405ns) {DropHandshakes:470(1701ns) {DropIncomingHandshake:442(385ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(954ns) PrepareNewSessionHandshake:477(3312ns) {SwitchToState:179(1405ns)}}}}}
2025-11-27T18:17:34.639352Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471018424870314:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(68286491ns) {Configure:104(68278801ns) {ProcessConfigured:135(1560592ns) {StartInitialHandshake:157(1558970ns) {DropHandshakes:470(360394ns) {DropIncomingHandshake:442(172ns) DropOutgoingHandshake:457(483ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:477(2404ns) {SwitchToState:179(93ns)}}}}}
2025-11-27T18:17:34.638254Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471016995235169:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(52504487ns) {Configure:104(48773396ns) {ProcessConfigured:135(2774419ns) {StartInitialHandshake:157(2771180ns) {DropHandshakes:470(2149ns) {DropIncomingHandshake:442(472ns) DropOutgoingHandshake:457(334ns)} GenerateSessionVirtualId:988(986ns) PrepareNewSessionHandshake:477(2012ns) {SwitchToState:179(865ns)}}}}}
2025-11-27T18:17:34.638413Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471020360730262:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(143816159ns) {Configure:104(143810092ns) {ProcessConfigured:135(3169972ns) {StartInitialHandshake:157(3165937ns) {DropHandshakes:470(2913ns) {DropIncomingHandshake:442(819ns) DropOutgoingHandshake:457(459ns)} GenerateSessionVirtualId:988(977ns) PrepareNewSessionHandshake:477(3530ns) {SwitchToState:179(1602ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.640542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471018424870315:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(152706000ns) {Configure:104(152697219ns) {ProcessConfigured:135(718558ns) {StartInitialHandshake:157(716893ns) {DropHandshakes:470(1486ns) {DropIncomingHandshake:442(337ns) DropOutgoingHandshake:457(170ns)} GenerateSessionVirtualId:988(666ns) PrepareNewSessionHandshake:477(1729ns) {SwitchToState:179(915ns)}}}}}
2025-11-27T18:17:34.646664Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471016995235232:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48693758ns) {Configure:104(48687908ns) {ProcessConfigured:135(4327507ns) {StartInitialHandshake:157(4325577ns) {DropHandshakes:470(1952ns) {DropIncomingHandshake:442(570ns) DropOutgoingHandshake:457(328ns)} GenerateSessionVirtualId:988(616ns) PrepareNewSessionHandshake:477(2983ns) {SwitchToState:179(1485ns)}}}}}
2025-11-27T18:17:34.640751Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471017069244291:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(54909234ns) {Configure:104(54901690ns) {ProcessConfigured:135(6883464ns) {StartInitialHandshake:157(6881223ns) {DropHandshakes:470(1502382ns) {DropIncomingHandshake:442(1811ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(1445ns) PrepareNewSessionHandshake:477(3871ns) {SwitchToState:179(1459ns)}}}}}
2025-11-27T18:17:34.641713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471018712876988:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(39403175ns) {Configure:104(39398206ns) {ProcessConfigured:135(64114ns) {StartInitialHandshake:157(58635ns) {DropHandshakes:470(2502ns) {DropIncomingHandshake:442(692ns) DropOutgoingHandshake:457(391ns)} GenerateSessionVirtualId:988(563ns) PrepareNewSessionHandshake:477(1324ns) {SwitchToState:179(516ns)}}}}}
2025-11-27T18:17:34.646906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471018424870313:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(58241259ns) {Configure:104(58237298ns) {ProcessConfigured:135(2402056ns) {StartInitialHandshake:157(2400482ns) {DropHandshakes:470(533ns) {DropIncomingHandshake:442(85ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:477(2039ns) {SwitchToState:179(165ns)}}}}}
2025-11-27T18:17:34.646878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471018712876986:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(57854793ns) {Configure:104(57849486ns) {ProcessConfigured:135(1221841ns) {StartInitialHandshake:157(1220440ns) {DropHandshakes:470(700ns) {DropIncomingHandshake:442(179ns) DropOutgoingHandshake:457(117ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:477(1748ns) {SwitchToState:179(341ns)}}}}}
2025-11-27T18:17:34.648685Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471020341921936:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(59957232ns) {Configure:104(59951432ns) {ProcessConfigured:135(887625ns) {StartInitialHandshake:157(885813ns) {DropHandshakes:470(1243ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(2188ns) {SwitchToState:179(851ns)}}}}}
2025-11-27T18:17:34.642166Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471017069244288:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(75424168ns) {Configure:104(75419336ns) {ProcessConfigured:135(8280141ns) {StartInitialHandshake:157(8278211ns) {DropHandshakes:470(1214ns) {DropIncomingHandshake:442(434ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(731ns) PrepareNewSessionHandshake:477(3184ns) {SwitchToState:179(1417ns)}}}}}
2025-11-27T18:17:34.650154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471016995235171:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(102754463ns) {Configure:104(102749557ns) {ProcessConfigured:135(3213972ns) {StartInitialHandshake:157(3212261ns) {DropHandshakes:470(1253ns) {DropIncomingHandshake:442(308ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(755ns) PrepareNewSessionHandshake:477(2381ns) {SwitchToState:179(1109ns)}}}}}
2025-11-27T18:17:34.650115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471018712876987:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(52779199ns) {Configure:104(52771052ns) {ProcessConfigured:135(3657740ns) {StartInitialHandshake:157(3655754ns) {DropHandshakes:470(756ns) {DropIncomingHandshake:442(142ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:477(2824ns) {SwitchToState:179(1188ns)}}}}}
2025-11-27T18:17:34.648801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471020341922001:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(47908163ns) {Configure:104(47900148ns) {ProcessConfigured:135(3435707ns) {StartInitialHandshake:157(3434716ns) {DropHandshakes:470(821ns) {DropIncomingHandshake:442(181ns) DropOutgoingHandshake:457(107ns)} GenerateSessionVirtualId:988(184ns) PrepareNewSessionHandshake:477(1518ns) {SwitchToState:179(81ns)}}}}}
2025-11-27T18:17:34.643354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471017069244302:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-27T18:17:34.654450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471018424870376:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(55263667ns) {Configure:104(55257744ns) {ProcessConfigured:135(6491879ns) {StartInitialHandshake:157(6489845ns) {DropHandshakes:470(1309ns) {DropIncomingHandshake:442(388ns) DropOutgoingHandshake:457(348ns)} GenerateSessionVirtualId:988(534ns) PrepareNewSessionHandshake:477(2203ns) {SwitchToState:179(599ns)}}}}}
2025-11-27T18:17:34.651444Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471020341922002:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39192127ns) {Configure:104(39187550ns) {ProcessConfigured:135(3224551ns) {StartInitialHandshake:157(3222659ns) {DropHandshakes:470(1644485ns) {DropIncomingHandshake:442(1689ns) DropOutgoingHandshake:457(190ns)} GenerateSessionVirtualId:988(1425ns) PrepareNewSessionHandshake:477(2564ns) {SwitchToState:179(1377ns)}}}}}
2025-11-27T18:17:34.649815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471017069244230:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(63899057ns) {Configure:104(63886421ns) {ProcessConfigured:135(5644774ns) {StartInitialHandshake:157(5642563ns) {DropHandshakes:470(2060ns) {DropIncomingHandshake:442(639ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(427ns) PrepareNewSessionHandshake:477(2744ns) {SwitchToState:179(295ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.654043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471017069244302:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.656039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471018712876999:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-27T18:17:34.655586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471017069244229:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.656820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471020360730330:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.656885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471018424870377:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16178568ns) {Configure:104(16170300ns) {ProcessConfigured:135(3859721ns) {StartInitialHandshake:157(3857088ns) {DropHandshakes:470(2468ns) {DropIncomingHandshake:442(724ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(951ns) PrepareNewSessionHandshake:477(3356ns) {SwitchToState:179(1565ns)}}}}}
2025-11-27T18:17:34.656868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471018712876999:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.657091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471016995235244:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18190
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.656874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471020360730330:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.657301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471020341922010:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45838
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.658632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471020341922008:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45838
2025-11-27T18:17:34.658661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471016995235244:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.659237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471017069244308:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-27T18:17:34.659132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471020360730330:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:34.660720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471020341922010:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.658726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471018424870385:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37699
2025-11-27T18:17:34.661104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471020341922008:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.659276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471017069244307:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-27T18:17:34.661102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471018424870387:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37699
2025-11-27T18:17:34.661182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471020341921936:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:34.665524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471018424870385:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.665967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471017069244231:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20922992ns) {Configure:104(20915459ns) {ProcessConfigured:135(2537448ns) {StartInitialHandshake:157(2535026ns) {DropHandshakes:470(2848ns) {DropIncomingHandshake:442(686ns) DropOutgoingHandshake:457(993ns)} GenerateSessionVirtualId:988(782ns) PrepareNewSessionHandshake:477(434840ns) {SwitchToState:179(2273ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.673417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471020360730337:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:17:34.675430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471018712876990:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.666243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471018424870387:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.665904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471020341921937:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:34.667910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471017069244307:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.675475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471018712876990:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.675307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471017069244230:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:34.681030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471018712877001:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-27T18:17:34.677147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471018424870313:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:34.673491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471020360730337:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.681053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471020341922018:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45838
2025-11-27T18:17:34.685408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471018424870392:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37699
2025-11-27T18:17:34.673579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471020360730325:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.681075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471018712876995:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.681296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471017069244308:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.691448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471018424870392:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.686988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471020341922018:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.681105Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471018712876995:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.674335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:34.692901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471018424870394:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37699
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.684329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471017069244311:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-27T18:17:34.693060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471016995235241:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.684321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471018712876990:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:34.691524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471017069244288:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:34.677308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471020360730325:2109] [node 6] ICP22 created new session: [1:7577471020360730346:2109]
2025-11-27T18:17:34.688706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471020341922020:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45838
2025-11-27T18:17:34.695231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471018424870394:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.693875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471020341922020:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.696134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471018424870315:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.704662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471020341922024:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45838
2025-11-27T18:17:34.693280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471018712876995:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:34.693116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471016995235241:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.695221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471017069244302:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:34.679583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471020360730346:2109] [node 6] ICS09 handshake done sender: [1:7577471020360730330:2111] self: [1:7577471020360730329:0] peer: [6:7577471017069244306:0] socket: 237 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.705276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471016995235241:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:34.705335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471020341922004:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.680352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471020360730346:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:34.704355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471017069244229:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:17:34.693860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471018712877002:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
::ffff:127.1.0.0
2025-11-27T18:17:34.693873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471018712877001:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.680397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:34.705363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471020341922004:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.708526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471018424870401:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37699
2025-11-27T18:17:34.704418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:34.710719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471016995235249:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18190
2025-11-27T18:17:34.706474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471020341922022:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45838
2025-11-27T18:17:34.684117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471020360730344:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26404
2025-11-27T18:17:34.704462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471017069244229:2048] [node 1] ICP22 created new session: [6:7577471017069244315:2048]
2025-11-27T18:17:34.693931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471018712876921:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.714879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471018424870398:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37699
2025-11-27T18:17:34.716581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471016995235232:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.707002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471020341922004:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:34.693973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:34.707570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471017069244315:2048] [node 1] ICS09 handshake done sender: [6:7577471017069244302:2114] self: [6:7577471017069244306:0] peer: [1:7577471020360730329:0] socket: 245 qp: -1
2025-11-27T18:17:34.684524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471020360730332:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.707470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471020341922008:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:34.716886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471016995235249:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.707853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471017069244315:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.684550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471020360730332:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.694018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471018712876921:2049] [node 3] ICP22 created new session: [2:7577471018712877005:2049]
2025-11-27T18:17:34.708700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471020341922000:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.718357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:34.685571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471020360730337:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:34.694262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471018712877005:2049] [node 3] ICS09 handshake done sender: [2:7577471018712876990:2111] self: [2:7577471018712876989:0] peer: [3:7577471020341922021:0] socket: 239 qp: -1
2025-11-27T18:17:34.708508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:34.722231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471018424870315:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3763) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471018424870394:2118] held: no
2025-11-27T18:17:34.708738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.725658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471018424870385:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:34.726313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471016995235232:2109] [node 6] ICP22 created new session: [5:7577471016995235252:2109]
2025-11-27T18:17:34.691820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471020360730344:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.717380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471017069244230:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7988) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471017069244307:2115] held: no
2025-11-27T18:17:34.694298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471018712877005:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:34.708771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471020341922000:2107] [node 4] ICP22 created new session: [3:7577471020341922027:2107]
2025-11-27T18:17:34.692761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471020360730263:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.726945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471016995235252:2109] [node 6] ICS09 handshake done sender: [5:7577471016995235241:2113] self: [5:7577471016995235240:0] peer: [6:7577471017069244313:0] socket: 256 qp: -1
2025-11-27T18:17:34.730658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471018424870394:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:34.730330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471017069244314:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-27T18:17:34.695253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.709675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471020341922027:2107] [node 4] ICS09 handshake done sender: [3:7577471020341922004:2111] self: [3:7577471020341922003:0] peer: [4:7577471018424870399:0] socket: 261 qp: -1
2025-11-27T18:17:34.692804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:34.731281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471018424870315:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.696283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471018712877002:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.729555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471016995235169:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.697408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471018712876988:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.731323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:34.733468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471017069244288:2103] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3655) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471017069244308:2116] held: no
2025-11-27T18:17:34.709707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471020341922027:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:34.729998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471016995235251:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18190
2025-11-27T18:17:34.693730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471020360730263:2049] [node 3] ICP22 created new session: [1:7577471020360730351:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.731381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471018424870315:2050] [node 3] ICP22 created new session: [4:7577471018424870402:2050]
2025-11-27T18:17:34.711132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471020341921937:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.701532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:34.738153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471017069244307:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:34.695167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471020360730351:2049] [node 3] ICS09 handshake done sender: [1:7577471020360730337:2113] self: [1:7577471020360730336:0] peer: [3:7577471020341922019:0] socket: 241 qp: -1
2025-11-27T18:17:34.731429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471018424870402:2050] [node 3] ICS09 handshake done sender: [4:7577471018424870394:2118] self: [4:7577471018424870399:0] peer: [3:7577471020341922003:0] socket: 268 qp: -1
2025-11-27T18:17:34.732070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471016995235252:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:34.711191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:34.742606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471017069244229:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25022976ns) {HandleHandshakeStatus:329(20790819ns) {DropIncomingHandshake:442(1525ns) DropOutgoingHandshake:457(6158ns) SwitchToState:179(907ns) ProcessPendingSessionEvents:495(9177103ns) {ForwardSessionEventToSession:612(3338147ns) ForwardSessionEventToSession:612(227390ns) ForwardSessionEventToSession:612(529725ns) ForwardSessionEventToSession:612(3344ns) ForwardSessionEventToSession:612(935727ns) ForwardSessionEventToSession:612(3673ns) ForwardSessionEventToSession:612(2638ns) ForwardSessionEventToSession:612(2899046ns) ForwardSessionEventToSession:612(3164ns) ForwardSessionEventToSession:612(1220827ns)}}}
2025-11-27T18:17:34.711460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:34.712424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471020341921937:2049] [node 2] ICP22 created new session: [3:7577471020341922028:2049]
2025-11-27T18:17:34.714029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471020341922028:2049] [node 2] ICS09 handshake done sender: [3:7577471020341922008:2113] self: [3:7577471020341922021:0] peer: [2:7577471018712876989:0] socket: 242 qp: -1
2025-11-27T18:17:34.714084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471020341922028:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:34.701595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471018712876988:2110] [node 6] ICP22 created new session: [2:7577471018712877008:2110]
2025-11-27T18:17:34.733709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471016995235253:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18190
2025-11-27T18:17:34.705820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471018712876921:2049] [node 3] ICP18 handshake (actor [2:7577471018712877001:2116]) is too old
2025-11-27T18:17:34.743960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471017069244317:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-27T18:17:34.731465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471018424870402:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:34.696203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471020360730351:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:34.714119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:34.707351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471018712877008:2110] [node 6] ICS09 handshake done sender: [2:7577471018712876995:2112] self: [2:7577471018712876994:0] peer: [6:7577471017069244312:0] socket: 253 qp: -1
2025-11-27T18:17:34.747691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471017069244230:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:34.696549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:34.734066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:34.731497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:34.707686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471018712877008:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:34.714972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471020341922010:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.747886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:34.744027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471016995235251:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.705572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471020360730332:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:34.731939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471018424870313:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:34.749106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471017069244230:2049] [node 2] ICP22 created new session: [6:7577471017069244319:2049]
2025-11-27T18:17:34.720352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471020341921936:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:34.708010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:34.707490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471020360730345:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26404
2025-11-27T18:17:34.747832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471016995235170:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.720397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:34.749843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471017069244308:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:34.758749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471016995235258:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18190
2025-11-27T18:17:34.737929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471018424870400:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37699
2025-11-27T18:17:34.739507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471018712876988:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(25893039ns) {HandleHandshakeStatus:329(25890631ns) {DropIncomingHandshake:442(14614ns) DropOutgoingHandshake:457(445ns) SwitchToState:179(339ns) ProcessPendingSessionEvents:495(8371610ns) {ForwardSessionEventToSession:612(3509361ns) ForwardSessionEventToSession:612(6351ns) ForwardSessionEventToSession:612(371328ns) ForwardSessionEventToSession:612(3218ns) ForwardSessionEventToSession:612(452272ns) ForwardSessionEventToSession:612(3717ns) ForwardSessionEventToSession:612(1922484ns) ForwardSessionEventToSession:612(2075944ns) ForwardSessionEventToSession:612(4218ns) ForwardSessionEventToSession:612(2629ns)}}}
2025-11-27T18:17:34.708087Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471020360730325:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(22886513ns) {HandleHandshakeStatus:329(22882264ns) {DropIncomingHandshake:442(575ns) DropOutgoingHandshake:457(861ns) SwitchToState:179(679ns) ProcessPendingSessionEvents:495(4543387ns) {ForwardSessionEventToSession:612(1066383ns) ForwardSessionEventToSession:612(1231165ns) ForwardSessionEventToSession:612(907146ns) ForwardSessionEventToSession:612(6165ns) ForwardSessionEventToSession:612(3424ns) ForwardSessionEventToSession:612(476967ns) ForwardSessionEventToSession:612(5088ns) ForwardSessionEventToSession:612(3640ns) ForwardSessionEventToSession:612(604488ns) ForwardSessionEventToSession:612(215969ns)}}}
2025-11-27T18:17:34.726072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471020341921936:2048] [node 1] ICP22 created new session: [3:7577471020341922032:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.741340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.750130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471017069244319:2049] [node 2] ICS09 handshake done sender: [6:7577471017069244307:2115] self: [6:7577471017069244312:0] peer: [2:7577471018712876994:0] socket: 254 qp: -1
2025-11-27T18:17:34.713346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471020360730326:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.747044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471018712877012:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-27T18:17:34.758883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471016995235253:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.726251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471020341921937:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:34.742856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471018424870313:2048] [node 1] ICP22 created new session: [4:7577471018424870404:2048]
2025-11-27T18:17:34.713602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471020360730345:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.733950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471020341922032:2048] [node 1] ICS09 handshake done sender: [3:7577471020341922010:2114] self: [3:7577471020341922019:0] peer: [1:7577471020360730336:0] socket: 243 qp: -1
2025-11-27T18:17:34.749100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471018712877012:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.750792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471017069244319:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:34.761057Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471016995235232:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(34537233ns) {HandleHandshakeStatus:329(34534843ns) {DropIncomingHandshake:442(14907ns) DropOutgoingHandshake:457(316ns) SwitchToState:179(190ns) ProcessPendingSessionEvents:495(5494869ns) {ForwardSessionEventToSession:612(3365846ns) ForwardSessionEventToSession:612(5882ns) ForwardSessionEventToSession:612(2595ns) ForwardSessionEventToSession:612(249636ns) ForwardSessionEventToSession:612(3467ns) ForwardSessionEventToSession:612(3234ns) ForwardSessionEventToSession:612(919447ns) ForwardSessionEventToSession:612(3687ns) ForwardSessionEventToSession:612(919920ns) ForwardSessionEventToSession:612(5693ns)}}}
2025-11-27T18:17:34.716902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:34.749987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471018712876997:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.750850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:34.747002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471018424870404:2048] [node 1] ICS09 handshake done sender: [4:7577471018424870385:2113] self: [4:7577471018424870395:0] peer: [1:7577471020360730331:0] socket: 244 qp: -1
2025-11-27T18:17:34.747357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471018424870400:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.747367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471018424870404:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:34.748235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:34.750868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471018424870314:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:34.761272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471018424870313:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24961119ns) {HandleHandshakeStatus:329(24959165ns) {DropIncomingHandshake:442(1460ns) DropOutgoingHandshake:457(5635ns) SwitchToState:179(207ns) ProcessPendingSessionEvents:495(2105088ns) {ForwardSessionEventToSession:612(735203ns) ForwardSessionEventToSession:612(4347ns) ForwardSessionEventToSession:612(3007ns) ForwardSessionEventToSession:612(501479ns) ForwardSessionEventToSession:612(5778ns) ForwardSessionEventToSession:612(483075ns) ForwardSessionEventToSession:612(4298ns) ForwardSessionEventToSession:612(3834ns) ForwardSessionEventToSession:612(345824ns) ForwardSessionEventToSession:612(3168ns)}}}
2025-11-27T18:17:34.767648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471018424870386:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.767673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471018424870386:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.772647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471018424870386:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:34.775307Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471018424870314:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7988) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471018424870400:2120] held: no
2025-11-27T18:17:34.775592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471018424870376:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.775646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:34.775691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471018424870376:2109] [node 6] ICP22 created new session: [4:7577471018424870411:2109]
2025-11-27T18:17:34.775746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471018424870411:2109] [node 6] ICS09 handshake done sender: [4:7577471018424870386:2114] self: [4:7577471018424870384:0] peer: [6:7577471017069244325:0] socket: 272 qp: -1
2025-11-27T18:17:34.775784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471018424870411:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:34.775819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:34.781893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471018424870409:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37699
2025-11-27T18:17:34.737869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471020341922032:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:34.761833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471016995235171:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:34.763509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471016995235249:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:34.754225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471017069244288:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:34.750027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471018712876997:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.786084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471018424870400:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:34.738331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:34.753683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471020341922035:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45838
2025-11-27T18:17:34.765653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471016995235169:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:34.717463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471020360730326:2107] [node 4] ICP22 created new session: [1:7577471020360730354:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.789428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471018424870314:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.754266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:34.750234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471018712876920:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.717989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471020360730263:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(21310318ns) {HandleHandshakeStatus:329(21308415ns) {DropIncomingHandshake:442(275ns) DropOutgoingHandshake:457(599ns) SwitchToState:179(601ns) ProcessPendingSessionEvents:495(6653341ns) {ForwardSessionEventToSession:612(1627123ns) ForwardSessionEventToSession:612(2290670ns) ForwardSessionEventToSession:612(4904ns) ForwardSessionEventToSession:612(3122ns) ForwardSessionEventToSession:612(12107ns) ForwardSessionEventToSession:612(2087ns) ForwardSessionEventToSession:612(1859ns) ForwardSessionEventToSession:612(728357ns) ForwardSessionEventToSession:612(2364ns) ForwardSessionEventToSession:612(642973ns)}}}
2025-11-27T18:17:34.766131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:34.757724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471020341921936:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32224826ns) {HandleHandshakeStatus:329(32222467ns) {DropIncomingHandshake:442(1911ns) DropOutgoingHandshake:457(5197ns) SwitchToState:179(482ns) ProcessPendingSessionEvents:495(3080363ns) {ForwardSessionEventToSession:612(1488912ns) ForwardSessionEventToSession:612(4016ns) ForwardSessionEventToSession:612(812256ns) ForwardSessionEventToSession:612(10522ns) ForwardSessionEventToSession:612(292025ns) ForwardSessionEventToSession:612(3493ns) ForwardSessionEventToSession:612(3554ns) ForwardSessionEventToSession:612(446750ns) ForwardSessionEventToSession:612(4019ns) ForwardSessionEventToSession:612(3277ns)}}}
2025-11-27T18:17:34.789475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:34.751692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471018712877013:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-27T18:17:34.760303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471020341922035:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.755429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471018712876996:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.755456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471018712876996:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.760738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471018712877013:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.766073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471018712876920:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471018712877012:2118] held: no
2025-11-27T18:17:34.766925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471018712876997:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:34.767024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471018712876996:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:34.767964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471018712877015:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13343
2025-11-27T18:17:34.769699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471018712876986:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.770079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:34.770634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471018712876987:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.770687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:34.770738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471018712876987:2109] [node 5] ICP22 created new session: [2:7577471018712877020:2109]
2025-11-27T18:17:34.770783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471018712877020:2109] [node 5] ICS09 handshake done sender: [2:7577471018712876997:2114] self: [2:7577471018712876993:0] peer: [5:7577471016995235257:0] socket: 292 qp: -1
2025-11-27T18:17:34.770819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471018712877020:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:34.770868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:34.720232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471020360730354:2107] [node 4] ICS09 handshake done sender: [1:7577471020360730332:2112] self: [1:7577471020360730331:0] peer: [4:7577471018424870395:0] socket: 238 qp: -1
2025-11-27T18:17:34.790329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471018424870314:2049] [node 2] ICP22 created new session: [4:7577471018424870413:2049]
2025-11-27T18:17:34.720231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471020360730263:2049] [node 3] ICP18 handshake (actor [1:7577471020360730344:2116]) is too old
2025-11-27T18:17:34.793447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471018424870389:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.722544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471020360730354:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:34.726990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:34.793483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471018424870389:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.743978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471020360730338:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.797342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471018424870413:2049] [node 2] ICS09 handshake done sender: [4:7577471018424870400:2120] self: [4:7577471018424870407:0] peer: [2:7577471018712876992:0] socket: 289 qp: -1
2025-11-27T18:17:34.744022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471020360730338:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.797386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471018424870413:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:34.750104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471020360730338:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:34.797780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471018424870389:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:34.752483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471020360730340:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.752508Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471020360730340:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.753689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471020360730327:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.753784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471020360730326:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(36731195ns) {HandleHandshakeStatus:329(36729108ns) {DropIncomingHandshake:442(240ns) DropOutgoingHandshake:457(290ns) SwitchToState:179(205ns) ProcessPendingSessionEvents:495(9506767ns) {ForwardSessionEventToSession:612(6803098ns) ForwardSessionEventToSession:612(610843ns) ForwardSessionEventToSession:612(4033ns) ForwardSessionEventToSession:612(3085ns) ForwardSessionEventToSession:612(606919ns) ForwardSessionEventToSession:612(4434ns) ForwardSessionEventToSession:612(3407ns) ForwardSessionEventToSession:612(528527ns) ForwardSessionEventToSession:612(444070ns) ForwardSessionEventToSession:612(477406ns)}}}
2025-11-27T18:17:34.753997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471020360730340:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:34.754311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471017069244288:2103] [node 5] ICP22 created new session: [6:7577471017069244323:2103]
2025-11-27T18:17:34.755398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:34.755455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471020360730327:2108] [node 5] ICP22 created new session: [1:7577471020360730358:2108]
2025-11-27T18:17:34.756021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471020360730262:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.754359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471017069244323:2103] [node 5] ICS09 handshake done sender: [6:7577471017069244308:2116] self: [6:7577471017069244313:0] peer: [5:7577471016995235240:0] socket: 258 qp: -1
2025-11-27T18:17:34.756075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:34.756135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471020360730262:2048] [node 2] ICP22 created new session: [1:7577471020360730359:2048]
2025-11-27T18:17:34.754402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471017069244323:2103] [node 5] ICS10 traffic start
2025-11-27T18:17:34.756183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471020360730359:2048] [node 2] ICS09 handshake done sender: [1:7577471020360730340:2115] self: [1:7577471020360730339:0] peer: [2:7577471018712877014:0] socket: 252 qp: -1
2025-11-27T18:17:34.756209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471020360730359:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:34.754433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:34.756244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:34.757227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471020360730358:2108] [node 5] ICS09 handshake done sender: [1:7577471020360730338:2114] self: [1:7577471020360730335:0] peer: [5:7577471016995235255:0] socket: 281 qp: -1
2025-11-27T18:17:34.754891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471017069244322:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-27T18:17:34.758504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471020360730326:2107] [node 4] ICP18 handshake (actor [1:7577471020360730345:2117]) is too old
2025-11-27T18:17:34.760946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471017069244320:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-27T18:17:34.758769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471020360730358:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:34.760330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:34.764923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471017069244320:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.765661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471017069244322:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.766012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471017069244291:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:34.767590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471017069244231:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:34.777897Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471017069244291:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17299) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471017069244320:2121] held: no
2025-11-27T18:17:34.778143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471017069244328:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-27T18:17:34.780777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471017069244322:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:34.784391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471017069244231:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:34.785330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:34.786318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471017069244231:2050] [node 3] ICP22 created new session: [6:7577471017069244329:2050]
2025-11-27T18:17:34.789360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471017069244329:2050] [node 3] ICS09 handshake done sender: [6:7577471017069244322:2122] self: [6:7577471017069244326:0] peer: [3:7577471020341922007:0] socket: 300 qp: -1
2025-11-27T18:17:34.789981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471017069244329:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:34.790012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:34.794908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471017069244327:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21810
2025-11-27T18:17:34.800971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471017069244231:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16462135ns) {HandleHandshakeStatus:329(16459685ns) {DropIncomingHandshake:442(2211ns) DropOutgoingHandshake:457(5692ns) SwitchToState:179(614ns) ProcessPendingSessionEvents:495(6085731ns) {ForwardSessionEventToSession:612(6031710ns) ForwardSessionEventToSession:612(10878ns) ForwardSessionEventToSession:612(3093ns) ForwardSessionEventToSession:612(4976ns) ForwardSessionEventToSession:612(2042ns) ForwardSessionEventToSession:612(2880ns) ForwardSessionEventToSession:612(7718ns) ForwardSessionEventToSession:612(2683ns) ForwardSessionEventToSession:612(4579ns) ForwardSessionEventToSession:612(5132ns)}}}
2025-11-27T18:17:34.801020Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471017069244231:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:34.767484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471016995235169:2048] [node 1] ICP22 created new session: [5:7577471016995235261:2048]
2025-11-27T18:17:34.767753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471016995235261:2048] [node 1] ICS09 handshake done sender: [5:7577471016995235249:2116] self: [5:7577471016995235255:0] peer: [1:7577471020360730335:0] socket: 282 qp: -1
2025-11-27T18:17:34.768050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471016995235261:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:34.769219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:34.772650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471016995235259:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18190
2025-11-27T18:17:34.777532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471016995235259:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.783248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471016995235233:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:34.784167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471016995235170:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7988) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471016995235251:2117] held: no
2025-11-27T18:17:34.788379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471016995235171:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3763) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471016995235253:2118] held: no
2025-11-27T18:17:34.792842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471016995235264:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18190
2025-11-27T18:17:34.797603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471016995235262:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18190
2025-11-27T18:17:34.800026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471016995235253:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:34.800279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471016995235266:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18190
2025-11-27T18:17:34.767242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471020341922014:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.767275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471020341922014:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.768763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471020341922012:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:34.768795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471020341922012:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:34.770254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471020341922014:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:34.774373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471020341922012:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:34.776801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471020341922002:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.777190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:34.778997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471020341922001:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.779047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:34.779081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471020341922001:2110] [node 6] ICP22 created new session: [3:7577471020341922040:2110]
2025-11-27T18:17:34.779131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471020341922040:2110] [node 6] ICS09 handshake done sender: [3:7577471020341922012:2116] self: [3:7577471020341922007:0] peer: [6:7577471017069244326:0] socket: 297 qp: -1
2025-11-27T18:17:34.779157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471020341922040:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:34.779186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:34.780618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471020341922002:2109] [node 5] ICP22 created new session: [3:7577471020341922039:2109]
2025-11-27T18:17:34.782704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471020341922039:2109] [node 5] ICS09 handshake done sender: [3:7577471020341922014:2117] self: [3:7577471020341922013:0] peer: [5:7577471016995235260:0] socket: 248 qp: -1
2025-11-27T18:17:34.782891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471020341922039:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:34.771732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471018712876986:2108] [node 4] ICP22 created new session: [2:7577471018712877019:2108]
2025-11-27T18:17:34.782938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:34.774260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471018712877012:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:34.776501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471018712877019:2108] [node 4] ICS09 handshake done sender: [2:7577471018712876996:2113] self: [2:7577471018712876992:0] peer: [4:7577471018424870407:0] socket: 288 qp: -1
2025-11-27T18:17:34.776968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471018712877019:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:34.777004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:34.780187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471018712876920:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:34.802133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471016995235171:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:34.802186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:34.802729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471016995235251:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:34.780220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:34.803095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471017069244320:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:34.801091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:34.782396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471018712876920:2048] [node 1] ICP22 created new session: [2:7577471018712877023:2048]
2025-11-27T18:17:34.782874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471018712877023:2048] [node 1] ICS09 handshake done sender: [2:7577471018712877012:2118] self: [2:7577471018712877014:0] peer: [1:7577471020360730339:0] socket: 266 qp: -1
2025-11-27T18:17:34.804796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471017069244291:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:34.804805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471018424870377:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:34.783620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471018712877023:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:34.783668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:34.798646Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471018712876986:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(19235195ns) {HandleHandshakeStatus:329(19232116ns) {DropIncomingHandshake:442(20588ns) DropOutgoingHandshake:457(476ns) SwitchToState:179(528ns) ProcessPendingSessionEvents:495(6218864ns) {ForwardSessionEventToSession:612(1618160ns) ForwardSessionEventToSession:612(972935ns) ForwardSessionEventToSession:612(604216ns) ForwardSessionEventToSession:612(3539ns) ForwardSessionEventToSession:612(3921ns) ForwardSessionEventToSession:612(271342ns) ForwardSessionEventToSession:612(2757ns) ForwardSessionEventToSession:612(2017527ns) ForwardSessionEventToSession:612(3366ns) ForwardSessionEventToSession:612(384744ns)}}}
2025-11-27T18:17:34.803263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471016995235171:2050] [node 3] ICP22 created new session: [5:7577471016995235267:2050]
2025-11-27T18:17:34.804868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:34.804846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:34.803436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471016995235170:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:34.804893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471017069244291:2107] [node 4] ICP22 created new session: [6:7577471017069244331:2107]
2025-11-27T18:17:34.803475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:34.803515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471016995235170:2049] [node 2] ICP22 created new session: [5:7577471016995235268:2049]
2025-11-27T18:17:34.803554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471016995235268:2049] [node 2] ICS09 handshake done sender: [5:7577471016995235251:2117] self: [5:7577471016995235257:0] peer: [2:7577471018712876993:0] socket: 295 qp: -1
2025-11-27T18:17:34.803598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471016995235268:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:34.803651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:34.804428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471016995235259:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:34.804586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471016995235267:2050] [node 3] ICS09 handshake done sender: [5:7577471016995235253:2118] self: [5:7577471016995235260:0] peer: [3:7577471020341922013:0] socket: 247 qp: -1
2025-11-27T18:17:34.806404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471016995235233:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:34.806475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:34.806515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471016995235233:2108] [node 4] ICP22 created new session: [5:7577471016995235270:2108]
2025-11-27T18:17:34.806562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471016995235270:2108] [node 4] ICS09 handshake done sender: [5:7577471016995235259:2120] self: [5:7577471016995235265:0] peer: [4:7577471018424870388:0] socket: 299 qp: -1
2025-11-27T18:17:34.806590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471016995235270:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:34.806623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:34.807532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471016995235267:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:34.807569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:34.808916Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471016995235233:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:34.804906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471018424870377:2108] [node 5] ICP22 created new session: [4:7577471018424870416:2108]
2025-11-27T18:17:34.804940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471017069244331:2107] [node 4] ICS09 handshake done sender: [6:7577471017069244320:2121] self: [6:7577471017069244325:0] peer: [4:7577471018424870384:0] socket: 273 qp: -1
2025-11-27T18:17:34.804974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471017069244331:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:34.805012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:34.807278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471018424870416:2108] [node 5] ICS09 handshake done sender: [4:7577471018424870389:2116] self: [4:7577471018424870388:0] peer: [5:7577471016995235265:0] socket: 298 qp: -1
2025-11-27T18:17:34.807974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471018424870416:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:34.808029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:34.852421Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:34.852718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471018712877023:2048] [node 1] ICS01 socket: 266 reason# EndOfStream
2025-11-27T18:17:34.852769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471018712876920:2048] [node 1] ICP30 unregister session Session# [2:7577471018712877023:2048] VirtualId# [2:7577471018712877014:0]
2025-11-27T18:17:34.852809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471018712877023:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.852904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:34.852972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:34.855046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:34.855301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471020341922032:2048] [node 1] ICS07 socket disconnect 243 reason# EndOfStream
2025-11-27T18:17:34.855340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471020341922032:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.855437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:34.855487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471020341922032:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:34.856381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471020341921936:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:34.856504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471020341922032:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:34.856571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471020341921936:2048] [node 1] ICP30 unregister session Session# [3:7577471020341922032:2048] VirtualId# [3:7577471020341922019:0]
2025-11-27T18:17:34.856623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:34.856652Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471020341921936:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:34.856666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:34.861484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:34.861728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471018424870404:2048] [node 1] ICS01 socket: 244 reason# EndOfStream
2025-11-27T18:17:34.861777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471018424870313:2048] [node 1] ICP30 unregister session Session# [4:7577471018424870404:2048] VirtualId# [4:7577471018424870395:0]
2025-11-27T18:17:34.861813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471018424870404:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.861943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:34.862032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:34.862499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:34.862849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471017069244315:2048] [node 1] ICS07 socket disconnect 245 reason# EndOfStream
2025-11-27T18:17:34.862902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471017069244315:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.863034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:34.863120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471017069244315:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:34.862286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:34.862595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471016995235261:2048] [node 1] ICS07 socket disconnect 282 reason# EndOfStream
2025-11-27T18:17:34.862642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471016995235261:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.862751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:34.862853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471016995235261:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:34.865041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471017069244229:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:34.866220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471020341921936:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:34.865119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471017069244315:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:34.865185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471017069244229:2048] [node 1] ICP30 unregister session Session# [6:7577471017069244315:2048] VirtualId# [6:7577471017069244306:0]
2025-11-27T18:17:34.869446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:34.869511Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471017069244229:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:34.869534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:34.869658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471016995235169:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:34.869725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471016995235261:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:34.869760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471016995235169:2048] [node 1] ICP30 unregister session Session# [5:7577471016995235261:2048] VirtualId# [5:7577471016995235255:0]
2025-11-27T18:17:34.869803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:34.869827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471016995235169:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:34.869839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:34.879076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471017069244229:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:34.879210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471016995235169:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:34.927684Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:34.927936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471018424870413:2049] [node 2] ICS01 socket: 289 reason# EndOfStream
2025-11-27T18:17:34.927980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471018424870314:2049] [node 2] ICP30 unregister session Session# [4:7577471018424870413:2049] VirtualId# [4:7577471018424870407:0]
2025-11-27T18:17:34.928020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471018424870413:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.928152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:34.928204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:34.928789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:34.927918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:34.928158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471020341922028:2049] [node 2] ICS07 socket disconnect 242 reason# EndOfStream
2025-11-27T18:17:34.928191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471020341922028:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.928291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:34.928346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471020341922028:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:34.929036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471016995235268:2049] [node 2] ICS01 socket: 295 reason# EndOfStream
2025-11-27T18:17:34.929086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471016995235170:2049] [node 2] ICP30 unregister session Session# [5:7577471016995235268:2049] VirtualId# [5:7577471016995235257:0]
2025-11-27T18:17:34.929117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471016995235268:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.929200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:34.929238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:34.931109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:34.931333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471017069244319:2049] [node 2] ICS01 socket: 254 reason# EndOfStream
2025-11-27T18:17:34.931372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471017069244230:2049] [node 2] ICP30 unregister session Session# [6:7577471017069244319:2049] VirtualId# [6:7577471017069244312:0]
2025-11-27T18:17:34.931399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471017069244319:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.931461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:34.931499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:34.975043Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:34.975314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471018424870402:2050] [node 3] ICS01 socket: 268 reason# EndOfStream
2025-11-27T18:17:34.975360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471018424870315:2050] [node 3] ICP30 unregister session Session# [4:7577471018424870402:2050] VirtualId# [4:7577471018424870399:0]
2025-11-27T18:17:34.975403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471018424870402:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.975477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:34.975515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:34.977021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:34.976757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:34.977016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471017069244329:2050] [node 3] ICS07 socket disconnect 300 reason# EndOfStream
2025-11-27T18:17:34.977045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471017069244329:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.977118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:34.977167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471017069244329:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:34.977254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471016995235267:2050] [node 3] ICS01 socket: 247 reason# EndOfStream
2025-11-27T18:17:34.977318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471016995235171:2050] [node 3] ICP30 unregister session Session# [5:7577471016995235267:2050] VirtualId# [5:7577471016995235260:0]
2025-11-27T18:17:34.977369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471016995235267:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:34.977449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:34.977491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:34.977957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471017069244231:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3763) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:34.978012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471017069244329:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:34.978087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471017069244231:2050] [node 3] ICP30 unregister session Session# [6:7577471017069244329:2050] VirtualId# [6:7577471017069244326:0]
2025-11-27T18:17:34.978132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:34.978160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471017069244231:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3763) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:34.978176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3763) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:34.987659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471017069244231:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:35.057094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:35.057385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471016995235270:2108] [node 4] ICS01 socket: 299 reason# EndOfStream
2025-11-27T18:17:35.057429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471016995235233:2108] [node 4] ICP30 unregister session Session# [5:7577471016995235270:2108] VirtualId# [5:7577471016995235265:0]
2025-11-27T18:17:35.057471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471016995235270:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:35.057542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:35.057585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:35.057094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:35.057384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471017069244331:2107] [node 4] ICS01 socket: 273 reason# EndOfStream
2025-11-27T18:17:35.057427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471017069244291:2107] [node 4] ICP30 unregister session Session# [6:7577471017069244331:2107] VirtualId# [6:7577471017069244325:0]
2025-11-27T18:17:35.057461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471017069244331:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:35.057542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:35.057585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:35.121445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:35.121690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471017069244323:2103] [node 5] ICS01 socket: 258 reason# EndOfStream
2025-11-27T18:17:35.121742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471017069244288:2103] [node 5] ICP30 unregister session Session# [6:7577471017069244323:2103] VirtualId# [6:7577471017069244313:0]
2025-11-27T18:17:35.121788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471017069244323:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:35.121882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:35.121935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:35.211888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471024094916370:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:35.211926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471024094916371:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:35.236912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471023915498798:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:35.236952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471023915498799:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:35.260549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471023072509181:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:35.260607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471023072509182:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:35.283030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471021851736832:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:35.283065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471021851736833:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:35.283092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471021851736834:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:35.305361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471023634718786:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:35.305378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471023634718788:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:35.305391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471023634718787:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:35.329673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471022773353862:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:35.329720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471022773353863:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:35.329751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471022773353864:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:35.390962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471022773353924:2106] [node 5] ICP01 ready to work
2025-11-27T18:17:35.503185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471023634718849:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:35.501706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471023915498862:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:35.505763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471023072509245:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:35.506509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471023072509244:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:35.506479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471021851736895:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.509874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471024094916436:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:35.509951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471024094916437:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:35.526298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471024094916438:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:35.508797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471023915498863:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:35.533852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471023072509246:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:35.507218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471021851736899:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:35.523378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471022773353864:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16462456ns) {Configure:104(16455888ns) {ProcessConfigured:135(79348ns) {StartInitialHandshake:157(75938ns) {DropHandshakes:470(3049ns) {DropIncomingHandshake:442(872ns) DropOutgoingHandshake:457(437ns)} GenerateSessionVirtualId:988(936ns) PrepareNewSessionHandshake:477(2034ns) {SwitchToState:179(771ns)}}}}}
2025-11-27T18:17:35.517627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471023634718853:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:35.523917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471022773353929:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29470
2025-11-27T18:17:35.510120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471023915498864:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:35.546986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471023634718788:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(140412033ns) {Configure:104(140409419ns) {ProcessConfigured:135(57681ns) {StartInitialHandshake:157(55540ns) {DropHandshakes:470(3307ns) {DropIncomingHandshake:442(774ns) DropOutgoingHandshake:457(513ns)} GenerateSessionVirtualId:988(976ns) PrepareNewSessionHandshake:477(1925ns) {SwitchToState:179(699ns)}}}}}
2025-11-27T18:17:35.529629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471022773353926:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.564774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471024094916436:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(30837953ns) {Configure:104(30831141ns) {ProcessConfigured:135(4086278ns) {StartInitialHandshake:157(4083910ns) {DropHandshakes:470(3081ns) {DropIncomingHandshake:442(730ns) DropOutgoingHandshake:457(537ns)} GenerateSessionVirtualId:988(1475ns) PrepareNewSessionHandshake:477(4607ns) {SwitchToState:179(2217ns)}}}}}
2025-11-27T18:17:35.569381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471023634718857:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11879
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.573093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471023915498862:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(57202321ns) {Configure:104(57198824ns) {ProcessConfigured:135(9622385ns) {StartInitialHandshake:157(9618854ns) {DropHandshakes:470(3089ns) {DropIncomingHandshake:442(644ns) DropOutgoingHandshake:457(655ns)} GenerateSessionVirtualId:988(667ns) PrepareNewSessionHandshake:477(5412ns) {SwitchToState:179(3159ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.581730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471022773353929:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.586891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471023634718857:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.621554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471021851736900:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
2025-11-27T18:17:35.626666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471023072509244:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(85255964ns) {Configure:104(84851850ns) {ProcessConfigured:135(60532ns) {StartInitialHandshake:157(58518ns) {DropHandshakes:470(3348ns) {DropIncomingHandshake:442(872ns) DropOutgoingHandshake:457(378ns)} GenerateSessionVirtualId:988(1021ns) PrepareNewSessionHandshake:477(2180ns) {SwitchToState:179(760ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.601928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471023915498799:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(100883749ns) {Configure:104(100876146ns) {ProcessConfigured:135(86048ns) {StartInitialHandshake:157(83934ns) {DropHandshakes:470(1068ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(196ns)} GenerateSessionVirtualId:988(664ns) PrepareNewSessionHandshake:477(2160ns) {SwitchToState:179(613ns)}}}}}
2025-11-27T18:17:35.606099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471023634718787:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(199737815ns) {Configure:104(199734761ns) {ProcessConfigured:135(85217ns) {StartInitialHandshake:157(82548ns) {DropHandshakes:470(2241ns) {DropIncomingHandshake:442(513ns) DropOutgoingHandshake:457(291ns)} GenerateSessionVirtualId:988(1007ns) PrepareNewSessionHandshake:477(1908ns) {SwitchToState:179(775ns)}}}}}
2025-11-27T18:17:35.586538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471022773353862:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(88972621ns) {Configure:104(88962491ns) {ProcessConfigured:135(87405ns) {StartInitialHandshake:157(84925ns) {DropHandshakes:470(2952ns) {DropIncomingHandshake:442(779ns) DropOutgoingHandshake:457(540ns)} GenerateSessionVirtualId:988(1542ns) PrepareNewSessionHandshake:477(2085ns) {SwitchToState:179(844ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.607056Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471024094916438:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48922820ns) {Configure:104(48917085ns) {ProcessConfigured:135(8920035ns) {StartInitialHandshake:157(8916421ns) {DropHandshakes:470(3842ns) {DropIncomingHandshake:442(1132ns) DropOutgoingHandshake:457(447ns)} GenerateSessionVirtualId:988(914ns) PrepareNewSessionHandshake:477(2763ns) {SwitchToState:179(1508ns)}}}}}
2025-11-27T18:17:35.621981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471023915498865:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
2025-11-27T18:17:35.609706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471024094916439:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45946
2025-11-27T18:17:35.623752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471023634718861:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11879
2025-11-27T18:17:35.635737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471022773353924:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(206212768ns) {Configure:104(206207570ns) {ProcessConfigured:135(24555012ns) {StartInitialHandshake:157(24553046ns) {DropHandshakes:470(2406ns) {DropIncomingHandshake:442(743ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(661ns) PrepareNewSessionHandshake:477(5378ns) {SwitchToState:179(2177ns)}}}}}
2025-11-27T18:17:35.657383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471023634718861:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.641738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471021851736900:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.648469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471022773353924:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.626199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471024094916439:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.643047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471023915498865:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.642029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471021851736903:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
2025-11-27T18:17:35.667433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471022773353939:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29470
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.666764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471024094916371:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(239733619ns) {Configure:104(239729991ns) {ProcessConfigured:135(27949362ns) {StartInitialHandshake:157(27946808ns) {DropHandshakes:470(1047ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(529ns) PrepareNewSessionHandshake:477(3865ns) {SwitchToState:179(2666ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.665653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471023072509181:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(161196141ns) {Configure:104(161183617ns) {ProcessConfigured:135(58200ns) {StartInitialHandshake:157(55758ns) {DropHandshakes:470(2931ns) {DropIncomingHandshake:442(716ns) DropOutgoingHandshake:457(487ns)} GenerateSessionVirtualId:988(1114ns) PrepareNewSessionHandshake:477(1913ns) {SwitchToState:179(615ns)}}}}}
2025-11-27T18:17:35.676786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471023634718786:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(243077211ns) {Configure:104(243073955ns) {ProcessConfigured:135(13757735ns) {StartInitialHandshake:157(13755631ns) {DropHandshakes:470(8423904ns) {DropIncomingHandshake:442(809ns) DropOutgoingHandshake:457(1623ns)} GenerateSessionVirtualId:988(1880ns) PrepareNewSessionHandshake:477(2121ns) {SwitchToState:179(806ns)}}}}}
2025-11-27T18:17:35.649390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471023915498874:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
2025-11-27T18:17:35.648901Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471021851736833:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(139959236ns) {Configure:104(139953941ns) {ProcessConfigured:135(5426923ns) {StartInitialHandshake:157(5423789ns) {DropHandshakes:470(2313ns) {DropIncomingHandshake:442(642ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(2385ns) PrepareNewSessionHandshake:477(1754ns) {SwitchToState:179(122ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.695298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471024094916437:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(128263828ns) {Configure:104(128258378ns) {ProcessConfigured:135(12176602ns) {StartInitialHandshake:157(8277466ns) {DropHandshakes:470(3702ns) {DropIncomingHandshake:442(696ns) DropOutgoingHandshake:457(405ns)} GenerateSessionVirtualId:988(2031ns) PrepareNewSessionHandshake:477(4175ns) {SwitchToState:179(1738ns)}}}}}
2025-11-27T18:17:35.674143Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471023072509245:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(164823716ns) {Configure:104(164818262ns) {ProcessConfigured:135(12319416ns) {StartInitialHandshake:157(12315231ns) {DropHandshakes:470(3089ns) {DropIncomingHandshake:442(600ns) DropOutgoingHandshake:457(758ns)} GenerateSessionVirtualId:988(1574ns) PrepareNewSessionHandshake:477(3837ns) {SwitchToState:179(2230ns)}}}}}
2025-11-27T18:17:35.676586Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471023915498798:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(240915733ns) {Configure:104(240908970ns) {ProcessConfigured:135(17645622ns) {StartInitialHandshake:157(17642745ns) {DropHandshakes:470(2144ns) {DropIncomingHandshake:442(533ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(509ns) PrepareNewSessionHandshake:477(4032ns) {SwitchToState:179(1839ns)}}}}}
2025-11-27T18:17:35.673561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471022773353939:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.681627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471023634718851:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.665479Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471021851736834:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(124221643ns) {Configure:104(124215973ns) {ProcessConfigured:135(29492441ns) {StartInitialHandshake:157(29485891ns) {DropHandshakes:470(4532ns) {DropIncomingHandshake:442(2040ns) DropOutgoingHandshake:457(372ns)} GenerateSessionVirtualId:988(3961ns) PrepareNewSessionHandshake:477(5059ns) {SwitchToState:179(1899ns)}}}}}
2025-11-27T18:17:35.690265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471023915498867:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.717873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471024094916451:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45946
2025-11-27T18:17:35.674270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471023072509247:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33884
2025-11-27T18:17:35.685478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471022773353863:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(163159026ns) {Configure:104(163153117ns) {ProcessConfigured:135(8339990ns) {StartInitialHandshake:157(8337165ns) {DropHandshakes:470(701ns) {DropIncomingHandshake:442(251ns) DropOutgoingHandshake:457(299ns)} GenerateSessionVirtualId:988(1982ns) PrepareNewSessionHandshake:477(2694ns) {SwitchToState:179(1365ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.676000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471021851736895:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(158206515ns) {Configure:104(158198731ns) {ProcessConfigured:135(14124134ns) {StartInitialHandshake:157(9131769ns) {DropHandshakes:470(1443ns) {DropIncomingHandshake:442(757ns) DropOutgoingHandshake:457(160ns)} GenerateSessionVirtualId:988(1281ns) PrepareNewSessionHandshake:477(2763ns) {SwitchToState:179(1828ns)}}}}}
2025-11-27T18:17:35.689633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471021851736913:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
2025-11-27T18:17:35.700135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471021851736913:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.700249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471021851736834:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.709107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471021851736899:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(140084975ns) {Configure:104(140072705ns) {ProcessConfigured:135(6683711ns) {StartInitialHandshake:157(6680951ns) {DropHandshakes:470(2151ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(658ns)} GenerateSessionVirtualId:988(618ns) PrepareNewSessionHandshake:477(2021ns) {SwitchToState:179(747ns)}}}}}
2025-11-27T18:17:35.754606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471021851736903:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.758401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471021851736833:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.760964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471021851736915:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
2025-11-27T18:17:35.685596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471022773353863:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.681677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471023634718851:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.674762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471023072509250:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33884
2025-11-27T18:17:35.690297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471023915498867:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.722226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471024094916451:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.695388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471023634718786:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.708344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471022773353924:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26414) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471022773353929:2111] held: no
2025-11-27T18:17:35.725458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471024094916441:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.699076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471023915498867:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:35.675456Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471023072509182:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(177887562ns) {Configure:104(177879664ns) {ProcessConfigured:135(9125921ns) {StartInitialHandshake:157(9124252ns) {DropHandshakes:470(3451ns) {DropIncomingHandshake:442(772ns) DropOutgoingHandshake:457(505ns)} GenerateSessionVirtualId:988(931ns) PrepareNewSessionHandshake:477(1653ns) {SwitchToState:179(449ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.706447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471023634718851:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:35.777302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471022773353944:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29470
2025-11-27T18:17:35.689996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471023072509247:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.701490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471023915498798:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.725520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471023915498874:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.733542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471023915498798:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18911208ns) {IncomingHandshake:261(18910137ns) {DropIncomingHandshake:442(1534ns) PrepareNewSessionHandshake:477(2734ns) {SwitchToState:179(528ns)} GenerateSessionVirtualId:988(671ns) IssueIncomingHandshakeReply:193(42683ns) {SwitchToState:179(394ns)}}}
2025-11-27T18:17:35.754557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471023915498864:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(147865061ns) {Configure:104(147857511ns) {ProcessConfigured:135(39919977ns) {StartInitialHandshake:157(39916636ns) {DropHandshakes:470(5176353ns) {DropIncomingHandshake:442(1614ns) DropOutgoingHandshake:457(653ns)} GenerateSessionVirtualId:988(926ns) PrepareNewSessionHandshake:477(2769ns) {SwitchToState:179(1574ns)}}}}}
2025-11-27T18:17:35.757625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471023915498863:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:35.758264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471023915498884:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
2025-11-27T18:17:35.761218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471023915498871:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.761236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471023915498871:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.761399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:35.769669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471023915498863:2109] [node 6] ICP22 created new session: [2:7577471023915498883:2109]
2025-11-27T18:17:35.706684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471023634718849:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:35.725502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471024094916441:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.795290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471022773353945:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29470
2025-11-27T18:17:35.773677Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471023915498871:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.796720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471021851736834:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23684) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471021851736913:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.749866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471024094916441:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.690134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471023072509261:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33884
2025-11-27T18:17:35.777609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471023915498869:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.706733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:35.825765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471021851736917:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.699466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471023072509263:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33884
2025-11-27T18:17:35.754745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471024094916431:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.850926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471022773353944:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.706771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471023634718849:2109] [node 6] ICP22 created new session: [5:7577471023634718868:2109]
2025-11-27T18:17:35.777646Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471023915498869:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.829431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471021851736913:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:35.857442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471022773353926:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.754777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471024094916431:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.706800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471023634718868:2109] [node 6] ICS09 handshake done sender: [5:7577471023634718851:2110] self: [5:7577471023634718850:0] peer: [6:7577471022773353940:0] socket: 242 qp: -1
2025-11-27T18:17:35.700058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471023072509263:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.777680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471023915498883:2109] [node 6] ICS09 handshake done sender: [2:7577471023915498867:2112] self: [2:7577471023915498866:0] peer: [6:7577471022773353943:0] socket: 257 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.833732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471021851736903:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.755782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471024094916455:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45946
2025-11-27T18:17:35.700482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471023072509256:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.868318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471022773353946:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29470
2025-11-27T18:17:35.706825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471023634718868:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:35.706849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:35.729906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471023634718853:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(172747718ns) {Configure:104(172740819ns) {ProcessConfigured:135(31653611ns) {StartInitialHandshake:157(18873890ns) {DropHandshakes:470(3749ns) {DropIncomingHandshake:442(1683ns) DropOutgoingHandshake:457(524ns)} GenerateSessionVirtualId:988(2259ns) PrepareNewSessionHandshake:477(2550ns) {SwitchToState:179(1208ns)}}}}}
2025-11-27T18:17:35.791216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471024094916457:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45946
2025-11-27T18:17:35.853880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471023634718872:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11879
2025-11-27T18:17:35.857401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471023634718857:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:35.795533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471024094916436:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:35.861563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471023634718786:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:35.795574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:35.870249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:35.870320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471023634718786:2048] [node 1] ICP22 created new session: [5:7577471023634718875:2048]
2025-11-27T18:17:35.801946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471024094916431:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:35.802205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471024094916436:2109] [node 5] ICP22 created new session: [1:7577471024094916459:2109]
2025-11-27T18:17:35.870383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471023634718875:2048] [node 1] ICS09 handshake done sender: [5:7577471023634718857:2112] self: [5:7577471023634718865:0] peer: [1:7577471024094916440:0] socket: 256 qp: -1
2025-11-27T18:17:35.813625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471024094916455:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.870420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471023634718875:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:35.870464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:35.829590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471024094916459:2109] [node 5] ICS09 handshake done sender: [1:7577471024094916441:2112] self: [1:7577471024094916440:0] peer: [5:7577471023634718865:0] socket: 255 qp: -1
2025-11-27T18:17:35.903610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471023634718874:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11879
2025-11-27T18:17:35.834446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471024094916370:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:35.909166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471023634718873:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11879
2025-11-27T18:17:35.834463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471024094916457:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.917504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471023634718878:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11879
2025-11-27T18:17:35.834504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:35.925091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471023634718873:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.834565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471024094916370:2048] [node 2] ICP22 created new session: [1:7577471024094916460:2048]
2025-11-27T18:17:35.834608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471024094916460:2048] [node 2] ICS09 handshake done sender: [1:7577471024094916431:2107] self: [1:7577471024094916430:0] peer: [2:7577471023915498881:0] socket: 240 qp: -1
2025-11-27T18:17:35.868657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471022773353939:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:35.834644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471024094916460:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:35.834681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:35.873602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471022773353948:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29470
2025-11-27T18:17:35.841542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471024094916459:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:35.894034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471022773353948:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.860034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:35.905498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471022773353863:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:35.905564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:35.876257Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471024094916370:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(41567142ns) {HandleHandshakeStatus:329(41564012ns) {DropIncomingHandshake:442(657ns) DropOutgoingHandshake:457(1039ns) SwitchToState:179(903ns) ProcessPendingSessionEvents:495(7891222ns) {ForwardSessionEventToSession:612(7816648ns) ForwardSessionEventToSession:612(7824ns) ForwardSessionEventToSession:612(6632ns) ForwardSessionEventToSession:612(7036ns) ForwardSessionEventToSession:612(3779ns) ForwardSessionEventToSession:612(2446ns) ForwardSessionEventToSession:612(8873ns) ForwardSessionEventToSession:612(2618ns) ForwardSessionEventToSession:612(15184ns) ForwardSessionEventToSession:612(3283ns)}}}
2025-11-27T18:17:35.908360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471022773353929:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:35.902267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471024094916436:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(107453779ns) {HandleHandshakeStatus:329(107450489ns) {DropIncomingHandshake:442(1193ns) DropOutgoingHandshake:457(592ns) SwitchToState:179(1281ns) ProcessPendingSessionEvents:495(8151982ns) {ForwardSessionEventToSession:612(8085976ns) ForwardSessionEventToSession:612(16650ns) ForwardSessionEventToSession:612(2899ns) ForwardSessionEventToSession:612(6185ns) ForwardSessionEventToSession:612(6058ns) ForwardSessionEventToSession:612(2908ns) ForwardSessionEventToSession:612(2225ns) ForwardSessionEventToSession:612(5679ns) ForwardSessionEventToSession:612(2656ns) ForwardSessionEventToSession:612(5739ns)}}}
2025-11-27T18:17:35.911530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471022773353863:2049] [node 2] ICP22 created new session: [6:7577471022773353951:2049]
2025-11-27T18:17:35.902347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471024094916436:2109] [node 5] ICP18 handshake (actor [1:7577471024094916457:2119]) is too old
2025-11-27T18:17:35.915370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471022773353924:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:35.917173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471024094916456:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45946
2025-11-27T18:17:35.915417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:35.933567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471024094916446:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.915465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471022773353924:2106] [node 5] ICP22 created new session: [6:7577471022773353953:2106]
2025-11-27T18:17:35.933600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471024094916446:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.915508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471022773353953:2106] [node 5] ICS09 handshake done sender: [6:7577471022773353929:2111] self: [6:7577471022773353940:0] peer: [5:7577471023634718850:0] socket: 245 qp: -1
2025-11-27T18:17:35.915544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471022773353953:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:35.915588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:35.916062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471022773353862:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.916361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471022773353952:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29470
2025-11-27T18:17:35.921540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471022773353949:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29470
2025-11-27T18:17:35.921863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471022773353944:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:35.930098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471022773353951:2049] [node 2] ICS09 handshake done sender: [6:7577471022773353939:2115] self: [6:7577471022773353943:0] peer: [2:7577471023915498866:0] socket: 259 qp: -1
2025-11-27T18:17:35.835797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471021851736833:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:35.835842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:35.835901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471021851736833:2049] [node 2] ICP22 created new session: [4:7577471021851736919:2049]
2025-11-27T18:17:35.835938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471021851736919:2049] [node 2] ICS09 handshake done sender: [4:7577471021851736903:2113] self: [4:7577471021851736916:0] peer: [2:7577471023915498868:0] socket: 267 qp: -1
2025-11-27T18:17:35.835964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471021851736919:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:35.835995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:35.877061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471021851736909:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.877091Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471021851736909:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.877883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471021851736834:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:35.877936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:35.877982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471021851736834:2050] [node 3] ICP22 created new session: [4:7577471021851736922:2050]
2025-11-27T18:17:35.878033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471021851736922:2050] [node 3] ICS09 handshake done sender: [4:7577471021851736913:2117] self: [4:7577471021851736914:0] peer: [3:7577471023072509253:0] socket: 277 qp: -1
2025-11-27T18:17:35.878069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471021851736922:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:35.878104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:35.879622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471021851736918:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
2025-11-27T18:17:35.885612Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471021851736909:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:35.700506Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471023072509256:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.708965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471023072509261:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.713869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471023072509182:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.730578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471023072509256:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:35.750038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471023072509250:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.754050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471023072509246:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(155253504ns) {Configure:104(155246234ns) {ProcessConfigured:135(42011537ns) {StartInitialHandshake:157(42009506ns) {DropHandshakes:470(3559507ns) {DropIncomingHandshake:442(1695ns) DropOutgoingHandshake:457(174ns)} GenerateSessionVirtualId:988(923ns) PrepareNewSessionHandshake:477(2801ns) {SwitchToState:179(852ns)}}}}}
2025-11-27T18:17:35.781568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471023072509269:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33884
2025-11-27T18:17:35.782019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471023072509245:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:35.782096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:35.782143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471023072509245:2107] [node 4] ICP22 created new session: [3:7577471023072509270:2107]
2025-11-27T18:17:35.793676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471023072509261:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:35.796168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471023072509182:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:35.796226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:35.802061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471023072509270:2107] [node 4] ICS09 handshake done sender: [3:7577471023072509256:2115] self: [3:7577471023072509253:0] peer: [4:7577471021851736914:0] socket: 271 qp: -1
2025-11-27T18:17:35.936364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471022773353951:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:35.785769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471023915498879:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.897793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471021851736833:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(24481901ns) {HandleHandshakeStatus:329(24478669ns) {DropIncomingHandshake:442(1248ns) DropOutgoingHandshake:457(4070ns) SwitchToState:179(702ns) ProcessPendingSessionEvents:495(24136374ns) {ForwardSessionEventToSession:612(59035ns) ForwardSessionEventToSession:612(6273ns) ForwardSessionEventToSession:612(2346ns) ForwardSessionEventToSession:612(5025ns) ForwardSessionEventToSession:612(2573ns) ForwardSessionEventToSession:612(1893ns) ForwardSessionEventToSession:612(6490744ns) ForwardSessionEventToSession:612(2924ns) ForwardSessionEventToSession:612(7987577ns) ForwardSessionEventToSession:612(9558421ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.936611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471024094916370:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:35.939921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471023634718874:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.786869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471023915498883:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:35.817464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471023072509270:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:35.911164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471021851736895:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:35.937845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471022773353926:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:35.937582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471024094916446:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:35.817534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:35.940019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471023634718878:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.937909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:35.911238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:35.829571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471023072509182:2049] [node 2] ICP22 created new session: [3:7577471023072509272:2049]
2025-11-27T18:17:35.850880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471023072509272:2049] [node 2] ICS09 handshake done sender: [3:7577471023072509261:2114] self: [3:7577471023072509266:0] peer: [2:7577471023915498870:0] socket: 269 qp: -1
2025-11-27T18:17:35.868485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471023072509272:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:35.868555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:35.885121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471023072509276:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33884
2025-11-27T18:17:35.901419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471023072509276:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.911885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471023072509181:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.917132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471023072509245:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(115329120ns) {HandleHandshakeStatus:329(111515052ns) {DropIncomingHandshake:442(19572ns) DropOutgoingHandshake:457(782ns) SwitchToState:179(887ns) ProcessPendingSessionEvents:495(9779169ns) {ForwardSessionEventToSession:612(9713093ns) ForwardSessionEventToSession:612(9604ns) ForwardSessionEventToSession:612(8253ns) ForwardSessionEventToSession:612(9424ns) ForwardSessionEventToSession:612(7523ns) ForwardSessionEventToSession:612(2744ns) ForwardSessionEventToSession:612(3440ns) ForwardSessionEventToSession:612(6058ns) ForwardSessionEventToSession:612(3598ns) ForwardSessionEventToSession:612(2558ns)}}}
2025-11-27T18:17:35.944370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471023072509249:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.944395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471023072509249:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.954846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471023072509279:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33884
2025-11-27T18:17:35.957384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471023072509276:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:35.959405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471023072509249:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:35.987142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471023072509265:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.987165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471023072509265:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.987878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471023072509181:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:35.987942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:35.987997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471023072509181:2048] [node 1] ICP22 created new session: [3:7577471023072509280:2048]
2025-11-27T18:17:35.988054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471023072509280:2048] [node 1] ICS09 handshake done sender: [3:7577471023072509276:2120] self: [3:7577471023072509277:0] peer: [1:7577471024094916444:0] socket: 306 qp: -1
2025-11-27T18:17:35.988089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471023072509280:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:35.988132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:36.001710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471023072509181:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:36.001835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471023072509182:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(140122156ns) {HandleHandshakeStatus:329(140119015ns) {DropIncomingHandshake:442(577ns) DropOutgoingHandshake:457(6747ns) SwitchToState:179(774ns) ProcessPendingSessionEvents:495(38668148ns) {ForwardSessionEventToSession:612(8418324ns) ForwardSessionEventToSession:612(3103306ns) ForwardSessionEventToSession:612(4625ns) ForwardSessionEventToSession:612(3306ns) ForwardSessionEventToSession:612(5711754ns) ForwardSessionEventToSession:612(3949ns) ForwardSessionEventToSession:612(7411628ns) ForwardSessionEventToSession:612(8453ns) ForwardSessionEventToSession:612(6115667ns) ForwardSessionEventToSession:612(5418412ns)}}}
2025-11-27T18:17:36.017709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471023072509244:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:36.017802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.938264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471024094916448:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.789841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471023915498799:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.789897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:35.937957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471022773353926:2109] [node 4] ICP22 created new session: [6:7577471022773353957:2109]
2025-11-27T18:17:35.942514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471023634718787:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.911412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471021851736918:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:36.035010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471023072509244:2109] [node 6] ICP22 created new session: [3:7577471027367476579:2109]
2025-11-27T18:17:35.938288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471024094916448:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.937999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471022773353957:2109] [node 4] ICS09 handshake done sender: [6:7577471022773353944:2117] self: [6:7577471022773353950:0] peer: [4:7577471021851736907:0] socket: 275 qp: -1
2025-11-27T18:17:35.938033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471022773353957:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:35.938073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:35.939408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:35.943538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471022773353949:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.943805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471022773353864:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.953833Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471022773353926:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:35.981673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471022773353960:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29470
2025-11-27T18:17:36.006733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471022773353863:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(112762283ns) {HandleHandshakeStatus:329(112757880ns) {DropIncomingHandshake:442(2062ns) DropOutgoingHandshake:457(7339ns) SwitchToState:179(621ns) ProcessPendingSessionEvents:495(14863635ns) {ForwardSessionEventToSession:612(14801675ns) ForwardSessionEventToSession:612(10103ns) ForwardSessionEventToSession:612(3289ns) ForwardSessionEventToSession:612(3159ns) ForwardSessionEventToSession:612(6005ns) ForwardSessionEventToSession:612(2912ns) ForwardSessionEventToSession:612(10381ns) ForwardSessionEventToSession:612(3509ns) ForwardSessionEventToSession:612(2639ns) ForwardSessionEventToSession:612(8553ns)}}}
2025-11-27T18:17:36.014368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471022773353949:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:36.045660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471022773353864:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:36.045757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:36.054069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471022773353864:2050] [node 3] ICP22 created new session: [6:7577471027068321258:2050]
2025-11-27T18:17:36.062256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471022773353862:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31411) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471022773353948:2120] held: no
2025-11-27T18:17:35.789949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471023915498799:2049] [node 3] ICP22 created new session: [2:7577471023915498889:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:35.946065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471024094916448:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:35.916697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471021851736895:2109] [node 6] ICP22 created new session: [4:7577471021851736927:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:36.045854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471023072509265:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:35.922424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471021851736926:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
2025-11-27T18:17:35.961663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471023634718787:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17338541ns) {IncomingHandshake:261(17336049ns) {DropIncomingHandshake:442(2035ns) PrepareNewSessionHandshake:477(1006ns) {SwitchToState:179(478ns)} GenerateSessionVirtualId:988(545ns) IssueIncomingHandshakeReply:193(5060798ns) {SwitchToState:179(1630ns)}}}
2025-11-27T18:17:36.065594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471027068321258:2050] [node 3] ICS09 handshake done sender: [6:7577471022773353949:2121] self: [6:7577471022773353959:0] peer: [3:7577471023072509248:0] socket: 302 qp: -1
2025-11-27T18:17:35.936286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471021851736927:2109] [node 6] ICS09 handshake done sender: [4:7577471021851736909:2115] self: [4:7577471021851736907:0] peer: [6:7577471022773353950:0] socket: 273 qp: -1
2025-11-27T18:17:35.937383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471021851736832:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.945612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471021851736927:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:35.945682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:35.945778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471021851736926:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.961483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471021851736832:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20577796ns) {IncomingHandshake:261(20576557ns) {DropIncomingHandshake:442(274ns) PrepareNewSessionHandshake:477(7216583ns) {SwitchToState:179(3924ns)} GenerateSessionVirtualId:988(1113ns) IssueIncomingHandshakeReply:193(70346ns) {SwitchToState:179(1181ns)}}}
2025-11-27T18:17:35.985887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471021851736912:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.985916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471021851736912:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.989596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471021851736912:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:35.989860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471021851736899:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:35.989919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:35.989982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471021851736899:2108] [node 5] ICP22 created new session: [4:7577471021851736933:2108]
2025-11-27T18:17:35.990034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471021851736933:2108] [node 5] ICS09 handshake done sender: [4:7577471021851736912:2116] self: [4:7577471021851736908:0] peer: [5:7577471023634718884:0] socket: 274 qp: -1
2025-11-27T18:17:35.990084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471021851736933:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:35.990122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:35.992202Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471021851736832:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31411) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471021851736918:2120] held: no
2025-11-27T18:17:36.007552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471021851736895:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(100974336ns) {HandleHandshakeStatus:329(100971054ns) {DropIncomingHandshake:442(28668ns) DropOutgoingHandshake:457(634ns) SwitchToState:179(660ns) ProcessPendingSessionEvents:495(15577642ns) {ForwardSessionEventToSession:612(15522358ns) ForwardSessionEventToSession:612(9341ns) ForwardSessionEventToSession:612(6661ns) ForwardSessionEventToSession:612(3507ns) ForwardSessionEventToSession:612(6253ns) ForwardSessionEventToSession:612(2793ns) ForwardSessionEventToSession:612(2873ns) ForwardSessionEventToSession:612(6621ns) ForwardSessionEventToSession:612(2512ns) ForwardSessionEventToSession:612(4962ns)}}}
2025-11-27T18:17:36.059186Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471021851736832:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(33012727ns) {HandleHandshakeStatus:395(33008312ns) {LogHandshakeFail:472(14000ns) {UpdateErrorStateLog:545(5413ns)} DropOutgoingHandshake:457(1500ns) ProcessPendingSessionEvents:495(25166845ns) {DropSessionEvent:512(10148ns) DropSessionEvent:512(3385ns) DropSessionEvent:512(4215ns) DropSessionEvent:512(3797ns) DropSessionEvent:512(9584445ns) DropSessionEvent:512(3902ns) DropSessionEvent:512(4134ns) DropSessionEvent:512(3900ns) DropSessionEvent:512(3965ns) DropSessionEvent:512(15533766ns)}}}
2025-11-27T18:17:36.089300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471021851736930:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
2025-11-27T18:17:36.059390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471027367476579:2109] [node 6] ICS09 handshake done sender: [3:7577471023072509249:2111] self: [3:7577471023072509248:0] peer: [6:7577471022773353959:0] socket: 301 qp: -1
2025-11-27T18:17:35.962022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471023634718881:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11879
2025-11-27T18:17:35.789995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471023915498889:2049] [node 3] ICS09 handshake done sender: [2:7577471023915498871:2114] self: [2:7577471023915498870:0] peer: [3:7577471023072509266:0] socket: 258 qp: -1
2025-11-27T18:17:35.790027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471023915498889:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:35.790068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:35.790805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471023915498869:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:35.791591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471023915498862:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:35.791638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:35.791687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471023915498862:2107] [node 4] ICP22 created new session: [2:7577471023915498893:2107]
2025-11-27T18:17:35.791730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471023915498893:2107] [node 4] ICS09 handshake done sender: [2:7577471023915498869:2113] self: [2:7577471023915498868:0] peer: [4:7577471021851736916:0] socket: 266 qp: -1
2025-11-27T18:17:35.791771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471023915498893:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:35.791802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:35.792107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:35.829767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471023915498879:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.834109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471023915498865:2111] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:35.836389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471023915498798:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:35.836430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:35.836471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471023915498798:2048] [node 1] ICP22 created new session: [2:7577471023915498895:2048]
2025-11-27T18:17:35.836504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471023915498895:2048] [node 1] ICS09 handshake done sender: [2:7577471023915498865:2111] self: [2:7577471023915498881:0] peer: [1:7577471024094916430:0] socket: 244 qp: -1
2025-11-27T18:17:35.836534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471023915498895:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:35.836561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:35.841740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471023915498799:2049] [node 3] ICP18 handshake (actor [2:7577471023915498879:2117]) is too old
2025-11-27T18:17:35.849714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471023915498863:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(94880659ns) {HandleHandshakeStatus:329(94877322ns) {DropIncomingHandshake:442(1297ns) DropOutgoingHandshake:457(584ns) SwitchToState:179(960ns) ProcessPendingSessionEvents:495(19778407ns) {ForwardSessionEventToSession:612(19690878ns) ForwardSessionEventToSession:612(15519ns) ForwardSessionEventToSession:612(3058ns) ForwardSessionEventToSession:612(7156ns) ForwardSessionEventToSession:612(3936ns) ForwardSessionEventToSession:612(6609ns) ForwardSessionEventToSession:612(10394ns) ForwardSessionEventToSession:612(2971ns) ForwardSessionEventToSession:612(3285ns) ForwardSessionEventToSession:612(5413ns)}}}
2025-11-27T18:17:35.863347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471023915498862:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(71518451ns) {HandleHandshakeStatus:329(71515121ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(394ns) SwitchToState:179(929ns) ProcessPendingSessionEvents:495(21665110ns) {ForwardSessionEventToSession:612(21581083ns) ForwardSessionEventToSession:612(7942ns) ForwardSessionEventToSession:612(19681ns) ForwardSessionEventToSession:612(4549ns) ForwardSessionEventToSession:612(3534ns) ForwardSessionEventToSession:612(6860ns) ForwardSessionEventToSession:612(3204ns) ForwardSessionEventToSession:612(13661ns) ForwardSessionEventToSession:612(2774ns) ForwardSessionEventToSession:612(3012ns)}}}
2025-11-27T18:17:35.945852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471023915498882:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:35.945886Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471023915498882:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.965749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471023915498882:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:35.999269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471023915498864:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:35.999348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:35.999411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471023915498864:2108] [node 5] ICP22 created new session: [2:7577471023915498898:2108]
2025-11-27T18:17:35.999457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471023915498898:2108] [node 5] ICS09 handshake done sender: [2:7577471023915498882:2118] self: [2:7577471023915498878:0] peer: [5:7577471023634718879:0] socket: 300 qp: -1
2025-11-27T18:17:35.999496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471023915498898:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:35.999530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:35.958773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471024094916445:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:36.059437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471027367476579:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:36.065703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:36.069360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471023072509246:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:36.069430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:36.069095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471027068321258:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:36.117709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471021851736918:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:36.118044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471021851736832:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:36.118095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:36.118152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471021851736832:2048] [node 1] ICP22 created new session: [4:7577471026146704231:2048]
2025-11-27T18:17:36.118194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471026146704231:2048] [node 1] ICS09 handshake done sender: [4:7577471021851736918:2120] self: [4:7577471021851736928:0] peer: [1:7577471024094916447:0] socket: 303 qp: -1
2025-11-27T18:17:36.118232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471026146704231:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:36.118275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:36.085471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471023072509246:2108] [node 5] ICP22 created new session: [3:7577471027367476581:2108]
2025-11-27T18:17:35.962917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471023634718873:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:35.958820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471024094916445:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:35.961479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471024094916371:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:35.969438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:35.980635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471024094916371:2049] [node 3] ICP22 created new session: [1:7577471024094916467:2049]
2025-11-27T18:17:35.980706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471024094916467:2049] [node 3] ICS09 handshake done sender: [1:7577471024094916446:2114] self: [1:7577471024094916444:0] peer: [3:7577471023072509277:0] socket: 272 qp: -1
2025-11-27T18:17:35.980757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471024094916467:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:35.980816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:35.981350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471024094916371:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(27757932ns) {HandleHandshakeStatus:329(27754890ns) {DropIncomingHandshake:442(22052ns) DropOutgoingHandshake:457(748ns) SwitchToState:179(1104ns) ProcessPendingSessionEvents:495(131265ns) {ForwardSessionEventToSession:612(79771ns) ForwardSessionEventToSession:612(2972ns) ForwardSessionEventToSession:612(2993ns) ForwardSessionEventToSession:612(6873ns) ForwardSessionEventToSession:612(6769ns) ForwardSessionEventToSession:612(2519ns) ForwardSessionEventToSession:612(6169ns) ForwardSessionEventToSession:612(2541ns) ForwardSessionEventToSession:612(9050ns) ForwardSessionEventToSession:612(2402ns)}}}
2025-11-27T18:17:35.981427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471024094916437:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:35.985416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:35.991929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471024094916445:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:36.005498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471024094916437:2108] [node 4] ICP22 created new session: [1:7577471024094916469:2108]
2025-11-27T18:17:36.005554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471024094916469:2108] [node 4] ICS09 handshake done sender: [1:7577471024094916448:2115] self: [1:7577471024094916447:0] peer: [4:7577471021851736928:0] socket: 299 qp: -1
2025-11-27T18:17:36.005601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471024094916469:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:36.005656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:36.006094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471024094916437:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(24492860ns) {HandleHandshakeStatus:329(24490198ns) {DropIncomingHandshake:442(15666ns) DropOutgoingHandshake:457(718ns) SwitchToState:179(799ns) ProcessPendingSessionEvents:495(118340ns) {ForwardSessionEventToSession:612(66814ns) ForwardSessionEventToSession:612(4078ns) ForwardSessionEventToSession:612(2980ns) ForwardSessionEventToSession:612(7910ns) ForwardSessionEventToSession:612(3082ns) ForwardSessionEventToSession:612(6289ns) ForwardSessionEventToSession:612(3667ns) ForwardSessionEventToSession:612(2552ns) ForwardSessionEventToSession:612(8917ns) ForwardSessionEventToSession:612(3212ns)}}}
2025-11-27T18:17:36.035244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471024094916438:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:36.035323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:36.050487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471024094916438:2110] [node 6] ICP22 created new session: [1:7577471028389883768:2110]
2025-11-27T18:17:36.077379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471028389883768:2110] [node 6] ICS09 handshake done sender: [1:7577471024094916445:2113] self: [1:7577471024094916443:0] peer: [6:7577471022773353955:0] socket: 262 qp: -1
2025-11-27T18:17:36.077457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471028389883768:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:36.077523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:36.090900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471024094916438:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(55469635ns) {HandleHandshakeStatus:329(55466044ns) {DropIncomingHandshake:442(25483ns) DropOutgoingHandshake:457(958ns) SwitchToState:179(869ns) ProcessPendingSessionEvents:495(144704ns) {ForwardSessionEventToSession:612(72535ns) ForwardSessionEventToSession:612(10357ns) ForwardSessionEventToSession:612(3145ns) ForwardSessionEventToSession:612(6632ns) ForwardSessionEventToSession:612(2436ns) ForwardSessionEventToSession:612(2881ns) ForwardSessionEventToSession:612(20200ns) ForwardSessionEventToSession:612(2113ns) ForwardSessionEventToSession:612(2889ns) ForwardSessionEventToSession:612(9243ns)}}}
2025-11-27T18:17:36.101826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471027367476581:2108] [node 5] ICS09 handshake done sender: [3:7577471023072509265:2118] self: [3:7577471023072509255:0] peer: [5:7577471023634718880:0] socket: 304 qp: -1
2025-11-27T18:17:36.101878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471027367476581:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:36.107569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:36.133482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471023072509244:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(83941216ns) {HandleHandshakeStatus:329(83935751ns) {DropIncomingHandshake:442(35233ns) DropOutgoingHandshake:457(615ns) SwitchToState:179(623ns) ProcessPendingSessionEvents:495(4043050ns) {ForwardSessionEventToSession:612(3962875ns) ForwardSessionEventToSession:612(10960ns) ForwardSessionEventToSession:612(3846ns) ForwardSessionEventToSession:612(9375ns) ForwardSessionEventToSession:612(3819ns) ForwardSessionEventToSession:612(19510ns) ForwardSessionEventToSession:612(6592ns) ForwardSessionEventToSession:612(2470ns) ForwardSessionEventToSession:612(3421ns) ForwardSessionEventToSession:612(6037ns)}}}
2025-11-27T18:17:36.133722Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471023072509246:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(64685955ns) {HandleHandshakeStatus:329(64682605ns) {DropIncomingHandshake:442(18980ns) DropOutgoingHandshake:457(755ns) SwitchToState:179(752ns) ProcessPendingSessionEvents:495(11688722ns) {ForwardSessionEventToSession:612(11610568ns) ForwardSessionEventToSession:612(13382ns) ForwardSessionEventToSession:612(5733ns) ForwardSessionEventToSession:612(19842ns) ForwardSessionEventToSession:612(3653ns) ForwardSessionEventToSession:612(2972ns) ForwardSessionEventToSession:612(7513ns) ForwardSessionEventToSession:612(2514ns) ForwardSessionEventToSession:612(3846ns) ForwardSessionEventToSession:612(5899ns)}}}
2025-11-27T18:17:36.077677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:36.097979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471027068321257:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29470
2025-11-27T18:17:36.103197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471022773353948:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:36.103375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471022773353862:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:36.103426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:36.103491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471022773353862:2048] [node 1] ICP22 created new session: [6:7577471027068321260:2048]
2025-11-27T18:17:36.103534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471027068321260:2048] [node 1] ICS09 handshake done sender: [6:7577471022773353948:2120] self: [6:7577471022773353955:0] peer: [1:7577471024094916443:0] socket: 286 qp: -1
2025-11-27T18:17:36.103572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471027068321260:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:36.103606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:36.110963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471022773353864:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(80628991ns) {HandleHandshakeStatus:329(80624750ns) {DropIncomingHandshake:442(3360ns) DropOutgoingHandshake:457(8080ns) SwitchToState:179(1089ns) ProcessPendingSessionEvents:495(9062126ns) {ForwardSessionEventToSession:612(9005195ns) ForwardSessionEventToSession:612(7211ns) ForwardSessionEventToSession:612(9091ns) ForwardSessionEventToSession:612(3094ns) ForwardSessionEventToSession:612(8074ns) ForwardSessionEventToSession:612(3089ns) ForwardSessionEventToSession:612(2608ns) ForwardSessionEventToSession:612(5197ns) ForwardSessionEventToSession:612(3113ns) ForwardSessionEventToSession:612(3308ns)}}}
2025-11-27T18:17:36.111055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471022773353864:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:35.963141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471023634718787:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:35.963193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:35.963239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471023634718787:2049] [node 2] ICP22 created new session: [5:7577471023634718882:2049]
2025-11-27T18:17:35.963283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471023634718882:2049] [node 2] ICS09 handshake done sender: [5:7577471023634718873:2118] self: [5:7577471023634718879:0] peer: [2:7577471023915498878:0] socket: 276 qp: -1
2025-11-27T18:17:35.963319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471023634718882:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:35.963367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:35.965577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471023634718788:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:35.975757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471023634718853:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:36.025751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471023634718853:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7765) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471023634718878:2120] held: no
2025-11-27T18:17:36.026070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471023634718853:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(24000679ns) {HandleHandshakeStatus:395(23996987ns) {LogHandshakeFail:472(13129ns) {UpdateErrorStateLog:545(4950ns)} DropOutgoingHandshake:457(2517ns) ProcessPendingSessionEvents:495(61627ns) {DropSessionEvent:512(12899ns) DropSessionEvent:512(4549ns) DropSessionEvent:512(4076ns) DropSessionEvent:512(4758ns) DropSessionEvent:512(3336ns) DropSessionEvent:512(3406ns) DropSessionEvent:512(7096ns) DropSessionEvent:512(7409ns) DropSessionEvent:512(3791ns) DropSessionEvent:512(3929ns)}}}
2025-11-27T18:17:36.034475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471027929686182:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11879
2025-11-27T18:17:36.037621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471023634718885:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11879
2025-11-27T18:17:36.067751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471023634718878:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:36.085844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471023634718853:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:36.085930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:36.085992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471023634718853:2108] [node 4] ICP22 created new session: [5:7577471027929686183:2108]
2025-11-27T18:17:36.086048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471027929686183:2108] [node 4] ICS09 handshake done sender: [5:7577471023634718878:2120] self: [5:7577471023634718884:0] peer: [4:7577471021851736908:0] socket: 309 qp: -1
2025-11-27T18:17:36.086086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471027929686183:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:36.086141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:36.089175Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471023634718874:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:36.101409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471023634718788:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23684) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471023634718874:2119] held: no
2025-11-27T18:17:36.101698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471023634718788:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:36.133876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:36.133938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471023634718788:2050] [node 3] ICP22 created new session: [5:7577471027929686185:2050]
2025-11-27T18:17:36.133986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471027929686185:2050] [node 3] ICS09 handshake done sender: [5:7577471023634718874:2119] self: [5:7577471023634718880:0] peer: [3:7577471023072509255:0] socket: 305 qp: -1
2025-11-27T18:17:36.134032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471027929686185:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:36.134100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:36.134349Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471023634718788:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(32571378ns) {HandleHandshakeStatus:329(32569665ns) {DropIncomingHandshake:442(3440ns) DropOutgoingHandshake:457(1876ns) SwitchToState:179(2815ns) ProcessPendingSessionEvents:495(673ns)}}
2025-11-27T18:17:36.189670Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:36.189916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471023915498895:2048] [node 1] ICS01 socket: 244 reason# EndOfStream
2025-11-27T18:17:36.189954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471023915498798:2048] [node 1] ICP30 unregister session Session# [2:7577471023915498895:2048] VirtualId# [2:7577471023915498881:0]
2025-11-27T18:17:36.189987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471023915498895:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.190096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:36.190143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:36.191714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:36.191922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471023072509280:2048] [node 1] ICS07 socket disconnect 306 reason# EndOfStream
2025-11-27T18:17:36.191953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471023072509280:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.192013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:36.192049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471023072509280:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:36.196168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471023072509181:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31411) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:36.196212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471023072509280:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:36.197104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471023072509181:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31411) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:36.197169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471023072509280:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:36.197278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471023072509181:2048] [node 1] ICP30 unregister session Session# [3:7577471023072509280:2048] VirtualId# [3:7577471023072509277:0]
2025-11-27T18:17:36.197336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:36.197375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471023072509181:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31411) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:36.197390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31411) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:36.201425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:36.201640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471026146704231:2048] [node 1] ICS01 socket: 303 reason# EndOfStream
2025-11-27T18:17:36.201681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471021851736832:2048] [node 1] ICP30 unregister session Session# [4:7577471026146704231:2048] VirtualId# [4:7577471021851736928:0]
2025-11-27T18:17:36.201715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471026146704231:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.201780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:36.201817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:36.201451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:36.201683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471027068321260:2048] [node 1] ICS01 socket: 286 reason# EndOfStream
2025-11-27T18:17:36.201739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471022773353862:2048] [node 1] ICP30 unregister session Session# [6:7577471027068321260:2048] VirtualId# [6:7577471022773353955:0]
2025-11-27T18:17:36.201787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471027068321260:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.201849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:36.201885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:36.205685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:36.205987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471023634718875:2048] [node 1] ICS01 socket: 256 reason# EndOfStream
2025-11-27T18:17:36.206027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471023634718786:2048] [node 1] ICP30 unregister session Session# [5:7577471023634718875:2048] VirtualId# [5:7577471023634718865:0]
2025-11-27T18:17:36.206060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471023634718875:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.206161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:36.206226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:36.208366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471023072509181:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:36.265462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:36.265712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471023072509272:2049] [node 2] ICS07 socket disconnect 269 reason# EndOfStream
2025-11-27T18:17:36.265742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471023072509272:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.265821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:36.265867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471023072509272:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:36.267000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471023072509182:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30038) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:36.267050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471023072509272:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:36.267085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471023072509182:2049] [node 2] ICP30 unregister session Session# [3:7577471023072509272:2049] VirtualId# [3:7577471023072509266:0]
2025-11-27T18:17:36.267123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:36.267157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471023072509182:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30038) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:36.267172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30038) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:36.268274Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:36.268472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471021851736919:2049] [node 2] ICS07 socket disconnect 267 reason# EndOfStream
2025-11-27T18:17:36.268506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471021851736919:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.268559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:36.268590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471021851736919:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:36.271882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:36.281831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471021851736833:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30038) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:36.281887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471021851736919:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:36.281915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471021851736833:2049] [node 2] ICP30 unregister session Session# [4:7577471021851736919:2049] VirtualId# [4:7577471021851736916:0]
2025-11-27T18:17:36.281959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:36.281985Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471021851736833:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30038) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:36.281995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30038) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:36.270079Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:36.270269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471022773353951:2049] [node 2] ICS07 socket disconnect 259 reason# EndOfStream
2025-11-27T18:17:36.270301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471022773353951:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.270367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:36.270419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471022773353951:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:36.271222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471022773353863:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30038) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:36.271267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471022773353951:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:36.271306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471022773353863:2049] [node 2] ICP30 unregister session Session# [6:7577471022773353951:2049] VirtualId# [6:7577471022773353943:0]
2025-11-27T18:17:36.271357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:36.271386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471022773353863:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30038) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:36.271401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30038) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:36.286929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471022773353863:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:36.272086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471023634718882:2049] [node 2] ICS01 socket: 276 reason# EndOfStream
2025-11-27T18:17:36.272124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471023634718787:2049] [node 2] ICP30 unregister session Session# [5:7577471023634718882:2049] VirtualId# [5:7577471023634718879:0]
2025-11-27T18:17:36.272150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471023634718882:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.272229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:36.272272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:36.295098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471021851736833:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:36.325442Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:36.325109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:36.325638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471027929686185:2050] [node 3] ICS01 socket: 305 reason# EndOfStream
2025-11-27T18:17:36.325402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471027068321258:2050] [node 3] ICS01 socket: 302 reason# EndOfStream
2025-11-27T18:17:36.325677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471023634718788:2050] [node 3] ICP30 unregister session Session# [5:7577471027929686185:2050] VirtualId# [5:7577471023634718880:0]
2025-11-27T18:17:36.325439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471022773353864:2050] [node 3] ICP30 unregister session Session# [6:7577471027068321258:2050] VirtualId# [6:7577471022773353959:0]
2025-11-27T18:17:36.325705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471027929686185:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.325475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471027068321258:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.325547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:36.325361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:36.325584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:36.325778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:36.325551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471021851736922:2050] [node 3] ICS01 socket: 277 reason# EndOfStream
2025-11-27T18:17:36.325825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:36.325581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471021851736834:2050] [node 3] ICP30 unregister session Session# [4:7577471021851736922:2050] VirtualId# [4:7577471021851736914:0]
2025-11-27T18:17:36.325608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471021851736922:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.325667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:36.325705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:36.405465Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:36.405721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471027929686183:2108] [node 4] ICS01 socket: 309 reason# EndOfStream
2025-11-27T18:17:36.405757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471023634718853:2108] [node 4] ICP30 unregister session Session# [5:7577471027929686183:2108] VirtualId# [5:7577471023634718884:0]
2025-11-27T18:17:36.405784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471027929686183:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.405867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:36.405465Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:36.405909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:36.405709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471022773353957:2109] [node 4] ICS01 socket: 275 reason# EndOfStream
2025-11-27T18:17:36.405747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471022773353926:2109] [node 4] ICP30 unregister session Session# [6:7577471022773353957:2109] VirtualId# [6:7577471022773353950:0]
2025-11-27T18:17:36.405784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471022773353957:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.405864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:36.405907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:36.469470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:36.469739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471022773353953:2106] [node 5] ICS01 socket: 245 reason# EndOfStream
2025-11-27T18:17:36.469796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471022773353924:2106] [node 5] ICP30 unregister session Session# [6:7577471022773353953:2106] VirtualId# [6:7577471022773353940:0]
2025-11-27T18:17:36.469833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471022773353953:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:36.469992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:36.470161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:36.622621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471025098204092:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:36.622673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471025098204093:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:36.641828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471026765613562:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:36.641866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471026765613563:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:36.665661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471025384125788:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:36.665722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471025384125789:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:36.687824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471027170646358:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:36.687863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471027170646359:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:36.687887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471027170646360:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:36.708039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471027823687056:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:36.708089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471027823687057:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:36.708139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471027823687058:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:36.728362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471026984151086:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:36.728413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471026984151087:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:36.728441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471026984151088:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:36.992144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471026984151148:2105] [node 5] ICP01 ready to work
2025-11-27T18:17:36.992139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471026765613623:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:36.996588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471026765613625:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:37.010797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471029679093145:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:37.007627Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471025098204092:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23527159ns) {Configure:104(20198685ns) {ProcessConfigured:135(2022555ns) {StartInitialHandshake:157(2020635ns) {DropHandshakes:470(2997ns) {DropIncomingHandshake:442(730ns) DropOutgoingHandshake:457(379ns)} GenerateSessionVirtualId:988(845ns) PrepareNewSessionHandshake:477(2389ns) {SwitchToState:179(1395ns)}}}}}
2025-11-27T18:17:37.008200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471029393171453:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:37.008477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471029393171454:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:37.096065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471031279118449:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:37.099229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471029679093145:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(88234907ns) {Configure:104(88230550ns) {ProcessConfigured:135(46621ns) {StartInitialHandshake:157(45283ns) {DropHandshakes:470(4153ns) {DropIncomingHandshake:442(515ns) DropOutgoingHandshake:457(181ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:477(1665ns) {SwitchToState:179(1144ns)}}}}}
2025-11-27T18:17:37.008853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471029393171455:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.097664Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471025098204093:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(88564949ns) {Configure:104(88559870ns) {ProcessConfigured:135(58588ns) {StartInitialHandshake:157(55533ns) {DropHandshakes:470(3193ns) {DropIncomingHandshake:442(701ns) DropOutgoingHandshake:457(370ns)} GenerateSessionVirtualId:988(630ns) PrepareNewSessionHandshake:477(1978ns) {SwitchToState:179(854ns)}}}}}
2025-11-27T18:17:37.105153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471029393171464:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48653
2025-11-27T18:17:37.107539Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471026765613563:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(99322517ns) {Configure:104(99317078ns) {ProcessConfigured:135(5047784ns) {StartInitialHandshake:157(5044451ns) {DropHandshakes:470(3935ns) {DropIncomingHandshake:442(938ns) DropOutgoingHandshake:457(614ns)} GenerateSessionVirtualId:988(1316ns) PrepareNewSessionHandshake:477(3920ns) {SwitchToState:179(2456ns)}}}}}
2025-11-27T18:17:37.106381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471029679093150:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:37.108666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471029679093151:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:37.114805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471032118654417:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:37.115710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471027170646359:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(87801073ns) {Configure:104(87796026ns) {ProcessConfigured:135(791194ns) {StartInitialHandshake:157(789484ns) {DropHandshakes:470(1021ns) {DropIncomingHandshake:442(306ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(302ns) PrepareNewSessionHandshake:477(3186ns) {SwitchToState:179(97ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.117276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471029679093161:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6442
2025-11-27T18:17:37.116129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471031465613722:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:37.116355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471031465613723:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:37.124184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471026984151086:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27707079ns) {Configure:104(27702503ns) {ProcessConfigured:135(50597ns) {StartInitialHandshake:157(48332ns) {DropHandshakes:470(4040ns) {DropIncomingHandshake:442(512ns) DropOutgoingHandshake:457(616ns)} GenerateSessionVirtualId:988(378ns) PrepareNewSessionHandshake:477(1464ns) {SwitchToState:179(419ns)}}}}}
2025-11-27T18:17:37.130461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471026765613623:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22633504ns) {Configure:104(22628906ns) {ProcessConfigured:135(10515086ns) {StartInitialHandshake:157(10511403ns) {DropHandshakes:470(2649ns) {DropIncomingHandshake:442(485ns) DropOutgoingHandshake:457(489ns)} GenerateSessionVirtualId:988(998ns) PrepareNewSessionHandshake:477(4909838ns) {SwitchToState:179(7760ns)}}}}}
2025-11-27T18:17:37.130948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471031060580927:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:37.120029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471032118654418:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.138504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471029393171468:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48653
2025-11-27T18:17:37.139682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471029393171468:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.138909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471029679093161:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.142434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471032118654413:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31855
2025-11-27T18:17:37.142735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471025384125788:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.143655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471029393171457:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.143692Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471029393171457:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.144664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471029393171457:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.146641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471029679093165:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6442
2025-11-27T18:17:37.151000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471025098204093:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.151097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:37.151131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471029679093161:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:37.151149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471025098204093:2049] [node 3] ICP22 created new session: [1:7577471029393171471:2049]
2025-11-27T18:17:37.151205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471029393171471:2049] [node 3] ICS09 handshake done sender: [1:7577471029393171457:2111] self: [1:7577471029393171456:0] peer: [3:7577471029679093164:0] socket: 251 qp: -1
2025-11-27T18:17:37.151263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471029393171471:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:37.151396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:37.153016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471025384125788:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.153385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:37.155110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471025384125788:2048] [node 1] ICP22 created new session: [3:7577471029679093166:2048]
2025-11-27T18:17:37.155151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471029679093166:2048] [node 1] ICS09 handshake done sender: [3:7577471029679093161:2115] self: [3:7577471029679093164:0] peer: [1:7577471029393171456:0] socket: 256 qp: -1
2025-11-27T18:17:37.155175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471029679093166:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:37.155539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.166606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471031279118458:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1038
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.171623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471032118654419:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31855
2025-11-27T18:17:37.178378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471031465613724:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11612
2025-11-27T18:17:37.177783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471031060580932:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41774
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.177965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471031060580933:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41774
2025-11-27T18:17:37.179964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471031060580936:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41774
2025-11-27T18:17:37.180174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031060580933:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.180435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471031060580940:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41774
2025-11-27T18:17:37.183419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471032118654419:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.183268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471031279118458:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.184753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471032118654413:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.186842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471029393171474:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48653
2025-11-27T18:17:37.187051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471029393171461:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.187084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471029393171461:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.188222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471029393171474:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.188362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471029393171475:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48653
2025-11-27T18:17:37.188733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471029393171461:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.183523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031060580932:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.183656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471026984151086:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.189884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471031465613724:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.190094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471031465613732:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11612
2025-11-27T18:17:37.190216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471031465613733:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11612
2025-11-27T18:17:37.190275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471031465613734:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11612
2025-11-27T18:17:37.190493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471027170646358:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.195196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471029679093169:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6442
2025-11-27T18:17:37.195761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471029393171454:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.195833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:37.197222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471031465613732:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.197332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471029393171454:2108] [node 6] ICP22 created new session: [1:7577471029393171478:2108]
2025-11-27T18:17:37.197387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471029393171478:2108] [node 6] ICS09 handshake done sender: [1:7577471029393171461:2113] self: [1:7577471029393171460:0] peer: [6:7577471031279118461:0] socket: 264 qp: -1
2025-11-27T18:17:37.197478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471027170646360:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.197445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471029393171478:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:37.197488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:37.198490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471029393171459:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.198512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471029393171459:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.200296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471032118654433:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31855
2025-11-27T18:17:37.200402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471027823687058:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.199898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471029679093157:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.207289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031060580936:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.199931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471029679093157:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.206301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471029393171459:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:37.207994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471026765613562:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.208646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471031465613733:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.208296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471031060580942:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41774
2025-11-27T18:17:37.209677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471029393171464:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.209800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471029393171451:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.203737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471029679093170:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6442
2025-11-27T18:17:37.209828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471029393171451:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.203858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471029679093157:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:37.210647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471029393171453:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.206392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471029679093150:2108] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.210679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:37.211699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031060580942:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.211059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471029393171453:2107] [node 4] ICP22 created new session: [1:7577471029393171482:2107]
2025-11-27T18:17:37.206423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:37.211506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471029393171482:2107] [node 4] ICS09 handshake done sender: [1:7577471029393171459:2112] self: [1:7577471029393171458:0] peer: [4:7577471031465613736:0] socket: 252 qp: -1
2025-11-27T18:17:37.206456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471029679093150:2108] [node 4] ICP22 created new session: [3:7577471029679093172:2108]
2025-11-27T18:17:37.211717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471029393171482:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:37.211761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:37.206508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471029679093172:2108] [node 4] ICS09 handshake done sender: [3:7577471029679093157:2113] self: [3:7577471029679093156:0] peer: [4:7577471031465613737:0] socket: 268 qp: -1
2025-11-27T18:17:37.206535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471029679093172:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:37.206563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:37.210797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471029679093169:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.210957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471029679093170:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.216553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471025384125789:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.216638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471029679093150:2108] [node 4] ICP18 handshake (actor [3:7577471029679093170:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.216784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471031465613738:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11612
2025-11-27T18:17:37.217602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471031279118462:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1038
2025-11-27T18:17:37.219713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471029393171453:2107] [node 4] ICP18 handshake (actor [1:7577471029393171464:2115]) is too old
2025-11-27T18:17:37.220441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471029679093149:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.218708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471026984151086:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3518) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471031279118458:2115] held: no
2025-11-27T18:17:37.220472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471029679093149:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.223086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471029679093149:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:37.230632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471029393171451:2109] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:37.218989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471031279118458:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.225583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471026984151086:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.231170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031060580924:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.233979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471031465613739:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11612
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.234916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471029679093145:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.234965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:37.235011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471029679093145:2107] [node 5] ICP22 created new session: [3:7577471029679093177:2107]
2025-11-27T18:17:37.235053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471029679093177:2107] [node 5] ICS09 handshake done sender: [3:7577471029679093149:2110] self: [3:7577471029679093148:0] peer: [5:7577471032118654434:0] socket: 254 qp: -1
2025-11-27T18:17:37.235088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471029679093177:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:37.235115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:37.235791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471029679093179:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6442
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.234653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471032118654433:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.225634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:37.225688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471026984151086:2048] [node 1] ICP22 created new session: [6:7577471031279118464:2048]
2025-11-27T18:17:37.225738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471031279118464:2048] [node 1] ICS09 handshake done sender: [6:7577471031279118458:2115] self: [6:7577471031279118461:0] peer: [1:7577471029393171460:0] socket: 257 qp: -1
2025-11-27T18:17:37.225770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471031279118464:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:37.225804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:37.226581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471031279118463:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1038
2025-11-27T18:17:37.236476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471031279118463:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.231223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471031060580924:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.236684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471027170646359:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.233806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471031060580946:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41774
2025-11-27T18:17:37.237769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471025098204092:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.236936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471026984151088:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.236579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471032118654436:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31855
2025-11-27T18:17:37.236360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471031060580924:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:37.236851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471032118654437:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31855
2025-11-27T18:17:37.238140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031060580940:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.237833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:37.240236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471029679093159:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.240368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471031465613734:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.240179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471025098204092:2048] [node 2] ICP22 created new session: [1:7577471029393171485:2048]
2025-11-27T18:17:37.240662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471027823687057:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.238179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471026765613563:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.240795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471027170646358:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3518) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471031465613724:2112] held: no
2025-11-27T18:17:37.240275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471029679093159:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.240723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471032118654419:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:37.238220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:37.242392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471031465613732:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:37.240246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471029393171485:2048] [node 2] ICS09 handshake done sender: [1:7577471029393171451:2109] self: [1:7577471025098204154:0] peer: [2:7577471031060580943:0] socket: 250 qp: -1
2025-11-27T18:17:37.238259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471026765613563:2049] [node 3] ICP22 created new session: [2:7577471031060580948:2049]
2025-11-27T18:17:37.240432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471029393171485:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:37.238301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471031060580948:2049] [node 3] ICS09 handshake done sender: [2:7577471031060580924:2110] self: [2:7577471031060580923:0] peer: [3:7577471029679093175:0] socket: 273 qp: -1
2025-11-27T18:17:37.240478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:37.238326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471031060580948:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:37.243704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471027823687058:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.244206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471029679093169:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:37.239699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:37.241940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471029393171475:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.245063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471027170646360:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10777) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471031465613732:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.246770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471031279118468:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1038
2025-11-27T18:17:37.244067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:37.242703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471031060580932:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:37.244171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471027823687058:2050] [node 3] ICP22 created new session: [5:7577471032118654439:2050]
2025-11-27T18:17:37.244129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031060580926:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.248032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471029393171463:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.248118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471031279118468:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.244152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471031060580926:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.248416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471027170646360:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.248073Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471029393171463:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.248785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471026984151148:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.248469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:37.244365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471032118654439:2050] [node 3] ICS09 handshake done sender: [5:7577471032118654419:2112] self: [5:7577471032118654434:0] peer: [3:7577471029679093148:0] socket: 261 qp: -1
2025-11-27T18:17:37.252538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471026765613562:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.248714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471027170646360:2050] [node 3] ICP22 created new session: [4:7577471031465613742:2050]
2025-11-27T18:17:37.250791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471029393171463:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:37.244404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471032118654439:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:37.244438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:37.245663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471032118654436:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.246100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471027823687056:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.249836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471032118654443:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31855
2025-11-27T18:17:37.250897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471032118654429:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.250919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471032118654429:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.255529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471029393171455:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.255585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:37.250237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471031465613742:2050] [node 3] ICS09 handshake done sender: [4:7577471031465613732:2116] self: [4:7577471031465613737:0] peer: [3:7577471029679093156:0] socket: 279 qp: -1
2025-11-27T18:17:37.250442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471031465613742:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:37.250910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:37.255647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471029393171455:2110] [node 5] ICP22 created new session: [1:7577471029393171489:2110]
2025-11-27T18:17:37.255692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471029393171489:2110] [node 5] ICS09 handshake done sender: [1:7577471029393171463:2114] self: [1:7577471029393171462:0] peer: [5:7577471032118654441:0] socket: 294 qp: -1
2025-11-27T18:17:37.255728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471029393171489:2110] [node 5] ICS10 traffic start
2025-11-27T18:17:37.254143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:37.256148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471032118654436:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:37.255757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:37.254267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471026765613562:2048] [node 1] ICP22 created new session: [2:7577471031060580950:2048]
2025-11-27T18:17:37.258069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471025384125789:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.258128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:37.258309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471025384125789:2049] [node 2] ICP22 created new session: [3:7577471029679093181:2049]
2025-11-27T18:17:37.258358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471029679093181:2049] [node 2] ICS09 handshake done sender: [3:7577471029679093169:2117] self: [3:7577471029679093175:0] peer: [2:7577471031060580923:0] socket: 281 qp: -1
2025-11-27T18:17:37.254326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471031060580950:2048] [node 1] ICS09 handshake done sender: [2:7577471031060580932:2114] self: [2:7577471031060580943:0] peer: [1:7577471025098204154:0] socket: 265 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.262098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471031279118470:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1038
2025-11-27T18:17:37.262173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471031279118471:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1038
2025-11-27T18:17:37.262584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471031279118463:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:37.262829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471026984151088:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.260318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471031465613724:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:37.258386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471029679093181:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:37.260070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471027823687056:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.262873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:37.262916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471026984151088:2050] [node 3] ICP22 created new session: [6:7577471031279118472:2050]
2025-11-27T18:17:37.260437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:37.258416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:37.260581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471027823687056:2048] [node 1] ICP22 created new session: [5:7577471032118654445:2048]
2025-11-27T18:17:37.262956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471031279118472:2050] [node 3] ICS09 handshake done sender: [6:7577471031279118463:2117] self: [6:7577471031279118467:0] peer: [3:7577471029679093158:0] socket: 271 qp: -1
2025-11-27T18:17:37.258606Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471025384125789:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:37.264853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471032118654445:2048] [node 1] ICS09 handshake done sender: [5:7577471032118654436:2119] self: [5:7577471032118654441:0] peer: [1:7577471029393171462:0] socket: 299 qp: -1
2025-11-27T18:17:37.262987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471031279118472:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:37.254710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471031060580950:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:37.260243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471029679093159:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:37.264883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471032118654445:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:37.267149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471029679093151:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.267175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:37.267209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471029679093151:2109] [node 6] ICP22 created new session: [3:7577471029679093183:2109]
2025-11-27T18:17:37.267237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471029679093183:2109] [node 6] ICS09 handshake done sender: [3:7577471029679093159:2114] self: [3:7577471029679093158:0] peer: [6:7577471031279118467:0] socket: 269 qp: -1
2025-11-27T18:17:37.267257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471029679093183:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:37.267277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:37.264912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:37.263021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:37.268530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471032118654429:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:37.254745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:37.271168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471027170646358:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.271217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:37.271480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471027170646358:2048] [node 1] ICP22 created new session: [4:7577471031465613745:2048]
2025-11-27T18:17:37.260393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031060580929:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.271771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471031465613745:2048] [node 1] ICS09 handshake done sender: [4:7577471031465613724:2112] self: [4:7577471031465613736:0] peer: [1:7577471029393171458:0] socket: 255 qp: -1
2025-11-27T18:17:37.271820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471031465613745:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:37.271856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:37.272162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471032118654418:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.272209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:37.272248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471032118654418:2111] [node 6] ICP22 created new session: [5:7577471032118654448:2111]
2025-11-27T18:17:37.272293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471032118654448:2111] [node 6] ICS09 handshake done sender: [5:7577471032118654429:2117] self: [5:7577471032118654428:0] peer: [6:7577471031279118469:0] socket: 266 qp: -1
2025-11-27T18:17:37.272325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471032118654448:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:37.272357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.272794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471031465613741:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11612
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.273651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471031465613744:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11612
2025-11-27T18:17:37.260423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471031060580929:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.266703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471031060580926:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:37.271630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471031060580929:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:37.274162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471027823687056:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:37.276231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471031465613741:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.275675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471031279118468:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:37.275875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471026984151148:2105] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.275910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:37.276237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471026984151148:2105] [node 5] ICP22 created new session: [6:7577471031279118475:2105]
2025-11-27T18:17:37.276284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471031279118475:2105] [node 5] ICS09 handshake done sender: [6:7577471031279118468:2118] self: [6:7577471031279118469:0] peer: [5:7577471032118654428:0] socket: 290 qp: -1
2025-11-27T18:17:37.278655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471032118654447:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31855
2025-11-27T18:17:37.276312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471031279118475:2105] [node 5] ICS10 traffic start
2025-11-27T18:17:37.276340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:37.276873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471031279118474:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1038
2025-11-27T18:17:37.278700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471026765613623:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.278999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471031465613733:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:37.280555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471027170646359:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.278758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:37.278799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471026765613623:2107] [node 4] ICP22 created new session: [2:7577471031060580957:2107]
2025-11-27T18:17:37.279098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471031060580957:2107] [node 4] ICS09 handshake done sender: [2:7577471031060580926:2111] self: [2:7577471031060580925:0] peer: [4:7577471031465613740:0] socket: 274 qp: -1
2025-11-27T18:17:37.279133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471031060580957:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:37.279164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:37.279603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471026765613625:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.279674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:37.279716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471026765613625:2108] [node 5] ICP22 created new session: [2:7577471031060580959:2108]
2025-11-27T18:17:37.279770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471031060580959:2108] [node 5] ICS09 handshake done sender: [2:7577471031060580929:2112] self: [2:7577471031060580928:0] peer: [5:7577471032118654438:0] socket: 278 qp: -1
2025-11-27T18:17:37.279795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471031060580959:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:37.279819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.281812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471031279118477:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1038
2025-11-27T18:17:37.280600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:37.281226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471027170646359:2049] [node 2] ICP22 created new session: [4:7577471031465613750:2049]
2025-11-27T18:17:37.282104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471032118654433:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:37.282225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471031465613750:2049] [node 2] ICS09 handshake done sender: [4:7577471031465613733:2117] self: [4:7577471031465613740:0] peer: [2:7577471031060580925:0] socket: 283 qp: -1
2025-11-27T18:17:37.282273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471031465613750:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:37.282307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.284512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471032118654450:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31855
2025-11-27T18:17:37.284672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471027823687057:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.284723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:37.284768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471027823687057:2049] [node 2] ICP22 created new session: [5:7577471032118654451:2049]
2025-11-27T18:17:37.284832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471032118654451:2049] [node 2] ICS09 handshake done sender: [5:7577471032118654433:2118] self: [5:7577471032118654438:0] peer: [2:7577471031060580928:0] socket: 286 qp: -1
2025-11-27T18:17:37.284876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471032118654451:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:37.284903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:37.289310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471031279118474:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.289420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471031279118477:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.289560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471031279118449:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.289621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471026984151087:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.290009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471031465613730:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.290042Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471031465613730:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.289951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031060580935:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.289987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471031060580935:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.291290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471031465613730:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:37.291456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471032118654450:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.291788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471031060580935:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:37.291794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471032118654417:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:37.292095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471031060580927:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.292149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:37.292193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471031060580927:2109] [node 6] ICP22 created new session: [2:7577471031060580962:2109]
2025-11-27T18:17:37.292231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471031060580962:2109] [node 6] ICS09 handshake done sender: [2:7577471031060580935:2116] self: [2:7577471031060580934:0] peer: [6:7577471031279118479:0] socket: 315 qp: -1
2025-11-27T18:17:37.292263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471031060580962:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:37.292297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:37.291310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471031279118480:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1038
2025-11-27T18:17:37.291395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471031279118481:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1038
2025-11-27T18:17:37.297611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471031465613723:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.297694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:37.297745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471031465613723:2110] [node 6] ICP22 created new session: [4:7577471031465613754:2110]
2025-11-27T18:17:37.297793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471031465613754:2110] [node 6] ICS09 handshake done sender: [4:7577471031465613730:2115] self: [4:7577471031465613729:0] peer: [6:7577471031279118478:0] socket: 313 qp: -1
2025-11-27T18:17:37.297830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471031465613754:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:37.297861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:37.298480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471031465613728:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:37.298510Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471031465613728:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:37.299948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471031279118474:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:37.300280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471026984151087:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11939) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471031279118477:2122] held: no
2025-11-27T18:17:37.300392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471031279118449:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.300438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:37.300478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471031279118449:2110] [node 4] ICP22 created new session: [6:7577471031279118482:2110]
2025-11-27T18:17:37.300523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471031279118482:2110] [node 4] ICS09 handshake done sender: [6:7577471031279118474:2121] self: [6:7577471031279118478:0] peer: [4:7577471031465613729:0] socket: 314 qp: -1
2025-11-27T18:17:37.300555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471031279118482:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:37.300588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:37.300695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471031465613728:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.302130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471032118654454:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31855
2025-11-27T18:17:37.302316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471031279118477:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:37.302503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471031465613722:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:37.302580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:37.302641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471031465613722:2109] [node 5] ICP22 created new session: [4:7577471031465613757:2109]
2025-11-27T18:17:37.302680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471031465613757:2109] [node 5] ICS09 handshake done sender: [4:7577471031465613728:2114] self: [4:7577471031465613727:0] peer: [5:7577471032118654453:0] socket: 312 qp: -1
2025-11-27T18:17:37.302712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471031465613757:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:37.302757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:37.302690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471032118654450:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:37.302699Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471031279118449:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:37.302818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471026984151087:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.302849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:37.302888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471026984151087:2049] [node 2] ICP22 created new session: [6:7577471031279118484:2049]
2025-11-27T18:17:37.302991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471031279118484:2049] [node 2] ICS09 handshake done sender: [6:7577471031279118477:2122] self: [6:7577471031279118479:0] peer: [2:7577471031060580934:0] socket: 246 qp: -1
2025-11-27T18:17:37.303151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471032118654417:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:37.303068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471031279118484:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:37.303195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:37.303106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:37.303228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471032118654417:2109] [node 4] ICP22 created new session: [5:7577471032118654455:2109]
2025-11-27T18:17:37.303266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471032118654455:2109] [node 4] ICS09 handshake done sender: [5:7577471032118654450:2123] self: [5:7577471032118654453:0] peer: [4:7577471031465613727:0] socket: 247 qp: -1
2025-11-27T18:17:37.303295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471032118654455:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:37.303329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:37.320435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:37.320704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471031060580950:2048] [node 1] ICS01 socket: 265 reason# EndOfStream
2025-11-27T18:17:37.320744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471026765613562:2048] [node 1] ICP30 unregister session Session# [2:7577471031060580950:2048] VirtualId# [2:7577471031060580943:0]
2025-11-27T18:17:37.320782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471031060580950:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.320865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:37.320935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:37.324901Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:37.325099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471031465613745:2048] [node 1] ICS01 socket: 255 reason# EndOfStream
2025-11-27T18:17:37.325151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471027170646358:2048] [node 1] ICP30 unregister session Session# [4:7577471031465613745:2048] VirtualId# [4:7577471031465613736:0]
2025-11-27T18:17:37.325182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471031465613745:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.325237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:37.325289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:37.326912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:37.327098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471031279118464:2048] [node 1] ICS01 socket: 257 reason# EndOfStream
2025-11-27T18:17:37.327161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471026984151086:2048] [node 1] ICP30 unregister session Session# [6:7577471031279118464:2048] VirtualId# [6:7577471031279118461:0]
2025-11-27T18:17:37.327186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471031279118464:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.327237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:37.327269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:37.327827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:37.328031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471032118654445:2048] [node 1] ICS01 socket: 299 reason# EndOfStream
2025-11-27T18:17:37.328077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471027823687056:2048] [node 1] ICP30 unregister session Session# [5:7577471032118654445:2048] VirtualId# [5:7577471032118654441:0]
2025-11-27T18:17:37.328110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471032118654445:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.328174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:37.328223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:37.325642Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:37.325841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471029679093166:2048] [node 1] ICS07 socket disconnect 256 reason# EndOfStream
2025-11-27T18:17:37.325871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471029679093166:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.325951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:37.326029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471029679093166:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:37.328595Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471025384125788:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3518) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:37.328644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471029679093166:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:37.328677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471025384125788:2048] [node 1] ICP30 unregister session Session# [3:7577471029679093166:2048] VirtualId# [3:7577471029679093164:0]
2025-11-27T18:17:37.328719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:37.328747Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471025384125788:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3518) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:37.328765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3518) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:37.341396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471025384125788:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:37.350729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:37.350945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471029679093181:2049] [node 2] ICS01 socket: 281 reason# EndOfStream
2025-11-27T18:17:37.350985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471025384125789:2049] [node 2] ICP30 unregister session Session# [3:7577471029679093181:2049] VirtualId# [3:7577471029679093175:0]
2025-11-27T18:17:37.351015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471029679093181:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.351097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:37.351156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:37.355658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:37.355891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471032118654451:2049] [node 2] ICS01 socket: 286 reason# EndOfStream
2025-11-27T18:17:37.355934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471027823687057:2049] [node 2] ICP30 unregister session Session# [5:7577471032118654451:2049] VirtualId# [5:7577471032118654438:0]
2025-11-27T18:17:37.355964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471032118654451:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.356033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:37.356129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:37.356537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:37.356731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471031279118484:2049] [node 2] ICS01 socket: 246 reason# EndOfStream
2025-11-27T18:17:37.356763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471026984151087:2049] [node 2] ICP30 unregister session Session# [6:7577471031279118484:2049] VirtualId# [6:7577471031279118479:0]
2025-11-27T18:17:37.356782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471031279118484:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.356830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:37.356866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:37.356893Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:37.357179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471031465613750:2049] [node 2] ICS01 socket: 283 reason# EndOfStream
2025-11-27T18:17:37.357221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471027170646359:2049] [node 2] ICP30 unregister session Session# [4:7577471031465613750:2049] VirtualId# [4:7577471031465613740:0]
2025-11-27T18:17:37.357245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471031465613750:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.357359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:37.357400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:37.401396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:37.401626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471032118654439:2050] [node 3] ICS01 socket: 261 reason# EndOfStream
2025-11-27T18:17:37.401672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471027823687058:2050] [node 3] ICP30 unregister session Session# [5:7577471032118654439:2050] VirtualId# [5:7577471032118654434:0]
2025-11-27T18:17:37.401716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471032118654439:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.401793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:37.401841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:37.402463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:37.402651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471031465613742:2050] [node 3] ICS01 socket: 279 reason# EndOfStream
2025-11-27T18:17:37.402688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471027170646360:2050] [node 3] ICP30 unregister session Session# [4:7577471031465613742:2050] VirtualId# [4:7577471031465613737:0]
2025-11-27T18:17:37.402715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471031465613742:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.402785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:37.402837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:37.404835Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:37.405025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471031279118472:2050] [node 3] ICS01 socket: 271 reason# EndOfStream
2025-11-27T18:17:37.405067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471026984151088:2050] [node 3] ICP30 unregister session Session# [6:7577471031279118472:2050] VirtualId# [6:7577471031279118467:0]
2025-11-27T18:17:37.405092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471031279118472:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.405177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:37.405232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:37.428583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:37.428857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471032118654455:2109] [node 4] ICS01 socket: 247 reason# EndOfStream
2025-11-27T18:17:37.428904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471032118654417:2109] [node 4] ICP30 unregister session Session# [5:7577471032118654455:2109] VirtualId# [5:7577471032118654453:0]
2025-11-27T18:17:37.428941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471032118654455:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.429025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:37.429068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:37.429908Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:37.430216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471031279118482:2110] [node 4] ICS01 socket: 314 reason# EndOfStream
2025-11-27T18:17:37.430349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471031279118449:2110] [node 4] ICP30 unregister session Session# [6:7577471031279118482:2110] VirtualId# [6:7577471031279118478:0]
2025-11-27T18:17:37.430386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471031279118482:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.430453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:37.430492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:37.473581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:37.473864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471031279118475:2105] [node 5] ICS01 socket: 290 reason# EndOfStream
2025-11-27T18:17:37.473905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471026984151148:2105] [node 5] ICP30 unregister session Session# [6:7577471031279118475:2105] VirtualId# [6:7577471031279118469:0]
2025-11-27T18:17:37.473947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471031279118475:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:37.474043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:37.474117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:37.546524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471032608452845:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:37.546580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471032608452846:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:37.570499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471031912970433:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:37.570545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471031912970434:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:37.594612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471031393118159:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:37.594688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471031393118160:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:37.617543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471030767605955:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:37.617587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471030767605956:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:37.617617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471030767605957:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:37.644728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471032962690152:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:37.644807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471032962690153:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:37.645006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471032962690154:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:37.672829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471031630527735:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:37.672886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471031630527736:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:37.672919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471031630527737:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:37.762852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471031630527796:2106] [node 5] ICP01 ready to work
2025-11-27T18:17:37.844800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471031393118223:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:37.848591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471032608452910:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:37.849959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471031912970496:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:37.849831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471032608452911:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:37.863148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471032962690215:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:37.853581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471030767606020:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:37.861797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471032608452912:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:37.852619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471031393118224:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:37.853913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471031912970498:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:37.855205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471031393118225:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:37.863162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471030767606019:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:37.863666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471031630527799:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:37.863138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471031912970497:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:37.877040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471032962690216:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:37.900133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471032962690152:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(112925042ns) {Configure:104(112918893ns) {ProcessConfigured:135(6335951ns) {StartInitialHandshake:157(6328725ns) {DropHandshakes:470(1250ns) {DropIncomingHandshake:442(283ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(887ns) PrepareNewSessionHandshake:477(3484ns) {SwitchToState:179(1517ns)}}}}}
2025-11-27T18:17:37.898193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471031630527796:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(122522091ns) {Configure:104(122512440ns) {ProcessConfigured:135(65042ns) {StartInitialHandshake:157(62261ns) {DropHandshakes:470(2160ns) {DropIncomingHandshake:442(546ns) DropOutgoingHandshake:457(410ns)} GenerateSessionVirtualId:988(1051ns) PrepareNewSessionHandshake:477(1695ns) {SwitchToState:179(428ns)}}}}}
2025-11-27T18:17:37.901498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471031393118159:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(37850905ns) {Configure:104(37838843ns) {ProcessConfigured:135(966421ns) {StartInitialHandshake:157(964464ns) {DropHandshakes:470(2213ns) {DropIncomingHandshake:442(372ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(635ns) PrepareNewSessionHandshake:477(1905ns) {SwitchToState:179(992ns)}}}}}
2025-11-27T18:17:37.901716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471031912970496:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(37151635ns) {Configure:104(37148146ns) {ProcessConfigured:135(51297ns) {StartInitialHandshake:157(46437ns) {DropHandshakes:470(1622ns) {DropIncomingHandshake:442(454ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(542ns) PrepareNewSessionHandshake:477(1595ns) {SwitchToState:179(490ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.913998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471032608452910:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50595840ns) {Configure:104(50590877ns) {ProcessConfigured:135(8065984ns) {StartInitialHandshake:157(8063988ns) {DropHandshakes:470(2311ns) {DropIncomingHandshake:442(606ns) DropOutgoingHandshake:457(354ns)} GenerateSessionVirtualId:988(550ns) PrepareNewSessionHandshake:477(6530ns) {SwitchToState:179(2052ns)}}}}}
2025-11-27T18:17:37.902349Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471031393118160:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46902759ns) {Configure:104(38857683ns) {ProcessConfigured:135(462140ns) {StartInitialHandshake:157(459884ns) {DropHandshakes:470(1398ns) {DropIncomingHandshake:442(444ns) DropOutgoingHandshake:457(165ns)} GenerateSessionVirtualId:988(593ns) PrepareNewSessionHandshake:477(1513ns) {SwitchToState:179(638ns)}}}}}
2025-11-27T18:17:37.910378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471032962690153:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20194178ns) {Configure:104(20185714ns) {ProcessConfigured:135(636882ns) {StartInitialHandshake:157(635240ns) {DropHandshakes:470(1245ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(405ns) PrepareNewSessionHandshake:477(3169ns) {SwitchToState:179(1277ns)}}}}}
2025-11-27T18:17:37.933711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471030767605956:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(70286157ns) {Configure:104(70282608ns) {ProcessConfigured:135(74879ns) {StartInitialHandshake:157(70694ns) {DropHandshakes:470(3558ns) {DropIncomingHandshake:442(769ns) DropOutgoingHandshake:457(516ns)} GenerateSessionVirtualId:988(1211ns) PrepareNewSessionHandshake:477(1850ns) {SwitchToState:179(664ns)}}}}}
2025-11-27T18:17:37.920649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471032608452846:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(65258143ns) {Configure:104(65248972ns) {ProcessConfigured:135(9682217ns) {StartInitialHandshake:157(9679826ns) {DropHandshakes:470(1213ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(701ns) PrepareNewSessionHandshake:477(3559ns) {SwitchToState:179(1897ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.944135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471031630527799:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(43317211ns) {Configure:104(43313007ns) {ProcessConfigured:135(7293175ns) {StartInitialHandshake:157(7291377ns) {DropHandshakes:470(945ns) {DropIncomingHandshake:442(235ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(4886ns) {SwitchToState:179(2840ns)}}}}}
2025-11-27T18:17:37.924094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471032608452921:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30822
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.944290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471031630527735:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(62504492ns) {Configure:104(62496793ns) {ProcessConfigured:135(7307674ns) {StartInitialHandshake:157(7305378ns) {DropHandshakes:470(1903ns) {DropIncomingHandshake:442(462ns) DropOutgoingHandshake:457(226ns)} GenerateSessionVirtualId:988(1487ns) PrepareNewSessionHandshake:477(4969ns) {SwitchToState:179(2874ns)}}}}}
2025-11-27T18:17:37.950955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471030767605955:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(51447116ns) {Configure:104(47013762ns) {ProcessConfigured:135(18443742ns) {StartInitialHandshake:157(18441280ns) {DropHandshakes:470(1509ns) {DropIncomingHandshake:442(313ns) DropOutgoingHandshake:457(160ns)} GenerateSessionVirtualId:988(906ns) PrepareNewSessionHandshake:477(10077ns) {SwitchToState:179(2972ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.924165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471032608452922:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30822
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:37.946579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471031630527808:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34393
2025-11-27T18:17:37.983855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471032962690228:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60487
2025-11-27T18:17:38.002642Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471032962690215:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(73750810ns) {Configure:104(73742687ns) {ProcessConfigured:135(73249ns) {StartInitialHandshake:157(70752ns) {DropHandshakes:470(3538ns) {DropIncomingHandshake:442(753ns) DropOutgoingHandshake:457(378ns)} GenerateSessionVirtualId:988(1070ns) PrepareNewSessionHandshake:477(2049ns) {SwitchToState:179(763ns)}}}}}
2025-11-27T18:17:37.997328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471031912970513:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32251
2025-11-27T18:17:37.983981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471031630527808:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.003831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471032962690229:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60487
2025-11-27T18:17:37.990115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471031630527736:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.030481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031912970504:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.032038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471030767606034:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30777
2025-11-27T18:17:38.032123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471030767606035:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30777
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.041705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471031393118225:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(128033302ns) {Configure:104(128027904ns) {ProcessConfigured:135(26786268ns) {StartInitialHandshake:157(26782805ns) {DropHandshakes:470(1378ns) {DropIncomingHandshake:442(321ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(282ns) PrepareNewSessionHandshake:477(5265ns) {SwitchToState:179(2664ns)}}}}}
2025-11-27T18:17:38.030697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471032962690229:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.030542Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471031912970504:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.044394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471032608452845:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(123954835ns) {Configure:104(123950128ns) {ProcessConfigured:135(6923708ns) {StartInitialHandshake:157(6920603ns) {DropHandshakes:470(2110ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(428ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(3712ns) {SwitchToState:179(1220ns)}}}}}
2025-11-27T18:17:38.044675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471032608452921:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.044935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471032608452927:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30822
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.056471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471035925495111:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34393
2025-11-27T18:17:38.057303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471031630527808:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:38.055921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031912970513:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.041822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471030767606035:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.056185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471031912970504:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:38.057801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471031393118223:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(131468138ns) {Configure:104(131460823ns) {ProcessConfigured:135(9372358ns) {StartInitialHandshake:157(9369936ns) {DropHandshakes:470(1395ns) {DropIncomingHandshake:442(416ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(579ns) PrepareNewSessionHandshake:477(3824716ns) {SwitchToState:179(4593ns)}}}}}
2025-11-27T18:17:38.061908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471031912970512:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32251
2025-11-27T18:17:38.062071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471032608452927:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.066586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471031630527736:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.067294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471032608452922:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.070237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471032608452926:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30822
2025-11-27T18:17:38.070291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471032962690228:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.071747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471035688085530:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47661
2025-11-27T18:17:38.071952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471035688085531:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47661
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.067340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:38.070753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471030767606034:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.070665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471032962690152:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.073431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471032608452915:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.073455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471032608452926:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.073466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471032608452915:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:38.074236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471032608452915:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:38.069820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471031912970498:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.075926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471037257657531:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60487
2025-11-27T18:17:38.067961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471031630527736:2049] [node 2] ICP22 created new session: [6:7577471035925495112:2049]
2025-11-27T18:17:38.077525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471035688085532:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47661
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.070726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:38.074484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471030767605955:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.078656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471037257657530:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60487
2025-11-27T18:17:38.073280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471035925495112:2049] [node 2] ICS09 handshake done sender: [6:7577471031630527808:2114] self: [6:7577471031630527814:0] peer: [2:7577471031912970503:0] socket: 268 qp: -1
2025-11-27T18:17:38.077749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471032608452910:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.079536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471037257657531:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.080153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471035688085533:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47661
2025-11-27T18:17:38.070784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471031912970498:2109] [node 6] ICP22 created new session: [2:7577471036207937812:2109]
2025-11-27T18:17:38.079679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471032962690153:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.080350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471032962690228:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:38.077550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471035925495112:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:38.077818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:38.074566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031912970512:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.078473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:38.078868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471032608452910:2109] [node 5] ICP22 created new session: [1:7577471036903420231:2109]
2025-11-27T18:17:38.080969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471032962690152:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.074858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471036207937812:2109] [node 6] ICS09 handshake done sender: [2:7577471031912970504:2112] self: [2:7577471031912970503:0] peer: [6:7577471031630527814:0] socket: 254 qp: -1
2025-11-27T18:17:38.074893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471036207937812:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:38.081020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:38.079611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471036903420231:2109] [node 5] ICS09 handshake done sender: [1:7577471032608452915:2112] self: [1:7577471032608452913:0] peer: [5:7577471037257657529:0] socket: 261 qp: -1
2025-11-27T18:17:38.083191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471032962690152:2048] [node 1] ICP22 created new session: [5:7577471037257657533:2048]
2025-11-27T18:17:38.079649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471036903420231:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:38.083794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471037257657533:2048] [node 1] ICS09 handshake done sender: [5:7577471032962690228:2114] self: [5:7577471037257657529:0] peer: [1:7577471032608452913:0] socket: 263 qp: -1
2025-11-27T18:17:38.079698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471032608452918:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.083827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471037257657533:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:38.079706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:38.079719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471032608452918:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:38.074929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:38.083863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:38.086383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471035688085531:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.086831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471035688085530:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.080676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031912970506:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.091745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471035062573335:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30777
2025-11-27T18:17:38.086914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471035688085533:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.088126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471031630527736:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17927207ns) {HandleHandshakeStatus:329(17923426ns) {DropIncomingHandshake:442(664ns) DropOutgoingHandshake:457(9340ns) SwitchToState:179(374ns) ProcessPendingSessionEvents:495(3623824ns) {ForwardSessionEventToSession:612(2334439ns) ForwardSessionEventToSession:612(3399ns) ForwardSessionEventToSession:612(99375ns) ForwardSessionEventToSession:612(2240ns) ForwardSessionEventToSession:612(413871ns) ForwardSessionEventToSession:612(2748ns) ForwardSessionEventToSession:612(2385ns) ForwardSessionEventToSession:612(747108ns) ForwardSessionEventToSession:612(3341ns) ForwardSessionEventToSession:612(2749ns)}}}
2025-11-27T18:17:38.092935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471032608452918:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:38.080709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471031912970506:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:38.087016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471035688085532:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.097823Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471032962690153:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64381) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471037257657531:2118] held: no
2025-11-27T18:17:38.082731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471031912970506:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:38.087949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471031393118224:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(43479369ns) {Configure:104(43473623ns) {ProcessConfigured:135(1946421ns) {StartInitialHandshake:157(1733518ns) {DropHandshakes:470(2115ns) {DropIncomingHandshake:442(568ns) DropOutgoingHandshake:457(430ns)} GenerateSessionVirtualId:988(1131ns) PrepareNewSessionHandshake:477(2401ns) {SwitchToState:179(1152ns)}}}}}
2025-11-27T18:17:38.085183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471031912970497:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.085232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:38.085287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471031912970497:2108] [node 5] ICP22 created new session: [2:7577471036207937817:2108]
2025-11-27T18:17:38.086688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471036207937817:2108] [node 5] ICS09 handshake done sender: [2:7577471031912970506:2113] self: [2:7577471031912970505:0] peer: [5:7577471037257657532:0] socket: 293 qp: -1
2025-11-27T18:17:38.087780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471036207937817:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:38.088278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:38.102255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471032608452911:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.104037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.104087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471032608452911:2107] [node 4] ICP22 created new session: [1:7577471036903420236:2107]
2025-11-27T18:17:38.091290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471035688085534:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47661
2025-11-27T18:17:38.097775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471031393118160:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.104290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471030767606034:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:38.104203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471037257657535:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60487
2025-11-27T18:17:38.098012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471031393118159:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.104394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471036903420236:2107] [node 4] ICS09 handshake done sender: [1:7577471032608452918:2114] self: [1:7577471032608452917:0] peer: [4:7577471035062573333:0] socket: 264 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.114955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471035925495116:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34393
2025-11-27T18:17:38.107532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471037257657531:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:38.107766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471032962690153:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.107809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:38.107847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471032962690153:2049] [node 2] ICP22 created new session: [5:7577471037257657538:2049]
2025-11-27T18:17:38.107887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471037257657538:2049] [node 2] ICS09 handshake done sender: [5:7577471037257657531:2118] self: [5:7577471037257657532:0] peer: [2:7577471031912970505:0] socket: 294 qp: -1
2025-11-27T18:17:38.107915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471037257657538:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:38.107949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.109704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031912970502:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.104424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471036903420236:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:38.103877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471035688085534:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.120810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471030767605955:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471030767606034:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.121942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471035062573334:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30777
2025-11-27T18:17:38.124993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471035925495116:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.122016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471035062573336:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30777
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.129784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471037257657540:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60487
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.104463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:38.109745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471031912970502:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:38.122562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471035688085545:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47661
2025-11-27T18:17:38.105807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471032608452916:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.105833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471032608452916:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:38.107007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471032608452916:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:38.111240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471032608452846:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.114299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:38.122556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471032608452846:2049] [node 3] ICP22 created new session: [1:7577471036903420240:2049]
2025-11-27T18:17:38.123545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471036903420240:2049] [node 3] ICS09 handshake done sender: [1:7577471032608452916:2113] self: [1:7577471032608452914:0] peer: [3:7577471035688085541:0] socket: 262 qp: -1
2025-11-27T18:17:38.124525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471036903420240:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:38.124568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.137885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471035688085546:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47661
2025-11-27T18:17:38.123581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471030767605955:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.126761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471035925495115:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34393
2025-11-27T18:17:38.133794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471031630527796:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.113588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471031912970497:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18183243ns) {HandleHandshakeStatus:329(18160062ns) {DropIncomingHandshake:442(11640ns) DropOutgoingHandshake:457(445ns) SwitchToState:179(426ns) ProcessPendingSessionEvents:495(9954463ns) {ForwardSessionEventToSession:612(4717511ns) ForwardSessionEventToSession:612(5762ns) ForwardSessionEventToSession:612(805230ns) ForwardSessionEventToSession:612(4379ns) ForwardSessionEventToSession:612(3448276ns) ForwardSessionEventToSession:612(3855ns) ForwardSessionEventToSession:612(3655ns) ForwardSessionEventToSession:612(942929ns) ForwardSessionEventToSession:612(3356ns) ForwardSessionEventToSession:612(2862ns)}}}
2025-11-27T18:17:38.125554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:38.125859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471035062573337:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30777
2025-11-27T18:17:38.126228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471035062573336:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.130450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471030767605955:2048] [node 1] ICP22 created new session: [4:7577471035062573339:2048]
2025-11-27T18:17:38.137479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471035062573339:2048] [node 1] ICS09 handshake done sender: [4:7577471030767606034:2116] self: [4:7577471035062573333:0] peer: [1:7577471032608452917:0] socket: 274 qp: -1
2025-11-27T18:17:38.137550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471035062573339:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:38.137601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:38.141346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471036903420235:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30822
2025-11-27T18:17:38.119094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471036207937819:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32251
2025-11-27T18:17:38.146771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471035925495115:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.146910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471031630527735:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.147931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471035062573337:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.154125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471035688085530:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:38.149923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471035062573334:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.147761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471032962690224:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.150027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471030767605956:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.147806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471032962690224:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:38.119250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471036207937822:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32251
2025-11-27T18:17:38.147209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471032608452846:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(35758953ns) {HandleHandshakeStatus:329(35755743ns) {DropIncomingHandshake:442(2524ns) DropOutgoingHandshake:457(2314ns) SwitchToState:179(870ns) ProcessPendingSessionEvents:495(3350242ns) {ForwardSessionEventToSession:612(3294842ns) ForwardSessionEventToSession:612(5249ns) ForwardSessionEventToSession:612(5866ns) ForwardSessionEventToSession:612(3036ns) ForwardSessionEventToSession:612(2116ns) ForwardSessionEventToSession:612(5734ns) ForwardSessionEventToSession:612(2548ns) ForwardSessionEventToSession:612(7335ns) ForwardSessionEventToSession:612(8260ns) ForwardSessionEventToSession:612(2462ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.162802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471035925495119:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34393
2025-11-27T18:17:38.165787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471035925495115:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.155494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471035688085532:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:38.166151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471031630527735:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.169981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:38.170047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471031630527735:2048] [node 1] ICP22 created new session: [6:7577471035925495120:2048]
2025-11-27T18:17:38.150000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471032608452920:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.150019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471032608452920:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:38.155707Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471036903420235:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.158990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471032608452920:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:38.169734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471036903420225:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.169777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471036903420225:2120] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.157425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471037257657540:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.125649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471031912970502:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:38.153773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471030767605957:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.176988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471032608452846:2049] [node 3] ICP18 handshake (actor [1:7577471036903420235:2121]) is too old
2025-11-27T18:17:38.179652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471035925495120:2048] [node 1] ICS09 handshake done sender: [6:7577471035925495115:2117] self: [6:7577471035925495118:0] peer: [1:7577471032608452919:0] socket: 296 qp: -1
2025-11-27T18:17:38.184091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471035925495120:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:38.184159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.157544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471032962690215:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.156349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471035688085527:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.179786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471032608452912:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.179852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:38.179904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471032608452912:2110] [node 6] ICP22 created new session: [1:7577471036903420243:2110]
2025-11-27T18:17:38.179950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471036903420243:2110] [node 6] ICS09 handshake done sender: [1:7577471032608452920:2115] self: [1:7577471032608452919:0] peer: [6:7577471035925495118:0] socket: 295 qp: -1
2025-11-27T18:17:38.179995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471036903420243:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:38.180035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:38.161339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471030767606028:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.142240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471031912970434:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.156372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471035688085527:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:38.148430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471036207937819:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.176476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471037257657542:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60487
2025-11-27T18:17:38.190009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471035925495122:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34393
2025-11-27T18:17:38.161367Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471030767606028:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:38.190318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471037257657540:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:38.157238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471031393118160:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.149432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:38.196457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471032962690224:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:38.204844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471032962690215:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.204904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:38.204959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471032962690215:2108] [node 4] ICP22 created new session: [5:7577471037257657544:2108]
2025-11-27T18:17:38.204995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471037257657544:2108] [node 4] ICS09 handshake done sender: [5:7577471037257657540:2120] self: [5:7577471037257657541:0] peer: [4:7577471030767606027:0] socket: 315 qp: -1
2025-11-27T18:17:38.205031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471037257657544:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:38.205065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:38.205331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471032962690215:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:38.205746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471032962690216:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.205800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:38.205851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471032962690216:2109] [node 6] ICP22 created new session: [5:7577471037257657546:2109]
2025-11-27T18:17:38.205882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471037257657546:2109] [node 6] ICS09 handshake done sender: [5:7577471032962690224:2113] self: [5:7577471032962690223:0] peer: [6:7577471035925495117:0] socket: 257 qp: -1
2025-11-27T18:17:38.205905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471037257657546:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:38.205930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:38.166709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471035062573344:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30777
2025-11-27T18:17:38.157322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:38.154376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471031912970434:2049] [node 3] ICP22 created new session: [2:7577471036207937823:2049]
2025-11-27T18:17:38.215481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471036903420225:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.167321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471035062573346:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30777
2025-11-27T18:17:38.167368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471035062573337:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:38.167643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471030767605956:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.167684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:38.167735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471030767605956:2049] [node 2] ICP22 created new session: [4:7577471035062573347:2049]
2025-11-27T18:17:38.167783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471035062573347:2049] [node 2] ICS09 handshake done sender: [4:7577471035062573337:2121] self: [4:7577471035062573342:0] peer: [2:7577471031912970510:0] socket: 313 qp: -1
2025-11-27T18:17:38.167815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471035062573347:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:38.167846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:38.169461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471030767606028:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:38.193236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471030767606019:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.193317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:38.193366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471030767606019:2108] [node 5] ICP22 created new session: [4:7577471035062573349:2108]
2025-11-27T18:17:38.193411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471035062573349:2108] [node 5] ICS09 handshake done sender: [4:7577471030767606028:2114] self: [4:7577471030767606027:0] peer: [5:7577471037257657541:0] socket: 314 qp: -1
2025-11-27T18:17:38.193447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471035062573349:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:38.193485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:38.203180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471035062573334:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:38.203790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471030767605957:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.207010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:38.211188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471030767605957:2050] [node 3] ICP22 created new session: [4:7577471035062573352:2050]
2025-11-27T18:17:38.217512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471035062573352:2050] [node 3] ICS09 handshake done sender: [4:7577471035062573334:2118] self: [4:7577471035062573343:0] peer: [3:7577471035688085526:0] socket: 302 qp: -1
2025-11-27T18:17:38.212256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471035925495116:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:38.221559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471035062573352:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:38.221639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.154444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471031912970511:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.157365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471031393118160:2049] [node 2] ICP22 created new session: [3:7577471035688085547:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.231450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471037257657548:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60487
2025-11-27T18:17:38.212633Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471031630527735:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(45287266ns) {HandleHandshakeStatus:329(45285078ns) {DropIncomingHandshake:442(578ns) DropOutgoingHandshake:457(4620ns) SwitchToState:179(402ns) ProcessPendingSessionEvents:495(15302032ns) {ForwardSessionEventToSession:612(3942643ns) ForwardSessionEventToSession:612(5313ns) ForwardSessionEventToSession:612(3724ns) ForwardSessionEventToSession:612(1501465ns) ForwardSessionEventToSession:612(4687ns) ForwardSessionEventToSession:612(2307ns) ForwardSessionEventToSession:612(5635305ns) ForwardSessionEventToSession:612(3171ns) ForwardSessionEventToSession:612(4183494ns) ForwardSessionEventToSession:612(3366ns)}}}
2025-11-27T18:17:38.154463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471031912970511:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:38.157412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471035688085547:2049] [node 2] ICS09 handshake done sender: [3:7577471035688085530:2115] self: [3:7577471035688085540:0] peer: [2:7577471031912970501:0] socket: 285 qp: -1
2025-11-27T18:17:38.230140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471032608452845:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.157442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471035688085547:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:38.157477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:38.163311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471031393118159:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.163364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:38.163408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471031393118159:2048] [node 1] ICP22 created new session: [3:7577471035688085550:2048]
2025-11-27T18:17:38.163451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471035688085550:2048] [node 1] ICS09 handshake done sender: [3:7577471035688085532:2117] self: [3:7577471035688085541:0] peer: [1:7577471032608452914:0] socket: 287 qp: -1
2025-11-27T18:17:38.163480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471035688085550:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:38.163508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:38.201926Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471031393118160:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(37414768ns) {HandleHandshakeStatus:329(37413103ns) {DropIncomingHandshake:442(526ns) DropOutgoingHandshake:457(4675ns) SwitchToState:179(322ns) ProcessPendingSessionEvents:495(37065164ns) {ForwardSessionEventToSession:612(12279013ns) ForwardSessionEventToSession:612(4138400ns) ForwardSessionEventToSession:612(3312926ns) ForwardSessionEventToSession:612(3372ns) ForwardSessionEventToSession:612(5068649ns) ForwardSessionEventToSession:612(5463ns) ForwardSessionEventToSession:612(3391ns) ForwardSessionEventToSession:612(3364947ns) ForwardSessionEventToSession:612(2404ns) ForwardSessionEventToSession:612(8857843ns)}}}
2025-11-27T18:17:38.202146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471035688085527:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:38.228895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471031393118225:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.228953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:38.228993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471031393118225:2107] [node 4] ICP22 created new session: [3:7577471035688085553:2107]
2025-11-27T18:17:38.229035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471035688085553:2107] [node 4] ICS09 handshake done sender: [3:7577471035688085527:2113] self: [3:7577471035688085526:0] peer: [4:7577471035062573343:0] socket: 299 qp: -1
2025-11-27T18:17:38.229066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471035688085553:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:38.229091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:38.224412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471035925495123:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34393
2025-11-27T18:17:38.249575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471030767606030:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.249607Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471030767606030:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.230201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:38.233326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471032608452845:2048] [node 2] ICP22 created new session: [1:7577471036903420246:2048]
2025-11-27T18:17:38.249443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471036903420246:2048] [node 2] ICS09 handshake done sender: [1:7577471036903420225:2120] self: [1:7577471036903420224:0] peer: [2:7577471036207937825:0] socket: 308 qp: -1
2025-11-27T18:17:38.249481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471036903420246:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:38.155040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471036207937822:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.155332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471031912970433:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.155587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471031912970511:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:38.155758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471036207937823:2049] [node 3] ICS09 handshake done sender: [2:7577471031912970502:2111] self: [2:7577471031912970501:0] peer: [3:7577471035688085540:0] socket: 253 qp: -1
2025-11-27T18:17:38.162226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471036207937823:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:38.162289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:38.175447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471031912970496:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.175512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:38.175564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471031912970496:2107] [node 4] ICP22 created new session: [2:7577471036207937827:2107]
2025-11-27T18:17:38.175605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471036207937827:2107] [node 4] ICS09 handshake done sender: [2:7577471031912970511:2114] self: [2:7577471031912970510:0] peer: [4:7577471035062573342:0] socket: 310 qp: -1
2025-11-27T18:17:38.175662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471036207937827:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:38.175701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:38.199293Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471031912970434:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(56505101ns) {HandleHandshakeStatus:329(56501700ns) {DropIncomingHandshake:442(329ns) DropOutgoingHandshake:457(555ns) SwitchToState:179(429ns) ProcessPendingSessionEvents:495(21349495ns) {ForwardSessionEventToSession:612(18638397ns) ForwardSessionEventToSession:612(662337ns) ForwardSessionEventToSession:612(2695ns) ForwardSessionEventToSession:612(2003062ns) ForwardSessionEventToSession:612(7284ns) ForwardSessionEventToSession:612(3145ns) ForwardSessionEventToSession:612(5351ns) ForwardSessionEventToSession:612(3026ns) ForwardSessionEventToSession:612(3939ns) ForwardSessionEventToSession:612(4943ns)}}}
2025-11-27T18:17:38.211230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471031912970434:2049] [node 3] ICP18 handshake (actor [2:7577471036207937819:2117]) is too old
2025-11-27T18:17:38.216487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471036207937829:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32251
2025-11-27T18:17:38.216991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471036207937822:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:38.218586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471031912970433:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.221743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:38.221801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471031912970433:2048] [node 1] ICP22 created new session: [2:7577471036207937830:2048]
2025-11-27T18:17:38.221849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471036207937830:2048] [node 1] ICS09 handshake done sender: [2:7577471036207937822:2118] self: [2:7577471036207937825:0] peer: [1:7577471036903420224:0] socket: 311 qp: -1
2025-11-27T18:17:38.221885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471036207937830:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:38.221921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:38.252623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:38.253686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471032608452845:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(23371163ns) {HandleHandshakeStatus:329(23369332ns) {DropIncomingHandshake:442(10967ns) DropOutgoingHandshake:457(415ns) SwitchToState:179(395ns) ProcessPendingSessionEvents:495(783806ns) {ForwardSessionEventToSession:612(640666ns) ForwardSessionEventToSession:612(6548ns) ForwardSessionEventToSession:612(96955ns) ForwardSessionEventToSession:612(5146ns) ForwardSessionEventToSession:612(3093ns) ForwardSessionEventToSession:612(3134ns) ForwardSessionEventToSession:612(5520ns) ForwardSessionEventToSession:612(6972ns) ForwardSessionEventToSession:612(2367ns) ForwardSessionEventToSession:612(5379ns)}}}
2025-11-27T18:17:38.250849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471030767605957:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(46928266ns) {HandleHandshakeStatus:329(46926634ns) {DropIncomingHandshake:442(696ns) DropOutgoingHandshake:457(5130ns) SwitchToState:179(626ns) ProcessPendingSessionEvents:495(12731248ns) {ForwardSessionEventToSession:612(12687382ns) ForwardSessionEventToSession:612(7437ns) ForwardSessionEventToSession:612(2125ns) ForwardSessionEventToSession:612(5928ns) ForwardSessionEventToSession:612(4341ns) ForwardSessionEventToSession:612(2087ns) ForwardSessionEventToSession:612(2135ns) ForwardSessionEventToSession:612(3676ns) ForwardSessionEventToSession:612(1300ns) ForwardSessionEventToSession:612(6729ns)}}}
2025-11-27T18:17:38.251628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471030767606030:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:38.251988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471030767606020:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.252050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:38.252082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471030767606020:2110] [node 6] ICP22 created new session: [4:7577471035062573355:2110]
2025-11-27T18:17:38.252119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471035062573355:2110] [node 6] ICS09 handshake done sender: [4:7577471030767606030:2115] self: [4:7577471030767606029:0] peer: [6:7577471035925495126:0] socket: 280 qp: -1
2025-11-27T18:17:38.252144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471035062573355:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:38.252959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:38.224569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471031630527796:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18412) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471035925495116:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.224685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471031630527796:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.224722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:38.224761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471031630527796:2106] [node 5] ICP22 created new session: [6:7577471035925495124:2106]
2025-11-27T18:17:38.224818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471035925495124:2106] [node 5] ICS09 handshake done sender: [6:7577471035925495116:2118] self: [6:7577471035925495117:0] peer: [5:7577471032962690223:0] socket: 307 qp: -1
2025-11-27T18:17:38.224851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471035925495124:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:38.224884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:38.241341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471035925495123:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.241656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471031630527799:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.254512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471035925495127:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34393
2025-11-27T18:17:38.256523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471035925495128:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34393
2025-11-27T18:17:38.256839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471035925495123:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:38.256997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471031630527799:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.257029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:38.257055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471031630527799:2109] [node 4] ICP22 created new session: [6:7577471035925495129:2109]
2025-11-27T18:17:38.257085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471035925495129:2109] [node 4] ICS09 handshake done sender: [6:7577471035925495123:2121] self: [6:7577471035925495126:0] peer: [4:7577471030767606029:0] socket: 279 qp: -1
2025-11-27T18:17:38.257107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471035925495129:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:38.257126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:38.263207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471037257657548:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.263373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471032962690154:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.263781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471035688085529:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.263814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471035688085529:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.264803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471037257657550:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60487
2025-11-27T18:17:38.265657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471037257657548:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:38.265165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471035688085529:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:38.267472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471031393118223:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.267532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:38.265989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471032962690154:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.267574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471031393118223:2108] [node 5] ICP22 created new session: [3:7577471035688085557:2108]
2025-11-27T18:17:38.266044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:38.267615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471035688085557:2108] [node 5] ICS09 handshake done sender: [3:7577471035688085529:2114] self: [3:7577471035688085528:0] peer: [5:7577471037257657549:0] socket: 252 qp: -1
2025-11-27T18:17:38.272754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471035925495127:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.267642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471035688085557:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:38.266084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471032962690154:2050] [node 3] ICP22 created new session: [5:7577471037257657551:2050]
2025-11-27T18:17:38.281714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471031630527737:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:38.266124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471037257657551:2050] [node 3] ICS09 handshake done sender: [5:7577471037257657548:2122] self: [5:7577471037257657549:0] peer: [3:7577471035688085528:0] socket: 256 qp: -1
2025-11-27T18:17:38.266152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471037257657551:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:38.266190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:38.267677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:38.283846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471035688085539:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:38.283863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471035688085539:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:38.285152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471035688085539:2120] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.287338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471035925495132:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34393
2025-11-27T18:17:38.287582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471031393118224:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:38.287653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:38.287695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471031393118224:2109] [node 6] ICP22 created new session: [3:7577471035688085560:2109]
2025-11-27T18:17:38.287739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471035688085560:2109] [node 6] ICS09 handshake done sender: [3:7577471035688085539:2120] self: [3:7577471035688085535:0] peer: [6:7577471035925495131:0] socket: 267 qp: -1
2025-11-27T18:17:38.287772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471035688085560:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:38.287802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:38.288697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471035925495127:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:38.288856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471031630527737:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:38.288929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:38.288978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471031630527737:2050] [node 3] ICP22 created new session: [6:7577471035925495133:2050]
2025-11-27T18:17:38.289017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471035925495133:2050] [node 3] ICS09 handshake done sender: [6:7577471035925495127:2122] self: [6:7577471035925495131:0] peer: [3:7577471035688085535:0] socket: 259 qp: -1
2025-11-27T18:17:38.289042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471035925495133:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:38.289074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:38.331403Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:38.331629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471036207937830:2048] [node 1] ICS01 socket: 311 reason# EndOfStream
2025-11-27T18:17:38.331676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471031912970433:2048] [node 1] ICP30 unregister session Session# [2:7577471036207937830:2048] VirtualId# [2:7577471036207937825:0]
2025-11-27T18:17:38.331719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471036207937830:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.331831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:38.331882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:38.336804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:38.337164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471035688085550:2048] [node 1] ICS01 socket: 287 reason# EndOfStream
2025-11-27T18:17:38.337216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471031393118159:2048] [node 1] ICP30 unregister session Session# [3:7577471035688085550:2048] VirtualId# [3:7577471035688085541:0]
2025-11-27T18:17:38.337296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471035688085550:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.337396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:38.337459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:38.346303Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:38.346516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471035925495120:2048] [node 1] ICS01 socket: 296 reason# EndOfStream
2025-11-27T18:17:38.346720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:38.346547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471031630527735:2048] [node 1] ICP30 unregister session Session# [6:7577471035925495120:2048] VirtualId# [6:7577471035925495118:0]
2025-11-27T18:17:38.346574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471035925495120:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.346954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471035062573339:2048] [node 1] ICS01 socket: 274 reason# EndOfStream
2025-11-27T18:17:38.347294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471030767605955:2048] [node 1] ICP30 unregister session Session# [4:7577471035062573339:2048] VirtualId# [4:7577471035062573333:0]
2025-11-27T18:17:38.346628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:38.347324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471035062573339:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.346674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:38.347404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:38.347443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:38.345656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:38.345907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471037257657533:2048] [node 1] ICS07 socket disconnect 263 reason# EndOfStream
2025-11-27T18:17:38.345934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471037257657533:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.346013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:38.346054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471037257657533:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:38.351869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471032962690152:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26232) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:38.352674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471037257657533:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:38.352724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471032962690152:2048] [node 1] ICP30 unregister session Session# [5:7577471037257657533:2048] VirtualId# [5:7577471037257657529:0]
2025-11-27T18:17:38.352777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:38.352806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471032962690152:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:38.352820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:38.362708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471032962690152:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:38.376054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:38.376293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471035688085547:2049] [node 2] ICS07 socket disconnect 285 reason# EndOfStream
2025-11-27T18:17:38.376334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471035688085547:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.376408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:38.376448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471035688085547:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:38.379013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471031393118160:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64381) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:38.379067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471035688085547:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:38.385054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:38.385319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471037257657538:2049] [node 2] ICS01 socket: 294 reason# EndOfStream
2025-11-27T18:17:38.385358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471032962690153:2049] [node 2] ICP30 unregister session Session# [5:7577471037257657538:2049] VirtualId# [5:7577471037257657532:0]
2025-11-27T18:17:38.385395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471037257657538:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.385463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:38.383339Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471031393118160:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64381) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:38.381510Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:38.382153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471035062573347:2049] [node 2] ICS01 socket: 313 reason# EndOfStream
2025-11-27T18:17:38.382192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471030767605956:2049] [node 2] ICP30 unregister session Session# [4:7577471035062573347:2049] VirtualId# [4:7577471035062573342:0]
2025-11-27T18:17:38.382231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471035062573347:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.382314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:38.384260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471035688085547:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:38.382372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:38.385501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:38.384300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471031393118160:2049] [node 2] ICP30 unregister session Session# [3:7577471035688085547:2049] VirtualId# [3:7577471035688085540:0]
2025-11-27T18:17:38.384362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:38.384397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471031393118160:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:38.384417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:38.387326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:38.387561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471035925495112:2049] [node 2] ICS07 socket disconnect 268 reason# EndOfStream
2025-11-27T18:17:38.387591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471035925495112:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.387660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:38.387706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471035925495112:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:38.388670Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471031630527736:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64381) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:38.388725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471035925495112:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:38.388758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471031630527736:2049] [node 2] ICP30 unregister session Session# [6:7577471035925495112:2049] VirtualId# [6:7577471031630527814:0]
2025-11-27T18:17:38.388797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:38.388828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471031630527736:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:38.388841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:38.396654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471031393118160:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:38.398529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471031630527736:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:38.447250Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:38.447503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471035062573352:2050] [node 3] ICS01 socket: 302 reason# EndOfStream
2025-11-27T18:17:38.447547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471030767605957:2050] [node 3] ICP30 unregister session Session# [4:7577471035062573352:2050] VirtualId# [4:7577471035062573343:0]
2025-11-27T18:17:38.447580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471035062573352:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.447673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:38.447966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:38.449461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:38.457030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471035925495133:2050] [node 3] ICS01 socket: 259 reason# EndOfStream
2025-11-27T18:17:38.457112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471031630527737:2050] [node 3] ICP30 unregister session Session# [6:7577471035925495133:2050] VirtualId# [6:7577471035925495131:0]
2025-11-27T18:17:38.457161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471035925495133:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.449380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:38.457283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:38.452817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471037257657551:2050] [node 3] ICS01 socket: 256 reason# EndOfStream
2025-11-27T18:17:38.457330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:38.452864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471032962690154:2050] [node 3] ICP30 unregister session Session# [5:7577471037257657551:2050] VirtualId# [5:7577471037257657549:0]
2025-11-27T18:17:38.452898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471037257657551:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.452990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:38.453034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:38.501516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:38.501755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471035925495129:2109] [node 4] ICS07 socket disconnect 279 reason# EndOfStream
2025-11-27T18:17:38.501792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471035925495129:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.501878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:38.501920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471035925495129:2109] [node 4] ICS15 start handshake
2025-11-27T18:17:38.503131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:38.503403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471037257657544:2108] [node 4] ICS01 socket: 315 reason# EndOfStream
2025-11-27T18:17:38.503460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471032962690215:2108] [node 4] ICP30 unregister session Session# [5:7577471037257657544:2108] VirtualId# [5:7577471037257657541:0]
2025-11-27T18:17:38.503505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471037257657544:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.503603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:38.503657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:38.505977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471031630527799:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14712) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:38.506059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471035925495129:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:38.506109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471031630527799:2109] [node 4] ICP30 unregister session Session# [6:7577471035925495129:2109] VirtualId# [6:7577471035925495126:0]
2025-11-27T18:17:38.506155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:38.506192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471031630527799:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14712) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:38.506207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14712) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:38.516335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471031630527799:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:17:38.553102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:38.553417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471035925495124:2106] [node 5] ICS01 socket: 307 reason# EndOfStream
2025-11-27T18:17:38.553460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471031630527796:2106] [node 5] ICP30 unregister session Session# [6:7577471035925495124:2106] VirtualId# [6:7577471035925495117:0]
2025-11-27T18:17:38.553497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471035925495124:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:38.553593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:38.553641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:38.643376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471033680263182:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:38.643490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471033680263183:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:38.663157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471035009934127:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:38.663175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471035009934126:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:38.684243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471035637931465:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:38.684308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471035637931466:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:38.717378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471035097350556:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:38.717460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471035097350558:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:38.718129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471035097350557:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:38.736414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471034058303293:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:38.736456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471034058303294:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:38.736506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471034058303295:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:38.762846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471036281300808:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:38.762886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471036281300809:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:38.762904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471036281300810:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:38.832316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471036281300868:2106] [node 5] ICP01 ready to work
2025-11-27T18:17:38.833997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471033680263243:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:38.941851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471035097350628:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:38.963787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471033680263182:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(130890644ns) {Configure:104(130883784ns) {ProcessConfigured:135(4412065ns) {StartInitialHandshake:157(85670ns) {DropHandshakes:470(3593ns) {DropIncomingHandshake:442(592ns) DropOutgoingHandshake:457(271ns)} GenerateSessionVirtualId:988(1993ns) PrepareNewSessionHandshake:477(2760ns) {SwitchToState:179(1733ns)}}}}}
2025-11-27T18:17:38.964229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471033680263248:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:38.942117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471034058303356:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:38.957561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471035637931528:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:38.964511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471033680263249:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:38.961512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471035637931529:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:38.970544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471035097350629:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:38.976766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471034058303357:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:38.976814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471035009934216:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:38.980113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471036281300871:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:38.961640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471035637931530:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:38.984411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471035009934217:2115] [node 5] ICP01 ready to work
2025-11-27T18:17:38.984501Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471036281300868:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(149369019ns) {Configure:104(149359654ns) {ProcessConfigured:135(4932872ns) {StartInitialHandshake:157(4930098ns) {DropHandshakes:470(4346258ns) {DropIncomingHandshake:442(713ns) DropOutgoingHandshake:457(2563ns)} GenerateSessionVirtualId:988(2178ns) PrepareNewSessionHandshake:477(3388ns) {SwitchToState:179(1762ns)}}}}}
2025-11-27T18:17:38.985870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471033680263183:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(151017210ns) {Configure:104(151011250ns) {ProcessConfigured:135(725760ns) {StartInitialHandshake:157(723449ns) {DropHandshakes:470(2552ns) {DropIncomingHandshake:442(654ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(1332ns) PrepareNewSessionHandshake:477(2411ns) {SwitchToState:179(768ns)}}}}}
2025-11-27T18:17:38.991708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471033680263243:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(151975212ns) {Configure:104(151971204ns) {ProcessConfigured:135(1562449ns) {StartInitialHandshake:157(1560359ns) {DropHandshakes:470(1642ns) {DropIncomingHandshake:442(400ns) DropOutgoingHandshake:457(352ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(1570ns) {SwitchToState:179(595ns)}}}}}
2025-11-27T18:17:38.990492Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471034058303293:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24054851ns) {Configure:104(24049256ns) {ProcessConfigured:135(644095ns) {StartInitialHandshake:157(641793ns) {DropHandshakes:470(1591ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:477(2557ns) {SwitchToState:179(1063ns)}}}}}
2025-11-27T18:17:38.989503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471035009934218:2124] [node 6] ICP01 ready to work
2025-11-27T18:17:38.990833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471035009934127:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(140812050ns) {Configure:104(140805475ns) {ProcessConfigured:135(60813ns) {StartInitialHandshake:157(58337ns) {DropHandshakes:470(1569ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(580ns) PrepareNewSessionHandshake:477(1231ns) {SwitchToState:179(613ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.003904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471036281300879:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-11-27T18:17:38.994155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471035009934126:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(147424362ns) {Configure:104(147412317ns) {ProcessConfigured:135(51633ns) {StartInitialHandshake:157(49529ns) {DropHandshakes:470(1378ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(490ns) PrepareNewSessionHandshake:477(4392ns) {SwitchToState:179(403ns)}}}}}
2025-11-27T18:17:39.006200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471036281300879:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.000041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471033680263261:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13119
2025-11-27T18:17:39.007158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471036281300808:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(133922304ns) {Configure:104(133913354ns) {ProcessConfigured:135(3417316ns) {StartInitialHandshake:157(3414549ns) {DropHandshakes:470(2722ns) {DropIncomingHandshake:442(596ns) DropOutgoingHandshake:457(275ns)} GenerateSessionVirtualId:988(1945ns) PrepareNewSessionHandshake:477(3120ns) {SwitchToState:179(1733ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:38.999068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471035009934221:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19812
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.013407Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471035097350558:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(127097000ns) {Configure:104(127085845ns) {ProcessConfigured:135(8121529ns) {StartInitialHandshake:157(8118868ns) {DropHandshakes:470(667481ns) {DropIncomingHandshake:442(1512ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(1952ns) PrepareNewSessionHandshake:477(3721ns) {SwitchToState:179(2145ns)}}}}}
2025-11-27T18:17:39.014117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471035637931466:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24496407ns) {Configure:104(24491352ns) {ProcessConfigured:135(50223ns) {StartInitialHandshake:157(47627ns) {DropHandshakes:470(1262ns) {DropIncomingHandshake:442(322ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(615ns) PrepareNewSessionHandshake:477(2093ns) {SwitchToState:179(588ns)}}}}}
2025-11-27T18:17:38.999691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471035009934221:2138] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.012397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471034058303294:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47724838ns) {Configure:104(47717515ns) {ProcessConfigured:135(11367897ns) {StartInitialHandshake:157(11366082ns) {DropHandshakes:470(494ns) {DropIncomingHandshake:442(192ns) DropOutgoingHandshake:457(36ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:477(2631ns) {SwitchToState:179(939ns)}}}}}
2025-11-27T18:17:39.020597Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471035097350557:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(124758359ns) {Configure:104(124749248ns) {ProcessConfigured:135(8288587ns) {StartInitialHandshake:157(8286426ns) {DropHandshakes:470(1346ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:477(3281ns) {SwitchToState:179(1779ns)}}}}}
2025-11-27T18:17:39.013572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471036281300808:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.015179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471037975230558:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13119
2025-11-27T18:17:39.008658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471035009934126:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.015054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471036281300810:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27496776ns) {Configure:104(27492821ns) {ProcessConfigured:135(2526066ns) {StartInitialHandshake:157(2523893ns) {DropHandshakes:470(1454ns) {DropIncomingHandshake:442(322ns) DropOutgoingHandshake:457(257ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:477(2839ns) {SwitchToState:179(1482ns)}}}}}
2025-11-27T18:17:39.020808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471035097350556:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38073009ns) {Configure:104(38061849ns) {ProcessConfigured:135(6185912ns) {StartInitialHandshake:157(6182921ns) {DropHandshakes:470(3944550ns) {DropIncomingHandshake:442(3050ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(2842ns) PrepareNewSessionHandshake:477(2341ns) {SwitchToState:179(1391ns)}}}}}
2025-11-27T18:17:39.023051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471034058303356:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32905820ns) {Configure:104(32898987ns) {ProcessConfigured:135(9328829ns) {StartInitialHandshake:157(9326388ns) {DropHandshakes:470(2341949ns) {DropIncomingHandshake:442(2409ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(1624ns) PrepareNewSessionHandshake:477(3106ns) {SwitchToState:179(1345ns)}}}}}
2025-11-27T18:17:39.016349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471033680263247:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.016013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471039932898836:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58943
2025-11-27T18:17:39.020886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471039304901528:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19812
2025-11-27T18:17:39.022095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471035637931465:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(41065665ns) {Configure:104(41058841ns) {ProcessConfigured:135(11905865ns) {StartInitialHandshake:157(11903968ns) {DropHandshakes:470(1923ns) {DropIncomingHandshake:442(508ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(448ns) PrepareNewSessionHandshake:477(2120ns) {SwitchToState:179(972ns)}}}}}
2025-11-27T18:17:39.022889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471035637931529:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(38125197ns) {Configure:104(38116361ns) {ProcessConfigured:135(2982857ns) {StartInitialHandshake:157(2980899ns) {DropHandshakes:470(1625ns) {DropIncomingHandshake:442(256ns) DropOutgoingHandshake:457(639ns)} GenerateSessionVirtualId:988(720ns) PrepareNewSessionHandshake:477(2545ns) {SwitchToState:179(370ns)}}}}}
2025-11-27T18:17:39.023530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471039932898836:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.016391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471033680263247:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.035233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471039392317932:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4460
2025-11-27T18:17:39.034214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471035009934220:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.022898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471033680263253:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.030477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471035637931466:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.041161Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471034058303295:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42861552ns) {Configure:104(42856909ns) {ProcessConfigured:135(9945047ns) {StartInitialHandshake:157(9941999ns) {DropHandshakes:470(2215717ns) {DropIncomingHandshake:442(1573ns) DropOutgoingHandshake:457(1658ns)} GenerateSessionVirtualId:988(1943ns) PrepareNewSessionHandshake:477(3767ns) {SwitchToState:179(1495ns)}}}}}
2025-11-27T18:17:39.022940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471033680263253:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.034255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471035009934220:2137] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.030498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471033680263261:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.041101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471039304901528:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.033433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471037975230558:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.043761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471035009934216:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(47996648ns) {Configure:104(47991846ns) {ProcessConfigured:135(1989869ns) {StartInitialHandshake:157(1987323ns) {DropHandshakes:470(3636ns) {DropIncomingHandshake:442(909ns) DropOutgoingHandshake:457(477ns)} GenerateSessionVirtualId:988(857ns) PrepareNewSessionHandshake:477(2798ns) {SwitchToState:179(893ns)}}}}}
2025-11-27T18:17:39.049601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471039392317932:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.049429Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471034058303357:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27730140ns) {Configure:104(25911790ns) {ProcessConfigured:135(849376ns) {StartInitialHandshake:157(847599ns) {DropHandshakes:470(2526ns) {DropIncomingHandshake:442(482ns) DropOutgoingHandshake:457(473ns)} GenerateSessionVirtualId:988(3332ns) PrepareNewSessionHandshake:477(1570ns) {SwitchToState:179(704ns)}}}}}
2025-11-27T18:17:39.049338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471039932898842:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58943
2025-11-27T18:17:39.049731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471039304901529:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19812
2025-11-27T18:17:39.036509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471033680263247:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:39.058688Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471035097350628:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(30402733ns) {Configure:104(30395140ns) {ProcessConfigured:135(6094729ns) {StartInitialHandshake:157(6092377ns) {DropHandshakes:470(1606ns) {DropIncomingHandshake:442(308ns) DropOutgoingHandshake:457(610ns)} GenerateSessionVirtualId:988(386ns) PrepareNewSessionHandshake:477(1884ns) {SwitchToState:179(734ns)}}}}}
2025-11-27T18:17:39.037727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471033680263253:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:39.050318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471039304901533:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19812
2025-11-27T18:17:39.058594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471040576268184:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.066315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471035097350629:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36486860ns) {Configure:104(36479827ns) {ProcessConfigured:135(741347ns) {StartInitialHandshake:157(738905ns) {DropHandshakes:470(1284ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(180ns)} GenerateSessionVirtualId:988(547ns) PrepareNewSessionHandshake:477(1870ns) {SwitchToState:179(924ns)}}}}}
2025-11-27T18:17:39.053540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471037975230560:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13119
2025-11-27T18:17:39.050376Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471035009934221:2138] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:39.068852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471039932898842:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.058051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471033680263182:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.068764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471035097350558:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.050600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471035009934126:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.070194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471039392317941:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4460
2025-11-27T18:17:39.071723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471036281300879:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:39.069013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471035637931534:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.050658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:39.069035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471035637931534:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.073866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471036281300808:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16179) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471036281300879:2113] held: no
2025-11-27T18:17:39.058223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471033680263249:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.050704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471035009934126:2048] [node 1] ICP22 created new session: [2:7577471039304901535:2048]
2025-11-27T18:17:39.070216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471035637931534:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:17:39.076133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471035097350558:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16358) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471039392317932:2122] held: no
::ffff:127.1.0.02025-11-27T18:17:39.058262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.058216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471039304901535:2048] [node 1] ICS09 handshake done sender: [2:7577471035009934221:2138] self: [2:7577471039304901527:0] peer: [1:7577471033680263246:0] socket: 262 qp: -1
2025-11-27T18:17:39.070434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471035637931530:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.058619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471039304901535:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:39.058670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:39.077338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471039392317932:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:39.072163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.058809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471035009934220:2137] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.058768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471033680263182:2048] [node 2] ICP22 created new session: [1:7577471037975230565:2048]
2025-11-27T18:17:39.076789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471036281300808:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.071265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471035009934127:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.058960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:39.078722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471035097350558:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.079145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471038353270661:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43573
2025-11-27T18:17:39.072834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471035637931530:2107] [node 4] ICP22 created new session: [3:7577471039932898846:2107]
2025-11-27T18:17:39.079493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:39.059396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471037975230565:2048] [node 2] ICS09 handshake done sender: [1:7577471033680263247:2110] self: [1:7577471033680263246:0] peer: [2:7577471039304901527:0] socket: 258 qp: -1
2025-11-27T18:17:39.079303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471038353270662:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43573
2025-11-27T18:17:39.061109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471037975230565:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:39.079813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471038353270665:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43573
2025-11-27T18:17:39.076841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.071310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:39.073170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471039932898846:2107] [node 4] ICS09 handshake done sender: [3:7577471035637931534:2111] self: [3:7577471035637931533:0] peer: [4:7577471039392317940:0] socket: 277 qp: -1
2025-11-27T18:17:39.080090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471035097350558:2050] [node 3] ICP22 created new session: [4:7577471039392317943:2050]
2025-11-27T18:17:39.076887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471036281300808:2048] [node 1] ICP22 created new session: [6:7577471040576268185:2048]
2025-11-27T18:17:39.078379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471040576268185:2048] [node 1] ICS09 handshake done sender: [6:7577471036281300879:2113] self: [6:7577471040576268182:0] peer: [1:7577471033680263252:0] socket: 263 qp: -1
2025-11-27T18:17:39.078749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471040576268185:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:39.078787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:39.061397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471033680263249:2109] [node 6] ICP22 created new session: [1:7577471037975230566:2109]
2025-11-27T18:17:39.072149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471035009934127:2049] [node 3] ICP22 created new session: [2:7577471039304901537:2049]
2025-11-27T18:17:39.080143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471039392317943:2050] [node 3] ICS09 handshake done sender: [4:7577471039392317932:2122] self: [4:7577471039392317940:0] peer: [3:7577471035637931533:0] socket: 289 qp: -1
2025-11-27T18:17:39.065537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471037975230566:2109] [node 6] ICS09 handshake done sender: [1:7577471033680263253:2112] self: [1:7577471033680263252:0] peer: [6:7577471040576268182:0] socket: 260 qp: -1
2025-11-27T18:17:39.082433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471038353270664:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43573
2025-11-27T18:17:39.080763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471039392317943:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:39.065680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:39.074166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471039932898846:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:39.073162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471039304901537:2049] [node 3] ICS09 handshake done sender: [2:7577471035009934220:2137] self: [2:7577471035009934219:0] peer: [3:7577471039932898841:0] socket: 264 qp: -1
2025-11-27T18:17:39.080805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:39.074210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:39.066037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471037975230566:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:39.083560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471038353270661:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.074203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471039304901537:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.083873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471038353270665:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.078490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471039932898843:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58943
2025-11-27T18:17:39.067436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:39.085543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471038353270662:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.076337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:39.074944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471037975230560:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.081592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471039932898836:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.102334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471039392317945:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4460
2025-11-27T18:17:39.076779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471035009934126:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21057064ns) {HandleHandshakeStatus:329(21055256ns) {DropIncomingHandshake:442(600ns) DropOutgoingHandshake:457(4799ns) SwitchToState:179(306ns) ProcessPendingSessionEvents:495(5635918ns) {ForwardSessionEventToSession:612(2188361ns) ForwardSessionEventToSession:612(11078ns) ForwardSessionEventToSession:612(3564ns) ForwardSessionEventToSession:612(59843ns) ForwardSessionEventToSession:612(2495589ns) ForwardSessionEventToSession:612(144005ns) ForwardSessionEventToSession:612(3751ns) ForwardSessionEventToSession:612(3086ns) ForwardSessionEventToSession:612(708455ns) ForwardSessionEventToSession:612(3175ns)}}}
2025-11-27T18:17:39.086880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471034058303293:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.082562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471035637931466:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.102482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471040576268188:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-11-27T18:17:39.078369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471037975230564:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13119
2025-11-27T18:17:39.102651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471039392317947:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4460
2025-11-27T18:17:39.080186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471039304901533:2145] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.090214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471034058303295:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.083695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:39.078892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471033680263249:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18645612ns) {HandleHandshakeStatus:329(18641869ns) {DropIncomingHandshake:442(14863ns) DropOutgoingHandshake:457(250ns) SwitchToState:179(617ns) ProcessPendingSessionEvents:495(4254542ns) {ForwardSessionEventToSession:612(1614836ns) ForwardSessionEventToSession:612(4835ns) ForwardSessionEventToSession:612(3794ns) ForwardSessionEventToSession:612(114385ns) ForwardSessionEventToSession:612(3266ns) ForwardSessionEventToSession:612(836641ns) ForwardSessionEventToSession:612(1075381ns) ForwardSessionEventToSession:612(3320ns) ForwardSessionEventToSession:612(576251ns) ForwardSessionEventToSession:612(3190ns)}}}
2025-11-27T18:17:39.104270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471039392317948:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4460
2025-11-27T18:17:39.093788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471038353270664:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.093779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471039304901541:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19812
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.106524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471039392317947:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.084064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471035637931466:2049] [node 2] ICP22 created new session: [3:7577471039932898850:2049]
2025-11-27T18:17:39.114514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471040576268189:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-11-27T18:17:39.081127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471037975230564:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.110938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471035009934127:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:39.106508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471038353270671:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43573
2025-11-27T18:17:39.084453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471039932898850:2049] [node 2] ICS09 handshake done sender: [3:7577471039932898836:2114] self: [3:7577471039932898841:0] peer: [2:7577471035009934219:0] socket: 275 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.108204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471039392317946:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4460
2025-11-27T18:17:39.120945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471035009934225:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.115619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471040576268189:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.084487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471039932898850:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:39.106518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471037975230572:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13119
2025-11-27T18:17:39.112317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471034058303294:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.109554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471039392317945:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.121003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471035009934225:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.084522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:39.115981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471036281300810:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.113528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471035097350556:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.112520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471033680263251:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.122080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471035009934225:2140] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:39.112551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471038353270671:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.114103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471039392317948:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.117377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471040576268188:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.114281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471034058303356:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.102598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471039932898849:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58943
2025-11-27T18:17:39.122330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471036281300809:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.112553Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471033680263251:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.126675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471039304901542:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19812
2025-11-27T18:17:39.125653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471034058303293:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16179) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471038353270661:2113] held: no
2025-11-27T18:17:39.108054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471035637931532:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.132032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471035009934217:2115] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.126141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471038353270678:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43573
2025-11-27T18:17:39.132578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471039304901526:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.132603Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471039304901526:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.135766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471038353270679:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43573
2025-11-27T18:17:39.114518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471039392317937:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.114172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471033680263260:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.108084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471035637931532:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.137233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471038353270680:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43573
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.137836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471036281300809:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25677) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471040576268188:2117] held: no
2025-11-27T18:17:39.133995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471039304901526:2141] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:39.114208Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471033680263260:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.114540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471039392317937:2123] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.113566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471039932898849:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.134246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471035009934218:2124] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.120875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471039392317937:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:39.114477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471033680263251:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:39.144134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471040576268192:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.115383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471035637931465:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.134297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:39.128128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471035097350628:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.114736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471037975230572:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.144670Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471034058303356:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27665) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471038353270671:2119] held: no
2025-11-27T18:17:39.145680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471040576268194:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-11-27T18:17:39.134353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471035009934218:2124] [node 6] ICP22 created new session: [2:7577471039304901546:2124]
2025-11-27T18:17:39.115624Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471035637931532:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:39.128886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:39.118432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471033680263248:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.147941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471040576268193:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-11-27T18:17:39.146182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471038353270677:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43573
2025-11-27T18:17:39.130442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471035097350628:2108] [node 5] ICP22 created new session: [4:7577471039392317953:2108]
2025-11-27T18:17:39.134400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471039304901546:2124] [node 6] ICS09 handshake done sender: [2:7577471039304901526:2141] self: [2:7577471039304901525:0] peer: [6:7577471040576268191:0] socket: 271 qp: -1
2025-11-27T18:17:39.118485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:39.122154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471039932898837:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.134428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471039304901546:2124] [node 6] ICS10 traffic start
2025-11-27T18:17:39.146884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471038353270671:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:39.118542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471033680263248:2108] [node 5] ICP22 created new session: [1:7577471037975230576:2108]
2025-11-27T18:17:39.149984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471040576268189:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:39.132423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471039392317946:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.122191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471039932898837:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.149491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471034058303356:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.134455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:39.151681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471040576268188:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:39.118594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471037975230576:2108] [node 5] ICS09 handshake done sender: [1:7577471033680263251:2111] self: [1:7577471033680263250:0] peer: [5:7577471038353270672:0] socket: 259 qp: -1
2025-11-27T18:17:39.132540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471039392317953:2108] [node 5] ICS09 handshake done sender: [4:7577471039392317937:2123] self: [4:7577471039392317936:0] peer: [5:7577471038353270676:0] socket: 290 qp: -1
2025-11-27T18:17:39.122534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471035637931528:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.135958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:39.149528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:39.122684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:39.151855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471040576268194:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.118628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471037975230576:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:39.136720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471035009934217:2115] [node 5] ICP22 created new session: [2:7577471039304901548:2115]
2025-11-27T18:17:39.149600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471038353270662:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:39.132583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471039392317953:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:39.118661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:39.132631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:39.137582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471035097350557:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.151898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471036281300809:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.128189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471035637931528:2108] [node 5] ICP22 created new session: [3:7577471039932898855:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.137804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471039304901548:2115] [node 5] ICS09 handshake done sender: [2:7577471035009934225:2140] self: [2:7577471035009934224:0] peer: [5:7577471038353270674:0] socket: 267 qp: -1
2025-11-27T18:17:39.141222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471039392317956:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4460
2025-11-27T18:17:39.149693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471038353270665:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:39.126677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471033680263258:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.137848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471039304901548:2115] [node 5] ICS10 traffic start
2025-11-27T18:17:39.146865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471035097350628:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17222718ns) {HandleHandshakeStatus:329(16507282ns) {DropIncomingHandshake:442(15587ns) DropOutgoingHandshake:457(3049ns) SwitchToState:179(578ns) ProcessPendingSessionEvents:495(6527482ns) {ForwardSessionEventToSession:612(877750ns) ForwardSessionEventToSession:612(10525ns) ForwardSessionEventToSession:612(181678ns) ForwardSessionEventToSession:612(3120ns) ForwardSessionEventToSession:612(2632ns) ForwardSessionEventToSession:612(2010366ns) ForwardSessionEventToSession:612(4639ns) ForwardSessionEventToSession:612(3413511ns) ForwardSessionEventToSession:612(4964ns) ForwardSessionEventToSession:612(2932ns)}}}
2025-11-27T18:17:39.151934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:39.150013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471034058303356:2108] [node 4] ICP22 created new session: [5:7577471038353270682:2108]
2025-11-27T18:17:39.130438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471039932898837:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:39.137886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:39.154462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471036281300809:2049] [node 2] ICP22 created new session: [6:7577471040576268196:2049]
2025-11-27T18:17:39.132640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471039932898855:2108] [node 5] ICS09 handshake done sender: [3:7577471035637931532:2110] self: [3:7577471035637931531:0] peer: [5:7577471038353270673:0] socket: 276 qp: -1
2025-11-27T18:17:39.140200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471039304901531:2144] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.151082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471038353270682:2108] [node 4] ICS09 handshake done sender: [5:7577471038353270671:2119] self: [5:7577471038353270676:0] peer: [4:7577471039392317936:0] socket: 306 qp: -1
2025-11-27T18:17:39.154980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471040576268196:2049] [node 2] ICS09 handshake done sender: [6:7577471040576268188:2117] self: [6:7577471040576268191:0] peer: [2:7577471039304901525:0] socket: 291 qp: -1
2025-11-27T18:17:39.140231Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471039304901531:2144] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.126716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471033680263258:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.155016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471040576268196:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:39.132681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471039932898855:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:39.151808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471038353270682:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:39.141439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471039304901542:2147] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.154401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:39.129171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471033680263260:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:39.148213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471035097350556:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16179) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471039392317945:2126] held: no
2025-11-27T18:17:39.154918Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471038353270661:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:39.132721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:39.136467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471033680263258:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:39.157872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471034058303295:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.149862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471039392317945:2126] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:39.158073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:39.152240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471035097350556:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.137839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471035637931529:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.158896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471034058303295:2050] [node 3] ICP22 created new session: [5:7577471038353270684:2050]
2025-11-27T18:17:39.155053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:39.152276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:39.137864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:39.159993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471034058303294:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.137907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471035637931529:2109] [node 6] ICP22 created new session: [3:7577471039932898859:2109]
2025-11-27T18:17:39.160201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471038353270684:2050] [node 3] ICS09 handshake done sender: [5:7577471038353270665:2117] self: [5:7577471038353270673:0] peer: [3:7577471035637931531:0] socket: 288 qp: -1
2025-11-27T18:17:39.155277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471036281300810:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.152323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471035097350556:2048] [node 1] ICP22 created new session: [4:7577471039392317959:2048]
2025-11-27T18:17:39.160236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471038353270684:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:39.155320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:39.139075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471039932898859:2109] [node 6] ICS09 handshake done sender: [3:7577471039932898837:2115] self: [3:7577471039932898833:0] peer: [6:7577471040576268190:0] socket: 315 qp: -1
2025-11-27T18:17:39.152358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471039392317959:2048] [node 1] ICS09 handshake done sender: [4:7577471039392317945:2126] self: [4:7577471039392317950:0] peer: [1:7577471033680263259:0] socket: 303 qp: -1
2025-11-27T18:17:39.140078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471039932898859:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:39.160337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:39.157367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471036281300810:2050] [node 3] ICP22 created new session: [6:7577471040576268198:2050]
2025-11-27T18:17:39.152388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471039392317959:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:39.140104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:39.160533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:39.158046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471040576268198:2050] [node 3] ICS09 handshake done sender: [6:7577471040576268189:2118] self: [6:7577471040576268190:0] peer: [3:7577471039932898833:0] socket: 266 qp: -1
2025-11-27T18:17:39.145867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471039932898858:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58943
2025-11-27T18:17:39.152425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:39.160685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471034058303294:2049] [node 2] ICP22 created new session: [5:7577471038353270685:2049]
2025-11-27T18:17:39.158092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471040576268198:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:39.157351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471035097350557:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25677) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471039392317946:2127] held: no
2025-11-27T18:17:39.151378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471035637931528:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(23344078ns) {HandleHandshakeStatus:329(23341161ns) {DropIncomingHandshake:442(212ns) DropOutgoingHandshake:457(105055ns) SwitchToState:179(332ns) ProcessPendingSessionEvents:495(7638131ns) {ForwardSessionEventToSession:612(2826808ns) ForwardSessionEventToSession:612(12773ns) ForwardSessionEventToSession:612(425594ns) ForwardSessionEventToSession:612(2758193ns) ForwardSessionEventToSession:612(6646ns) ForwardSessionEventToSession:612(386130ns) ForwardSessionEventToSession:612(2392ns) ForwardSessionEventToSession:612(1195666ns) ForwardSessionEventToSession:612(4048ns) ForwardSessionEventToSession:612(2397ns)}}}
2025-11-27T18:17:39.160998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471038353270685:2049] [node 2] ICS09 handshake done sender: [5:7577471038353270662:2114] self: [5:7577471038353270674:0] peer: [2:7577471035009934224:0] socket: 282 qp: -1
2025-11-27T18:17:39.159130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:39.161620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471039392317958:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4460
2025-11-27T18:17:39.154963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471039932898849:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:39.161792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471038353270685:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:39.163342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471036281300871:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.163149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471039392317946:2127] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:39.161991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:39.157325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471035637931465:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16179) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471039932898849:2118] held: no
2025-11-27T18:17:39.136564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471033680263243:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.145780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471039304901531:2144] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:39.162138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471034058303293:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.158040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471035637931465:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.136765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.162192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:39.159082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:39.137199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471033680263243:2107] [node 4] ICP22 created new session: [1:7577471037975230580:2107]
2025-11-27T18:17:39.160000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471035637931465:2048] [node 1] ICP22 created new session: [3:7577471039932898862:2048]
2025-11-27T18:17:39.138015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471037975230580:2107] [node 4] ICS09 handshake done sender: [1:7577471033680263260:2114] self: [1:7577471033680263259:0] peer: [4:7577471039392317950:0] socket: 302 qp: -1
2025-11-27T18:17:39.162230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471034058303293:2048] [node 1] ICP22 created new session: [5:7577471038353270689:2048]
2025-11-27T18:17:39.160667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471039932898862:2048] [node 1] ICS09 handshake done sender: [3:7577471039932898849:2118] self: [3:7577471039932898854:0] peer: [1:7577471033680263257:0] socket: 309 qp: -1
2025-11-27T18:17:39.163060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471038353270689:2048] [node 1] ICS09 handshake done sender: [5:7577471038353270661:2113] self: [5:7577471038353270672:0] peer: [1:7577471033680263250:0] socket: 280 qp: -1
2025-11-27T18:17:39.163102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471038353270689:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:39.149691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471035009934216:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.163708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471035097350557:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.160933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471039932898862:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:39.138460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471037975230580:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:39.163135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:39.150134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:39.163869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.138507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:39.164378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471035097350557:2049] [node 2] ICP22 created new session: [4:7577471039392317961:2049]
2025-11-27T18:17:39.160965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:39.151039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471035009934216:2107] [node 4] ICP22 created new session: [2:7577471039304901551:2107]
2025-11-27T18:17:39.144674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471033680263183:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.151355Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471035009934217:2115] [node 5] ICP35 event processing took too much time PendingConnection:279(16298470ns) {HandleHandshakeStatus:329(16295662ns) {DropIncomingHandshake:442(819ns) DropOutgoingHandshake:457(1075ns) SwitchToState:179(642ns) ProcessPendingSessionEvents:495(5776623ns) {ForwardSessionEventToSession:612(3413183ns) ForwardSessionEventToSession:612(6977ns) ForwardSessionEventToSession:612(291795ns) ForwardSessionEventToSession:612(3265ns) ForwardSessionEventToSession:612(1310280ns) ForwardSessionEventToSession:612(4821ns) ForwardSessionEventToSession:612(3366ns) ForwardSessionEventToSession:612(225634ns) ForwardSessionEventToSession:612(493447ns) ForwardSessionEventToSession:612(6192ns)}}}
2025-11-27T18:17:39.164543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471039392317939:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.164561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471039932898861:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58943
2025-11-27T18:17:39.144712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:39.166050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471035637931529:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:39.144740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471033680263183:2049] [node 3] ICP22 created new session: [1:7577471037975230582:2049]
2025-11-27T18:17:39.166737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471039932898864:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58943
2025-11-27T18:17:39.164566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471039392317939:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.154149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471039304901551:2107] [node 4] ICS09 handshake done sender: [2:7577471039304901531:2144] self: [2:7577471039304901530:0] peer: [4:7577471039392317957:0] socket: 301 qp: -1
2025-11-27T18:17:39.164712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471039392317961:2049] [node 2] ICS09 handshake done sender: [4:7577471039392317946:2127] self: [4:7577471039392317957:0] peer: [2:7577471039304901530:0] socket: 307 qp: -1
2025-11-27T18:17:39.154363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471039304901551:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:39.147658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471037975230582:2049] [node 3] ICS09 handshake done sender: [1:7577471033680263258:2113] self: [1:7577471033680263257:0] peer: [3:7577471039932898854:0] socket: 269 qp: -1
2025-11-27T18:17:39.154914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:39.165267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471039392317961:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:39.160120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471035009934217:2115] [node 5] ICP18 handshake (actor [2:7577471039304901542:2147]) is too old
2025-11-27T18:17:39.165307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:39.168828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471039392317939:2124] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.148026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471037975230582:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:39.148076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:39.169436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471040576268201:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-11-27T18:17:39.169533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471035097350629:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.169609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:39.169679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471035097350629:2110] [node 6] ICP22 created new session: [4:7577471039392317964:2110]
2025-11-27T18:17:39.169826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471039392317964:2110] [node 6] ICS09 handshake done sender: [4:7577471039392317939:2124] self: [4:7577471039392317938:0] peer: [6:7577471040576268200:0] socket: 327 qp: -1
2025-11-27T18:17:39.169833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471040576268202:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-11-27T18:17:39.169873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471039392317964:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:39.169918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:39.170381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471040576268194:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:39.171141Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471036281300871:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27665) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471040576268194:2121] held: no
2025-11-27T18:17:39.171459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471036281300871:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.171506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:39.171545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471036281300871:2107] [node 4] ICP22 created new session: [6:7577471040576268203:2107]
2025-11-27T18:17:39.171641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471040576268203:2107] [node 4] ICS09 handshake done sender: [6:7577471040576268194:2121] self: [6:7577471040576268200:0] peer: [4:7577471039392317938:0] socket: 328 qp: -1
2025-11-27T18:17:39.171680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471040576268203:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:39.171727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:39.171888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471040576268201:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.172045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471036281300868:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:39.172260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471038353270667:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:39.172304Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471038353270667:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:39.173071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471038353270667:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.173226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471040576268206:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-11-27T18:17:39.173288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471034058303357:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:39.173455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:39.173500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471040576268201:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:39.173511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471034058303357:2109] [node 6] ICP22 created new session: [5:7577471038353270692:2109]
2025-11-27T18:17:39.173625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471038353270692:2109] [node 6] ICS09 handshake done sender: [5:7577471038353270667:2118] self: [5:7577471038353270666:0] peer: [6:7577471040576268205:0] socket: 265 qp: -1
2025-11-27T18:17:39.173653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471038353270692:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:39.173752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471036281300868:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:39.173826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:39.173835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:39.173957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471036281300868:2106] [node 5] ICP22 created new session: [6:7577471040576268207:2106]
2025-11-27T18:17:39.174013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471040576268207:2106] [node 5] ICS09 handshake done sender: [6:7577471040576268201:2122] self: [6:7577471040576268205:0] peer: [5:7577471038353270666:0] socket: 279 qp: -1
2025-11-27T18:17:39.174812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471040576268207:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:39.174930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:39.209868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:39.210172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471039304901535:2048] [node 1] ICS01 socket: 262 reason# EndOfStream
2025-11-27T18:17:39.210354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471035009934126:2048] [node 1] ICP30 unregister session Session# [2:7577471039304901535:2048] VirtualId# [2:7577471039304901527:0]
2025-11-27T18:17:39.210419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471039304901535:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.210528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:39.210577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:39.212569Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:39.212795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471039932898862:2048] [node 1] ICS01 socket: 309 reason# EndOfStream
2025-11-27T18:17:39.212840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471035637931465:2048] [node 1] ICP30 unregister session Session# [3:7577471039932898862:2048] VirtualId# [3:7577471039932898854:0]
2025-11-27T18:17:39.212874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471039932898862:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.212941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:39.212982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:39.216441Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:39.216661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471038353270689:2048] [node 1] ICS01 socket: 280 reason# EndOfStream
2025-11-27T18:17:39.216716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471034058303293:2048] [node 1] ICP30 unregister session Session# [5:7577471038353270689:2048] VirtualId# [5:7577471038353270672:0]
2025-11-27T18:17:39.216749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471038353270689:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.216822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:39.216868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:39.217406Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:39.217404Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:39.217617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471040576268185:2048] [node 1] ICS01 socket: 263 reason# EndOfStream
2025-11-27T18:17:39.217600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471039392317959:2048] [node 1] ICS01 socket: 303 reason# EndOfStream
2025-11-27T18:17:39.217640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471035097350556:2048] [node 1] ICP30 unregister session Session# [4:7577471039392317959:2048] VirtualId# [4:7577471039392317950:0]
2025-11-27T18:17:39.217659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471036281300808:2048] [node 1] ICP30 unregister session Session# [6:7577471040576268185:2048] VirtualId# [6:7577471040576268182:0]
2025-11-27T18:17:39.217681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471039392317959:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.217752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:39.217696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471040576268185:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.217790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:39.217759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:39.217805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:39.275116Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:39.269278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:39.269531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471039932898850:2049] [node 2] ICS07 socket disconnect 275 reason# EndOfStream
2025-11-27T18:17:39.269568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471039932898850:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.269668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:39.269717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471039932898850:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:39.272247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471035637931466:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25677) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:39.272329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471039932898850:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:39.272370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471035637931466:2049] [node 2] ICP30 unregister session Session# [3:7577471039932898850:2049] VirtualId# [3:7577471039932898841:0]
2025-11-27T18:17:39.272536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:39.272569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471035637931466:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25677) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:39.272581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25677) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:39.275345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471039392317961:2049] [node 2] ICS01 socket: 307 reason# EndOfStream
2025-11-27T18:17:39.276710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:39.276927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471038353270685:2049] [node 2] ICS01 socket: 282 reason# EndOfStream
2025-11-27T18:17:39.275421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471035097350557:2049] [node 2] ICP30 unregister session Session# [4:7577471039392317961:2049] VirtualId# [4:7577471039392317957:0]
2025-11-27T18:17:39.275463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471039392317961:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.275550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:39.278681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471034058303294:2049] [node 2] ICP30 unregister session Session# [5:7577471038353270685:2049] VirtualId# [5:7577471038353270674:0]
2025-11-27T18:17:39.275638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:39.278740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471038353270685:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.278830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:39.278900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:39.281418Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:39.281657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471040576268196:2049] [node 2] ICS01 socket: 291 reason# EndOfStream
2025-11-27T18:17:39.281705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471036281300809:2049] [node 2] ICP30 unregister session Session# [6:7577471040576268196:2049] VirtualId# [6:7577471040576268191:0]
2025-11-27T18:17:39.281744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471040576268196:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.281837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:39.281888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:39.285729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471035637931466:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:39.337157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:39.337356Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:39.337360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:39.337464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471039392317943:2050] [node 3] ICS01 socket: 289 reason# EndOfStream
2025-11-27T18:17:39.337511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471035097350558:2050] [node 3] ICP30 unregister session Session# [4:7577471039392317943:2050] VirtualId# [4:7577471039392317940:0]
2025-11-27T18:17:39.337551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471039392317943:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.337573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471038353270684:2050] [node 3] ICS07 socket disconnect 288 reason# EndOfStream
2025-11-27T18:17:39.337579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471040576268198:2050] [node 3] ICS07 socket disconnect 266 reason# EndOfStream
2025-11-27T18:17:39.337606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471038353270684:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.337616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471040576268198:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.337653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:39.337701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:39.337695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:39.337710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:39.337748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471038353270684:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:39.337756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471040576268198:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:39.340050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471034058303295:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16358) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:39.340119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471038353270684:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:39.340166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471034058303295:2050] [node 3] ICP30 unregister session Session# [5:7577471038353270684:2050] VirtualId# [5:7577471038353270673:0]
2025-11-27T18:17:39.340237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:39.340335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471034058303295:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16358) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:39.340404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16358) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:39.340514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471036281300810:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16358) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:39.340598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471040576268198:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:39.340718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471036281300810:2050] [node 3] ICP30 unregister session Session# [6:7577471040576268198:2050] VirtualId# [6:7577471040576268190:0]
2025-11-27T18:17:39.340775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:39.340822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471036281300810:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16358) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:39.340849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16358) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:39.350827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471036281300810:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:39.353453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471034058303295:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:39.427256Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:39.427505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471040576268203:2107] [node 4] ICS01 socket: 328 reason# EndOfStream
2025-11-27T18:17:39.427564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471036281300871:2107] [node 4] ICP30 unregister session Session# [6:7577471040576268203:2107] VirtualId# [6:7577471040576268200:0]
2025-11-27T18:17:39.427609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471040576268203:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.427740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:39.427804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:39.429661Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:39.430043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471038353270682:2108] [node 4] ICS01 socket: 306 reason# EndOfStream
2025-11-27T18:17:39.430106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471034058303356:2108] [node 4] ICP30 unregister session Session# [5:7577471038353270682:2108] VirtualId# [5:7577471038353270676:0]
2025-11-27T18:17:39.430142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471038353270682:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.430253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:39.430345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:39.507311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:39.507592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471040576268207:2106] [node 5] ICS07 socket disconnect 279 reason# EndOfStream
2025-11-27T18:17:39.507637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471040576268207:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:39.507723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:39.507794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471040576268207:2106] [node 5] ICS15 start handshake
2025-11-27T18:17:39.509027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471036281300868:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:39.509122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471040576268207:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:39.509183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471036281300868:2106] [node 5] ICP30 unregister session Session# [6:7577471040576268207:2106] VirtualId# [6:7577471040576268205:0]
2025-11-27T18:17:39.509236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:17:39.509289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471036281300868:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:39.509304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:39.598910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471040642105366:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:39.598964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471040642105367:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:39.621934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471041383705140:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:39.621994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471041383705141:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:39.646596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471040649874678:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:39.646625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471040649874677:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:39.667134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471041854140107:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:39.667172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471041854140108:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:39.667206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471041854140109:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:39.692673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471038845176033:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:39.692732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471038845176035:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:39.694114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471038845176034:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:39.713096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471040269709077:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:39.713150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471040269709078:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:39.713176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471040269709079:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:39.893396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471038845176095:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:39.898053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471040269709141:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:39.891073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471041383705205:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:39.897822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471040642105432:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:39.897867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471041854140173:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:39.902565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471040649874740:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:39.915082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471041383705206:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:39.909395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471041854140174:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:39.905819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471040642105433:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:39.916827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471040649874678:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(117599092ns) {Configure:104(110566276ns) {ProcessConfigured:135(72172ns) {StartInitialHandshake:157(69661ns) {DropHandshakes:470(3085ns) {DropIncomingHandshake:442(891ns) DropOutgoingHandshake:457(403ns)} GenerateSessionVirtualId:988(956ns) PrepareNewSessionHandshake:477(2022ns) {SwitchToState:179(764ns)}}}}}
2025-11-27T18:17:39.921619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471038845176096:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:39.925036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471040269709142:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:39.909749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471040642105434:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:39.918547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471040649874741:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:39.921925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471040649874744:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:39.929339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471041383705208:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:39.937434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471038845176035:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(130845381ns) {Configure:104(130837963ns) {ProcessConfigured:135(46241ns) {StartInitialHandshake:157(44643ns) {DropHandshakes:470(1362ns) {DropIncomingHandshake:442(326ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(486ns) PrepareNewSessionHandshake:477(961ns) {SwitchToState:179(265ns)}}}}}
2025-11-27T18:17:39.943994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471040642105433:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18483184ns) {Configure:104(18476826ns) {ProcessConfigured:135(484605ns) {StartInitialHandshake:157(477566ns) {DropHandshakes:470(3012ns) {DropIncomingHandshake:442(832ns) DropOutgoingHandshake:457(485ns)} GenerateSessionVirtualId:988(721ns) PrepareNewSessionHandshake:477(1597ns) {SwitchToState:179(531ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.950737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471041854140173:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24711807ns) {Configure:104(24705647ns) {ProcessConfigured:135(41780ns) {StartInitialHandshake:157(37834ns) {DropHandshakes:470(746ns) {DropIncomingHandshake:442(220ns) DropOutgoingHandshake:457(98ns)} GenerateSessionVirtualId:988(279ns) PrepareNewSessionHandshake:477(1945ns) {SwitchToState:179(470ns)}}}}}
2025-11-27T18:17:39.952864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471041854140180:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9493
2025-11-27T18:17:39.952925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471041854140181:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9493
2025-11-27T18:17:39.965359Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471040649874677:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(148425640ns) {Configure:104(148420693ns) {ProcessConfigured:135(7392673ns) {StartInitialHandshake:157(7390830ns) {DropHandshakes:470(1405ns) {DropIncomingHandshake:442(427ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:477(7043ns) {SwitchToState:179(2272ns)}}}}}
2025-11-27T18:17:39.982227Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471041383705206:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(50110651ns) {Configure:104(50104463ns) {ProcessConfigured:135(66904ns) {StartInitialHandshake:157(64722ns) {DropHandshakes:470(1601ns) {DropIncomingHandshake:442(518ns) DropOutgoingHandshake:457(231ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:477(2079ns) {SwitchToState:179(718ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.982771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471040269709077:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(80145324ns) {Configure:104(80139651ns) {ProcessConfigured:135(8564534ns) {StartInitialHandshake:157(8562548ns) {DropHandshakes:470(3137ns) {DropIncomingHandshake:442(1033ns) DropOutgoingHandshake:457(382ns)} GenerateSessionVirtualId:988(1192ns) PrepareNewSessionHandshake:477(2762ns) {SwitchToState:179(1553ns)}}}}}
2025-11-27T18:17:39.947953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471040642105367:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(146928663ns) {Configure:104(146922368ns) {ProcessConfigured:135(3406395ns) {StartInitialHandshake:157(3402578ns) {DropHandshakes:470(3809ns) {DropIncomingHandshake:442(2064ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(893ns) PrepareNewSessionHandshake:477(4582ns) {SwitchToState:179(2217ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.969587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471041854140107:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(78620373ns) {Configure:104(78613435ns) {ProcessConfigured:135(19324688ns) {StartInitialHandshake:157(19321882ns) {DropHandshakes:470(1320ns) {DropIncomingHandshake:442(399ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(602ns) PrepareNewSessionHandshake:477(4308ns) {SwitchToState:179(2014ns)}}}}}
2025-11-27T18:17:39.984899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471040269709149:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.992002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471040642105432:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(58730921ns) {Configure:104(58723354ns) {ProcessConfigured:135(113038ns) {StartInitialHandshake:157(110666ns) {DropHandshakes:470(5753ns) {DropIncomingHandshake:442(555ns) DropOutgoingHandshake:457(425ns)} GenerateSessionVirtualId:988(624ns) PrepareNewSessionHandshake:477(2337ns) {SwitchToState:179(749ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.999749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471040649874751:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-11-27T18:17:39.978069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471041854140181:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.993300Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471041383705205:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(64426040ns) {Configure:104(64420459ns) {ProcessConfigured:135(51074ns) {StartInitialHandshake:157(49112ns) {DropHandshakes:470(3094ns) {DropIncomingHandshake:442(302ns) DropOutgoingHandshake:457(209ns)} GenerateSessionVirtualId:988(496ns) PrepareNewSessionHandshake:477(533ns) {SwitchToState:179(96ns)}}}}}
2025-11-27T18:17:39.995692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471038845176106:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50557
2025-11-27T18:17:39.989641Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471041854140109:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(83743785ns) {Configure:104(83725443ns) {ProcessConfigured:135(11412305ns) {StartInitialHandshake:157(11409314ns) {DropHandshakes:470(1224ns) {DropIncomingHandshake:442(348ns) DropOutgoingHandshake:457(266ns)} GenerateSessionVirtualId:988(1739ns) PrepareNewSessionHandshake:477(4209ns) {SwitchToState:179(2237ns)}}}}}
2025-11-27T18:17:39.992353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471040642105438:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.002307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471040269709079:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(83834100ns) {Configure:104(83826431ns) {ProcessConfigured:135(24618069ns) {StartInitialHandshake:157(24614735ns) {DropHandshakes:470(3496ns) {DropIncomingHandshake:442(832ns) DropOutgoingHandshake:457(400ns)} GenerateSessionVirtualId:988(855ns) PrepareNewSessionHandshake:477(2059ns) {SwitchToState:179(154ns)}}}}}
2025-11-27T18:17:40.001900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471038845176033:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(88293322ns) {Configure:104(88283948ns) {ProcessConfigured:135(36710826ns) {StartInitialHandshake:157(36708121ns) {DropHandshakes:470(2926ns) {DropIncomingHandshake:442(811ns) DropOutgoingHandshake:457(426ns)} GenerateSessionVirtualId:988(727ns) PrepareNewSessionHandshake:477(3113ns) {SwitchToState:179(1911ns)}}}}}
2025-11-27T18:17:40.002901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471040649874752:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-11-27T18:17:40.002674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471038845176106:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.003710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471040649874741:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(72779225ns) {Configure:104(72774166ns) {ProcessConfigured:135(9891807ns) {StartInitialHandshake:157(9890031ns) {DropHandshakes:470(1500ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(384ns)} GenerateSessionVirtualId:988(549ns) PrepareNewSessionHandshake:477(3481ns) {SwitchToState:179(1509ns)}}}}}
2025-11-27T18:17:40.003316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471041383705141:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(93400580ns) {Configure:104(93392665ns) {ProcessConfigured:135(5502638ns) {StartInitialHandshake:157(5500657ns) {DropHandshakes:470(1302ns) {DropIncomingHandshake:442(344ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(509ns) PrepareNewSessionHandshake:477(2496ns) {SwitchToState:179(985ns)}}}}}
2025-11-27T18:17:39.990399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471041854140107:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:39.992388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471040642105438:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.003551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471040269709078:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(70331243ns) {Configure:104(70323264ns) {ProcessConfigured:135(42065933ns) {StartInitialHandshake:157(42062481ns) {DropHandshakes:470(615ns) {DropIncomingHandshake:442(118ns) DropOutgoingHandshake:457(101ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:477(3363ns) {SwitchToState:179(1530ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.003597Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471038845176034:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(82866604ns) {Configure:104(82858247ns) {ProcessConfigured:135(30248683ns) {StartInitialHandshake:157(30246574ns) {DropHandshakes:470(1362ns) {DropIncomingHandshake:442(385ns) DropOutgoingHandshake:457(157ns)} GenerateSessionVirtualId:988(476ns) PrepareNewSessionHandshake:477(2132ns) {SwitchToState:179(1005ns)}}}}}
2025-11-27T18:17:40.006787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471038845176096:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.003625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471041383705140:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(66040914ns) {Configure:104(66033714ns) {ProcessConfigured:135(1352746ns) {StartInitialHandshake:157(1350555ns) {DropHandshakes:470(1951ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(695ns)} GenerateSessionVirtualId:988(522ns) PrepareNewSessionHandshake:477(1516ns) {SwitchToState:179(578ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.006577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471040649874751:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.006844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471038845176095:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(76316640ns) {Configure:104(76310285ns) {ProcessConfigured:135(3296516ns) {StartInitialHandshake:157(3294520ns) {DropHandshakes:470(2605ns) {DropIncomingHandshake:442(572ns) DropOutgoingHandshake:457(485ns)} GenerateSessionVirtualId:988(460ns) PrepareNewSessionHandshake:477(1976ns) {SwitchToState:179(726ns)}}}}}
2025-11-27T18:17:39.990622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471041854140190:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9493
2025-11-27T18:17:40.003557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471040269709149:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.004939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471040642105447:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3144
2025-11-27T18:17:40.006925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471040649874752:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.005390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471040642105447:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.021670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471041383705207:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57918
2025-11-27T18:17:40.012903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471040642105438:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:40.014131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471043140143409:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50557
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.000159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471041854140180:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.021431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471040269709141:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(79113418ns) {Configure:104(79104523ns) {ProcessConfigured:135(3547735ns) {StartInitialHandshake:157(3262181ns) {DropHandshakes:470(1200ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(1682ns) {SwitchToState:179(128ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.022432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471040642105433:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.026913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471038845176096:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5413) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471038845176106:2112] held: no
2025-11-27T18:17:40.012672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471040649874744:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(76264086ns) {Configure:104(76258086ns) {ProcessConfigured:135(1595553ns) {StartInitialHandshake:157(1593509ns) {DropHandshakes:470(1299ns) {DropIncomingHandshake:442(284ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(271ns) PrepareNewSessionHandshake:477(1935ns) {SwitchToState:179(109ns)}}}}}
2025-11-27T18:17:40.025631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471040269709142:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.048560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471044564676455:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-27T18:17:40.002833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471041854140108:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(88029277ns) {Configure:104(83567370ns) {ProcessConfigured:135(11897224ns) {StartInitialHandshake:157(11895135ns) {DropHandshakes:470(666ns) {DropIncomingHandshake:442(200ns) DropOutgoingHandshake:457(131ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(2569ns) {SwitchToState:179(1390ns)}}}}}
2025-11-27T18:17:40.032751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471041383705207:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.023441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471040649874677:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.012905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471041854140179:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.027898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471038845176106:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:40.022485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:40.045092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471041383705140:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.012933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471041854140179:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.032374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471038845176096:2107] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.023927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471040642105446:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3144
2025-11-27T18:17:40.050879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471040269709149:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:40.041987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471044944842056:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-11-27T18:17:40.014466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471041854140179:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:40.014778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471041854140174:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.014859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:40.014911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471041854140174:2108] [node 5] ICP22 created new session: [4:7577471046149107492:2108]
2025-11-27T18:17:40.014968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471046149107492:2108] [node 5] ICS09 handshake done sender: [4:7577471041854140179:2114] self: [4:7577471041854140178:0] peer: [5:7577471043140143408:0] socket: 274 qp: -1
2025-11-27T18:17:40.015001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471046149107492:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:40.015040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:40.030678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471046149107489:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9493
2025-11-27T18:17:40.032291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471041854140177:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.032336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471041854140177:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.034093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471041854140177:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:40.035196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471041854140173:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.037227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:40.024252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471040642105436:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.032416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:40.052405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471041383705217:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57918
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.059474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471044944842058:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-11-27T18:17:40.062272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471040269709142:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:40.042027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471041854140173:2109] [node 6] ICP22 created new session: [4:7577471046149107497:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.062066Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471041383705208:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(58539509ns) {Configure:104(58533601ns) {ProcessConfigured:135(6883833ns) {StartInitialHandshake:157(6880507ns) {DropHandshakes:470(2858ns) {DropIncomingHandshake:442(690ns) DropOutgoingHandshake:457(408ns)} GenerateSessionVirtualId:988(696ns) PrepareNewSessionHandshake:477(5240ns) {SwitchToState:179(2408ns)}}}}}
2025-11-27T18:17:40.043188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471038845176096:2107] [node 4] ICP22 created new session: [5:7577471043140143410:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.059605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471044944842059:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-11-27T18:17:40.024279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471040642105436:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.042115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471046149107490:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9493
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.068150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471044564676454:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-27T18:17:40.074664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471045678672515:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57918
2025-11-27T18:17:40.047787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471043140143410:2107] [node 4] ICS09 handshake done sender: [5:7577471038845176106:2112] self: [5:7577471043140143408:0] peer: [4:7577471041854140178:0] socket: 275 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.069930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:40.050108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471046149107497:2109] [node 6] ICS09 handshake done sender: [4:7577471041854140177:2113] self: [4:7577471041854140176:0] peer: [6:7577471044564676452:0] socket: 273 qp: -1
2025-11-27T18:17:40.070378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471040649874752:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.025186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471040642105436:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:40.076107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471041383705217:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.047833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471043140143410:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:40.053279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471046149107497:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:40.070581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471040269709142:2109] [node 4] ICP22 created new session: [6:7577471044564676456:2109]
2025-11-27T18:17:40.025530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471040642105433:2108] [node 4] ICP22 created new session: [1:7577471044937072746:2108]
2025-11-27T18:17:40.074055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471044944842058:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.078823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471040649874677:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.080733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471045678672521:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57918
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.047877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:40.053563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:40.073944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471044564676454:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.082672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:40.082696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471045678672515:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.026891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471040642105445:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3144
2025-11-27T18:17:40.055181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471041854140190:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.074754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471044564676456:2109] [node 4] ICS09 handshake done sender: [6:7577471040269709149:2113] self: [6:7577471044564676452:0] peer: [4:7577471041854140176:0] socket: 281 qp: -1
2025-11-27T18:17:40.082490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471043140143413:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50557
2025-11-27T18:17:40.083390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471040649874677:2048] [node 1] ICP22 created new session: [3:7577471044944842060:2048]
2025-11-27T18:17:40.027943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471044937072746:2108] [node 4] ICS09 handshake done sender: [1:7577471040642105438:2112] self: [1:7577471040642105437:0] peer: [4:7577471041854140191:0] socket: 270 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.071832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471041854140173:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(37303337ns) {HandleHandshakeStatus:329(37300369ns) {DropIncomingHandshake:442(2166ns) DropOutgoingHandshake:457(545ns) SwitchToState:179(756ns) ProcessPendingSessionEvents:495(1707242ns) {ForwardSessionEventToSession:612(1656351ns) ForwardSessionEventToSession:612(8812ns) ForwardSessionEventToSession:612(2957ns) ForwardSessionEventToSession:612(2224ns) ForwardSessionEventToSession:612(5442ns) ForwardSessionEventToSession:612(2189ns) ForwardSessionEventToSession:612(4204ns) ForwardSessionEventToSession:612(4962ns) ForwardSessionEventToSession:612(2668ns) ForwardSessionEventToSession:612(5253ns)}}}
2025-11-27T18:17:40.030264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471044937072746:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:40.087680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471045678672521:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.075410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471040269709077:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.090699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471043140143413:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.085340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471044944842057:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-11-27T18:17:40.089148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471045678672522:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57918
2025-11-27T18:17:40.030312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:40.072235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471041854140173:2109] [node 6] ICP18 handshake (actor [4:7577471041854140190:2120]) is too old
2025-11-27T18:17:40.076113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471044564676456:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:40.096863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471038845176033:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.086935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471044944842060:2048] [node 1] ICS09 handshake done sender: [3:7577471040649874752:2114] self: [3:7577471044944842055:0] peer: [1:7577471040642105435:0] socket: 284 qp: -1
2025-11-27T18:17:40.040865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471040642105367:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.078310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:40.088540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471044564676460:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-27T18:17:40.093555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471044564676454:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:40.097738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471040269709077:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18444) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471044564676454:2117] held: no
2025-11-27T18:17:40.102415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471040269709077:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:40.102462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:40.102514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471040269709077:2048] [node 1] ICP22 created new session: [6:7577471044564676461:2048]
2025-11-27T18:17:40.102563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471044564676461:2048] [node 1] ICS09 handshake done sender: [6:7577471044564676454:2117] self: [6:7577471044564676457:0] peer: [1:7577471040642105439:0] socket: 304 qp: -1
2025-11-27T18:17:40.102598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471044564676461:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:40.102640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:40.107580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471040269709142:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(30347257ns) {HandleHandshakeStatus:329(28391892ns) {DropIncomingHandshake:442(1035ns) DropOutgoingHandshake:457(14798ns) SwitchToState:179(1287ns) ProcessPendingSessionEvents:495(7340418ns) {ForwardSessionEventToSession:612(4269297ns) ForwardSessionEventToSession:612(7908ns) ForwardSessionEventToSession:612(1127622ns) ForwardSessionEventToSession:612(2808ns) ForwardSessionEventToSession:612(393961ns) ForwardSessionEventToSession:612(4657ns) ForwardSessionEventToSession:612(3096ns) ForwardSessionEventToSession:612(262078ns) ForwardSessionEventToSession:612(606600ns) ForwardSessionEventToSession:612(2874ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.040917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:40.087683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471044944842059:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.090795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471041383705207:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:40.072973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471041854140181:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:40.040972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471040642105367:2049] [node 3] ICP22 created new session: [1:7577471044937072751:2049]
2025-11-27T18:17:40.091154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471045678672524:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57918
2025-11-27T18:17:40.078737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471046149107498:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9493
2025-11-27T18:17:40.087969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471044944842060:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:40.116623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471043140143415:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50557
2025-11-27T18:17:40.041011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471044937072751:2049] [node 3] ICS09 handshake done sender: [1:7577471040642105436:2111] self: [1:7577471040642105435:0] peer: [3:7577471044944842055:0] socket: 269 qp: -1
2025-11-27T18:17:40.080669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471041854140107:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:40.088017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.092513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471041383705213:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.082637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.122056Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471038845176033:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18444) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471043140143413:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.041044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471044937072751:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:40.089080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471040649874678:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.092582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471041383705213:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.090110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471044944842057:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.094185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471040649874746:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.094214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471040649874746:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.108217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471040649874746:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:40.114835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471040649874677:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(28705741ns) {HandleHandshakeStatus:329(28703273ns) {DropIncomingHandshake:442(2810ns) DropOutgoingHandshake:457(5547ns) SwitchToState:179(442ns) ProcessPendingSessionEvents:495(12056540ns) {ForwardSessionEventToSession:612(4332280ns) ForwardSessionEventToSession:612(5513ns) ForwardSessionEventToSession:612(3402ns) ForwardSessionEventToSession:612(789732ns) ForwardSessionEventToSession:612(5828ns) ForwardSessionEventToSession:612(13175ns) ForwardSessionEventToSession:612(3443697ns) ForwardSessionEventToSession:612(3073ns) ForwardSessionEventToSession:612(3434187ns) ForwardSessionEventToSession:612(5569ns)}}}
2025-11-27T18:17:40.115503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471040649874740:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.117055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:40.117506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471040649874740:2107] [node 4] ICP22 created new session: [3:7577471044944842067:2107]
2025-11-27T18:17:40.118986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471044944842067:2107] [node 4] ICS09 handshake done sender: [3:7577471040649874746:2111] self: [3:7577471040649874745:0] peer: [4:7577471046149107501:0] socket: 291 qp: -1
2025-11-27T18:17:40.120134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471044944842067:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:40.120182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:40.122791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471044564676465:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-27T18:17:40.083177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471041854140107:2048] [node 1] ICP22 created new session: [4:7577471046149107500:2048]
2025-11-27T18:17:40.083431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471046149107489:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.086302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471046149107500:2048] [node 1] ICS09 handshake done sender: [4:7577471041854140181:2116] self: [4:7577471041854140191:0] peer: [1:7577471040642105437:0] socket: 272 qp: -1
2025-11-27T18:17:40.086878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471046149107500:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:40.086990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471041854140109:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.087674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:40.041077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:40.043330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471040642105445:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.049547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471040642105446:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.057159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471040642105427:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.057204Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471040642105427:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.067863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471040642105433:2108] [node 4] ICP18 handshake (actor [1:7577471040642105445:2115]) is too old
2025-11-27T18:17:40.069884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471040642105427:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:40.076496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471044937072750:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3144
2025-11-27T18:17:40.076689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471040642105366:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.076722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:40.076754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471040642105366:2048] [node 2] ICP22 created new session: [1:7577471044937072757:2048]
2025-11-27T18:17:40.076784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471044937072757:2048] [node 2] ICS09 handshake done sender: [1:7577471040642105427:2107] self: [1:7577471040642105426:0] peer: [2:7577471045678672519:0] socket: 264 qp: -1
2025-11-27T18:17:40.076809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471044937072757:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:40.076840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:40.077213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471040642105440:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.077245Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471040642105440:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.083191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471040642105440:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:40.085439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471044937072750:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.086823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471044937072754:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3144
2025-11-27T18:17:40.086916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471040642105367:2049] [node 3] ICP18 handshake (actor [1:7577471044937072750:2118]) is too old
2025-11-27T18:17:40.089504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471040642105434:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.089985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471044937072754:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.090425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:40.090844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471040642105434:2110] [node 6] ICP22 created new session: [1:7577471044937072760:2110]
2025-11-27T18:17:40.091532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471040642105366:2048] [node 2] ICP18 handshake (actor [1:7577471044937072754:2119]) is too old
2025-11-27T18:17:40.092803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471044937072760:2110] [node 6] ICS09 handshake done sender: [1:7577471040642105440:2113] self: [1:7577471040642105439:0] peer: [6:7577471044564676457:0] socket: 302 qp: -1
2025-11-27T18:17:40.092842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471044937072760:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:40.092899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:40.106876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471040642105444:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.106928Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471040642105444:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.116575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471040642105444:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:40.117578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471040642105432:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.117624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:40.118658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471040642105432:2109] [node 5] ICP22 created new session: [1:7577471044937072763:2109]
2025-11-27T18:17:40.120732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471044937072763:2109] [node 5] ICS09 handshake done sender: [1:7577471040642105444:2114] self: [1:7577471040642105443:0] peer: [5:7577471043140143414:0] socket: 319 qp: -1
2025-11-27T18:17:40.121059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471044937072763:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:40.121095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:40.093550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471041383705210:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.126684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471043140143418:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50557
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.088779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471046149107498:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.093584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471041383705210:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.089953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471041854140108:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.094431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471041383705140:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18444) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471041383705207:2112] held: no
2025-11-27T18:17:40.096130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471046149107504:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9493
2025-11-27T18:17:40.100846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471041383705140:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.127337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471043140143416:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50557
2025-11-27T18:17:40.106057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471046149107498:2123] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.134352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471044564676465:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.134316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471040649874740:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16299798ns) {HandleHandshakeStatus:329(16297126ns) {DropIncomingHandshake:442(11486ns) DropOutgoingHandshake:457(539ns) SwitchToState:179(224ns) ProcessPendingSessionEvents:495(3608856ns) {ForwardSessionEventToSession:612(3240974ns) ForwardSessionEventToSession:612(323648ns) ForwardSessionEventToSession:612(3435ns) ForwardSessionEventToSession:612(2818ns) ForwardSessionEventToSession:612(6226ns) ForwardSessionEventToSession:612(5188ns) ForwardSessionEventToSession:612(5784ns) ForwardSessionEventToSession:612(3098ns) ForwardSessionEventToSession:612(2481ns) ForwardSessionEventToSession:612(4806ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.136177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471040269709141:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.133118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471043140143416:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.135705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471044944842068:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-11-27T18:17:40.141557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471040649874754:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.107645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471041854140108:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.101146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471041383705210:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:40.134418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471043140143415:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.105066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:40.138507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471044564676466:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-27T18:17:40.105542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471045678672524:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.108274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:40.141594Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471040649874754:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.137707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471040642105432:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(19936002ns) {HandleHandshakeStatus:329(19933003ns) {DropIncomingHandshake:442(11056ns) DropOutgoingHandshake:457(347ns) SwitchToState:179(313ns) ProcessPendingSessionEvents:495(5861652ns) {ForwardSessionEventToSession:612(3343160ns) ForwardSessionEventToSession:612(781461ns) ForwardSessionEventToSession:612(3019ns) ForwardSessionEventToSession:612(576068ns) ForwardSessionEventToSession:612(4216ns) ForwardSessionEventToSession:612(2098ns) ForwardSessionEventToSession:612(398366ns) ForwardSessionEventToSession:612(3610ns) ForwardSessionEventToSession:612(726944ns) ForwardSessionEventToSession:612(9167ns)}}}
2025-11-27T18:17:40.147398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471044564676464:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.149988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471044564676466:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.105776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471041383705140:2048] [node 1] ICP22 created new session: [2:7577471045678672528:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.108806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471041854140107:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16213965ns) {HandleHandshakeStatus:329(16210853ns) {DropIncomingHandshake:442(464ns) DropOutgoingHandshake:457(6031ns) SwitchToState:179(353ns) ProcessPendingSessionEvents:495(6889394ns) {ForwardSessionEventToSession:612(1314439ns) ForwardSessionEventToSession:612(6052ns) ForwardSessionEventToSession:612(1248671ns) ForwardSessionEventToSession:612(10819ns) ForwardSessionEventToSession:612(4037ns) ForwardSessionEventToSession:612(1521042ns) ForwardSessionEventToSession:612(4065ns) ForwardSessionEventToSession:612(2727ns) ForwardSessionEventToSession:612(2753579ns) ForwardSessionEventToSession:612(2771ns)}}}
2025-11-27T18:17:40.142769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471040649874678:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16098) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471044944842059:2120] held: no
2025-11-27T18:17:40.151671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471040269709079:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.135108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471038845176035:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.114675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471041854140108:2049] [node 2] ICP22 created new session: [4:7577471046149107505:2049]
2025-11-27T18:17:40.107591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471045678672528:2048] [node 1] ICS09 handshake done sender: [2:7577471041383705207:2112] self: [2:7577471045678672519:0] peer: [1:7577471040642105426:0] socket: 266 qp: -1
2025-11-27T18:17:40.136764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471043140143417:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50557
2025-11-27T18:17:40.144160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471044944842059:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:40.152132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471044564676468:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-27T18:17:40.114940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471041854140107:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:40.108212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471045678672528:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:40.152497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471044564676464:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.147934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471040649874678:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:40.138169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471043140143407:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.114662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471041383705206:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.114718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:40.115064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:40.117293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471041383705206:2107] [node 4] ICP22 created new session: [2:7577471045678672532:2107]
2025-11-27T18:17:40.156815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471040269709078:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.116746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471046149107505:2049] [node 2] ICS09 handshake done sender: [4:7577471046149107498:2123] self: [4:7577471046149107503:0] peer: [2:7577471041383705209:0] socket: 312 qp: -1
2025-11-27T18:17:40.138215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471043140143407:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.116896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471046149107505:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:40.147987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:40.157730Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471044564676465:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:40.118092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471045678672532:2107] [node 4] ICS09 handshake done sender: [2:7577471041383705210:2113] self: [2:7577471041383705209:0] peer: [4:7577471046149107503:0] socket: 310 qp: -1
2025-11-27T18:17:40.149225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471043140143407:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:40.117186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:40.134339Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471041854140108:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(19060469ns) {HandleHandshakeStatus:329(19057460ns) {DropIncomingHandshake:442(778ns) DropOutgoingHandshake:457(14681ns) SwitchToState:179(926ns) ProcessPendingSessionEvents:495(8596599ns) {ForwardSessionEventToSession:612(2264563ns) ForwardSessionEventToSession:612(482805ns) ForwardSessionEventToSession:612(9039ns) ForwardSessionEventToSession:612(2494ns) ForwardSessionEventToSession:612(2280ns) ForwardSessionEventToSession:612(202307ns) ForwardSessionEventToSession:612(4404461ns) ForwardSessionEventToSession:612(16906ns) ForwardSessionEventToSession:612(1181398ns) ForwardSessionEventToSession:612(4916ns)}}}
2025-11-27T18:17:40.136058Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471041854140109:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471046149107489:2121] held: no
2025-11-27T18:17:40.137392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471046149107507:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9493
2025-11-27T18:17:40.148699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471046149107489:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:40.151713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471041854140109:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:40.152579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:40.152628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471041854140109:2050] [node 3] ICP22 created new session: [4:7577471046149107508:2050]
2025-11-27T18:17:40.161032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471046149107508:2050] [node 3] ICS09 handshake done sender: [4:7577471046149107489:2121] self: [4:7577471046149107501:0] peer: [3:7577471040649874745:0] socket: 292 qp: -1
2025-11-27T18:17:40.161877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471046149107508:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:40.161925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.148038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471040649874678:2049] [node 2] ICP22 created new session: [3:7577471044944842072:2049]
2025-11-27T18:17:40.118230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471041383705140:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19305851ns) {HandleHandshakeStatus:329(19304337ns) {DropIncomingHandshake:442(2354ns) DropOutgoingHandshake:457(462ns) SwitchToState:179(684ns) ProcessPendingSessionEvents:495(757ns)}}
2025-11-27T18:17:40.118383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471041383705213:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:40.161136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471040269709141:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32197) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471044564676465:2121] held: no
2025-11-27T18:17:40.150320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471043140143413:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.148085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471044944842072:2049] [node 2] ICS09 handshake done sender: [3:7577471044944842059:2120] self: [3:7577471044944842063:0] peer: [2:7577471041383705211:0] socket: 321 qp: -1
2025-11-27T18:17:40.118776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471045678672532:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:40.163682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471040269709141:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:40.148121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471044944842072:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:40.164065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:40.153002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471043140143418:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.118813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:40.148151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: 2025-11-27T18:17:40.153054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471038845176033:2048] [node 1] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:17:40.126759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471041383705141:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.166091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471040269709141:2108] [node 5] ICP22 created new session: [6:7577471044564676471:2108]
2025-11-27T18:17:40.168133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471044564676471:2108] [node 5] ICS09 handshake done sender: [6:7577471044564676465:2121] self: [6:7577471044564676467:0] peer: [5:7577471038845176110:0] socket: 290 qp: -1
2025-11-27T18:17:40.126821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:40.153085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:40.150665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471040649874754:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:40.168178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471044564676471:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:40.168503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:40.150995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471040649874741:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.176050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471044564676472:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-27T18:17:40.153120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471038845176033:2048] [node 1] ICP22 created new session: [5:7577471043140143424:2048]
2025-11-27T18:17:40.177281Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471040269709079:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471044564676466:2122] held: no
2025-11-27T18:17:40.128072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471041383705141:2049] [node 3] ICP22 created new session: [2:7577471045678672536:2049]
2025-11-27T18:17:40.151048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:40.153172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471043140143424:2048] [node 1] ICS09 handshake done sender: [5:7577471043140143413:2117] self: [5:7577471043140143414:0] peer: [1:7577471040642105443:0] socket: 320 qp: -1
2025-11-27T18:17:40.179690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471044564676466:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:40.153210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471043140143424:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:40.134382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471045678672536:2049] [node 3] ICS09 handshake done sender: [2:7577471041383705213:2114] self: [2:7577471041383705211:0] peer: [3:7577471044944842063:0] socket: 311 qp: -1
2025-11-27T18:17:40.151079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471040649874741:2108] [node 5] ICP22 created new session: [3:7577471044944842074:2108]
2025-11-27T18:17:40.135602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471045678672536:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:40.152487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471044944842074:2108] [node 5] ICS09 handshake done sender: [3:7577471040649874754:2115] self: [3:7577471040649874753:0] peer: [5:7577471043140143420:0] socket: 271 qp: -1
2025-11-27T18:17:40.180692Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471040269709078:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16098) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471044564676464:2120] held: no
2025-11-27T18:17:40.153243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:40.157824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471044944842074:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:40.135643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:40.181015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471044564676474:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-27T18:17:40.153785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471043140143423:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50557
2025-11-27T18:17:40.157876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:40.147369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471041383705206:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(21884882ns) {HandleHandshakeStatus:329(21882299ns) {DropIncomingHandshake:442(18860ns) DropOutgoingHandshake:457(519ns) SwitchToState:179(692ns) ProcessPendingSessionEvents:495(6678040ns) {ForwardSessionEventToSession:612(4808104ns) ForwardSessionEventToSession:612(5581ns) ForwardSessionEventToSession:612(4534ns) ForwardSessionEventToSession:612(1104582ns) ForwardSessionEventToSession:612(3346ns) ForwardSessionEventToSession:612(2508ns) ForwardSessionEventToSession:612(608937ns) ForwardSessionEventToSession:612(3453ns) ForwardSessionEventToSession:612(122845ns) ForwardSessionEventToSession:612(2804ns)}}}
2025-11-27T18:17:40.181157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471040269709079:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:40.181405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:40.155029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471038845176095:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.181587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471040269709079:2050] [node 3] ICP22 created new session: [6:7577471044564676475:2050]
2025-11-27T18:17:40.155077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:40.181783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471044564676464:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:40.163800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471041383705216:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.155117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471038845176095:2108] [node 6] ICP22 created new session: [5:7577471043140143426:2108]
2025-11-27T18:17:40.182083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471044564676475:2050] [node 3] ICS09 handshake done sender: [6:7577471044564676466:2122] self: [6:7577471044564676469:0] peer: [3:7577471044944842051:0] socket: 331 qp: -1
2025-11-27T18:17:40.155157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471043140143426:2108] [node 6] ICS09 handshake done sender: [5:7577471043140143407:2115] self: [5:7577471038845176110:0] peer: [6:7577471044564676467:0] socket: 289 qp: -1
2025-11-27T18:17:40.182690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471044564676475:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:40.163838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471041383705216:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.155190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471043140143426:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:40.182700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471040269709078:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:40.164169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471045678672520:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.155231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:40.182950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:40.164196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471045678672520:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.155703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471038845176034:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.182958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:40.157542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471043140143415:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:40.183207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471040269709078:2049] [node 2] ICP22 created new session: [6:7577471044564676476:2049]
2025-11-27T18:17:40.168753Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471045678672520:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:40.183422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471044564676476:2049] [node 2] ICS09 handshake done sender: [6:7577471044564676464:2120] self: [6:7577471044564676470:0] peer: [2:7577471045678672517:0] socket: 317 qp: -1
2025-11-27T18:17:40.161507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471038845176035:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471043140143415:2118] held: no
2025-11-27T18:17:40.183605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471044564676476:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:40.169293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471041383705216:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:40.161895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471038845176035:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:40.183646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:40.175856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471041383705205:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.163636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:40.175926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:40.163715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471038845176035:2050] [node 3] ICP22 created new session: [5:7577471043140143429:2050]
2025-11-27T18:17:40.164597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471043140143429:2050] [node 3] ICS09 handshake done sender: [5:7577471043140143415:2118] self: [5:7577471043140143420:0] peer: [3:7577471040649874753:0] socket: 279 qp: -1
2025-11-27T18:17:40.176144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471041383705208:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.176194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471041383705205:2109] [node 5] ICP22 created new session: [2:7577471045678672540:2109]
2025-11-27T18:17:40.176613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:40.164657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471043140143429:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:40.176870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471045678672540:2109] [node 5] ICS09 handshake done sender: [2:7577471041383705216:2116] self: [2:7577471041383705215:0] peer: [5:7577471043140143428:0] socket: 330 qp: -1
2025-11-27T18:17:40.164691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:40.176969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471043140143431:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50557
2025-11-27T18:17:40.176968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471041383705208:2111] [node 6] ICP22 created new session: [2:7577471045678672541:2111]
2025-11-27T18:17:40.178898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471045678672540:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:40.177363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471038845176034:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16098) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471043140143418:2121] held: no
2025-11-27T18:17:40.178948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:40.179415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471043140143418:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:40.179446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471045678672541:2111] [node 6] ICS09 handshake done sender: [2:7577471045678672520:2119] self: [2:7577471045678672517:0] peer: [6:7577471044564676470:0] socket: 303 qp: -1
2025-11-27T18:17:40.180376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471038845176034:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:40.179477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471045678672541:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:40.179651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:40.180547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:40.180599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471038845176034:2049] [node 2] ICP22 created new session: [5:7577471043140143432:2049]
2025-11-27T18:17:40.181201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471043140143432:2049] [node 2] ICS09 handshake done sender: [5:7577471043140143418:2121] self: [5:7577471043140143428:0] peer: [2:7577471041383705215:0] socket: 298 qp: -1
2025-11-27T18:17:40.158585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471044944842052:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.181242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471043140143432:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:40.158611Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471044944842052:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.181297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:40.160791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471044944842052:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:40.164147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471040649874744:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.165910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:40.167883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471040649874744:2109] [node 6] ICP22 created new session: [3:7577471044944842077:2109]
2025-11-27T18:17:40.168880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471044944842077:2109] [node 6] ICS09 handshake done sender: [3:7577471044944842052:2116] self: [3:7577471044944842051:0] peer: [6:7577471044564676469:0] socket: 307 qp: -1
2025-11-27T18:17:40.170712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471044944842077:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:40.170754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:40.176648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471040649874741:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(22726810ns) {HandleHandshakeStatus:329(22720822ns) {DropIncomingHandshake:442(16129ns) DropOutgoingHandshake:457(791ns) SwitchToState:179(417ns) ProcessPendingSessionEvents:495(10095428ns) {ForwardSessionEventToSession:612(2093533ns) ForwardSessionEventToSession:612(901553ns) ForwardSessionEventToSession:612(3600ns) ForwardSessionEventToSession:612(3323ns) ForwardSessionEventToSession:612(1268460ns) ForwardSessionEventToSession:612(6648ns) ForwardSessionEventToSession:612(607245ns) ForwardSessionEventToSession:612(3529ns) ForwardSessionEventToSession:612(2293ns) ForwardSessionEventToSession:612(4646172ns)}}}
2025-11-27T18:17:40.202669Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:40.202918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471045678672528:2048] [node 1] ICS01 socket: 266 reason# EndOfStream
2025-11-27T18:17:40.202961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471041383705140:2048] [node 1] ICP30 unregister session Session# [2:7577471045678672528:2048] VirtualId# [2:7577471045678672519:0]
2025-11-27T18:17:40.202992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471045678672528:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.203094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:40.203162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:40.204731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:40.204937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471044944842060:2048] [node 1] ICS07 socket disconnect 284 reason# EndOfStream
2025-11-27T18:17:40.204966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471044944842060:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.205035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:40.205085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471044944842060:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:40.205996Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471040649874677:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18444) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:40.206073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471044944842060:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:40.206109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471040649874677:2048] [node 1] ICP30 unregister session Session# [3:7577471044944842060:2048] VirtualId# [3:7577471044944842055:0]
2025-11-27T18:17:40.206155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:40.206194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471040649874677:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18444) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:40.206208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18444) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:40.208116Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:40.208349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471046149107500:2048] [node 1] ICS07 socket disconnect 272 reason# EndOfStream
2025-11-27T18:17:40.208393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471046149107500:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.208482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:40.208539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471046149107500:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:40.208902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:40.209138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471043140143424:2048] [node 1] ICS01 socket: 320 reason# EndOfStream
2025-11-27T18:17:40.209190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471038845176033:2048] [node 1] ICP30 unregister session Session# [5:7577471043140143424:2048] VirtualId# [5:7577471043140143414:0]
2025-11-27T18:17:40.209227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471043140143424:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.209350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:40.209412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:40.209512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471041854140107:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18444) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:40.209689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471046149107500:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:40.209731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471041854140107:2048] [node 1] ICP30 unregister session Session# [4:7577471046149107500:2048] VirtualId# [4:7577471041854140191:0]
2025-11-27T18:17:40.209772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:40.209820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471041854140107:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18444) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:40.209839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18444) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:40.212436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:40.212778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471044564676461:2048] [node 1] ICS01 socket: 304 reason# EndOfStream
2025-11-27T18:17:40.212864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471040269709077:2048] [node 1] ICP30 unregister session Session# [6:7577471044564676461:2048] VirtualId# [6:7577471044564676457:0]
2025-11-27T18:17:40.212933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471044564676461:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.213070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:40.213247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:40.216115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471040649874677:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:40.219184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471041854140107:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:40.287608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:40.287888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471044944842072:2049] [node 2] ICS01 socket: 321 reason# EndOfStream
2025-11-27T18:17:40.287936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471040649874678:2049] [node 2] ICP30 unregister session Session# [3:7577471044944842072:2049] VirtualId# [3:7577471044944842063:0]
2025-11-27T18:17:40.287975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471044944842072:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.288062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:40.288105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:40.293500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:40.293801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471046149107505:2049] [node 2] ICS01 socket: 312 reason# EndOfStream
2025-11-27T18:17:40.293862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471041854140108:2049] [node 2] ICP30 unregister session Session# [4:7577471046149107505:2049] VirtualId# [4:7577471046149107503:0]
2025-11-27T18:17:40.293900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471046149107505:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.297488Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:40.307769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:40.308109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471043140143432:2049] [node 2] ICS01 socket: 298 reason# EndOfStream
2025-11-27T18:17:40.311320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471038845176034:2049] [node 2] ICP30 unregister session Session# [5:7577471043140143432:2049] VirtualId# [5:7577471043140143428:0]
2025-11-27T18:17:40.294010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:40.297807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471044564676476:2049] [node 2] ICS01 socket: 317 reason# EndOfStream
2025-11-27T18:17:40.294061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:40.297855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471040269709078:2049] [node 2] ICP30 unregister session Session# [6:7577471044564676476:2049] VirtualId# [6:7577471044564676470:0]
2025-11-27T18:17:40.297885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471044564676476:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.297984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:40.298042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:40.314685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471043140143432:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.314822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:40.314967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:40.359192Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:40.359454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471044564676475:2050] [node 3] ICS01 socket: 331 reason# EndOfStream
2025-11-27T18:17:40.359491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471040269709079:2050] [node 3] ICP30 unregister session Session# [6:7577471044564676475:2050] VirtualId# [6:7577471044564676469:0]
2025-11-27T18:17:40.359525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471044564676475:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.359693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:40.359881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471043140143429:2050] [node 3] ICS01 socket: 279 reason# EndOfStream
2025-11-27T18:17:40.359913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471038845176035:2050] [node 3] ICP30 unregister session Session# [5:7577471043140143429:2050] VirtualId# [5:7577471043140143420:0]
2025-11-27T18:17:40.359940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471043140143429:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.360009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:40.360048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:40.360479Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:40.360683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471046149107508:2050] [node 3] ICS01 socket: 292 reason# EndOfStream
2025-11-27T18:17:40.360747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471041854140109:2050] [node 3] ICP30 unregister session Session# [4:7577471046149107508:2050] VirtualId# [4:7577471046149107501:0]
2025-11-27T18:17:40.360792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471046149107508:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.360411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:40.360484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:40.360876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:40.360930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:40.432913Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:40.433292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471044564676456:2109] [node 4] ICS07 socket disconnect 281 reason# EndOfStream
2025-11-27T18:17:40.433323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471044564676456:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.433427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:40.433476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471044564676456:2109] [node 4] ICS15 start handshake
2025-11-27T18:17:40.433500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:40.433718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471043140143410:2107] [node 4] ICS01 socket: 275 reason# EndOfStream
2025-11-27T18:17:40.433761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471038845176096:2107] [node 4] ICP30 unregister session Session# [5:7577471043140143410:2107] VirtualId# [5:7577471043140143408:0]
2025-11-27T18:17:40.433791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471043140143410:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.433863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:40.433908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:40.434611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471040269709142:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5413) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:40.434713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471044564676456:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:40.434778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471040269709142:2109] [node 4] ICP30 unregister session Session# [6:7577471044564676456:2109] VirtualId# [6:7577471044564676452:0]
2025-11-27T18:17:40.434832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:40.434875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471040269709142:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:40.434897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5413) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:40.444615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471040269709142:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:17:40.477436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:40.477669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471044564676471:2108] [node 5] ICS01 socket: 290 reason# EndOfStream
2025-11-27T18:17:40.477728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471040269709141:2108] [node 5] ICP30 unregister session Session# [6:7577471044564676471:2108] VirtualId# [6:7577471044564676467:0]
2025-11-27T18:17:40.477766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471044564676471:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:40.477850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:40.477902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:40.591579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471045965932275:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:40.591636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471045965932274:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:40.613568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471045097180633:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:40.613607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471045097180634:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:40.636375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471043509909391:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:40.636420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471043509909392:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:40.655585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471044575280388:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:40.655627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471044575280389:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:40.655659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471044575280390:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:40.677384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471046156708686:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:40.677410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471046156708687:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:40.677452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471046156708688:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:40.704414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471042160931013:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:40.704461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471042160931014:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:40.704529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471042160931015:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:40.778936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471042160931074:2104] [node 5] ICP01 ready to work
2025-11-27T18:17:40.803941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471045097180697:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:40.789782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471045965932338:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:40.816961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471043509909453:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:40.790230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471045965932339:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:40.818338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471043509909455:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:40.846331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471045097180698:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:40.812988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471045965932340:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:40.864829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471043509909459:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:40.875443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471044575280452:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:40.875231Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471045965932338:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(61870066ns) {Configure:104(61864674ns) {ProcessConfigured:135(15364574ns) {StartInitialHandshake:157(15361085ns) {DropHandshakes:470(2259ns) {DropIncomingHandshake:442(633ns) DropOutgoingHandshake:457(353ns)} GenerateSessionVirtualId:988(1114ns) PrepareNewSessionHandshake:477(4612ns) {SwitchToState:179(2365ns)}}}}}
2025-11-27T18:17:40.871112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471045097180699:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:40.876308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471042160931078:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:40.876681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471045965932274:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(76037341ns) {Configure:104(76031523ns) {ProcessConfigured:135(25638246ns) {StartInitialHandshake:157(25635203ns) {DropHandshakes:470(2450ns) {DropIncomingHandshake:442(831ns) DropOutgoingHandshake:457(348ns)} GenerateSessionVirtualId:988(1128ns) PrepareNewSessionHandshake:477(3878ns) {SwitchToState:179(2042ns)}}}}}
2025-11-27T18:17:40.876296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471044575280453:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:40.873404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471046156708760:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:40.878129Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471045097180697:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(72208691ns) {Configure:104(72204278ns) {ProcessConfigured:135(19523808ns) {StartInitialHandshake:157(19520405ns) {DropHandshakes:470(8111389ns) {DropIncomingHandshake:442(647ns) DropOutgoingHandshake:457(2140ns)} GenerateSessionVirtualId:988(2145ns) PrepareNewSessionHandshake:477(2537ns) {SwitchToState:179(1116ns)}}}}}
2025-11-27T18:17:40.876052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471046156708759:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: 2025-11-27T18:17:40.890061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471042160931074:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(105498154ns) {Configure:104(105485749ns) {ProcessConfigured:135(3174471ns) {StartInitialHandshake:157(3171273ns) {DropHandshakes:470(13065ns) {DropIncomingHandshake:442(1041ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(866ns) PrepareNewSessionHandshake:477(3626ns) {SwitchToState:179(1513ns)}}}}}
::ffff:127.1.0.0
2025-11-27T18:17:40.902324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471046156708759:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25161577ns) {Configure:104(25157975ns) {ProcessConfigured:135(49635ns) {StartInitialHandshake:157(48277ns) {DropHandshakes:470(2089ns) {DropIncomingHandshake:442(621ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(612ns) PrepareNewSessionHandshake:477(1154ns) {SwitchToState:179(726ns)}}}}}
2025-11-27T18:17:40.893532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471042160931076:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43583
2025-11-27T18:17:40.913768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471045965932339:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(88357029ns) {Configure:104(88350872ns) {ProcessConfigured:135(4643655ns) {StartInitialHandshake:157(4641185ns) {DropHandshakes:470(2443ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(485ns)} GenerateSessionVirtualId:988(695ns) PrepareNewSessionHandshake:477(4385ns) {SwitchToState:179(2403ns)}}}}}
2025-11-27T18:17:40.911153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471045097180634:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52997804ns) {Configure:104(52983831ns) {ProcessConfigured:135(117771ns) {StartInitialHandshake:157(115184ns) {DropHandshakes:470(2844ns) {DropIncomingHandshake:442(734ns) DropOutgoingHandshake:457(362ns)} GenerateSessionVirtualId:988(1614ns) PrepareNewSessionHandshake:477(2279ns) {SwitchToState:179(747ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.914028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471045097180633:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(68317989ns) {Configure:104(68311287ns) {ProcessConfigured:135(3210741ns) {StartInitialHandshake:157(3208281ns) {DropHandshakes:470(3031ns) {DropIncomingHandshake:442(469ns) DropOutgoingHandshake:457(689ns)} GenerateSessionVirtualId:988(875ns) PrepareNewSessionHandshake:477(8902ns) {SwitchToState:179(7112ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.940628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471045965932354:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62490
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.929481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471043509909453:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(80904063ns) {Configure:104(80889133ns) {ProcessConfigured:135(5726583ns) {StartInitialHandshake:157(5723058ns) {DropHandshakes:470(2807ns) {DropIncomingHandshake:442(541ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(1221ns) PrepareNewSessionHandshake:477(3199ns) {SwitchToState:179(1327ns)}}}}}
2025-11-27T18:17:40.942803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471045965932340:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(84201534ns) {Configure:104(84192722ns) {ProcessConfigured:135(8773277ns) {StartInitialHandshake:157(8769563ns) {DropHandshakes:470(2408ns) {DropIncomingHandshake:442(757ns) DropOutgoingHandshake:457(318ns)} GenerateSessionVirtualId:988(1134ns) PrepareNewSessionHandshake:477(3359ns) {SwitchToState:179(1597ns)}}}}}
2025-11-27T18:17:40.943136Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471042160931015:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(72007881ns) {Configure:104(72002277ns) {ProcessConfigured:135(12255342ns) {StartInitialHandshake:157(12253438ns) {DropHandshakes:470(1026ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(1438ns) PrepareNewSessionHandshake:477(3866ns) {SwitchToState:179(2519ns)}}}}}
2025-11-27T18:17:40.953303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471045097180709:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40306
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.954690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471044575280390:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(89646882ns) {Configure:104(89637116ns) {ProcessConfigured:135(6785432ns) {StartInitialHandshake:157(6781966ns) {DropHandshakes:470(3286ns) {DropIncomingHandshake:442(920ns) DropOutgoingHandshake:457(588ns)} GenerateSessionVirtualId:988(1041ns) PrepareNewSessionHandshake:477(4027ns) {SwitchToState:179(2224ns)}}}}}
2025-11-27T18:17:40.929832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471043509909462:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64592
2025-11-27T18:17:40.947795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471045965932275:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(115297736ns) {Configure:104(115289371ns) {ProcessConfigured:135(6442249ns) {StartInitialHandshake:157(6440262ns) {DropHandshakes:470(1619ns) {DropIncomingHandshake:442(382ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(558ns) PrepareNewSessionHandshake:477(2319ns) {SwitchToState:179(547ns)}}}}}
2025-11-27T18:17:40.934378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471043509909391:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(88663214ns) {Configure:104(88652859ns) {ProcessConfigured:135(8481374ns) {StartInitialHandshake:157(8478448ns) {DropHandshakes:470(1499ns) {DropIncomingHandshake:442(440ns) DropOutgoingHandshake:457(140ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:477(2404ns) {SwitchToState:179(840ns)}}}}}
2025-11-27T18:17:40.963133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471044575280388:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(96279888ns) {Configure:104(96271005ns) {ProcessConfigured:135(3271699ns) {StartInitialHandshake:157(3268740ns) {DropHandshakes:470(1191ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(173ns)} GenerateSessionVirtualId:988(796ns) PrepareNewSessionHandshake:477(3074ns) {SwitchToState:179(1723ns)}}}}}
2025-11-27T18:17:40.947861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471045965932355:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62490
2025-11-27T18:17:40.955857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471042160931076:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.949813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471046156708760:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(53619790ns) {Configure:104(53615213ns) {ProcessConfigured:135(8515189ns) {StartInitialHandshake:157(8512943ns) {DropHandshakes:470(1645ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(483ns) PrepareNewSessionHandshake:477(3327ns) {SwitchToState:179(1664ns)}}}}}
2025-11-27T18:17:40.980016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471045097180698:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(91280250ns) {Configure:104(91270685ns) {ProcessConfigured:135(18540308ns) {StartInitialHandshake:157(18537751ns) {DropHandshakes:470(1398ns) {DropIncomingHandshake:442(432ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(490ns) PrepareNewSessionHandshake:477(4218ns) {SwitchToState:179(2055ns)}}}}}
2025-11-27T18:17:40.969809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471046156708688:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(92664086ns) {Configure:104(84532247ns) {ProcessConfigured:135(12430059ns) {StartInitialHandshake:157(12428110ns) {DropHandshakes:470(1206ns) {DropIncomingHandshake:442(426ns) DropOutgoingHandshake:457(184ns)} GenerateSessionVirtualId:988(443ns) PrepareNewSessionHandshake:477(3853ns) {SwitchToState:179(915ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.977511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471042160931013:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(71535860ns) {Configure:104(71533328ns) {ProcessConfigured:135(13202763ns) {StartInitialHandshake:157(8672591ns) {DropHandshakes:470(1855ns) {DropIncomingHandshake:442(155ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(1852ns) PrepareNewSessionHandshake:477(3696ns) {SwitchToState:179(1442ns)}}}}}
2025-11-27T18:17:40.963202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471045965932355:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.953969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471043509909466:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64592
2025-11-27T18:17:40.992728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471042160931014:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(93145048ns) {Configure:104(93140295ns) {ProcessConfigured:135(14117969ns) {StartInitialHandshake:157(14115744ns) {DropHandshakes:470(6196652ns) {DropIncomingHandshake:442(2602ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(1624ns) PrepareNewSessionHandshake:477(3918ns) {SwitchToState:179(2052ns)}}}}}
2025-11-27T18:17:40.989355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471045097180704:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.993298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471042160931015:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.989431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471045097180704:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.989604Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471044575280389:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(91445436ns) {Configure:104(91430240ns) {ProcessConfigured:135(12723691ns) {StartInitialHandshake:157(12721501ns) {DropHandshakes:470(2441ns) {DropIncomingHandshake:442(612ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(989ns) PrepareNewSessionHandshake:477(3269ns) {SwitchToState:179(1863ns)}}}}}
2025-11-27T18:17:40.963342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471043509909462:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.977865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471046156708686:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(70118752ns) {Configure:104(70115207ns) {ProcessConfigured:135(12961608ns) {StartInitialHandshake:157(12958393ns) {DropHandshakes:470(4216214ns) {DropIncomingHandshake:442(1783ns) DropOutgoingHandshake:457(411ns)} GenerateSessionVirtualId:988(2578ns) PrepareNewSessionHandshake:477(3769ns) {SwitchToState:179(2326ns)}}}}}
2025-11-27T18:17:40.995942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471045965932356:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62490
2025-11-27T18:17:40.995942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471042160931078:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(85384812ns) {Configure:104(85378360ns) {ProcessConfigured:135(7139237ns) {StartInitialHandshake:157(7136845ns) {DropHandshakes:470(2254ns) {DropIncomingHandshake:442(632ns) DropOutgoingHandshake:457(395ns)} GenerateSessionVirtualId:988(1321ns) PrepareNewSessionHandshake:477(16285ns) {SwitchToState:179(2618ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.007158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471042160931089:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43583
2025-11-27T18:17:40.979953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471046156708687:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(105124831ns) {Configure:104(105119801ns) {ProcessConfigured:135(18146180ns) {StartInitialHandshake:157(13439534ns) {DropHandshakes:470(4381ns) {DropIncomingHandshake:442(1115ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(1874ns) PrepareNewSessionHandshake:477(4141ns) {SwitchToState:179(1729ns)}}}}}
2025-11-27T18:17:40.996007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471044575280452:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(89956227ns) {Configure:104(89952754ns) {ProcessConfigured:135(15900130ns) {StartInitialHandshake:157(15896681ns) {DropHandshakes:470(9321ns) {DropIncomingHandshake:442(410ns) DropOutgoingHandshake:457(7866ns)} GenerateSessionVirtualId:988(845ns) PrepareNewSessionHandshake:477(3515ns) {SwitchToState:179(1357ns)}}}}}
2025-11-27T18:17:40.977626Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471043509909392:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(84778558ns) {Configure:104(84768779ns) {ProcessConfigured:135(16101619ns) {StartInitialHandshake:157(16099128ns) {DropHandshakes:470(1888ns) {DropIncomingHandshake:442(500ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(486ns) PrepareNewSessionHandshake:477(4651ns) {SwitchToState:179(1930ns)}}}}}
2025-11-27T18:17:40.994494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471045097180699:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(93500814ns) {Configure:104(93491244ns) {ProcessConfigured:135(13401233ns) {StartInitialHandshake:157(13397734ns) {DropHandshakes:470(2347ns) {DropIncomingHandshake:442(607ns) DropOutgoingHandshake:457(341ns)} GenerateSessionVirtualId:988(778ns) PrepareNewSessionHandshake:477(4703ns) {SwitchToState:179(2239ns)}}}}}
2025-11-27T18:17:40.997879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471045965932349:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.980033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471043509909391:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:40.988738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471046156708773:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-27T18:17:40.994556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471045097180709:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.002005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471044575280453:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(111721400ns) {Configure:104(111716141ns) {ProcessConfigured:135(26287975ns) {StartInitialHandshake:157(26284763ns) {DropHandshakes:470(2558ns) {DropIncomingHandshake:442(491ns) DropOutgoingHandshake:457(656ns)} GenerateSessionVirtualId:988(857ns) PrepareNewSessionHandshake:477(4127ns) {SwitchToState:179(1677ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.997908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471045965932349:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:40.995422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471046156708773:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:40.980188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471043509909459:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(74410961ns) {Configure:104(73817880ns) {ProcessConfigured:135(5994034ns) {StartInitialHandshake:157(5991791ns) {DropHandshakes:470(2069ns) {DropIncomingHandshake:442(543ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(994ns) PrepareNewSessionHandshake:477(2584ns) {SwitchToState:179(1107ns)}}}}}
2025-11-27T18:17:40.997804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471045097180633:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.001409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471045965932349:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.983131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471043509909466:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.001897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471045097180704:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:41.023062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471042160931090:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43583
2025-11-27T18:17:40.995599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471046156708688:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:40.997521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471046156708778:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-27T18:17:41.029570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471042160931090:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.006916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471045965932344:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.027986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471045097180634:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:40.983347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471043509909392:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.028787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471048870247763:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4392
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.000706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471043509909472:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64592
2025-11-27T18:17:41.006955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471045965932344:2112] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.028755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:41.037111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471048870247763:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.013775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471046156708764:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-27T18:17:41.030610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471042160931089:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.025063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471045965932344:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.038152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471044575280464:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.038178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471044575280464:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:41.038891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471044575280464:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:41.041164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471048870247766:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4392
2025-11-27T18:17:41.029219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471049392148013:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40306
2025-11-27T18:17:41.024443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471046156708766:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-27T18:17:41.031120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471042160931074:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:41.003047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471043509909457:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.029313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471045097180634:2049] [node 3] ICP22 created new session: [2:7577471049392148015:2049]
2025-11-27T18:17:41.026623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471046156708764:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.037921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471042160931078:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:41.003083Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471043509909457:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:41.027300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471045965932354:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.041414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471046455898391:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43583
2025-11-27T18:17:41.027827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471046156708686:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:41.027909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471045965932275:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.030643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471049392148015:2049] [node 3] ICS09 handshake done sender: [2:7577471045097180704:2112] self: [2:7577471045097180703:0] peer: [3:7577471043509909474:0] socket: 283 qp: -1
2025-11-27T18:17:41.006859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471043509909461:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.058156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471044575280452:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.056507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471046455898395:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43583
2025-11-27T18:17:41.031169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471045097180716:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40306
2025-11-27T18:17:41.027976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:41.006902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471043509909461:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.028372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471046156708766:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.065645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471042160931076:2109] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:41.058237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:41.029359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471046156708778:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.017687Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471043509909457:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:41.028959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471045965932275:2049] [node 3] ICP22 created new session: [1:7577471050260899657:2049]
2025-11-27T18:17:41.037110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471049392148015:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.060661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471048870247766:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.037178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:41.025226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471047804876773:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64592
2025-11-27T18:17:41.029986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471050260899657:2049] [node 3] ICS09 handshake done sender: [1:7577471045965932349:2113] self: [1:7577471045965932347:0] peer: [3:7577471043509909473:0] socket: 277 qp: -1
2025-11-27T18:17:41.065789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471046455898394:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43583
2025-11-27T18:17:41.029536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471046156708687:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.060978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471044575280452:2109] [node 6] ICP22 created new session: [4:7577471048870247768:2109]
2025-11-27T18:17:41.039097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471046156708762:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.069406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471042160931089:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.030623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471045965932274:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.037729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471045097180701:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.030550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471043509909466:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:41.068599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471048870247768:2109] [node 6] ICS09 handshake done sender: [4:7577471044575280464:2114] self: [4:7577471044575280458:0] peer: [6:7577471046455898393:0] socket: 295 qp: -1
2025-11-27T18:17:41.037760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471045097180701:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:41.030961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471043509909461:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:41.030663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:41.070661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471046455898394:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.039128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471046156708762:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:41.069447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471044575280388:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.030753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471050260899657:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:41.071228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471042160931015:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.039105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471049392148013:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.053489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050451676077:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-27T18:17:41.037477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471043509909455:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.041462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471045097180716:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.069537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471048870247768:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:41.071622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471042160931013:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:41.053600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471046156708762:2120] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:41.039238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:41.031098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.071577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:41.072749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471042160931078:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.031137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471045965932356:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.040560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471043509909392:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.057035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050451676080:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-27T18:17:41.053503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471049392148014:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40306
2025-11-27T18:17:41.083089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471048870247767:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4392
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.041197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:41.072798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:41.053551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471045097180701:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:41.069694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471046156708766:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:41.092545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471048870247767:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.036317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471045965932274:2048] [node 2] ICP22 created new session: [1:7577471050260899658:2048]
2025-11-27T18:17:41.072831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471042160931078:2110] [node 4] ICP22 created new session: [6:7577471046455898399:2110]
2025-11-27T18:17:41.071191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471046156708773:2127] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.041346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471043509909455:2109] [node 6] ICP22 created new session: [3:7577471047804876775:2109]
2025-11-27T18:17:41.036837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471050260899656:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62490
2025-11-27T18:17:41.092636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471044575280388:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6900) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471048870247766:2117] held: no
2025-11-27T18:17:41.058343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471045097180634:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(18420727ns) {HandleHandshakeStatus:329(18418270ns) {DropIncomingHandshake:442(1430ns) DropOutgoingHandshake:457(543ns) SwitchToState:179(743ns) ProcessPendingSessionEvents:495(7502150ns) {ForwardSessionEventToSession:612(59441ns) ForwardSessionEventToSession:612(2703ns) ForwardSessionEventToSession:612(2262174ns) ForwardSessionEventToSession:612(5497ns) ForwardSessionEventToSession:612(967366ns) ForwardSessionEventToSession:612(1006149ns) ForwardSessionEventToSession:612(13374ns) ForwardSessionEventToSession:612(2962ns) ForwardSessionEventToSession:612(3161774ns) ForwardSessionEventToSession:612(3630ns)}}}
2025-11-27T18:17:41.099131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471044575280452:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(35085579ns) {HandleHandshakeStatus:329(35080977ns) {DropIncomingHandshake:442(1234ns) DropOutgoingHandshake:457(948ns) SwitchToState:179(413ns) ProcessPendingSessionEvents:495(10017253ns) {ForwardSessionEventToSession:612(5014819ns) ForwardSessionEventToSession:612(11588ns) ForwardSessionEventToSession:612(2679ns) ForwardSessionEventToSession:612(6699ns) ForwardSessionEventToSession:612(1301239ns) ForwardSessionEventToSession:612(3475ns) ForwardSessionEventToSession:612(2538ns) ForwardSessionEventToSession:612(2892969ns) ForwardSessionEventToSession:612(3966ns) ForwardSessionEventToSession:612(760986ns)}}}
2025-11-27T18:17:41.060261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471045097180709:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:41.041507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471043509909472:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.072869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471046455898399:2110] [node 4] ICS09 handshake done sender: [6:7577471042160931089:2116] self: [6:7577471046455898393:0] peer: [4:7577471044575280458:0] socket: 298 qp: -1
2025-11-27T18:17:41.037274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471050260899658:2048] [node 2] ICS09 handshake done sender: [1:7577471045965932344:2112] self: [1:7577471045965932341:0] peer: [2:7577471045097180714:0] socket: 274 qp: -1
2025-11-27T18:17:41.061097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471045097180697:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.045088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471043509909453:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.072890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471046455898399:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:41.100504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471044575280389:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:41.071706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050451676081:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-27T18:17:41.039193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471045965932343:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.045171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:41.061172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:41.104432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471048870247773:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4392
2025-11-27T18:17:41.075198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:41.073087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471046156708759:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.039217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471045965932343:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:41.105049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471048870247773:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.047751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471043509909453:2108] [node 5] ICP22 created new session: [3:7577471047804876777:2108]
2025-11-27T18:17:41.065360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471045097180633:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6900) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471045097180709:2114] held: no
2025-11-27T18:17:41.040314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471050260899658:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:41.075251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471042160931015:2050] [node 3] ICP22 created new session: [6:7577471046455898398:2050]
2025-11-27T18:17:41.065761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471045097180697:2109] [node 5] ICP22 created new session: [2:7577471049392148021:2109]
2025-11-27T18:17:41.073118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:41.047809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471047804876777:2108] [node 5] ICS09 handshake done sender: [3:7577471043509909461:2111] self: [3:7577471043509909460:0] peer: [5:7577471046156708777:0] socket: 282 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.075874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:41.073159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471046156708759:2110] [node 6] ICP22 created new session: [5:7577471050451676082:2110]
2025-11-27T18:17:41.040354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:41.105213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471048870247772:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4392
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.047850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471047804876777:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:41.080704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471046455898398:2050] [node 3] ICS09 handshake done sender: [6:7577471042160931076:2109] self: [6:7577471042160931092:0] peer: [3:7577471043509909456:0] socket: 271 qp: -1
2025-11-27T18:17:41.048363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471050260899656:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.073186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471050451676082:2110] [node 6] ICS09 handshake done sender: [5:7577471046156708762:2120] self: [5:7577471046156708761:0] peer: [6:7577471046455898392:0] socket: 296 qp: -1
2025-11-27T18:17:41.082635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471046455898398:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:41.082981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:41.083252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471042160931074:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471042160931090:2117] held: no
2025-11-27T18:17:41.083512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471046455898396:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43583
2025-11-27T18:17:41.087863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471042160931090:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:41.088883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471046455898402:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43583
2025-11-27T18:17:41.089028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471046455898403:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43583
2025-11-27T18:17:41.094506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471046455898402:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.095091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471046455898394:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:41.095919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471042160931014:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:41.095974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471042160931074:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.096989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:41.097310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471042160931013:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.097347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:41.097396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471042160931013:2048] [node 1] ICP22 created new session: [6:7577471046455898406:2048]
2025-11-27T18:17:41.097449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471046455898406:2048] [node 1] ICS09 handshake done sender: [6:7577471046455898394:2119] self: [6:7577471046455898397:0] peer: [1:7577471045965932348:0] socket: 324 qp: -1
2025-11-27T18:17:41.097477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471046455898406:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:41.097501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:41.100409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471042160931074:2104] [node 5] ICP22 created new session: [6:7577471046455898405:2104]
2025-11-27T18:17:41.102728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471042160931013:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:41.103046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471046455898405:2104] [node 5] ICS09 handshake done sender: [6:7577471042160931090:2117] self: [6:7577471046455898392:0] peer: [5:7577471046156708761:0] socket: 299 qp: -1
2025-11-27T18:17:41.103089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471046455898405:2104] [node 5] ICS10 traffic start
2025-11-27T18:17:41.103124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:41.105086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471042160931078:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(23056741ns) {HandleHandshakeStatus:329(23054742ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(3257ns) SwitchToState:179(187ns) ProcessPendingSessionEvents:495(7963672ns) {ForwardSessionEventToSession:612(4279344ns) ForwardSessionEventToSession:612(6274ns) ForwardSessionEventToSession:612(974513ns) ForwardSessionEventToSession:612(3435ns) ForwardSessionEventToSession:612(16827ns) ForwardSessionEventToSession:612(1586440ns) ForwardSessionEventToSession:612(3642ns) ForwardSessionEventToSession:612(1071337ns) ForwardSessionEventToSession:612(4420ns) ForwardSessionEventToSession:612(2822ns)}}}
2025-11-27T18:17:41.106336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471042160931015:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(26019182ns) {HandleHandshakeStatus:329(25491787ns) {DropIncomingHandshake:442(889ns) DropOutgoingHandshake:457(7334ns) SwitchToState:179(251ns) ProcessPendingSessionEvents:495(4922256ns) {ForwardSessionEventToSession:612(2282273ns) ForwardSessionEventToSession:612(5788ns) ForwardSessionEventToSession:612(2853ns) ForwardSessionEventToSession:612(437369ns) ForwardSessionEventToSession:612(908545ns) ForwardSessionEventToSession:612(3011ns) ForwardSessionEventToSession:612(305589ns) ForwardSessionEventToSession:612(2621ns) ForwardSessionEventToSession:612(955572ns) ForwardSessionEventToSession:612(3579ns)}}}
2025-11-27T18:17:41.135631Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471042160931014:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29341) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471046455898402:2122] held: no
2025-11-27T18:17:41.141625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471046455898409:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43583
2025-11-27T18:17:41.143409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471046455898402:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:41.143618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471042160931014:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.143686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:41.143743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471042160931014:2049] [node 2] ICP22 created new session: [6:7577471046455898410:2049]
2025-11-27T18:17:41.143801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471046455898410:2049] [node 2] ICS09 handshake done sender: [6:7577471046455898402:2122] self: [6:7577471046455898404:0] peer: [2:7577471045097180712:0] socket: 337 qp: -1
2025-11-27T18:17:41.143836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471046455898410:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:41.143873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:41.052612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471045965932274:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(21802419ns) {HandleHandshakeStatus:329(21798801ns) {DropIncomingHandshake:442(446ns) DropOutgoingHandshake:457(594ns) SwitchToState:179(267ns) ProcessPendingSessionEvents:495(4304180ns) {ForwardSessionEventToSession:612(4227746ns) ForwardSessionEventToSession:612(10022ns) ForwardSessionEventToSession:612(3132ns) ForwardSessionEventToSession:612(3344ns) ForwardSessionEventToSession:612(5562ns) ForwardSessionEventToSession:612(2476ns) ForwardSessionEventToSession:612(11559ns) ForwardSessionEventToSession:612(2765ns) ForwardSessionEventToSession:612(12694ns) ForwardSessionEventToSession:612(8314ns)}}}
2025-11-27T18:17:41.052750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471045965932274:2048] [node 2] ICP18 handshake (actor [1:7577471045965932354:2116]) is too old
2025-11-27T18:17:41.053890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471045965932343:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:41.058127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471045965932338:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.065573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:41.069459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471045965932275:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(29476899ns) {HandleHandshakeStatus:329(29471885ns) {DropIncomingHandshake:442(15220ns) DropOutgoingHandshake:457(544ns) SwitchToState:179(331ns) ProcessPendingSessionEvents:495(15961270ns) {ForwardSessionEventToSession:612(4828816ns) ForwardSessionEventToSession:612(484637ns) ForwardSessionEventToSession:612(5161ns) ForwardSessionEventToSession:612(3283ns) ForwardSessionEventToSession:612(2942724ns) ForwardSessionEventToSession:612(4334ns) ForwardSessionEventToSession:612(3555ns) ForwardSessionEventToSession:612(5475014ns) ForwardSessionEventToSession:612(5653ns) ForwardSessionEventToSession:612(2189293ns)}}}
2025-11-27T18:17:41.069583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471045965932338:2108] [node 5] ICP22 created new session: [1:7577471050260899666:2108]
2025-11-27T18:17:41.070565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471050260899666:2108] [node 5] ICS09 handshake done sender: [1:7577471045965932343:2111] self: [1:7577471045965932342:0] peer: [5:7577471050451676075:0] socket: 273 qp: -1
2025-11-27T18:17:41.070600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471050260899666:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:41.073275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:41.109630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471048870247766:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:41.073661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471045965932353:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.073693Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471045965932353:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:41.112058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471048870247777:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4392
2025-11-27T18:17:41.075731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471045965932353:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:41.113345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471044575280390:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:41.076483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471045965932350:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.048249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471047804876771:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64592
2025-11-27T18:17:41.115719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471044575280388:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.069515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471049392148021:2109] [node 5] ICS09 handshake done sender: [2:7577471045097180701:2111] self: [2:7577471045097180700:0] peer: [5:7577471050451676076:0] socket: 275 qp: -1
2025-11-27T18:17:41.069951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471049392148021:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:41.048308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471043509909392:2049] [node 2] ICP22 created new session: [3:7577471047804876776:2049]
2025-11-27T18:17:41.076522Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471045965932350:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:41.070711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471045097180633:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.116169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471048870247767:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:41.071388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:41.116518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:41.071726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471045097180633:2048] [node 1] ICP22 created new session: [2:7577471049392148023:2048]
2025-11-27T18:17:41.048667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471047804876775:2109] [node 6] ICS09 handshake done sender: [3:7577471043509909457:2110] self: [3:7577471043509909456:0] peer: [6:7577471042160931092:0] socket: 270 qp: -1
2025-11-27T18:17:41.117555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471044575280388:2048] [node 1] ICP22 created new session: [4:7577471048870247778:2048]
2025-11-27T18:17:41.074712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471050451676082:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:41.073242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:41.073303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471046156708687:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.075824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471049392148023:2048] [node 1] ICS09 handshake done sender: [2:7577471045097180709:2114] self: [2:7577471045097180714:0] peer: [1:7577471045965932341:0] socket: 280 qp: -1
2025-11-27T18:17:41.087782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471045965932340:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.121478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471044575280389:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.076496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471049392148023:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:41.087815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471045965932350:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:41.080741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:41.087090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471045097180633:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16275042ns) {HandleHandshakeStatus:329(16272896ns) {DropIncomingHandshake:442(750ns) DropOutgoingHandshake:457(878ns) SwitchToState:179(908ns) ProcessPendingSessionEvents:495(858ns)}}
2025-11-27T18:17:41.091792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:41.096970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471045097180697:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(29697906ns) {HandleHandshakeStatus:329(29695890ns) {DropIncomingHandshake:442(17167ns) DropOutgoingHandshake:457(15010ns) SwitchToState:179(798ns) ProcessPendingSessionEvents:495(12483912ns) {ForwardSessionEventToSession:612(5993382ns) ForwardSessionEventToSession:612(6113ns) ForwardSessionEventToSession:612(770582ns) ForwardSessionEventToSession:612(2849ns) ForwardSessionEventToSession:612(2806325ns) ForwardSessionEventToSession:612(4616ns) ForwardSessionEventToSession:612(3581ns) ForwardSessionEventToSession:612(2870109ns) ForwardSessionEventToSession:612(4217ns) ForwardSessionEventToSession:612(3551ns)}}}
2025-11-27T18:17:41.092207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471045965932340:2107] [node 4] ICP22 created new session: [1:7577471050260899670:2107]
2025-11-27T18:17:41.102681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471049392148024:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40306
2025-11-27T18:17:41.093568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471045965932339:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:17:41.103767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471045097180713:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:17:41.094735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471050260899670:2107] [node 4] ICS09 handshake done sender: [1:7577471045965932353:2115] self: [1:7577471045965932352:0] peer: [4:7577471048870247770:0] socket: 322 qp: -1
2025-11-27T18:17:41.103815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471045097180713:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:41.094815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471050260899670:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:41.108237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471049392148024:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.094866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:41.108806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471045097180711:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.096796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:41.099036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471045965932339:2110] [node 6] ICP22 created new session: [1:7577471050260899672:2110]
2025-11-27T18:17:41.108846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471045097180711:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:41.103379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471050260899672:2110] [node 6] ICS09 handshake done sender: [1:7577471045965932350:2114] self: [1:7577471045965932348:0] peer: [6:7577471046455898397:0] socket: 289 qp: -1
2025-11-27T18:17:41.112070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471045097180711:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:41.103390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471045965932338:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(35531723ns) {HandleHandshakeStatus:329(35528605ns) {DropIncomingHandshake:442(567ns) DropOutgoingHandshake:457(739ns) SwitchToState:179(4064ns) ProcessPendingSessionEvents:495(10915254ns) {ForwardSessionEventToSession:612(4220745ns) ForwardSessionEventToSession:612(4492ns) ForwardSessionEventToSession:612(1081032ns) ForwardSessionEventToSession:612(4382ns) ForwardSessionEventToSession:612(3625ns) ForwardSessionEventToSession:612(3926068ns) ForwardSessionEventToSession:612(8672ns) ForwardSessionEventToSession:612(3022ns) ForwardSessionEventToSession:612(1639272ns) ForwardSessionEventToSession:612(2900ns)}}}
2025-11-27T18:17:41.112313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471045097180698:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.103725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471050260899672:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:41.112353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:41.104263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:41.112393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471045097180698:2107] [node 4] ICP22 created new session: [2:7577471049392148028:2107]
2025-11-27T18:17:41.131993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471045965932339:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(22369843ns) {HandleHandshakeStatus:329(22367504ns) {DropIncomingHandshake:442(15044ns) DropOutgoingHandshake:457(368ns) SwitchToState:179(1139ns) ProcessPendingSessionEvents:495(6816373ns) {ForwardSessionEventToSession:612(3098007ns) ForwardSessionEventToSession:612(14022ns) ForwardSessionEventToSession:612(1936083ns) ForwardSessionEventToSession:612(5224ns) ForwardSessionEventToSession:612(290910ns) ForwardSessionEventToSession:612(2392ns) ForwardSessionEventToSession:612(3805ns) ForwardSessionEventToSession:612(1444959ns) ForwardSessionEventToSession:612(2582ns) ForwardSessionEventToSession:612(2940ns)}}}
2025-11-27T18:17:41.112442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471049392148028:2107] [node 4] ICS09 handshake done sender: [2:7577471045097180711:2115] self: [2:7577471045097180710:0] peer: [4:7577471048870247774:0] socket: 327 qp: -1
2025-11-27T18:17:41.112477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471049392148028:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:41.112509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:41.113759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471045097180698:2107] [node 4] ICP18 handshake (actor [2:7577471049392148024:2120]) is too old
2025-11-27T18:17:41.049925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471047804876775:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:41.123062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471045097180713:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:41.049966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:41.074840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:41.131117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471045097180699:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.049778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:41.053683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471047804876776:2049] [node 2] ICS09 handshake done sender: [3:7577471043509909466:2113] self: [3:7577471043509909474:0] peer: [2:7577471045097180703:0] socket: 285 qp: -1
2025-11-27T18:17:41.055837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471047804876776:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:41.056510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:41.061218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471043509909391:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6900) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471043509909462:2112] held: no
2025-11-27T18:17:41.076543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471043509909462:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:41.083846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471043509909391:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.084121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471043509909453:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(31774584ns) {HandleHandshakeStatus:329(31772000ns) {DropIncomingHandshake:442(1868ns) DropOutgoingHandshake:457(1322ns) SwitchToState:179(1254ns) ProcessPendingSessionEvents:495(12157222ns) {ForwardSessionEventToSession:612(8206222ns) ForwardSessionEventToSession:612(5664ns) ForwardSessionEventToSession:612(592227ns) ForwardSessionEventToSession:612(4304ns) ForwardSessionEventToSession:612(3516ns) ForwardSessionEventToSession:612(664191ns) ForwardSessionEventToSession:612(678155ns) ForwardSessionEventToSession:612(3537ns) ForwardSessionEventToSession:612(1977264ns) ForwardSessionEventToSession:612(3604ns)}}}
2025-11-27T18:17:41.087715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:41.088080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471043509909455:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(38114601ns) {HandleHandshakeStatus:329(38111016ns) {DropIncomingHandshake:442(1631ns) DropOutgoingHandshake:457(1846ns) SwitchToState:179(218ns) ProcessPendingSessionEvents:495(12290876ns) {ForwardSessionEventToSession:612(8882684ns) ForwardSessionEventToSession:612(5186ns) ForwardSessionEventToSession:612(718923ns) ForwardSessionEventToSession:612(520705ns) ForwardSessionEventToSession:612(6694ns) ForwardSessionEventToSession:612(347281ns) ForwardSessionEventToSession:612(9183ns) ForwardSessionEventToSession:612(1775741ns) ForwardSessionEventToSession:612(3248ns) ForwardSessionEventToSession:612(2858ns)}}}
2025-11-27T18:17:41.092352Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471043509909392:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(42021976ns) {HandleHandshakeStatus:329(42018270ns) {DropIncomingHandshake:442(686ns) DropOutgoingHandshake:457(5172ns) SwitchToState:179(399ns) ProcessPendingSessionEvents:495(13949722ns) {ForwardSessionEventToSession:612(2961790ns) ForwardSessionEventToSession:612(2002340ns) ForwardSessionEventToSession:612(3754ns) ForwardSessionEventToSession:612(1911639ns) ForwardSessionEventToSession:612(3771ns) ForwardSessionEventToSession:612(1060942ns) ForwardSessionEventToSession:612(3584ns) ForwardSessionEventToSession:612(1522ns) ForwardSessionEventToSession:612(4125978ns) ForwardSessionEventToSession:612(1850918ns)}}}
2025-11-27T18:17:41.093093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471043509909391:2048] [node 1] ICP22 created new session: [3:7577471047804876784:2048]
2025-11-27T18:17:41.096022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471047804876784:2048] [node 1] ICS09 handshake done sender: [3:7577471043509909462:2112] self: [3:7577471043509909473:0] peer: [1:7577471045965932347:0] socket: 279 qp: -1
2025-11-27T18:17:41.096903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471047804876784:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:41.099293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:41.108086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471043509909391:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21210100ns) {HandleHandshakeStatus:329(21207131ns) {DropIncomingHandshake:442(1243ns) DropOutgoingHandshake:457(280ns) SwitchToState:179(967ns) ProcessPendingSessionEvents:495(1086ns)}}
2025-11-27T18:17:41.116602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471043509909471:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.116631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471043509909471:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:41.127348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471043509909471:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:41.130664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471043509909459:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.131336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:41.131777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471043509909459:2107] [node 4] ICP22 created new session: [3:7577471047804876787:2107]
2025-11-27T18:17:41.133960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471047804876787:2107] [node 4] ICS09 handshake done sender: [3:7577471043509909471:2116] self: [3:7577471043509909470:0] peer: [4:7577471048870247776:0] socket: 335 qp: -1
2025-11-27T18:17:41.134390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471047804876787:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:41.134437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:41.149110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471043509909459:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16104212ns) {HandleHandshakeStatus:329(15235527ns) {DropIncomingHandshake:442(14095ns) DropOutgoingHandshake:457(782ns) SwitchToState:179(839ns) ProcessPendingSessionEvents:495(6463305ns) {ForwardSessionEventToSession:612(1671323ns) ForwardSessionEventToSession:612(9959ns) ForwardSessionEventToSession:612(575718ns) ForwardSessionEventToSession:612(3829ns) ForwardSessionEventToSession:612(3552ns) ForwardSessionEventToSession:612(319902ns) ForwardSessionEventToSession:612(3339ns) ForwardSessionEventToSession:612(3686ns) ForwardSessionEventToSession:612(3429270ns) ForwardSessionEventToSession:612(430068ns)}}}
2025-11-27T18:17:41.121545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:41.131213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:41.131259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471045097180699:2110] [node 6] ICP22 created new session: [2:7577471049392148031:2110]
2025-11-27T18:17:41.133365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471049392148031:2110] [node 6] ICS09 handshake done sender: [2:7577471045097180713:2116] self: [2:7577471045097180712:0] peer: [6:7577471046455898404:0] socket: 305 qp: -1
2025-11-27T18:17:41.134095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471049392148031:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:41.135325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:41.075475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:41.121598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471044575280389:2049] [node 2] ICP22 created new session: [4:7577471048870247779:2049]
2025-11-27T18:17:41.076536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471046156708687:2049] [node 2] ICP22 created new session: [5:7577471050451676083:2049]
2025-11-27T18:17:41.121642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471048870247779:2049] [node 2] ICS09 handshake done sender: [4:7577471048870247767:2118] self: [4:7577471048870247774:0] peer: [2:7577471045097180710:0] socket: 288 qp: -1
2025-11-27T18:17:41.121680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471048870247779:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:41.121712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:41.123278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471048870247778:2048] [node 1] ICS09 handshake done sender: [4:7577471048870247766:2117] self: [4:7577471048870247770:0] peer: [1:7577471045965932352:0] socket: 323 qp: -1
2025-11-27T18:17:41.125974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471048870247778:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:41.126445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:41.130380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471044575280389:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:41.134319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471044575280466:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.134351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471044575280466:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:41.139349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471044575280390:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20732) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471048870247773:2120] held: no
2025-11-27T18:17:41.145180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471044575280466:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:41.145578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471044575280453:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:41.146142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471048870247782:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4392
2025-11-27T18:17:41.147662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:41.147786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471048870247773:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:41.148047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471044575280453:2108] [node 5] ICP22 created new session: [4:7577471048870247784:2108]
2025-11-27T18:17:41.149547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471048870247784:2108] [node 5] ICS09 handshake done sender: [4:7577471044575280466:2115] self: [4:7577471044575280463:0] peer: [5:7577471050451676093:0] socket: 290 qp: -1
2025-11-27T18:17:41.149684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471044575280390:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.149727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:41.149730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471048870247784:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:41.149996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471044575280390:2050] [node 3] ICP22 created new session: [4:7577471048870247785:2050]
2025-11-27T18:17:41.150016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:41.151499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471048870247785:2050] [node 3] ICS09 handshake done sender: [4:7577471048870247773:2120] self: [4:7577471048870247776:0] peer: [3:7577471043509909470:0] socket: 336 qp: -1
2025-11-27T18:17:41.151681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471048870247785:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:41.151721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:41.077522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471046156708688:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.077582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:41.077623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471046156708688:2050] [node 3] ICP22 created new session: [5:7577471050451676085:2050]
2025-11-27T18:17:41.077664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471050451676085:2050] [node 3] ICS09 handshake done sender: [5:7577471046156708773:2127] self: [5:7577471046156708777:0] peer: [3:7577471043509909460:0] socket: 294 qp: -1
2025-11-27T18:17:41.077698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471050451676085:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:41.077749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:41.079953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471046156708764:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:41.080931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471050451676083:2049] [node 2] ICS09 handshake done sender: [5:7577471046156708766:2122] self: [5:7577471050451676076:0] peer: [2:7577471045097180700:0] socket: 281 qp: -1
2025-11-27T18:17:41.083808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471050451676083:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:41.083852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:41.089557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471046156708686:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.092500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:41.092557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471046156708686:2048] [node 1] ICP22 created new session: [5:7577471050451676089:2048]
2025-11-27T18:17:41.095520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471050451676089:2048] [node 1] ICS09 handshake done sender: [5:7577471046156708764:2121] self: [5:7577471050451676075:0] peer: [1:7577471045965932342:0] socket: 278 qp: -1
2025-11-27T18:17:41.095782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471050451676089:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:41.097037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:41.105010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471046156708759:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(25850507ns) {HandleHandshakeStatus:329(25849092ns) {DropIncomingHandshake:442(9699ns) DropOutgoingHandshake:457(148ns) SwitchToState:179(188ns) ProcessPendingSessionEvents:495(7120149ns) {ForwardSessionEventToSession:612(1238564ns) ForwardSessionEventToSession:612(1400696ns) ForwardSessionEventToSession:612(4844ns) ForwardSessionEventToSession:612(3283ns) ForwardSessionEventToSession:612(978419ns) ForwardSessionEventToSession:612(4389ns) ForwardSessionEventToSession:612(1434386ns) ForwardSessionEventToSession:612(20452ns) ForwardSessionEventToSession:612(3949ns) ForwardSessionEventToSession:612(2001540ns)}}}
2025-11-27T18:17:41.109585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471046156708687:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(27241640ns) {HandleHandshakeStatus:329(27240023ns) {DropIncomingHandshake:442(4860ns) DropOutgoingHandshake:457(6690ns) SwitchToState:179(387ns) ProcessPendingSessionEvents:495(6746519ns) {ForwardSessionEventToSession:612(3227627ns) ForwardSessionEventToSession:612(3335ns) ForwardSessionEventToSession:612(1740ns) ForwardSessionEventToSession:612(281956ns) ForwardSessionEventToSession:612(963860ns) ForwardSessionEventToSession:612(3470ns) ForwardSessionEventToSession:612(257956ns) ForwardSessionEventToSession:612(1969ns) ForwardSessionEventToSession:612(1988826ns) ForwardSessionEventToSession:612(3997ns)}}}
2025-11-27T18:17:41.116962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050451676092:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-27T18:17:41.116871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471046156708686:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20934275ns) {HandleHandshakeStatus:329(19190477ns) {DropIncomingHandshake:442(690ns) DropOutgoingHandshake:457(4309ns) SwitchToState:179(767ns) ProcessPendingSessionEvents:495(5295122ns) {ForwardSessionEventToSession:612(1581024ns) ForwardSessionEventToSession:612(9024ns) ForwardSessionEventToSession:612(3667ns) ForwardSessionEventToSession:612(3007ns) ForwardSessionEventToSession:612(884783ns) ForwardSessionEventToSession:612(403605ns) ForwardSessionEventToSession:612(1634848ns) ForwardSessionEventToSession:612(3429ns) ForwardSessionEventToSession:612(2643ns) ForwardSessionEventToSession:612(751090ns)}}}
2025-11-27T18:17:41.129417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471050451676092:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.130528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471046156708760:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:41.148040Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471046156708760:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10257) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471050451676092:2132] held: no
2025-11-27T18:17:41.148951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050451676094:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-27T18:17:41.149632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471050451676092:2132] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:41.151148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471046156708760:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:41.151187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:41.151575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471046156708760:2108] [node 4] ICP22 created new session: [5:7577471050451676095:2108]
2025-11-27T18:17:41.152225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471050451676095:2108] [node 4] ICS09 handshake done sender: [5:7577471050451676092:2132] self: [5:7577471050451676093:0] peer: [4:7577471044575280463:0] socket: 292 qp: -1
2025-11-27T18:17:41.152294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471050451676095:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:41.152342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:41.213624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:41.213834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471049392148023:2048] [node 1] ICS01 socket: 280 reason# EndOfStream
2025-11-27T18:17:41.213881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471045097180633:2048] [node 1] ICP30 unregister session Session# [2:7577471049392148023:2048] VirtualId# [2:7577471045097180714:0]
2025-11-27T18:17:41.213914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471049392148023:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.213986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:41.214039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:41.214967Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:41.215160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471047804876784:2048] [node 1] ICS01 socket: 279 reason# EndOfStream
2025-11-27T18:17:41.215194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471043509909391:2048] [node 1] ICP30 unregister session Session# [3:7577471047804876784:2048] VirtualId# [3:7577471043509909473:0]
2025-11-27T18:17:41.215237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471047804876784:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.215318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:41.215360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:41.217820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:41.218026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471048870247778:2048] [node 1] ICS01 socket: 323 reason# EndOfStream
2025-11-27T18:17:41.218095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471044575280388:2048] [node 1] ICP30 unregister session Session# [4:7577471048870247778:2048] VirtualId# [4:7577471048870247770:0]
2025-11-27T18:17:41.218121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471048870247778:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.218187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:41.218225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:41.218849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:41.219046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471050451676089:2048] [node 1] ICS07 socket disconnect 278 reason# EndOfStream
2025-11-27T18:17:41.219077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471050451676089:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.219150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:41.219197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471050451676089:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:41.219794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:41.220009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471046455898406:2048] [node 1] ICS01 socket: 324 reason# EndOfStream
2025-11-27T18:17:41.220059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471042160931013:2048] [node 1] ICP30 unregister session Session# [6:7577471046455898406:2048] VirtualId# [6:7577471046455898397:0]
2025-11-27T18:17:41.220104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471046455898406:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.220132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471046156708686:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6900) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:41.220208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471050451676089:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:41.220170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:41.220252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471046156708686:2048] [node 1] ICP30 unregister session Session# [5:7577471050451676089:2048] VirtualId# [5:7577471050451676075:0]
2025-11-27T18:17:41.220326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:41.220461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:41.220541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471046156708686:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6900) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.220609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6900) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.230924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471046156708686:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:41.272538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:41.272784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471047804876776:2049] [node 2] ICS07 socket disconnect 285 reason# EndOfStream
2025-11-27T18:17:41.272824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471047804876776:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.272917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:41.272978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471047804876776:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:41.275110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:41.275356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471048870247779:2049] [node 2] ICS07 socket disconnect 288 reason# EndOfStream
2025-11-27T18:17:41.275247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471043509909392:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29341) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:41.275398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471048870247779:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.275292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471047804876776:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:41.275478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:41.275512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471048870247779:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:41.276537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471043509909392:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29341) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:41.276618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471044575280389:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29341) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:41.276725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471047804876776:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:41.276996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471048870247779:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:41.276978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471043509909392:2049] [node 2] ICP30 unregister session Session# [3:7577471047804876776:2049] VirtualId# [3:7577471043509909474:0]
2025-11-27T18:17:41.277114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:41.277150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471044575280389:2049] [node 2] ICP30 unregister session Session# [4:7577471048870247779:2049] VirtualId# [4:7577471048870247774:0]
2025-11-27T18:17:41.277172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471043509909392:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29341) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.277215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:41.277187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29341) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.277251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471044575280389:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29341) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.277290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29341) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.278031Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:41.278272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471050451676083:2049] [node 2] ICS07 socket disconnect 281 reason# EndOfStream
2025-11-27T18:17:41.278307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471050451676083:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.278321Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:41.278413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:41.278495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471050451676083:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:41.278498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471046455898410:2049] [node 2] ICS01 socket: 337 reason# EndOfStream
2025-11-27T18:17:41.278549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471042160931014:2049] [node 2] ICP30 unregister session Session# [6:7577471046455898410:2049] VirtualId# [6:7577471046455898404:0]
2025-11-27T18:17:41.278580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471046455898410:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.278696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:41.278751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:41.279276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471046156708687:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29341) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:41.279329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471050451676083:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:41.279418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471046156708687:2049] [node 2] ICP30 unregister session Session# [5:7577471050451676083:2049] VirtualId# [5:7577471050451676076:0]
2025-11-27T18:17:41.279467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:41.279492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471046156708687:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29341) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.279508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29341) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.287212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471044575280389:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:41.287213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471043509909392:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:41.289277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471046156708687:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:41.337931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:41.338198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471048870247785:2050] [node 3] ICS01 socket: 336 reason# EndOfStream
2025-11-27T18:17:41.338229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471044575280390:2050] [node 3] ICP30 unregister session Session# [4:7577471048870247785:2050] VirtualId# [4:7577471048870247776:0]
2025-11-27T18:17:41.338258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471048870247785:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.338339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:41.338385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:41.339017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:41.339244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471050451676085:2050] [node 3] ICS01 socket: 294 reason# EndOfStream
2025-11-27T18:17:41.339280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471046156708688:2050] [node 3] ICP30 unregister session Session# [5:7577471050451676085:2050] VirtualId# [5:7577471046156708777:0]
2025-11-27T18:17:41.339302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471050451676085:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.339387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:41.339453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:41.339966Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:41.340162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471046455898398:2050] [node 3] ICS07 socket disconnect 271 reason# EndOfStream
2025-11-27T18:17:41.340209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471046455898398:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.340273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:41.340321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471046455898398:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:41.341145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471042160931015:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20732) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:41.341247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471046455898398:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:41.341308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471042160931015:2050] [node 3] ICP30 unregister session Session# [6:7577471046455898398:2050] VirtualId# [6:7577471042160931092:0]
2025-11-27T18:17:41.341369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:41.341406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471042160931015:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20732) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.341419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20732) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.350719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471042160931015:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:41.395766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:41.396036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471050451676095:2108] [node 4] ICS01 socket: 292 reason# EndOfStream
2025-11-27T18:17:41.396115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471046156708760:2108] [node 4] ICP30 unregister session Session# [5:7577471050451676095:2108] VirtualId# [5:7577471050451676093:0]
2025-11-27T18:17:41.396164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471050451676095:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.396259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:41.396316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:41.397455Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:41.397660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471046455898399:2110] [node 4] ICS07 socket disconnect 298 reason# EndOfStream
2025-11-27T18:17:41.397703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471046455898399:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.397796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:41.397835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471046455898399:2110] [node 4] ICS15 start handshake
2025-11-27T18:17:41.401792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471042160931078:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10257) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:41.405415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471046455898399:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:41.405499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471042160931078:2110] [node 4] ICP30 unregister session Session# [6:7577471046455898399:2110] VirtualId# [6:7577471046455898393:0]
2025-11-27T18:17:41.405611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:41.405692Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471042160931078:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10257) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.405729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10257) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:41.416262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471042160931078:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:17:41.480925Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:41.481194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471046455898405:2104] [node 5] ICS01 socket: 299 reason# EndOfStream
2025-11-27T18:17:41.481253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471042160931074:2104] [node 5] ICP30 unregister session Session# [6:7577471046455898405:2104] VirtualId# [6:7577471046455898392:0]
2025-11-27T18:17:41.481313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471046455898405:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:41.481527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:41.481597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:41.583665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471046836801317:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:41.583710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471046836801318:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:41.629744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471047389109996:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:41.629798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471047389109997:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:41.653515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471048984784897:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:41.653547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471048984784898:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:41.678459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471046757091778:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:41.678517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471046757091779:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:41.678546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471046757091780:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:41.701560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471046703282323:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:41.701612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471046703282324:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:41.701639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471046703282325:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:41.721271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471050547265683:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:41.721314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471050547265684:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:41.721344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471050547265685:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:41.830312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471046757091840:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:41.878064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471050547265745:2103] [node 5] ICP01 ready to work
2025-11-27T18:17:41.922589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471048984784897:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(70446410ns) {Configure:104(70438301ns) {ProcessConfigured:135(74216ns) {StartInitialHandshake:157(66770ns) {DropHandshakes:470(2277ns) {DropIncomingHandshake:442(801ns) DropOutgoingHandshake:457(457ns)} GenerateSessionVirtualId:988(748ns) PrepareNewSessionHandshake:477(1741ns) {SwitchToState:179(567ns)}}}}}
2025-11-27T18:17:41.923046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471048984784971:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:41.923331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471048984784972:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:41.923590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471048984784973:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:41.921083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471046703282388:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:41.920923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471047389110060:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:41.930320Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471046757091840:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(99714985ns) {Configure:104(99710285ns) {ProcessConfigured:135(82349ns) {StartInitialHandshake:157(79734ns) {DropHandshakes:470(1672ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(669ns) PrepareNewSessionHandshake:477(1908ns) {SwitchToState:179(689ns)}}}}}
2025-11-27T18:17:41.930486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471050547265748:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:41.930745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471046836801317:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(102734620ns) {Configure:104(102659125ns) {ProcessConfigured:135(9525678ns) {StartInitialHandshake:157(9302719ns) {DropHandshakes:470(6060ns) {DropIncomingHandshake:442(3075ns) DropOutgoingHandshake:457(486ns)} GenerateSessionVirtualId:988(1557ns) PrepareNewSessionHandshake:477(3108ns) {SwitchToState:179(1729ns)}}}}}
2025-11-27T18:17:41.930602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471047389110061:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:41.930973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471046757091845:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:41.930975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471047389110062:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:41.931504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471046836801382:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:41.933038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471046703282391:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:41.932021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471046836801383:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:41.932450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471046836801384:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:41.959546Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471047389109997:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18376100ns) {Configure:104(18371137ns) {ProcessConfigured:135(74223ns) {StartInitialHandshake:157(71360ns) {DropHandshakes:470(3022ns) {DropIncomingHandshake:442(700ns) DropOutgoingHandshake:457(730ns)} GenerateSessionVirtualId:988(564ns) PrepareNewSessionHandshake:477(1667ns) {SwitchToState:179(653ns)}}}}}
2025-11-27T18:17:41.963420Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471048984784973:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27380663ns) {Configure:104(27377223ns) {ProcessConfigured:135(3181191ns) {StartInitialHandshake:157(3178528ns) {DropHandshakes:470(1201ns) {DropIncomingHandshake:442(241ns) DropOutgoingHandshake:457(326ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:477(4048ns) {SwitchToState:179(844ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.964707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471046836801393:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4357
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.970639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471046703282400:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4674
2025-11-27T18:17:41.971839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471046757091780:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17621060ns) {Configure:104(17616173ns) {ProcessConfigured:135(11980966ns) {StartInitialHandshake:157(11978694ns) {DropHandshakes:470(1449ns) {DropIncomingHandshake:442(280ns) DropOutgoingHandshake:457(300ns)} GenerateSessionVirtualId:988(438ns) PrepareNewSessionHandshake:477(2782ns) {SwitchToState:179(232ns)}}}}}
2025-11-27T18:17:41.985468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471047389110060:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21280516ns) {Configure:104(21275508ns) {ProcessConfigured:135(1884388ns) {StartInitialHandshake:157(1881430ns) {DropHandshakes:470(3370ns) {DropIncomingHandshake:442(848ns) DropOutgoingHandshake:457(447ns)} GenerateSessionVirtualId:988(699ns) PrepareNewSessionHandshake:477(4678ns) {SwitchToState:179(2003ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.995558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471047389110069:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55821
2025-11-27T18:17:41.995736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471047389110070:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55821
2025-11-27T18:17:41.995846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471047389110071:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55821
2025-11-27T18:17:41.989387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471046703282400:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:41.998802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471047389110069:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.999259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471046757091845:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27173447ns) {Configure:104(27168007ns) {ProcessConfigured:135(95802ns) {StartInitialHandshake:157(92575ns) {DropHandshakes:470(3669ns) {DropIncomingHandshake:442(360ns) DropOutgoingHandshake:457(408ns)} GenerateSessionVirtualId:988(1167ns) PrepareNewSessionHandshake:477(1491ns) {SwitchToState:179(787ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.999514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471048984784983:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34799
No RDMA context found for GID: 2025-11-27T18:17:41.999652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471048984784984:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34799
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:41.999751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471046836801397:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4357
2025-11-27T18:17:41.999756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471048984784985:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34799
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.000362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471046836801398:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4357
2025-11-27T18:17:42.000430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471046836801399:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4357
2025-11-27T18:17:42.000625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471047389110071:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.000619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471046836801393:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.000743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471046836801397:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.000813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471047389110070:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.001487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471046757091854:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43811
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.001726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471046757091855:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43811
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.002493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471051131768698:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4357
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.002564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471046757091854:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.002663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471051052059153:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43811
2025-11-27T18:17:42.001721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471048984784983:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.001820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471048984784984:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.005428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050998249701:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4674
2025-11-27T18:17:42.006193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471046836801398:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.006406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471048984784897:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.006399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471047389109996:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.006621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471047389110080:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55821
2025-11-27T18:17:42.005111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471046757091778:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.006608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471046836801399:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.017536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471051684077377:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55821
2025-11-27T18:17:42.017787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471047389110080:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.018033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471051684077380:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55821
2025-11-27T18:17:42.006788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471046836801386:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.006809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471046836801386:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.007308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471046836801381:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.007321Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471046836801381:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.008148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471046836801386:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:42.009660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471048984784985:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.017916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471048984784898:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.018016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471053279752286:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34799
2025-11-27T18:17:42.019984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471048984784983:2124] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:42.020135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471047389110066:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.020225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471048984784897:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.020169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471047389110066:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.021039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471047389110070:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:42.020468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:42.021043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471048984784897:2048] [node 1] ICP22 created new session: [3:7577471053279752288:2048]
2025-11-27T18:17:42.024413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471053279752288:2048] [node 1] ICS09 handshake done sender: [3:7577471048984784983:2124] self: [3:7577471053279752284:0] peer: [1:7577471046836801385:0] socket: 297 qp: -1
2025-11-27T18:17:42.024462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471053279752288:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:42.024515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:42.024884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471047389109996:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.024936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:42.024977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471047389109996:2048] [node 1] ICP22 created new session: [2:7577471051684077383:2048]
2025-11-27T18:17:42.019350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471046836801381:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:42.025021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471051684077383:2048] [node 1] ICS09 handshake done sender: [2:7577471047389110070:2115] self: [2:7577471051684077379:0] peer: [1:7577471046836801380:0] socket: 292 qp: -1
2025-11-27T18:17:42.020032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471046836801388:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.025051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471051684077383:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:42.020057Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471046836801388:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.025083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:42.024281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471046836801318:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.024353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:42.024417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471046836801318:2049] [node 3] ICP22 created new session: [1:7577471051131768703:2049]
2025-11-27T18:17:42.024480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471051131768703:2049] [node 3] ICS09 handshake done sender: [1:7577471046836801386:2111] self: [1:7577471046836801385:0] peer: [3:7577471053279752284:0] socket: 283 qp: -1
2025-11-27T18:17:42.024514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471051131768703:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:42.024547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:42.029132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471046836801317:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.029199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:42.029247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471046836801317:2048] [node 2] ICP22 created new session: [1:7577471051131768705:2048]
2025-11-27T18:17:42.029326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471051131768705:2048] [node 2] ICS09 handshake done sender: [1:7577471046836801381:2110] self: [1:7577471046836801380:0] peer: [2:7577471051684077379:0] socket: 282 qp: -1
2025-11-27T18:17:42.029364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471051131768705:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:42.029418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.038391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471053279752290:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34799
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.039027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050998249702:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4674
2025-11-27T18:17:42.039580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471046757091855:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.039703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471051052059153:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.039994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471046757091779:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.040053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471046757091780:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.040878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471047389110066:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:42.042167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471047389110068:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.043036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471046757091844:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.043067Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471046757091844:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.044455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471048984784977:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.044489Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471048984784977:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.053211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471048984784898:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3546) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471048984784985:2126] held: no
2025-11-27T18:17:42.041095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471050998249701:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.042198Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471047389110068:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.042914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471047389109997:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.042982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:42.043030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471047389109997:2049] [node 3] ICP22 created new session: [2:7577471051684077385:2049]
2025-11-27T18:17:42.043073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471051684077385:2049] [node 3] ICS09 handshake done sender: [2:7577471047389110066:2112] self: [2:7577471047389110065:0] peer: [3:7577471053279752287:0] socket: 295 qp: -1
2025-11-27T18:17:42.043605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471051684077385:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:42.043654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:42.047516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471047389110068:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:42.047831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471047389110060:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.047868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:42.047909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471047389110060:2107] [node 4] ICP22 created new session: [2:7577471051684077390:2107]
2025-11-27T18:17:42.047966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471051684077390:2107] [node 4] ICS09 handshake done sender: [2:7577471047389110068:2113] self: [2:7577471047389110067:0] peer: [4:7577471051052059157:0] socket: 296 qp: -1
2025-11-27T18:17:42.041235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471046703282388:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.053516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471051131768698:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.054172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471046836801388:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:42.054409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471046836801382:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.086046Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471046757091778:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1297) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471046757091854:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.086712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471051052059159:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43811
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.087180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471051052059160:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43811
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.087576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471051052059161:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43811
2025-11-27T18:17:42.087752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471046757091844:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:42.088165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471046757091840:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.088213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:42.073747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471048984784979:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.047997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471051684077390:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:42.054451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:42.047156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050998249704:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4674
2025-11-27T18:17:42.054483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471046836801382:2107] [node 4] ICP22 created new session: [1:7577471051131768711:2107]
2025-11-27T18:17:42.054530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471051131768711:2107] [node 4] ICS09 handshake done sender: [1:7577471046836801388:2112] self: [1:7577471046836801387:0] peer: [4:7577471051052059156:0] socket: 284 qp: -1
2025-11-27T18:17:42.054564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471051131768711:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:42.054599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:42.048031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:42.058115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471047389110073:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.058138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471047389110073:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.059608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471047389110073:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:42.074359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471047389110061:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.100800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471046757091840:2108] [node 5] ICP22 created new session: [4:7577471051052059165:2108]
2025-11-27T18:17:42.073809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471048984784979:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.101428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471051052059165:2108] [node 5] ICS09 handshake done sender: [4:7577471046757091844:2111] self: [4:7577471046757091843:0] peer: [5:7577471050998249703:0] socket: 309 qp: -1
2025-11-27T18:17:42.052573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471050998249702:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.074429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.052711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471046703282325:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.105050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471048984784985:2126] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:42.101764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471051052059165:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:42.053989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471050998249704:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.057678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471046703282324:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.057791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050998249706:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4674
2025-11-27T18:17:42.102480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:42.074499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471047389110061:2109] [node 5] ICP22 created new session: [2:7577471051684077393:2109]
2025-11-27T18:17:42.105773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471048984784898:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.074553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471051684077393:2109] [node 5] ICS09 handshake done sender: [2:7577471047389110073:2117] self: [2:7577471047389110072:0] peer: [5:7577471050998249707:0] socket: 310 qp: -1
2025-11-27T18:17:42.074593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471051684077393:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:42.074628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:42.106122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471046757091840:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17601834ns) {HandleHandshakeStatus:329(17598669ns) {DropIncomingHandshake:442(676ns) DropOutgoingHandshake:457(701ns) SwitchToState:179(558ns) ProcessPendingSessionEvents:495(2367137ns) {ForwardSessionEventToSession:612(1058930ns) ForwardSessionEventToSession:612(3158ns) ForwardSessionEventToSession:612(430867ns) ForwardSessionEventToSession:612(3081ns) ForwardSessionEventToSession:612(590612ns) ForwardSessionEventToSession:612(4350ns) ForwardSessionEventToSession:612(2569ns) ForwardSessionEventToSession:612(252261ns) ForwardSessionEventToSession:612(3331ns) ForwardSessionEventToSession:612(2176ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.106432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471051052059163:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43811
2025-11-27T18:17:42.059429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050998249708:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4674
2025-11-27T18:17:42.103876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471051684077377:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.105849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:42.106779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471046757091854:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.060085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471050998249704:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:42.103982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471047389110060:2107] [node 4] ICP18 handshake (actor [2:7577471051684077377:2120]) is too old
2025-11-27T18:17:42.106753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471048984784898:2049] [node 2] ICP22 created new session: [3:7577471053279752294:2049]
2025-11-27T18:17:42.060249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471046703282324:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.060296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:42.060352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471046703282324:2049] [node 2] ICP22 created new session: [5:7577471050998249709:2049]
2025-11-27T18:17:42.107775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471054842233058:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51529
2025-11-27T18:17:42.060391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471050998249709:2049] [node 2] ICS09 handshake done sender: [5:7577471050998249704:2120] self: [5:7577471050998249707:0] peer: [2:7577471047389110072:0] socket: 325 qp: -1
2025-11-27T18:17:42.108428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471046757091855:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:42.060422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471050998249709:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:42.106962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471053279752294:2049] [node 2] ICS09 handshake done sender: [3:7577471048984784985:2126] self: [3:7577471053279752287:0] peer: [2:7577471047389110065:0] socket: 299 qp: -1
2025-11-27T18:17:42.060452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.106995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471053279752294:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:42.101804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471050998249706:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.107351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:42.104848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050998249712:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4674
2025-11-27T18:17:42.109248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471046836801390:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.110634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471047389110075:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.110662Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471047389110075:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.109350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471054842233059:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51529
2025-11-27T18:17:42.107533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471046703282323:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.111335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471047389110075:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:42.109303Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471046836801390:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.109833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471048984784977:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:42.113229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471046836801390:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:42.109753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471054842233058:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.114074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471046757091778:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.114135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:42.114166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471046757091778:2048] [node 1] ICP22 created new session: [4:7577471051052059167:2048]
2025-11-27T18:17:42.114196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471051052059167:2048] [node 1] ICS09 handshake done sender: [4:7577471046757091854:2116] self: [4:7577471051052059156:0] peer: [1:7577471046836801387:0] socket: 304 qp: -1
2025-11-27T18:17:42.112353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471047389110062:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.116177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471046836801383:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.109184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471050998249701:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:42.116232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:42.116270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471046836801383:2108] [node 5] ICP22 created new session: [1:7577471051131768714:2108]
2025-11-27T18:17:42.114469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471046703282388:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.114511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:42.114548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471046703282388:2108] [node 4] ICP22 created new session: [5:7577471050998249714:2108]
2025-11-27T18:17:42.115206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471050998249714:2108] [node 4] ICS09 handshake done sender: [5:7577471050998249701:2118] self: [5:7577471050998249703:0] peer: [4:7577471046757091843:0] socket: 316 qp: -1
2025-11-27T18:17:42.116307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471051131768714:2108] [node 5] ICS09 handshake done sender: [1:7577471046836801390:2113] self: [1:7577471046836801389:0] peer: [5:7577471050998249713:0] socket: 306 qp: -1
2025-11-27T18:17:42.115246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471050998249714:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:42.116335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471051131768714:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:42.115279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:42.116360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:42.112412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:42.109853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471050547265684:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.112710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471047389110062:2110] [node 6] ICP22 created new session: [2:7577471051684077396:2110]
2025-11-27T18:17:42.115825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471048984784971:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.114214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471051052059167:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:42.114241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:42.116503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471051052059163:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.116556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471046757091779:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.116943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:42.116995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471046757091779:2049] [node 2] ICP22 created new session: [4:7577471051052059169:2049]
2025-11-27T18:17:42.117033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471051052059169:2049] [node 2] ICS09 handshake done sender: [4:7577471046757091855:2117] self: [4:7577471051052059157:0] peer: [2:7577471047389110067:0] socket: 305 qp: -1
2025-11-27T18:17:42.117072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471051052059169:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:42.117098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.113682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471051684077396:2110] [node 6] ICS09 handshake done sender: [2:7577471047389110075:2118] self: [2:7577471047389110074:0] peer: [6:7577471054842233060:0] socket: 324 qp: -1
2025-11-27T18:17:42.115872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:42.114721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471054842233059:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.117297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471048984784971:2107] [node 4] ICP22 created new session: [3:7577471053279752298:2107]
2025-11-27T18:17:42.113733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471051684077396:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:42.114425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:42.117358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471053279752298:2107] [node 4] ICS09 handshake done sender: [3:7577471048984784977:2121] self: [3:7577471048984784976:0] peer: [4:7577471051052059158:0] socket: 307 qp: -1
2025-11-27T18:17:42.117384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471053279752298:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:42.121616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471054842233061:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51529
2025-11-27T18:17:42.121788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471046757091779:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:42.117459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:42.125500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471051052059153:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:42.126405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471046757091840:2108] [node 5] ICP18 handshake (actor [4:7577471051052059163:2122]) is too old
2025-11-27T18:17:42.126492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471046757091780:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.129208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471048984784979:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:42.126545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:42.126587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471046757091780:2050] [node 3] ICP22 created new session: [4:7577471051052059172:2050]
2025-11-27T18:17:42.129290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471050547265683:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.126628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471051052059172:2050] [node 3] ICS09 handshake done sender: [4:7577471051052059153:2118] self: [4:7577471051052059158:0] peer: [3:7577471048984784976:0] socket: 311 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.130300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471046703282323:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1297) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471050998249706:2121] held: no
2025-11-27T18:17:42.130646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050998249716:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4674
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.134607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471050998249717:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4674
2025-11-27T18:17:42.126660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471051052059172:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:42.126692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:42.135896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471050998249706:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:42.136731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471050547265684:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3546) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471054842233058:2116] held: no
2025-11-27T18:17:42.140860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471054842233058:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:42.141041Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471050998249702:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:42.141483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471046836801392:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.141321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471048984784972:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.146061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471050547265684:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.146123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:42.146166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471050547265684:2049] [node 2] ICP22 created new session: [6:7577471054842233066:2049]
2025-11-27T18:17:42.146205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471054842233066:2049] [node 2] ICS09 handshake done sender: [6:7577471054842233058:2116] self: [6:7577471054842233060:0] peer: [2:7577471047389110074:0] socket: 326 qp: -1
2025-11-27T18:17:42.146234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471054842233066:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:42.146256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:42.141512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471046836801392:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.141401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:42.141446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471048984784972:2108] [node 5] ICP22 created new session: [3:7577471053279752300:2108]
2025-11-27T18:17:42.141488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471053279752300:2108] [node 5] ICS09 handshake done sender: [3:7577471048984784979:2122] self: [3:7577471048984784978:0] peer: [5:7577471050998249705:0] socket: 300 qp: -1
2025-11-27T18:17:42.141528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471053279752300:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:42.141555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.146981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471046703282323:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.147036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:42.147077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471046703282323:2048] [node 1] ICP22 created new session: [5:7577471050998249719:2048]
2025-11-27T18:17:42.147112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471050998249719:2048] [node 1] ICS09 handshake done sender: [5:7577471050998249706:2121] self: [5:7577471050998249713:0] peer: [1:7577471046836801389:0] socket: 328 qp: -1
2025-11-27T18:17:42.147136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471050998249719:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:42.147168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:42.147672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471046703282325:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.147920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:42.147953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471046703282325:2050] [node 3] ICP22 created new session: [5:7577471050998249721:2050]
2025-11-27T18:17:42.147990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471050998249721:2050] [node 3] ICS09 handshake done sender: [5:7577471050998249702:2119] self: [5:7577471050998249705:0] peer: [3:7577471048984784978:0] socket: 303 qp: -1
2025-11-27T18:17:42.148018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471050998249721:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.148042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:42.148051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471053279752297:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34799
2025-11-27T18:17:42.147457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471054842233063:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51529
2025-11-27T18:17:42.147562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471054842233064:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51529
2025-11-27T18:17:42.147613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471054842233065:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51529
2025-11-27T18:17:42.148517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471054842233063:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.148594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471054842233068:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51529
2025-11-27T18:17:42.148738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471050547265748:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.149584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471046836801392:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:42.150978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471046757091853:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.151010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471046757091853:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.151167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471046836801384:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.151229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:42.152604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471046757091853:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:42.152851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471048984784971:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:42.152148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471046836801384:2109] [node 6] ICP22 created new session: [1:7577471051131768717:2109]
2025-11-27T18:17:42.152950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471051131768717:2109] [node 6] ICS09 handshake done sender: [1:7577471046836801392:2114] self: [1:7577471046836801391:0] peer: [6:7577471054842233062:0] socket: 327 qp: -1
2025-11-27T18:17:42.152984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471051131768717:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:42.153016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:42.153039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471054842233059:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:42.153791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471046757091845:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.153839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:42.153895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471046757091845:2109] [node 6] ICP22 created new session: [4:7577471051052059175:2109]
2025-11-27T18:17:42.154015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471051052059175:2109] [node 6] ICS09 handshake done sender: [4:7577471046757091853:2115] self: [4:7577471046757091852:0] peer: [6:7577471054842233070:0] socket: 286 qp: -1
2025-11-27T18:17:42.154047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471051052059175:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:42.154078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.155532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471053279752302:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34799
2025-11-27T18:17:42.154108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471050547265683:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.154161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:42.154208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471050547265683:2048] [node 1] ICP22 created new session: [6:7577471054842233072:2048]
2025-11-27T18:17:42.154502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471054842233072:2048] [node 1] ICS09 handshake done sender: [6:7577471054842233059:2117] self: [6:7577471054842233062:0] peer: [1:7577471046836801391:0] socket: 331 qp: -1
2025-11-27T18:17:42.154530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471054842233072:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:42.154562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:42.156897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471054842233065:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.157056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471054842233071:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51529
2025-11-27T18:17:42.157091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471053279752302:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.157546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471054842233064:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.157710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471054842233063:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:42.158036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471050547265745:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.158112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471050547265685:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.158148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471050547265748:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.158183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:42.158226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471050547265748:2109] [node 4] ICP22 created new session: [6:7577471054842233077:2109]
2025-11-27T18:17:42.158270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471054842233077:2109] [node 4] ICS09 handshake done sender: [6:7577471054842233063:2119] self: [6:7577471054842233070:0] peer: [4:7577471046757091852:0] socket: 289 qp: -1
2025-11-27T18:17:42.158306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471054842233077:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:42.158351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.159374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471051052059177:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43811
2025-11-27T18:17:42.159559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471048984784981:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.159585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471048984784981:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.159541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471046703282399:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.159576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471046703282399:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.160754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471046703282399:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.161454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471046703282391:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.161515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:42.161501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471048984784981:2123] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:42.161555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471046703282391:2111] [node 6] ICP22 created new session: [5:7577471050998249724:2111]
2025-11-27T18:17:42.161520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471054842233079:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51529
2025-11-27T18:17:42.161617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471050998249724:2111] [node 6] ICS09 handshake done sender: [5:7577471046703282399:2116] self: [5:7577471046703282398:0] peer: [6:7577471054842233075:0] socket: 278 qp: -1
2025-11-27T18:17:42.161695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471050998249724:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:42.161726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:42.161797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471048984784973:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.161840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:42.161877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471048984784973:2109] [node 6] ICP22 created new session: [3:7577471053279752305:2109]
2025-11-27T18:17:42.161934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471053279752305:2109] [node 6] ICS09 handshake done sender: [3:7577471048984784981:2123] self: [3:7577471048984784980:0] peer: [6:7577471054842233076:0] socket: 285 qp: -1
2025-11-27T18:17:42.161975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471053279752305:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:42.162005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:42.162182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471054842233065:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:42.163176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471050547265745:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.163257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:42.163299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471050547265745:2103] [node 5] ICP22 created new session: [6:7577471054842233081:2103]
2025-11-27T18:17:42.163336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471054842233081:2103] [node 5] ICS09 handshake done sender: [6:7577471054842233065:2121] self: [6:7577471054842233075:0] peer: [5:7577471046703282398:0] socket: 312 qp: -1
2025-11-27T18:17:42.163359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471054842233081:2103] [node 5] ICS10 traffic start
2025-11-27T18:17:42.163399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.165529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471054842233080:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51529
2025-11-27T18:17:42.165915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471050547265745:2103] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:42.167341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471050547265685:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471054842233064:2120] held: no
2025-11-27T18:17:42.167448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471054842233064:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:42.167638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471050547265685:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.167682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:42.167727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471050547265685:2050] [node 3] ICP22 created new session: [6:7577471054842233083:2050]
2025-11-27T18:17:42.167765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471054842233083:2050] [node 3] ICS09 handshake done sender: [6:7577471054842233064:2120] self: [6:7577471054842233076:0] peer: [3:7577471048984784980:0] socket: 291 qp: -1
2025-11-27T18:17:42.167798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471054842233083:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:42.167824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:42.183566Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:42.183843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471051684077383:2048] [node 1] ICS01 socket: 292 reason# EndOfStream
2025-11-27T18:17:42.183898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471047389109996:2048] [node 1] ICP30 unregister session Session# [2:7577471051684077383:2048] VirtualId# [2:7577471051684077379:0]
2025-11-27T18:17:42.183933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471051684077383:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.184010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:42.184071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:42.184746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:42.184956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471053279752288:2048] [node 1] ICS01 socket: 297 reason# EndOfStream
2025-11-27T18:17:42.184999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471048984784897:2048] [node 1] ICP30 unregister session Session# [3:7577471053279752288:2048] VirtualId# [3:7577471053279752284:0]
2025-11-27T18:17:42.185023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471053279752288:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.185097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:42.185159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:42.187276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:42.187420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471051052059167:2048] [node 1] ICS01 socket: 304 reason# EndOfStream
2025-11-27T18:17:42.187450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471046757091778:2048] [node 1] ICP30 unregister session Session# [4:7577471051052059167:2048] VirtualId# [4:7577471051052059156:0]
2025-11-27T18:17:42.187466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471051052059167:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.187527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:42.187572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:42.188114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:42.188305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471050998249719:2048] [node 1] ICS01 socket: 328 reason# EndOfStream
2025-11-27T18:17:42.188342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471046703282323:2048] [node 1] ICP30 unregister session Session# [5:7577471050998249719:2048] VirtualId# [5:7577471050998249713:0]
2025-11-27T18:17:42.188368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471050998249719:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.188419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:42.188457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:42.188998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:42.189173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471054842233072:2048] [node 1] ICS01 socket: 331 reason# EndOfStream
2025-11-27T18:17:42.189204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471050547265683:2048] [node 1] ICP30 unregister session Session# [6:7577471054842233072:2048] VirtualId# [6:7577471054842233062:0]
2025-11-27T18:17:42.189228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471054842233072:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.189309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:42.189342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:42.217798Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:42.218044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471053279752294:2049] [node 2] ICS01 socket: 299 reason# EndOfStream
2025-11-27T18:17:42.218091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471048984784898:2049] [node 2] ICP30 unregister session Session# [3:7577471053279752294:2049] VirtualId# [3:7577471053279752287:0]
2025-11-27T18:17:42.218135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471053279752294:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.218220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:42.218271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:42.220214Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:42.220405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471051052059169:2049] [node 2] ICS01 socket: 305 reason# EndOfStream
2025-11-27T18:17:42.220448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471046757091779:2049] [node 2] ICP30 unregister session Session# [4:7577471051052059169:2049] VirtualId# [4:7577471051052059157:0]
2025-11-27T18:17:42.220478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471051052059169:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.220540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:42.220610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:42.220999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:42.221198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471050998249709:2049] [node 2] ICS01 socket: 325 reason# EndOfStream
2025-11-27T18:17:42.221239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471046703282324:2049] [node 2] ICP30 unregister session Session# [5:7577471050998249709:2049] VirtualId# [5:7577471050998249707:0]
2025-11-27T18:17:42.221277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471050998249709:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.221333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:42.221377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:42.221953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:42.222167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471054842233066:2049] [node 2] ICS01 socket: 326 reason# EndOfStream
2025-11-27T18:17:42.222205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471050547265684:2049] [node 2] ICP30 unregister session Session# [6:7577471054842233066:2049] VirtualId# [6:7577471054842233060:0]
2025-11-27T18:17:42.222229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471054842233066:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.222292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:42.222330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:42.243656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:42.243850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471051052059172:2050] [node 3] ICS01 socket: 311 reason# EndOfStream
2025-11-27T18:17:42.243885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471046757091780:2050] [node 3] ICP30 unregister session Session# [4:7577471051052059172:2050] VirtualId# [4:7577471051052059158:0]
2025-11-27T18:17:42.243912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471051052059172:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.243982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:42.244022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:42.244671Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:42.244863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471050998249721:2050] [node 3] ICS01 socket: 303 reason# EndOfStream
2025-11-27T18:17:42.244896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471046703282325:2050] [node 3] ICP30 unregister session Session# [5:7577471050998249721:2050] VirtualId# [5:7577471050998249705:0]
2025-11-27T18:17:42.244921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471050998249721:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.244992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:42.245052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:42.245687Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:42.245863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471054842233083:2050] [node 3] ICS01 socket: 291 reason# EndOfStream
2025-11-27T18:17:42.245908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471050547265685:2050] [node 3] ICP30 unregister session Session# [6:7577471054842233083:2050] VirtualId# [6:7577471054842233076:0]
2025-11-27T18:17:42.245939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471054842233083:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.245995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:42.246037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:42.299932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:42.301389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:42.300295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471054842233077:2109] [node 4] ICS07 socket disconnect 289 reason# EndOfStream
2025-11-27T18:17:42.301719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471050998249714:2108] [node 4] ICS01 socket: 316 reason# EndOfStream
2025-11-27T18:17:42.300339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471054842233077:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.301769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471046703282388:2108] [node 4] ICP30 unregister session Session# [5:7577471050998249714:2108] VirtualId# [5:7577471050998249703:0]
2025-11-27T18:17:42.300463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:42.300580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471054842233077:2109] [node 4] ICS15 start handshake
2025-11-27T18:17:42.301822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471050998249714:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.301933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:42.302037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:42.306423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471050547265748:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9131) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:42.306493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471054842233077:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:42.306527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471050547265748:2109] [node 4] ICP30 unregister session Session# [6:7577471054842233077:2109] VirtualId# [6:7577471054842233070:0]
2025-11-27T18:17:42.306571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:42.306605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471050547265748:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9131) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:42.306623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9131) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:42.317467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471050547265748:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:17:42.357058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:42.357323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471054842233081:2103] [node 5] ICS07 socket disconnect 312 reason# EndOfStream
2025-11-27T18:17:42.357365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471054842233081:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:42.357434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:42.357483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471054842233081:2103] [node 5] ICS15 start handshake
2025-11-27T18:17:42.358445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471050547265745:2103] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16914) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:42.358506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471054842233081:2103] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:42.358538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471050547265745:2103] [node 5] ICP30 unregister session Session# [6:7577471054842233081:2103] VirtualId# [6:7577471054842233075:0]
2025-11-27T18:17:42.358586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:17:42.358618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471050547265745:2103] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16914) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:42.358633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16914) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:42.373381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471050547265745:2103] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:42.450820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471053441197077:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:42.450859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471053441197078:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:42.472859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471052456531538:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:42.472904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471052456531539:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:42.496240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471051867657740:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:42.496298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471051867657741:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:42.518200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471053024651563:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:42.518245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471053024651564:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:42.518324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471053024651565:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:42.540011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471054172831297:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:42.540050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471054172831298:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:42.540255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471054172831296:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:42.559559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471053275428948:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:42.559599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471053275428949:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:42.559626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471053275428950:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:42.667456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471053275429008:2104] [node 5] ICP01 ready to work
2025-11-27T18:17:42.805814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471053024651628:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:42.760997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471053441197140:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:42.803121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471051867657804:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:42.805461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471052456531601:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:42.809938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471052456531602:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:42.806076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471054172831361:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:42.804848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471051867657803:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:42.819036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471053024651630:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:42.809794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471054172831360:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:42.819196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471052456531603:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:42.825579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471053441197077:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(161014006ns) {Configure:104(160997163ns) {ProcessConfigured:135(47618ns) {StartInitialHandshake:157(46193ns) {DropHandshakes:470(2304ns) {DropIncomingHandshake:442(400ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(699ns) PrepareNewSessionHandshake:477(674ns) {SwitchToState:179(230ns)}}}}}
2025-11-27T18:17:42.825934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471053441197143:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:42.826166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471053441197144:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:42.813747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471051867657805:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:42.826864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471053024651563:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(113963273ns) {Configure:104(113957558ns) {ProcessConfigured:135(12568822ns) {StartInitialHandshake:157(12565528ns) {DropHandshakes:470(3754ns) {DropIncomingHandshake:442(1213ns) DropOutgoingHandshake:457(737ns)} GenerateSessionVirtualId:988(1249ns) PrepareNewSessionHandshake:477(4587ns) {SwitchToState:179(2533ns)}}}}}
2025-11-27T18:17:42.824574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471053275429012:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:42.828635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471053275429008:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(156388483ns) {Configure:104(156382060ns) {ProcessConfigured:135(4832756ns) {StartInitialHandshake:157(4827871ns) {DropHandshakes:470(3378ns) {DropIncomingHandshake:442(934ns) DropOutgoingHandshake:457(393ns)} GenerateSessionVirtualId:988(2271ns) PrepareNewSessionHandshake:477(4619ns) {SwitchToState:179(1738ns)}}}}}
2025-11-27T18:17:42.846329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471053441197140:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19896250ns) {Configure:104(19893800ns) {ProcessConfigured:135(3102705ns) {StartInitialHandshake:157(3100849ns) {DropHandshakes:470(4516ns) {DropIncomingHandshake:442(523ns) DropOutgoingHandshake:457(209ns)} GenerateSessionVirtualId:988(941ns) PrepareNewSessionHandshake:477(1259ns) {SwitchToState:179(43ns)}}}}}
2025-11-27T18:17:42.848133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471053441197078:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(85036731ns) {Configure:104(63249161ns) {ProcessConfigured:135(6664752ns) {StartInitialHandshake:157(6661338ns) {DropHandshakes:470(2799ns) {DropIncomingHandshake:442(598ns) DropOutgoingHandshake:457(355ns)} GenerateSessionVirtualId:988(1807ns) PrepareNewSessionHandshake:477(3120ns) {SwitchToState:179(1407ns)}}}}}
2025-11-27T18:17:42.852040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471052456531538:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32515482ns) {Configure:104(32509848ns) {ProcessConfigured:135(39163ns) {StartInitialHandshake:157(37171ns) {DropHandshakes:470(1396ns) {DropIncomingHandshake:442(375ns) DropOutgoingHandshake:457(289ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:477(518ns) {SwitchToState:179(324ns)}}}}}
2025-11-27T18:17:42.850519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471051867657740:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30744835ns) {Configure:104(30735284ns) {ProcessConfigured:135(808663ns) {StartInitialHandshake:157(806883ns) {DropHandshakes:470(1261ns) {DropIncomingHandshake:442(423ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(428ns) PrepareNewSessionHandshake:477(1405ns) {SwitchToState:179(644ns)}}}}}
2025-11-27T18:17:42.854983Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471053024651628:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23646646ns) {Configure:104(22007661ns) {ProcessConfigured:135(1938857ns) {StartInitialHandshake:157(1937187ns) {DropHandshakes:470(1104ns) {DropIncomingHandshake:442(308ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(267ns) PrepareNewSessionHandshake:477(2067ns) {SwitchToState:179(1105ns)}}}}}
2025-11-27T18:17:42.855206Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471052456531539:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27380237ns) {Configure:104(27375695ns) {ProcessConfigured:135(2051267ns) {StartInitialHandshake:157(2048350ns) {DropHandshakes:470(1898ns) {DropIncomingHandshake:442(473ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(769ns) PrepareNewSessionHandshake:477(3046ns) {SwitchToState:179(790ns)}}}}}
2025-11-27T18:17:42.864910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471054172831298:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(30880691ns) {Configure:104(30875581ns) {ProcessConfigured:135(2700698ns) {StartInitialHandshake:157(2698901ns) {DropHandshakes:470(1129ns) {DropIncomingHandshake:442(316ns) DropOutgoingHandshake:457(143ns)} GenerateSessionVirtualId:988(770ns) PrepareNewSessionHandshake:477(1543ns) {SwitchToState:179(599ns)}}}}}
2025-11-27T18:17:42.885093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471053275428950:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35005592ns) {Configure:104(35000238ns) {ProcessConfigured:135(66382ns) {StartInitialHandshake:157(58964ns) {DropHandshakes:470(1957ns) {DropIncomingHandshake:442(541ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(1039ns) PrepareNewSessionHandshake:477(1429ns) {SwitchToState:179(459ns)}}}}}
2025-11-27T18:17:42.877611Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471054172831296:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(59819646ns) {Configure:104(59813160ns) {ProcessConfigured:135(22688090ns) {StartInitialHandshake:157(22685168ns) {DropHandshakes:470(336028ns) {DropIncomingHandshake:442(816ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(1225ns) PrepareNewSessionHandshake:477(2827ns) {SwitchToState:179(1337ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.892509Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471052456531603:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36029327ns) {Configure:104(36024403ns) {ProcessConfigured:135(36917ns) {StartInitialHandshake:157(35887ns) {DropHandshakes:470(1043ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(156ns)} GenerateSessionVirtualId:988(283ns) PrepareNewSessionHandshake:477(544ns) {SwitchToState:179(129ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.889146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471051867657803:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36258630ns) {Configure:104(36250849ns) {ProcessConfigured:135(38969ns) {StartInitialHandshake:157(37545ns) {DropHandshakes:470(1237ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(167ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(789ns) {SwitchToState:179(155ns)}}}}}
2025-11-27T18:17:42.896392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471053024651639:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33547
2025-11-27T18:17:42.896462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471053024651640:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33547
2025-11-27T18:17:42.896516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471053024651641:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33547
2025-11-27T18:17:42.897004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471053024651630:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(55030161ns) {Configure:104(55024471ns) {ProcessConfigured:135(17458505ns) {StartInitialHandshake:157(17454541ns) {DropHandshakes:470(2708ns) {DropIncomingHandshake:442(574ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(696ns) PrepareNewSessionHandshake:477(3284255ns) {SwitchToState:179(3189ns)}}}}}
2025-11-27T18:17:42.895076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471053441197143:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(38750201ns) {Configure:104(38744798ns) {ProcessConfigured:135(59358ns) {StartInitialHandshake:157(57484ns) {DropHandshakes:470(2013ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(720ns) PrepareNewSessionHandshake:477(1323ns) {SwitchToState:179(509ns)}}}}}
2025-11-27T18:17:42.891032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471051867657804:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49584544ns) {Configure:104(49578245ns) {ProcessConfigured:135(20623511ns) {StartInitialHandshake:157(20621275ns) {DropHandshakes:470(1134ns) {DropIncomingHandshake:442(406ns) DropOutgoingHandshake:457(184ns)} GenerateSessionVirtualId:988(702ns) PrepareNewSessionHandshake:477(4066ns) {SwitchToState:179(1840ns)}}}}}
2025-11-27T18:17:42.897237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471052456531617:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10505
2025-11-27T18:17:42.897869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471054172831297:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17564656ns) {Configure:104(17559939ns) {ProcessConfigured:135(1634771ns) {StartInitialHandshake:157(1633173ns) {DropHandshakes:470(1106ns) {DropIncomingHandshake:442(293ns) DropOutgoingHandshake:457(166ns)} GenerateSessionVirtualId:988(356ns) PrepareNewSessionHandshake:477(1419ns) {SwitchToState:179(616ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.909098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471052456531618:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10505
2025-11-27T18:17:42.908666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471053024651640:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.912437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471051867657814:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26740
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.910577Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471054172831361:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(35166740ns) {Configure:104(35156824ns) {ProcessConfigured:135(9257475ns) {StartInitialHandshake:157(9254521ns) {DropHandshakes:470(1962ns) {DropIncomingHandshake:442(533ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(352ns) PrepareNewSessionHandshake:477(3482ns) {SwitchToState:179(1829ns)}}}}}
2025-11-27T18:17:42.910006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471053024651639:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.912596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471051867657815:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26740
2025-11-27T18:17:42.911432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471052456531618:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.910127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471053024651564:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.914110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471053441197156:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1318
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.911456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471052456531609:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.912659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471051867657816:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26740
2025-11-27T18:17:42.914265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471053441197157:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1318
2025-11-27T18:17:42.915902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471053441197157:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.911487Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471052456531609:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.928355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471052456531609:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:42.928722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471052456531601:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.928787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:42.928844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471052456531601:2107] [node 4] ICP22 created new session: [2:7577471052456531621:2107]
2025-11-27T18:17:42.915733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471051867657816:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.928885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471052456531621:2107] [node 4] ICS09 handshake done sender: [2:7577471052456531609:2112] self: [2:7577471052456531608:0] peer: [4:7577471053024651645:0] socket: 284 qp: -1
2025-11-27T18:17:42.932168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471053441197156:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.932260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471053441197159:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1318
2025-11-27T18:17:42.932318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471053441197158:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1318
2025-11-27T18:17:42.945584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471053441197158:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.928918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471052456531621:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:42.928957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:42.945358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471052456531617:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.949449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471052456531538:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.961893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471053441197142:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.961942Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471053441197142:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.949990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471051867657814:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.961793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471053024651641:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.957470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471051867657815:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.961905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471053024651564:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2345) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471053024651639:2116] held: no
2025-11-27T18:17:42.963618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471052456531605:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.961586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471051867657741:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.961684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471051867657740:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.961883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471051867657805:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(56671840ns) {Configure:104(56660802ns) {ProcessConfigured:135(467065ns) {StartInitialHandshake:157(464152ns) {DropHandshakes:470(2154ns) {DropIncomingHandshake:442(538ns) DropOutgoingHandshake:457(316ns)} GenerateSessionVirtualId:988(1033ns) PrepareNewSessionHandshake:477(1805ns) {SwitchToState:179(541ns)}}}}}
2025-11-27T18:17:42.963814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471053441197147:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.963672Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471052456531605:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.963252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471053024651647:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33547
2025-11-27T18:17:42.963776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471053024651648:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33547
2025-11-27T18:17:42.963923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471053024651563:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.964882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471053024651647:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.967871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471053024651639:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:42.968234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471053024651565:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:42.969794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471053024651651:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33547
2025-11-27T18:17:42.963846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471053441197147:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.967954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471052456531605:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:42.970447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471053024651564:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.970502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:42.966042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471053441197159:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.966198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471053441197148:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.970740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471051867657811:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.970738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471053024651564:2049] [node 2] ICP22 created new session: [4:7577471053024651653:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.971403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471054172831360:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(30393123ns) {Configure:104(30385659ns) {ProcessConfigured:135(57130ns) {StartInitialHandshake:157(53966ns) {DropHandshakes:470(1388ns) {DropIncomingHandshake:442(338ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(572ns) PrepareNewSessionHandshake:477(909ns) {SwitchToState:179(373ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.970778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471051867657811:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.978217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471051867657811:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:42.966217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471053441197148:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:42.967587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471053441197142:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:42.971986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471053441197148:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:42.981788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471053441197140:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.981851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.989918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471054172831371:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45905
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:42.992047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471053275429026:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6163
2025-11-27T18:17:42.981904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471053441197140:2107] [node 4] ICP22 created new session: [1:7577471053441197167:2107]
2025-11-27T18:17:42.990120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471054172831372:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45905
2025-11-27T18:17:42.977779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471052456531625:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10505
2025-11-27T18:17:42.971080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471053024651653:2049] [node 2] ICS09 handshake done sender: [4:7577471053024651639:2116] self: [4:7577471053024651645:0] peer: [2:7577471052456531608:0] socket: 295 qp: -1
2025-11-27T18:17:42.990731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471054172831371:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:42.987759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471051867657825:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26740
2025-11-27T18:17:42.981950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471053441197167:2107] [node 4] ICS09 handshake done sender: [1:7577471053441197148:2112] self: [1:7577471053441197146:0] peer: [4:7577471053024651649:0] socket: 292 qp: -1
2025-11-27T18:17:42.981987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471053441197167:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:42.982022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:42.982445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471053441197077:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.982507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:42.982542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471053441197077:2048] [node 2] ICP22 created new session: [1:7577471053441197169:2048]
2025-11-27T18:17:42.982592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471053441197169:2048] [node 2] ICS09 handshake done sender: [1:7577471053441197142:2110] self: [1:7577471053441197141:0] peer: [2:7577471052456531624:0] socket: 285 qp: -1
2025-11-27T18:17:42.982630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471053441197169:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:42.982652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:42.990807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471054172831376:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45905
2025-11-27T18:17:42.971399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471053024651653:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:42.971449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:42.992879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471053024651641:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:42.978130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471052456531539:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:42.984965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:43.018244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471051867657803:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:43.021793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471053024651563:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:42.987715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471052456531539:2049] [node 3] ICP22 created new session: [2:7577471052456531627:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.018329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:43.025696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:43.034751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471054172831376:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.039355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471054172831372:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.018375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471051867657803:2107] [node 4] ICP22 created new session: [3:7577471056162625123:2107]
2025-11-27T18:17:43.018441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471056162625123:2107] [node 4] ICS09 handshake done sender: [3:7577471051867657811:2112] self: [3:7577471051867657810:0] peer: [4:7577471053024651652:0] socket: 310 qp: -1
2025-11-27T18:17:43.018481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471056162625123:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:43.018521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:43.042858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471051867657803:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(19667651ns) {HandleHandshakeStatus:329(19658519ns) {DropIncomingHandshake:442(16370ns) DropOutgoingHandshake:457(464ns) SwitchToState:179(458ns) ProcessPendingSessionEvents:495(19260509ns) {ForwardSessionEventToSession:612(53406ns) ForwardSessionEventToSession:612(2761559ns) ForwardSessionEventToSession:612(2959ns) ForwardSessionEventToSession:612(2593480ns) ForwardSessionEventToSession:612(5176ns) ForwardSessionEventToSession:612(3782ns) ForwardSessionEventToSession:612(1887453ns) ForwardSessionEventToSession:612(4574ns) ForwardSessionEventToSession:612(2464554ns) ForwardSessionEventToSession:612(9462016ns)}}}
2025-11-27T18:17:43.044065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471056162625125:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26740
2025-11-27T18:17:42.991487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471052456531617:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.066353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471053441197147:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:43.025758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471053024651563:2048] [node 1] ICP22 created new session: [4:7577471057319618952:2048]
2025-11-27T18:17:42.995911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471052456531627:2049] [node 3] ICS09 handshake done sender: [2:7577471052456531605:2110] self: [2:7577471052456531604:0] peer: [3:7577471051867657823:0] socket: 282 qp: -1
2025-11-27T18:17:43.057432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471054172831375:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45905
2025-11-27T18:17:42.999917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471052456531627:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:43.044128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471056162625126:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26740
2025-11-27T18:17:43.001967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:43.039210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471052456531538:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.042527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:43.073411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471057570396323:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6163
2025-11-27T18:17:43.038229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471057319618952:2048] [node 1] ICS09 handshake done sender: [4:7577471053024651641:2118] self: [4:7577471053024651649:0] peer: [1:7577471053441197146:0] socket: 297 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.049810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471052456531538:2048] [node 1] ICP22 created new session: [2:7577471056751498927:2048]
2025-11-27T18:17:43.060555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471056751498927:2048] [node 1] ICS09 handshake done sender: [2:7577471052456531617:2115] self: [2:7577471052456531624:0] peer: [1:7577471053441197141:0] socket: 293 qp: -1
2025-11-27T18:17:43.063759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471056751498927:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:43.066485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:43.073276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471052456531539:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(93146123ns) {HandleHandshakeStatus:329(88398249ns) {DropIncomingHandshake:442(444ns) DropOutgoingHandshake:457(5213ns) SwitchToState:179(303ns) ProcessPendingSessionEvents:495(40656076ns) {ForwardSessionEventToSession:612(15845045ns) ForwardSessionEventToSession:612(11384ns) ForwardSessionEventToSession:612(6024999ns) ForwardSessionEventToSession:612(4662ns) ForwardSessionEventToSession:612(2589ns) ForwardSessionEventToSession:612(2143558ns) ForwardSessionEventToSession:612(3935ns) ForwardSessionEventToSession:612(16581081ns) ForwardSessionEventToSession:612(9898ns) ForwardSessionEventToSession:612(3090ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.077313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471054172831360:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:43.077482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471054172831298:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:43.078112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471058467798678:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45905
2025-11-27T18:17:43.057543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471051867657814:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:43.078967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471052456531629:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10505
2025-11-27T18:17:43.041753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471057319618952:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:43.041805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:43.071743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471053024651655:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33547
2025-11-27T18:17:43.066255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471051867657741:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.068756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:43.073243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471051867657741:2049] [node 2] ICP22 created new session: [3:7577471056162625128:2049]
2025-11-27T18:17:43.073859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471056162625128:2049] [node 2] ICS09 handshake done sender: [3:7577471051867657814:2114] self: [3:7577471051867657823:0] peer: [2:7577471052456531604:0] socket: 305 qp: -1
2025-11-27T18:17:43.076824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471053441197078:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:43.075426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471051867657815:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:43.079856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471052456531538:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(35074809ns) {HandleHandshakeStatus:329(35071765ns) {DropIncomingHandshake:442(1312ns) DropOutgoingHandshake:457(4918ns) SwitchToState:179(529ns) ProcessPendingSessionEvents:495(1920788ns) {ForwardSessionEventToSession:612(1875220ns) ForwardSessionEventToSession:612(3775ns) ForwardSessionEventToSession:612(5894ns) ForwardSessionEventToSession:612(2008ns) ForwardSessionEventToSession:612(2071ns) ForwardSessionEventToSession:612(7027ns) ForwardSessionEventToSession:612(2931ns) ForwardSessionEventToSession:612(4439ns) ForwardSessionEventToSession:612(5181ns) ForwardSessionEventToSession:612(2290ns)}}}
2025-11-27T18:17:43.078835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:43.080060Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471053024651563:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(62711793ns) {HandleHandshakeStatus:329(62708488ns) {DropIncomingHandshake:442(2511ns) DropOutgoingHandshake:457(7323ns) SwitchToState:179(2771ns) ProcessPendingSessionEvents:495(9739334ns) {ForwardSessionEventToSession:612(5913186ns) ForwardSessionEventToSession:612(4951ns) ForwardSessionEventToSession:612(989474ns) ForwardSessionEventToSession:612(3098ns) ForwardSessionEventToSession:612(273201ns) ForwardSessionEventToSession:612(1473184ns) ForwardSessionEventToSession:612(2767ns) ForwardSessionEventToSession:612(2511ns) ForwardSessionEventToSession:612(1059048ns) ForwardSessionEventToSession:612(2367ns)}}}
2025-11-27T18:17:43.079009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471053441197078:2049] [node 3] ICP22 created new session: [1:7577471057736164470:2049]
2025-11-27T18:17:43.076107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471056162625128:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:43.079260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471057736164470:2049] [node 3] ICS09 handshake done sender: [1:7577471053441197147:2111] self: [1:7577471053441197145:0] peer: [3:7577471051867657824:0] socket: 290 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.082409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471053024651565:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471053024651647:2119] held: no
2025-11-27T18:17:43.076159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:43.089249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471057570396323:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.089461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471053275428948:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:43.082488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471054172831375:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.084219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471053024651563:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.079261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471051867657740:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.090676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471057570396324:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6163
2025-11-27T18:17:43.079302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:43.079356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471051867657740:2048] [node 1] ICP22 created new session: [3:7577471056162625130:2048]
2025-11-27T18:17:43.079402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471056162625130:2048] [node 1] ICS09 handshake done sender: [3:7577471051867657815:2115] self: [3:7577471051867657824:0] peer: [1:7577471053441197145:0] socket: 306 qp: -1
2025-11-27T18:17:43.079433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471056162625130:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:43.086336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471054172831297:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:43.079470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471056162625126:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.079300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471057736164470:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:43.085518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471053024651636:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.079471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.079337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:43.085547Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471053024651636:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:43.080449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471051867657809:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.092122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471057570396324:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.086698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471057736164469:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1318
2025-11-27T18:17:43.080476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471051867657809:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:43.085963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471051867657809:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:43.090769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471053024651647:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:43.092409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471058467798681:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45905
2025-11-27T18:17:43.094585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471053275428950:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:43.093574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471053024651565:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.091457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471057736164469:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.090060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471051867657804:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.093612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:43.093649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471053024651565:2050] [node 3] ICP22 created new session: [4:7577471057319618956:2050]
2025-11-27T18:17:43.093682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471057319618956:2050] [node 3] ICS09 handshake done sender: [4:7577471053024651647:2119] self: [4:7577471053024651652:0] peer: [3:7577471051867657810:0] socket: 311 qp: -1
2025-11-27T18:17:43.096567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471058467798678:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.092246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471053441197150:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.090143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:43.093705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471057319618956:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:43.092275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471053441197150:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:43.097846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471052456531629:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.098052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471052456531611:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.098070Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471052456531611:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:43.099963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471056751498929:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10505
2025-11-27T18:17:43.102085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471052456531539:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:43.104598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471052456531611:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:43.096922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471054172831296:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:43.090203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471051867657804:2108] [node 5] ICP22 created new session: [3:7577471056162625135:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.090242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471056162625135:2108] [node 5] ICS09 handshake done sender: [3:7577471051867657809:2111] self: [3:7577471051867657808:0] peer: [5:7577471058467798679:0] socket: 309 qp: -1
2025-11-27T18:17:43.096989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471058467798682:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45905
2025-11-27T18:17:43.102341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471053275429026:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.113613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471052456531602:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:43.113724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:43.113774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471052456531602:2108] [node 5] ICP22 created new session: [2:7577471056751498932:2108]
2025-11-27T18:17:43.113817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471056751498932:2108] [node 5] ICS09 handshake done sender: [2:7577471052456531611:2113] self: [2:7577471052456531610:0] peer: [5:7577471058467798680:0] socket: 302 qp: -1
2025-11-27T18:17:43.113854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471056751498932:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:43.113899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:43.093728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:43.109459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471053275428949:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:43.097220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471054172831298:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471054172831372:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.090898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471056162625135:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:43.094717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471053024651636:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:43.109794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471054172831372:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:43.117658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471053441197150:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:43.115148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471057570396327:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6163
2025-11-27T18:17:43.091232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:43.097119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471051867657822:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.097149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471051867657822:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:43.117203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471051867657822:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.123468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471051867657805:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:43.123500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:43.123528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471051867657805:2109] [node 6] ICP22 created new session: [3:7577471056162625138:2109]
2025-11-27T18:17:43.123552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471056162625138:2109] [node 6] ICS09 handshake done sender: [3:7577471051867657822:2117] self: [3:7577471051867657821:0] peer: [6:7577471057570396326:0] socket: 300 qp: -1
2025-11-27T18:17:43.123570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471056162625138:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:43.123589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.105390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471053024651628:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:43.118659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471052456531616:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.118684Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471052456531616:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:43.120367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471052456531616:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:43.121190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471052456531603:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:43.121474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:43.123079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471052456531603:2109] [node 6] ICP22 created new session: [2:7577471056751498935:2109]
2025-11-27T18:17:43.126124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471056751498935:2109] [node 6] ICS09 handshake done sender: [2:7577471052456531616:2114] self: [2:7577471052456531615:0] peer: [6:7577471057570396329:0] socket: 334 qp: -1
2025-11-27T18:17:43.126161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471056751498935:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:43.126212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.117736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471053441197155:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.117754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471053441197155:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:43.121871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471053441197155:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:43.122004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471053441197144:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:43.122041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:43.122071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471053441197144:2109] [node 6] ICP22 created new session: [1:7577471057736164475:2109]
2025-11-27T18:17:43.122108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471057736164475:2109] [node 6] ICS09 handshake done sender: [1:7577471053441197150:2113] self: [1:7577471053441197149:0] peer: [6:7577471057570396325:0] socket: 338 qp: -1
2025-11-27T18:17:43.122152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471057736164475:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:43.122172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:43.122431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471053441197143:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:43.122460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:43.122496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471053441197143:2108] [node 5] ICP22 created new session: [1:7577471057736164477:2108]
2025-11-27T18:17:43.122542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471057736164477:2108] [node 5] ICS09 handshake done sender: [1:7577471053441197155:2114] self: [1:7577471053441197154:0] peer: [5:7577471058467798684:0] socket: 339 qp: -1
2025-11-27T18:17:43.122561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471057736164477:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:43.122580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:43.118080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471057570396327:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.116418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471054172831298:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.105440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:43.105488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471053024651628:2108] [node 5] ICP22 created new session: [4:7577471057319618958:2108]
2025-11-27T18:17:43.105539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471057319618958:2108] [node 5] ICS09 handshake done sender: [4:7577471053024651636:2114] self: [4:7577471053024651635:0] peer: [5:7577471058467798677:0] socket: 299 qp: -1
2025-11-27T18:17:43.105573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471057319618958:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:43.105601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:43.124894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471053024651638:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.124908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471053024651638:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:43.125776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471053024651638:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:43.125941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471053024651630:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:43.127496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:43.130997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471053024651630:2109] [node 6] ICP22 created new session: [4:7577471057319618961:2109]
2025-11-27T18:17:43.131030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471057319618961:2109] [node 6] ICS09 handshake done sender: [4:7577471053024651638:2115] self: [4:7577471053024651637:0] peer: [6:7577471057570396332:0] socket: 308 qp: -1
2025-11-27T18:17:43.131052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471057319618961:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:43.131081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:43.116445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:43.116482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471054172831298:2050] [node 3] ICP22 created new session: [5:7577471058467798686:2050]
2025-11-27T18:17:43.116513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471058467798686:2050] [node 3] ICS09 handshake done sender: [5:7577471054172831372:2117] self: [5:7577471058467798679:0] peer: [3:7577471051867657808:0] socket: 318 qp: -1
2025-11-27T18:17:43.116530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471058467798686:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:43.116550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:43.116835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471054172831376:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:43.126742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471058467798685:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45905
2025-11-27T18:17:43.127958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471054172831297:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2345) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471054172831375:2119] held: no
2025-11-27T18:17:43.128062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471054172831360:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.128101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:43.128145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471054172831360:2107] [node 4] ICP22 created new session: [5:7577471058467798689:2107]
2025-11-27T18:17:43.128187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471058467798689:2107] [node 4] ICS09 handshake done sender: [5:7577471054172831376:2120] self: [5:7577471058467798677:0] peer: [4:7577471053024651635:0] socket: 326 qp: -1
2025-11-27T18:17:43.128215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471058467798689:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:43.128245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:43.130080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471054172831375:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:43.118361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471057570396330:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6163
2025-11-27T18:17:43.123242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471057570396331:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6163
2025-11-27T18:17:43.124355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471053275429012:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:43.124706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471057570396328:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6163
2025-11-27T18:17:43.124742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471057570396324:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:43.127191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471057570396333:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6163
2025-11-27T18:17:43.133401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471057570396323:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.134009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471058467798688:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45905
2025-11-27T18:17:43.133795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471053275428950:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.133856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:43.133897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471053275428950:2050] [node 3] ICP22 created new session: [6:7577471057570396335:2050]
2025-11-27T18:17:43.133939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471057570396335:2050] [node 3] ICS09 handshake done sender: [6:7577471057570396324:2117] self: [6:7577471057570396326:0] peer: [3:7577471051867657821:0] socket: 312 qp: -1
2025-11-27T18:17:43.133971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471057570396335:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:43.133999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:43.137135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471054172831296:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471058467798678:2121] held: no
2025-11-27T18:17:43.137940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471054172831297:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.135867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471053275428948:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.137979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:43.137885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:43.137934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471053275428948:2048] [node 1] ICP22 created new session: [6:7577471057570396337:2048]
2025-11-27T18:17:43.138030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471057570396337:2048] [node 1] ICS09 handshake done sender: [6:7577471057570396323:2116] self: [6:7577471057570396325:0] peer: [1:7577471053441197149:0] socket: 340 qp: -1
2025-11-27T18:17:43.138052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471057570396337:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:43.138019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471054172831297:2049] [node 2] ICP22 created new session: [5:7577471058467798691:2049]
2025-11-27T18:17:43.138070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:43.138375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471058467798678:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:43.138583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471058467798691:2049] [node 2] ICS09 handshake done sender: [5:7577471054172831375:2119] self: [5:7577471058467798680:0] peer: [2:7577471052456531610:0] socket: 325 qp: -1
2025-11-27T18:17:43.138795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471058467798691:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:43.138836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:43.138709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471053275429026:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:43.139558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471054172831296:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.141219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471057570396334:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6163
2025-11-27T18:17:43.139634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:43.139679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471054172831296:2048] [node 1] ICP22 created new session: [5:7577471058467798693:2048]
2025-11-27T18:17:43.139714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471058467798693:2048] [node 1] ICS09 handshake done sender: [5:7577471058467798678:2121] self: [5:7577471058467798684:0] peer: [1:7577471053441197154:0] socket: 294 qp: -1
2025-11-27T18:17:43.139735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471058467798693:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:43.139765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:43.141778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471053275429012:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2947) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471057570396327:2118] held: no
2025-11-27T18:17:43.143106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471057570396328:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.143176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471053275428949:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.143204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:43.143231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471053275428949:2049] [node 2] ICP22 created new session: [6:7577471057570396339:2049]
2025-11-27T18:17:43.143257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471057570396339:2049] [node 2] ICS09 handshake done sender: [6:7577471053275429026:2115] self: [6:7577471057570396329:0] peer: [2:7577471052456531615:0] socket: 335 qp: -1
2025-11-27T18:17:43.143278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471057570396339:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:43.143297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:43.144089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471057570396327:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:43.144462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471053275428948:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:43.144699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471053275429008:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:43.145063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471054172831370:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:43.145100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471054172831370:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:43.145068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471053275429012:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.145140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:43.145175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471053275429012:2107] [node 4] ICP22 created new session: [6:7577471057570396343:2107]
2025-11-27T18:17:43.145345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471057570396343:2107] [node 4] ICS09 handshake done sender: [6:7577471057570396327:2118] self: [6:7577471057570396332:0] peer: [4:7577471053024651637:0] socket: 337 qp: -1
2025-11-27T18:17:43.145387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471057570396343:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:43.145422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:43.146363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471054172831370:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.146655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471057570396345:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6163
2025-11-27T18:17:43.146863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471057570396328:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:43.146594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471054172831361:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:43.146662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:43.146719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471054172831361:2108] [node 6] ICP22 created new session: [5:7577471058467798696:2108]
2025-11-27T18:17:43.146767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471058467798696:2108] [node 6] ICS09 handshake done sender: [5:7577471054172831370:2115] self: [5:7577471054172831369:0] peer: [6:7577471057570396342:0] socket: 298 qp: -1
2025-11-27T18:17:43.146798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471058467798696:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:43.146838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:43.147310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471053275429008:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:43.147365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:43.147414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471053275429008:2104] [node 5] ICP22 created new session: [6:7577471057570396346:2104]
2025-11-27T18:17:43.147452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471057570396346:2104] [node 5] ICS09 handshake done sender: [6:7577471057570396328:2119] self: [6:7577471057570396342:0] peer: [5:7577471054172831369:0] socket: 344 qp: -1
2025-11-27T18:17:43.147480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471057570396346:2104] [node 5] ICS10 traffic start
2025-11-27T18:17:43.147541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:43.148763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471053275429008:2104] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:43.185809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:43.186049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471056751498927:2048] [node 1] ICS07 socket disconnect 293 reason# EndOfStream
2025-11-27T18:17:43.186084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471056751498927:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.186189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:43.186230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471056751498927:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:43.187325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:43.187550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471056162625130:2048] [node 1] ICS01 socket: 306 reason# EndOfStream
2025-11-27T18:17:43.187604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471051867657740:2048] [node 1] ICP30 unregister session Session# [3:7577471056162625130:2048] VirtualId# [3:7577471051867657824:0]
2025-11-27T18:17:43.187677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471056162625130:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.187747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:43.187788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:43.189238Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471052456531538:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:43.189472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471056751498927:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:43.190296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471052456531538:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:43.190444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471056751498927:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:43.190479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471052456531538:2048] [node 1] ICP30 unregister session Session# [2:7577471056751498927:2048] VirtualId# [2:7577471052456531624:0]
2025-11-27T18:17:43.190529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:43.190558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471052456531538:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:43.190573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:43.191837Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:43.192057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471057319618952:2048] [node 1] ICS07 socket disconnect 297 reason# EndOfStream
2025-11-27T18:17:43.192083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471057319618952:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.192147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:43.192181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471057319618952:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:43.195451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:43.195656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471057570396337:2048] [node 1] ICS07 socket disconnect 340 reason# EndOfStream
2025-11-27T18:17:43.195994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:43.195686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471057570396337:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.195756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:43.195797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471057570396337:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:43.196288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471058467798693:2048] [node 1] ICS01 socket: 294 reason# EndOfStream
2025-11-27T18:17:43.196379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471054172831296:2048] [node 1] ICP30 unregister session Session# [5:7577471058467798693:2048] VirtualId# [5:7577471058467798684:0]
2025-11-27T18:17:43.196440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471058467798693:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.196500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:43.196534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:43.193831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471053024651563:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:43.193884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471057319618952:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:43.200545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471053275428948:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:43.200627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471057570396337:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:43.200658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471053275428948:2048] [node 1] ICP30 unregister session Session# [6:7577471057570396337:2048] VirtualId# [6:7577471057570396325:0]
2025-11-27T18:17:43.200712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:43.200741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471053275428948:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:43.200755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:43.200854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471052456531538:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:43.193924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471053024651563:2048] [node 1] ICP30 unregister session Session# [4:7577471057319618952:2048] VirtualId# [4:7577471053024651649:0]
2025-11-27T18:17:43.193982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:43.194028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471053024651563:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:43.194044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:43.203197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471053024651563:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:43.210315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471053275428948:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:43.228550Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:43.228778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471056162625128:2049] [node 2] ICS01 socket: 305 reason# EndOfStream
2025-11-27T18:17:43.228815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471051867657741:2049] [node 2] ICP30 unregister session Session# [3:7577471056162625128:2049] VirtualId# [3:7577471051867657823:0]
2025-11-27T18:17:43.228845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471056162625128:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.228915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:43.228956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:43.231068Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:43.231306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471053024651653:2049] [node 2] ICS01 socket: 295 reason# EndOfStream
2025-11-27T18:17:43.231354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471053024651564:2049] [node 2] ICP30 unregister session Session# [4:7577471053024651653:2049] VirtualId# [4:7577471053024651645:0]
2025-11-27T18:17:43.231385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471053024651653:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.231468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:43.231527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:43.232543Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:43.232757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471057570396339:2049] [node 2] ICS07 socket disconnect 335 reason# EndOfStream
2025-11-27T18:17:43.232792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471057570396339:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.232851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:43.232892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471057570396339:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:43.233517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:43.233709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471058467798691:2049] [node 2] ICS01 socket: 325 reason# EndOfStream
2025-11-27T18:17:43.233743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471054172831297:2049] [node 2] ICP30 unregister session Session# [5:7577471058467798691:2049] VirtualId# [5:7577471058467798680:0]
2025-11-27T18:17:43.233770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471058467798691:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.233832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:43.233871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:43.234170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471053275428949:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2345) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:43.234255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471057570396339:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:43.234290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471053275428949:2049] [node 2] ICP30 unregister session Session# [6:7577471057570396339:2049] VirtualId# [6:7577471057570396329:0]
2025-11-27T18:17:43.234339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:43.234378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471053275428949:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2345) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:43.234392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2345) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:43.242192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471053275428949:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:43.283369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:43.283617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471057319618956:2050] [node 3] ICS01 socket: 311 reason# EndOfStream
2025-11-27T18:17:43.283658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471053024651565:2050] [node 3] ICP30 unregister session Session# [4:7577471057319618956:2050] VirtualId# [4:7577471053024651652:0]
2025-11-27T18:17:43.283710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471057319618956:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.283795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:43.283839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:43.284754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:43.284951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471058467798686:2050] [node 3] ICS01 socket: 318 reason# EndOfStream
2025-11-27T18:17:43.284991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471054172831298:2050] [node 3] ICP30 unregister session Session# [5:7577471058467798686:2050] VirtualId# [5:7577471058467798679:0]
2025-11-27T18:17:43.285017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471058467798686:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.285095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:43.285147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:43.286713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:43.286902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471057570396335:2050] [node 3] ICS01 socket: 312 reason# EndOfStream
2025-11-27T18:17:43.286940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471053275428950:2050] [node 3] ICP30 unregister session Session# [6:7577471057570396335:2050] VirtualId# [6:7577471057570396326:0]
2025-11-27T18:17:43.286964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471057570396335:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.287039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:43.287106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:43.329443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:43.329696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471057570396343:2107] [node 4] ICS01 socket: 337 reason# EndOfStream
2025-11-27T18:17:43.329735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471053275429012:2107] [node 4] ICP30 unregister session Session# [6:7577471057570396343:2107] VirtualId# [6:7577471057570396332:0]
2025-11-27T18:17:43.329768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471057570396343:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.329858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:43.329903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:43.329566Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:43.329768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471058467798689:2107] [node 4] ICS01 socket: 326 reason# EndOfStream
2025-11-27T18:17:43.329809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471054172831360:2107] [node 4] ICP30 unregister session Session# [5:7577471058467798689:2107] VirtualId# [5:7577471058467798677:0]
2025-11-27T18:17:43.329839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471058467798689:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.329927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:43.329975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:43.378413Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:43.378647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471057570396346:2104] [node 5] ICS07 socket disconnect 344 reason# EndOfStream
2025-11-27T18:17:43.378676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471057570396346:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:43.378753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:43.378801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471057570396346:2104] [node 5] ICS15 start handshake
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:43.466513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471056245372281:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:43.466555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471056245372282:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:43.491286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471055695166973:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:43.491333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471055695166974:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:43.511514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471056322534336:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:43.511550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471056322534337:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:43.531708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471058085547578:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:43.531745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471058085547579:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:43.531931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471058085547577:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:43.565425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471058636372284:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:43.565504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471058636372285:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:43.565542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471058636372286:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:43.574758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471055396700481:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:43.574793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471055396700482:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:43.574812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471055396700483:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:43.818821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471055396700542:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.824977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471055695167033:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:43.839699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471058636372345:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8967
2025-11-27T18:17:43.854585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471056245372347:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:43.854900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471056245372348:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:43.860901Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471056245372282:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29608188ns) {Configure:104(29602637ns) {ProcessConfigured:135(61265ns) {StartInitialHandshake:157(58446ns) {DropHandshakes:470(1949ns) {DropIncomingHandshake:442(480ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(1175ns) PrepareNewSessionHandshake:477(1179ns) {SwitchToState:179(311ns)}}}}}
2025-11-27T18:17:43.969461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471058085547653:2110] [node 5] ICP01 ready to work
2025-11-27T18:17:43.927762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471055695167039:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:43.874889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471056245372349:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:44.017540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471062380514952:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:43.927943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471055695167038:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:44.008766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471056322534412:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:44.025932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471056322534413:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:44.025410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471059990134336:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:44.034873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471062931339644:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:44.034726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471059691667844:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.054374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471062931339645:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:44.045643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471060617501710:2116] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.085547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471056245372348:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(209730367ns) {Configure:104(209724584ns) {ProcessConfigured:135(56601ns) {StartInitialHandshake:157(53796ns) {DropHandshakes:470(2729ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(1021ns) PrepareNewSessionHandshake:477(1730ns) {SwitchToState:179(467ns)}}}}}
2025-11-27T18:17:44.087656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471060540339652:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24675
2025-11-27T18:17:44.076623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471058085547653:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(45873119ns) {Configure:104(45864524ns) {ProcessConfigured:135(66929ns) {StartInitialHandshake:157(64010ns) {DropHandshakes:470(1747ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(1021ns) PrepareNewSessionHandshake:477(1829ns) {SwitchToState:179(903ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.116380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471060540339652:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.080240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471058085547579:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(147145694ns) {Configure:104(147139242ns) {ProcessConfigured:135(72872ns) {StartInitialHandshake:157(70594ns) {DropHandshakes:470(2513ns) {DropIncomingHandshake:442(1142ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(804ns) PrepareNewSessionHandshake:477(2089ns) {SwitchToState:179(869ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.093450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471055695166974:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(165225147ns) {Configure:104(165219078ns) {ProcessConfigured:135(76712ns) {StartInitialHandshake:157(74341ns) {DropHandshakes:470(3432ns) {DropIncomingHandshake:442(531ns) DropOutgoingHandshake:457(575ns)} GenerateSessionVirtualId:988(958ns) PrepareNewSessionHandshake:477(1506ns) {SwitchToState:179(728ns)}}}}}
2025-11-27T18:17:44.118579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471058636372286:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(64612978ns) {Configure:104(64602741ns) {ProcessConfigured:135(68703ns) {StartInitialHandshake:157(66494ns) {DropHandshakes:470(3809ns) {DropIncomingHandshake:442(963ns) DropOutgoingHandshake:457(383ns)} GenerateSessionVirtualId:988(999ns) PrepareNewSessionHandshake:477(2038ns) {SwitchToState:179(749ns)}}}}}
2025-11-27T18:17:44.118950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471062931339655:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8967
2025-11-27T18:17:44.080801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471062380514958:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45335
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.139384Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471058636372284:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(197328819ns) {Configure:104(149160275ns) {ProcessConfigured:135(94899ns) {StartInitialHandshake:157(91859ns) {DropHandshakes:470(4216ns) {DropIncomingHandshake:442(1001ns) DropOutgoingHandshake:457(665ns)} GenerateSessionVirtualId:988(1449ns) PrepareNewSessionHandshake:477(1812ns) {SwitchToState:179(572ns)}}}}}
2025-11-27T18:17:44.113405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471059990134343:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-27T18:17:44.133348Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471055396700483:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(54135324ns) {Configure:104(54129273ns) {ProcessConfigured:135(16064169ns) {StartInitialHandshake:157(16061737ns) {DropHandshakes:470(889ns) {DropIncomingHandshake:442(298ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:477(2748ns) {SwitchToState:179(1200ns)}}}}}
2025-11-27T18:17:44.084258Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471062380514952:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(64272423ns) {Configure:104(64264875ns) {ProcessConfigured:135(14326623ns) {StartInitialHandshake:157(14323997ns) {DropHandshakes:470(1778ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(573ns)} GenerateSessionVirtualId:988(580ns) PrepareNewSessionHandshake:477(3428ns) {SwitchToState:179(1707ns)}}}}}
2025-11-27T18:17:44.113527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471059990134344:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.162263Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471055396700481:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(201732747ns) {Configure:104(201727693ns) {ProcessConfigured:135(70126ns) {StartInitialHandshake:157(64163ns) {DropHandshakes:470(3044ns) {DropIncomingHandshake:442(674ns) DropOutgoingHandshake:457(535ns)} GenerateSessionVirtualId:988(866ns) PrepareNewSessionHandshake:477(2018ns) {SwitchToState:179(759ns)}}}}}
2025-11-27T18:17:44.163231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471059691667849:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24569
2025-11-27T18:17:44.163312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471059691667850:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24569
2025-11-27T18:17:44.163456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471059691667851:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24569
2025-11-27T18:17:44.146042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471056245372347:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(252218959ns) {Configure:104(252214151ns) {ProcessConfigured:135(18576005ns) {StartInitialHandshake:157(18572766ns) {DropHandshakes:470(4033ns) {DropIncomingHandshake:442(439ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(757ns) PrepareNewSessionHandshake:477(4648ns) {SwitchToState:179(2341ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.173956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471056322534337:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(287612168ns) {Configure:104(287600374ns) {ProcessConfigured:135(81377ns) {StartInitialHandshake:157(77712ns) {DropHandshakes:470(2752ns) {DropIncomingHandshake:442(595ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(1031ns) PrepareNewSessionHandshake:477(1423ns) {SwitchToState:179(524ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.179035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471060540339658:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24675
2025-11-27T18:17:44.155111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471062931339655:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.131348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471055695167033:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.141238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471062380514961:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45335
2025-11-27T18:17:44.193340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471060540339658:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.193496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471060540339649:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.193523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471060540339649:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.178260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471059691667850:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.155333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471062931339644:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.142067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471059990134343:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.157754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471062380514955:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.178460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471055396700481:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.190411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471056322534336:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(127989288ns) {Configure:104(127982822ns) {ProcessConfigured:135(7692237ns) {StartInitialHandshake:157(7690090ns) {DropHandshakes:470(2532ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(349ns)} GenerateSessionVirtualId:988(579ns) PrepareNewSessionHandshake:477(2752ns) {SwitchToState:179(1591ns)}}}}}
2025-11-27T18:17:44.161954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471058636372345:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.181700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471055695166973:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(303684309ns) {Configure:104(301868707ns) {ProcessConfigured:135(10495898ns) {StartInitialHandshake:157(10493092ns) {DropHandshakes:470(3021ns) {DropIncomingHandshake:442(748ns) DropOutgoingHandshake:457(415ns)} GenerateSessionVirtualId:988(1101ns) PrepareNewSessionHandshake:477(2235ns) {SwitchToState:179(139ns)}}}}}
2025-11-27T18:17:44.157782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471062380514955:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.158802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471062380514958:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.170101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471062380514955:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:44.198148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471058085547653:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(16726853ns) {IncomingHandshake:261(16723048ns) {DropIncomingHandshake:442(2422ns) PrepareNewSessionHandshake:477(10976ns) {SwitchToState:179(8672ns)} GenerateSessionVirtualId:988(872ns) IssueIncomingHandshakeReply:193(9801ns)}}
2025-11-27T18:17:44.237753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471058085547653:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.237826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:44.217598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471060540339649:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:44.237346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471056245372342:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.225868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471056322534413:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(51622681ns) {Configure:104(51617150ns) {ProcessConfigured:135(72336ns) {StartInitialHandshake:157(69315ns) {DropHandshakes:470(3295ns) {DropIncomingHandshake:442(943ns) DropOutgoingHandshake:457(759ns)} GenerateSessionVirtualId:988(746ns) PrepareNewSessionHandshake:477(2478ns) {SwitchToState:179(909ns)}}}}}
2025-11-27T18:17:44.175241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471062931339661:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8967
2025-11-27T18:17:44.195575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471062931339655:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:44.205786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471062931339644:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.205862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:44.219448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471062931339647:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.219472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471062931339647:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.220439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471062931339647:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:44.220690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471062931339645:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.220754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:44.220803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471062931339645:2110] [node 6] ICP22 created new session: [5:7577471062931339665:2110]
2025-11-27T18:17:44.220858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471062931339665:2110] [node 6] ICS09 handshake done sender: [5:7577471062931339647:2111] self: [5:7577471062931339646:0] peer: [6:7577471059691667858:0] socket: 298 qp: -1
2025-11-27T18:17:44.220898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471062931339665:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:44.220934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:44.232010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471062931339644:2109] [node 4] ICP22 created new session: [5:7577471062931339663:2109]
2025-11-27T18:17:44.242433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471062931339663:2109] [node 4] ICS09 handshake done sender: [5:7577471062931339655:2114] self: [5:7577471062931339660:0] peer: [4:7577471062380514954:0] socket: 303 qp: -1
2025-11-27T18:17:44.246476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471062931339667:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8967
2025-11-27T18:17:44.190009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471059691667844:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(33601361ns) {Configure:104(33594279ns) {ProcessConfigured:135(78205ns) {StartInitialHandshake:157(76224ns) {DropHandshakes:470(2993ns) {DropIncomingHandshake:442(900ns) DropOutgoingHandshake:457(381ns)} GenerateSessionVirtualId:988(657ns) PrepareNewSessionHandshake:477(1969ns) {SwitchToState:179(770ns)}}}}}
2025-11-27T18:17:44.216324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471055695166973:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:44.216578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471059990134344:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.233830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471059990134354:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-27T18:17:44.237433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471055695167038:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(95806956ns) {Configure:104(95803328ns) {ProcessConfigured:135(11980944ns) {StartInitialHandshake:157(11978645ns) {DropHandshakes:470(2648ns) {DropIncomingHandshake:442(507ns) DropOutgoingHandshake:457(412ns)} GenerateSessionVirtualId:988(566ns) PrepareNewSessionHandshake:477(3394ns) {SwitchToState:179(316ns)}}}}}
2025-11-27T18:17:44.237405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471056245372342:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.250879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471058085547653:2110] [node 5] ICP22 created new session: [4:7577471062380514964:2110]
2025-11-27T18:17:44.226619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471060617501717:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64357
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.277830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471059990134354:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.277979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471059990134342:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.278004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471059990134342:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.279927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471059990134356:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-27T18:17:44.279988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471059990134357:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7213
2025-11-27T18:17:44.280539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471059990134342:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:44.280836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471059990134336:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.280892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:44.280941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471059990134336:2110] [node 6] ICP22 created new session: [2:7577471059990134360:2110]
2025-11-27T18:17:44.280991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471059990134360:2110] [node 6] ICS09 handshake done sender: [2:7577471059990134342:2114] self: [2:7577471059990134341:0] peer: [6:7577471059691667866:0] socket: 311 qp: -1
2025-11-27T18:17:44.281032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471059990134360:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:44.281072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:44.253645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471062931339667:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.206188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471059691667857:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24569
2025-11-27T18:17:44.261781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471062380514964:2110] [node 5] ICS09 handshake done sender: [4:7577471062380514955:2121] self: [4:7577471062380514954:0] peer: [5:7577471062931339660:0] socket: 302 qp: -1
2025-11-27T18:17:44.218802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471059691667849:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.219012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471055396700542:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:44.227342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471059691667850:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:44.228095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471055396700481:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.228150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:44.228198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471055396700481:2048] [node 1] ICP22 created new session: [6:7577471059691667860:2048]
2025-11-27T18:17:44.228262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471059691667860:2048] [node 1] ICS09 handshake done sender: [6:7577471059691667850:2114] self: [6:7577471059691667854:0] peer: [1:7577471060540339648:0] socket: 316 qp: -1
2025-11-27T18:17:44.228305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471059691667860:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:44.228345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:44.252030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471055396700482:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(182178797ns) {Configure:104(182171528ns) {ProcessConfigured:135(9344709ns) {StartInitialHandshake:157(9341818ns) {DropHandshakes:470(2928ns) {DropIncomingHandshake:442(822ns) DropOutgoingHandshake:457(532ns)} GenerateSessionVirtualId:988(1145ns) PrepareNewSessionHandshake:477(4669ns) {SwitchToState:179(2293ns)}}}}}
2025-11-27T18:17:44.271736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471059691667851:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.272010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471055396700542:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1827) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471059691667849:2113] held: no
2025-11-27T18:17:44.272395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471055396700482:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.297357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471059691667867:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24569
2025-11-27T18:17:44.247684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471060617501723:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64357
2025-11-27T18:17:44.254974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471062931339663:2109] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.300536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471059691667851:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:44.300746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471059691667859:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24569
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.303339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471055396700482:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.303402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:44.303463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471055396700482:2049] [node 2] ICP22 created new session: [6:7577471059691667868:2049]
2025-11-27T18:17:44.303509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471059691667868:2049] [node 2] ICS09 handshake done sender: [6:7577471059691667851:2115] self: [6:7577471059691667866:0] peer: [2:7577471059990134341:0] socket: 317 qp: -1
2025-11-27T18:17:44.303547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471059691667868:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:44.303579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:44.305789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471059691667849:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:44.306842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471055396700542:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.306898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:44.306948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471055396700542:2106] [node 5] ICP22 created new session: [6:7577471059691667870:2106]
2025-11-27T18:17:44.307004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471059691667870:2106] [node 5] ICS09 handshake done sender: [6:7577471059691667849:2113] self: [6:7577471059691667858:0] peer: [5:7577471062931339646:0] socket: 315 qp: -1
2025-11-27T18:17:44.307064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471059691667870:2106] [node 5] ICS10 traffic start
2025-11-27T18:17:44.307110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:44.269384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471062380514964:2110] [node 5] ICS10 traffic start
2025-11-27T18:17:44.247940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471060617501725:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64357
2025-11-27T18:17:44.287379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471056245372348:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.310563Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471056322534412:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(191995687ns) {Configure:104(181954026ns) {ProcessConfigured:135(6491548ns) {StartInitialHandshake:157(6489056ns) {DropHandshakes:470(2995ns) {DropIncomingHandshake:442(761ns) DropOutgoingHandshake:457(532ns)} GenerateSessionVirtualId:988(398ns) PrepareNewSessionHandshake:477(5730ns) {SwitchToState:179(2747ns)}}}}}
2025-11-27T18:17:44.269480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:44.270012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471058085547653:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(32037107ns) {HandleHandshakeStatus:329(32033051ns) {DropIncomingHandshake:442(18668ns) DropOutgoingHandshake:457(531ns) SwitchToState:179(862ns) ProcessPendingSessionEvents:495(123424ns) {ForwardSessionEventToSession:612(71755ns) ForwardSessionEventToSession:612(6683ns) ForwardSessionEventToSession:612(3530ns) ForwardSessionEventToSession:612(2182ns) ForwardSessionEventToSession:612(5933ns) ForwardSessionEventToSession:612(2634ns) ForwardSessionEventToSession:612(8719ns) ForwardSessionEventToSession:612(2843ns) ForwardSessionEventToSession:612(3369ns) ForwardSessionEventToSession:612(5531ns)}}}
2025-11-27T18:17:44.285005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471062380514967:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45335
2025-11-27T18:17:44.306509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471062380514967:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.308907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471062380514961:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.311513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471058085547578:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:44.308827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471055695167033:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:44.263147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:44.287471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:44.319940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471059691667872:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24569
2025-11-27T18:17:44.321906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471059990134340:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.287526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471056245372348:2110] [node 6] ICP22 created new session: [1:7577471060540339664:2110]
2025-11-27T18:17:44.311808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471060617501723:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.271299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471058636372286:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.287598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471060540339664:2110] [node 6] ICS09 handshake done sender: [1:7577471060540339649:2112] self: [1:7577471060540339648:0] peer: [6:7577471059691667854:0] socket: 306 qp: -1
2025-11-27T18:17:44.287640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471060540339664:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:44.287677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:44.288406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471056245372342:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:44.301729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471060540339663:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24675
2025-11-27T18:17:44.314644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471056245372281:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.314687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:44.321944Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471059990134340:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.322519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471055695166973:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.322570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:44.322631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471055695166973:2048] [node 1] ICP22 created new session: [2:7577471059990134363:2048]
2025-11-27T18:17:44.322680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471059990134363:2048] [node 1] ICS09 handshake done sender: [2:7577471055695167033:2107] self: [2:7577471059990134353:0] peer: [1:7577471056245372341:0] socket: 290 qp: -1
2025-11-27T18:17:44.322722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471059990134363:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:44.322762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:44.329193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471059990134338:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.329218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471059990134338:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.332528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471059990134340:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:44.340293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471059990134338:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:44.345327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471059691667872:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.311895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471060617501717:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.354593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471055695167039:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.354678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:44.354737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471055695167039:2108] [node 4] ICP22 created new session: [2:7577471059990134367:2108]
2025-11-27T18:17:44.285352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471062931339672:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8967
2025-11-27T18:17:44.357824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471059990134367:2108] [node 4] ICS09 handshake done sender: [2:7577471059990134340:2113] self: [2:7577471059990134339:0] peer: [4:7577471062380514969:0] socket: 310 qp: -1
2025-11-27T18:17:44.357898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471059990134367:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:44.314743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471056245372281:2048] [node 2] ICP22 created new session: [1:7577471060540339667:2048]
2025-11-27T18:17:44.343777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471062380514971:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45335
2025-11-27T18:17:44.361509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471055396700483:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:44.312930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471060617501716:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.353447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471062380514961:2125] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:44.311349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471062931339673:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8967
2025-11-27T18:17:44.358865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.314802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471060540339667:2048] [node 2] ICS09 handshake done sender: [1:7577471056245372342:2107] self: [1:7577471056245372341:0] peer: [2:7577471059990134353:0] socket: 289 qp: -1
2025-11-27T18:17:44.314841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471060540339667:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:44.314880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:44.312957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471060617501716:2125] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.313789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471060617501724:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64357
2025-11-27T18:17:44.314081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471056322534337:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:44.314151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471056322534336:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:44.317473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471060617501716:2125] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:44.317925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471060617501710:2116] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.317964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:44.318011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471060617501710:2116] [node 5] ICP22 created new session: [3:7577471060617501732:2116]
2025-11-27T18:17:44.318063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471060617501732:2116] [node 5] ICS09 handshake done sender: [3:7577471060617501716:2125] self: [3:7577471060617501715:0] peer: [5:7577471062931339671:0] socket: 330 qp: -1
2025-11-27T18:17:44.318098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471060617501732:2116] [node 5] ICS10 traffic start
2025-11-27T18:17:44.318130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:44.318712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471060617501731:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64357
2025-11-27T18:17:44.321821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471060617501725:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.323247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471060617501710:2116] [node 5] ICP18 handshake (actor [3:7577471060617501725:2130]) is too old
2025-11-27T18:17:44.325813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471060617501724:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.330569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471060617501736:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64357
2025-11-27T18:17:44.340885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471060617501723:2128] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:44.341395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471056322534337:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.341460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:44.341523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471056322534337:2049] [node 2] ICP22 created new session: [3:7577471060617501737:2049]
2025-11-27T18:17:44.341578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471060617501737:2049] [node 2] ICS09 handshake done sender: [3:7577471060617501723:2128] self: [3:7577471060617501729:0] peer: [2:7577471059990134337:0] socket: 327 qp: -1
2025-11-27T18:17:44.341609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471060617501737:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:44.341645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:44.355517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471060617501731:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.371687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471056322534337:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(24798583ns) {HandleHandshakeStatus:329(24796243ns) {DropIncomingHandshake:442(1485ns) DropOutgoingHandshake:457(6624ns) SwitchToState:179(708ns) ProcessPendingSessionEvents:495(16581322ns) {ForwardSessionEventToSession:612(442671ns) ForwardSessionEventToSession:612(4267ns) ForwardSessionEventToSession:612(6119483ns) ForwardSessionEventToSession:612(5513ns) ForwardSessionEventToSession:612(2259565ns) ForwardSessionEventToSession:612(2995310ns) ForwardSessionEventToSession:612(4278ns) ForwardSessionEventToSession:612(3069ns) ForwardSessionEventToSession:612(4726082ns) ForwardSessionEventToSession:612(3771ns)}}}
2025-11-27T18:17:44.379984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471056322534336:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471060617501717:2126] held: no
2025-11-27T18:17:44.381908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471060617501741:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64357
2025-11-27T18:17:44.386841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471060617501719:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.386877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471060617501719:2127] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.359329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471055695166974:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.354353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471058085547578:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.358193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471062931339644:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(115453566ns) {HandleHandshakeStatus:329(115449636ns) {DropIncomingHandshake:442(883ns) DropOutgoingHandshake:457(22217ns) SwitchToState:179(465ns) ProcessPendingSessionEvents:495(27472782ns) {ForwardSessionEventToSession:612(6737664ns) ForwardSessionEventToSession:612(10067429ns) ForwardSessionEventToSession:612(3816ns) ForwardSessionEventToSession:612(1211548ns) ForwardSessionEventToSession:612(4197ns) ForwardSessionEventToSession:612(3398ns) ForwardSessionEventToSession:612(8709876ns) ForwardSessionEventToSession:612(4336ns) ForwardSessionEventToSession:612(3568ns) ForwardSessionEventToSession:612(709040ns)}}}
2025-11-27T18:17:44.320177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471060540339663:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.359379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:44.357741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:44.326830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471056245372351:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.393401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471060617501717:2126] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.358353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471058636372286:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26107) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471062931339667:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.413120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471056322534336:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.326867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471056245372351:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.359423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471055695166974:2049] [node 3] ICP22 created new session: [2:7577471059990134369:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.358262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471062380514970:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45335
2025-11-27T18:17:44.372953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471062931339673:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.360789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471058085547578:2049] [node 2] ICP22 created new session: [4:7577471062380514972:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.425848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471059691667875:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24569
2025-11-27T18:17:44.343369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471060540339662:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24675
2025-11-27T18:17:44.359479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471059990134369:2049] [node 3] ICS09 handshake done sender: [2:7577471059990134338:2112] self: [2:7577471059990134337:0] peer: [3:7577471060617501729:0] socket: 309 qp: -1
2025-11-27T18:17:44.373145Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471058636372286:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16213674ns) {HandleHandshakeStatus:395(16210681ns) {LogHandshakeFail:472(8303ns) {UpdateErrorStateLog:545(3134ns)} DropOutgoingHandshake:457(1957ns) ProcessPendingSessionEvents:495(7717963ns) {DropSessionEvent:512(8274ns) DropSessionEvent:512(5545ns) DropSessionEvent:512(3371016ns) DropSessionEvent:512(4289790ns) DropSessionEvent:512(5714ns) DropSessionEvent:512(5821ns) DropSessionEvent:512(4992ns) DropSessionEvent:512(5163ns) DropSessionEvent:512(5056ns) DropSessionEvent:512(5828ns)}}}
2025-11-27T18:17:44.413178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:44.358529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471060540339662:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.359510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471059990134369:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:44.447817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471059691667872:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:44.413227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471056322534336:2048] [node 1] ICP22 created new session: [3:7577471060617501743:2048]
2025-11-27T18:17:44.370891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471062380514972:2049] [node 2] ICS09 handshake done sender: [4:7577471062380514961:2125] self: [4:7577471062380514969:0] peer: [2:7577471059990134339:0] socket: 312 qp: -1
2025-11-27T18:17:44.366753Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471056245372351:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:44.373354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471062931339672:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.359546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:44.385597Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471059990134336:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:44.397564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471059990134351:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.397599Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471059990134351:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.408704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471059990134351:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:44.443004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471055695167038:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.443067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:44.454027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471055695167038:2109] [node 5] ICP22 created new session: [2:7577471059990134372:2109]
2025-11-27T18:17:44.471031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471059990134372:2109] [node 5] ICS09 handshake done sender: [2:7577471059990134351:2118] self: [2:7577471059990134350:0] peer: [5:7577471062931339676:0] socket: 343 qp: -1
2025-11-27T18:17:44.475307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471059990134372:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:44.482244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:44.449209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471055396700483:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.413290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471060617501743:2048] [node 1] ICS09 handshake done sender: [3:7577471060617501717:2126] self: [3:7577471060617501730:0] peer: [1:7577471056245372350:0] socket: 322 qp: -1
2025-11-27T18:17:44.367072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471056245372282:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.373921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471058636372284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:44.371363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471062380514972:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.373982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471058636372285:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:44.449252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:44.413327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471060617501743:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:44.391256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471062931339674:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8967
2025-11-27T18:17:44.393803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471062931339677:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8967
2025-11-27T18:17:44.409410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471062931339673:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:44.418932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471062931339667:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:44.441478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471058636372284:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.441555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:44.441619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471058636372284:2048] [node 1] ICP22 created new session: [5:7577471062931339679:2048]
2025-11-27T18:17:44.441673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471062931339679:2048] [node 1] ICS09 handshake done sender: [5:7577471062931339673:2120] self: [5:7577471062931339675:0] peer: [1:7577471060540339650:0] socket: 346 qp: -1
2025-11-27T18:17:44.441718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471062931339679:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:44.441756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:44.442367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471058636372286:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.442402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:44.442449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471058636372286:2050] [node 3] ICP22 created new session: [5:7577471062931339681:2050]
2025-11-27T18:17:44.442531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471062931339681:2050] [node 3] ICS09 handshake done sender: [5:7577471062931339667:2118] self: [5:7577471062931339671:0] peer: [3:7577471060617501715:0] socket: 291 qp: -1
2025-11-27T18:17:44.442567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471062931339681:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:44.442598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:44.442843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471062931339678:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8967
2025-11-27T18:17:44.469503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471058636372284:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:44.469993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471058636372285:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11548) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471062931339672:2119] held: no
2025-11-27T18:17:44.473775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471062931339672:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:44.486343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471058636372285:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.486400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:44.499897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471058636372285:2049] [node 2] ICP22 created new session: [5:7577471062931339683:2049]
2025-11-27T18:17:44.500712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471062931339683:2049] [node 2] ICS09 handshake done sender: [5:7577471062931339672:2119] self: [5:7577471062931339676:0] peer: [2:7577471059990134350:0] socket: 344 qp: -1
2025-11-27T18:17:44.501003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471062931339683:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:44.501068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:44.367133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:44.367184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471056245372282:2049] [node 3] ICP22 created new session: [1:7577471060540339673:2049]
2025-11-27T18:17:44.367228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471060540339673:2049] [node 3] ICS09 handshake done sender: [1:7577471056245372351:2111] self: [1:7577471056245372350:0] peer: [3:7577471060617501730:0] socket: 305 qp: -1
2025-11-27T18:17:44.367268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471060540339673:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:44.367304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:44.385399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471060540339651:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.385449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471060540339651:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.391093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471060540339651:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:44.414356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471056245372349:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.414434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:44.414484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471056245372349:2109] [node 5] ICP22 created new session: [1:7577471060540339676:2109]
2025-11-27T18:17:44.414526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471060540339676:2109] [node 5] ICS09 handshake done sender: [1:7577471060540339651:2113] self: [1:7577471060540339650:0] peer: [5:7577471062931339675:0] socket: 321 qp: -1
2025-11-27T18:17:44.414559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471060540339676:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:44.414588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:44.417643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471060540339656:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.417675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471060540339656:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.454177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471060540339656:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:44.466192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471056245372347:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.466267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:44.466321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471056245372347:2108] [node 4] ICP22 created new session: [1:7577471060540339679:2108]
2025-11-27T18:17:44.466389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471060540339679:2108] [node 4] ICS09 handshake done sender: [1:7577471060540339656:2115] self: [1:7577471060540339655:0] peer: [4:7577471062380514976:0] socket: 349 qp: -1
2025-11-27T18:17:44.466433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471060540339679:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:44.466469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:44.371409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:44.371433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471062380514970:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.386076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471058085547578:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(31550466ns) {HandleHandshakeStatus:329(31547182ns) {DropIncomingHandshake:442(1033ns) DropOutgoingHandshake:457(35670ns) SwitchToState:179(425ns) ProcessPendingSessionEvents:495(126157ns) {ForwardSessionEventToSession:612(67222ns) ForwardSessionEventToSession:612(3481ns) ForwardSessionEventToSession:612(21808ns) ForwardSessionEventToSession:612(2415ns) ForwardSessionEventToSession:612(2272ns) ForwardSessionEventToSession:612(4363ns) ForwardSessionEventToSession:612(4014ns) ForwardSessionEventToSession:612(4879ns) ForwardSessionEventToSession:612(2666ns) ForwardSessionEventToSession:612(3441ns)}}}
2025-11-27T18:17:44.408304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471058085547577:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:44.410252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471058085547577:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22803140ns) {IncomingHandshake:261(21384388ns) {DropIncomingHandshake:442(1366ns) PrepareNewSessionHandshake:477(3447ns) {SwitchToState:179(2222ns)} GenerateSessionVirtualId:988(616ns) IssueIncomingHandshakeReply:193(17006243ns) {SwitchToState:179(3476ns)}}}
2025-11-27T18:17:44.448884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471062380514977:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45335
2025-11-27T18:17:44.478498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471062380514974:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45335
2025-11-27T18:17:44.484366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471062380514970:2127] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:44.498717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471058085547577:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471062380514970:2127] held: no
2025-11-27T18:17:44.498848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471058085547577:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.498896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:44.498950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471058085547577:2048] [node 1] ICP22 created new session: [4:7577471062380514978:2048]
2025-11-27T18:17:44.499001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471062380514978:2048] [node 1] ICS09 handshake done sender: [4:7577471062380514970:2127] self: [4:7577471062380514976:0] peer: [1:7577471060540339655:0] socket: 350 qp: -1
2025-11-27T18:17:44.499037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471062380514978:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:44.499071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:44.500466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471062380514974:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.449316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471055396700483:2050] [node 3] ICP22 created new session: [6:7577471059691667876:2050]
2025-11-27T18:17:44.413369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:44.414155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471060617501719:2127] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:44.447474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471056322534413:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.447538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:44.447589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471056322534413:2110] [node 6] ICP22 created new session: [3:7577471060617501745:2110]
2025-11-27T18:17:44.447645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471060617501745:2110] [node 6] ICS09 handshake done sender: [3:7577471060617501719:2127] self: [3:7577471060617501718:0] peer: [6:7577471059691667873:0] socket: 351 qp: -1
2025-11-27T18:17:44.447686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471060617501745:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:44.458161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:44.489775Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471056322534413:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(42067959ns) {HandleHandshakeStatus:329(42065372ns) {DropIncomingHandshake:442(19616ns) DropOutgoingHandshake:457(537ns) SwitchToState:179(330ns) ProcessPendingSessionEvents:495(5383429ns) {ForwardSessionEventToSession:612(5317759ns) ForwardSessionEventToSession:612(17037ns) ForwardSessionEventToSession:612(3436ns) ForwardSessionEventToSession:612(2150ns) ForwardSessionEventToSession:612(5378ns) ForwardSessionEventToSession:612(2071ns) ForwardSessionEventToSession:612(3084ns) ForwardSessionEventToSession:612(5125ns) ForwardSessionEventToSession:612(2693ns) ForwardSessionEventToSession:612(13904ns)}}}
2025-11-27T18:17:44.504374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471060617501727:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.504400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471060617501727:2131] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:44.504002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471058085547579:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:44.449368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471059691667876:2050] [node 3] ICS09 handshake done sender: [6:7577471059691667872:2121] self: [6:7577471059691667873:0] peer: [3:7577471060617501718:0] socket: 352 qp: -1
2025-11-27T18:17:44.449398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471059691667876:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:44.449430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:44.484271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471055396700483:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:44.501367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471059691667874:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24569
2025-11-27T18:17:44.511148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471059691667874:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.511330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471059691667844:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:44.512830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471062380514960:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:44.512868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471062380514960:2124] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.515678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471059691667879:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24569
2025-11-27T18:17:44.515239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471062380514960:2124] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:44.516349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471060617501727:2131] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:44.516638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471056322534412:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.516701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:44.516755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471056322534412:2107] [node 4] ICP22 created new session: [3:7577471060617501748:2107]
2025-11-27T18:17:44.516828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471060617501748:2107] [node 4] ICS09 handshake done sender: [3:7577471060617501727:2131] self: [3:7577471060617501726:0] peer: [4:7577471062380514980:0] socket: 297 qp: -1
2025-11-27T18:17:44.516870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471060617501748:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:44.516912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:44.515346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471055695167038:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(62995533ns) {HandleHandshakeStatus:329(62992201ns) {DropIncomingHandshake:442(16305ns) DropOutgoingHandshake:457(308ns) SwitchToState:179(485ns) ProcessPendingSessionEvents:495(8462671ns) {ForwardSessionEventToSession:612(2641019ns) ForwardSessionEventToSession:612(517021ns) ForwardSessionEventToSession:612(2932ns) ForwardSessionEventToSession:612(266221ns) ForwardSessionEventToSession:612(467486ns) ForwardSessionEventToSession:612(3500ns) ForwardSessionEventToSession:612(4065555ns) ForwardSessionEventToSession:612(5854ns) ForwardSessionEventToSession:612(3621ns) ForwardSessionEventToSession:612(473558ns)}}}
2025-11-27T18:17:44.517640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471059691667874:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:44.516279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471062380514982:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45335
2025-11-27T18:17:44.518186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471062380514952:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:44.518257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:44.521581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471062380514952:2111] [node 6] ICP22 created new session: [4:7577471062380514983:2111]
2025-11-27T18:17:44.522462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471059691667844:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.522569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:44.522614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471059691667844:2110] [node 4] ICP22 created new session: [6:7577471059691667880:2110]
2025-11-27T18:17:44.522658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471059691667880:2110] [node 4] ICS09 handshake done sender: [6:7577471059691667874:2122] self: [6:7577471059691667878:0] peer: [4:7577471062380514953:0] socket: 329 qp: -1
2025-11-27T18:17:44.522700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471059691667880:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:44.522740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:44.525451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471062380514983:2111] [node 6] ICS09 handshake done sender: [4:7577471062380514960:2124] self: [4:7577471062380514953:0] peer: [6:7577471059691667878:0] socket: 296 qp: -1
2025-11-27T18:17:44.526131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471062380514983:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:44.527632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:44.529412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471062380514974:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:44.530388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471058085547579:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26107) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471062380514974:2129] held: no
2025-11-27T18:17:44.530506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471058085547579:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:44.530551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:44.530610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471058085547579:2050] [node 3] ICP22 created new session: [4:7577471062380514985:2050]
2025-11-27T18:17:44.530656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471062380514985:2050] [node 3] ICS09 handshake done sender: [4:7577471062380514974:2129] self: [4:7577471062380514980:0] peer: [3:7577471060617501726:0] socket: 319 qp: -1
2025-11-27T18:17:44.530697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471062380514985:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:44.530755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:44.567960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:44.568197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471060617501743:2048] [node 1] ICS01 socket: 322 reason# EndOfStream
2025-11-27T18:17:44.568233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471056322534336:2048] [node 1] ICP30 unregister session Session# [3:7577471060617501743:2048] VirtualId# [3:7577471060617501730:0]
2025-11-27T18:17:44.568267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471060617501743:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.568343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:44.568380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:44.569381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:44.569567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471059990134363:2048] [node 1] ICS07 socket disconnect 290 reason# EndOfStream
2025-11-27T18:17:44.569603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471059990134363:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.569696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:44.569737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471059990134363:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:44.573678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:44.573890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471059691667860:2048] [node 1] ICS01 socket: 316 reason# EndOfStream
2025-11-27T18:17:44.573936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471055396700481:2048] [node 1] ICP30 unregister session Session# [6:7577471059691667860:2048] VirtualId# [6:7577471059691667854:0]
2025-11-27T18:17:44.573970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471059691667860:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.574051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:44.574095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:44.574188Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:44.574361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471062380514978:2048] [node 1] ICS01 socket: 350 reason# EndOfStream
2025-11-27T18:17:44.574393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471058085547577:2048] [node 1] ICP30 unregister session Session# [4:7577471062380514978:2048] VirtualId# [4:7577471062380514976:0]
2025-11-27T18:17:44.574415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471062380514978:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.574453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:44.574475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:44.574540Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:44.576150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471055695166973:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25440) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:44.574657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471062931339679:2048] [node 1] ICS01 socket: 346 reason# EndOfStream
2025-11-27T18:17:44.574682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471058636372284:2048] [node 1] ICP30 unregister session Session# [5:7577471062931339679:2048] VirtualId# [5:7577471062931339675:0]
2025-11-27T18:17:44.574705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471062931339679:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.574821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:44.574888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:44.581299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471059990134363:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:44.582031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471055695166973:2048] [node 1] ICP30 unregister session Session# [2:7577471059990134363:2048] VirtualId# [2:7577471059990134353:0]
2025-11-27T18:17:44.582105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:44.593579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471055695166973:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25440) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:44.593627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25440) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:44.671396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:44.671700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471060617501737:2049] [node 2] ICS07 socket disconnect 327 reason# EndOfStream
2025-11-27T18:17:44.671751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471060617501737:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.671896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:44.671984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471060617501737:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:44.673511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:44.673744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471062380514972:2049] [node 2] ICS07 socket disconnect 312 reason# EndOfStream
2025-11-27T18:17:44.673785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471062380514972:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.673849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:44.673886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471062380514972:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:44.674856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471058085547578:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11548) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:44.674909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471062380514972:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:44.674942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471058085547578:2049] [node 2] ICP30 unregister session Session# [4:7577471062380514972:2049] VirtualId# [4:7577471062380514969:0]
2025-11-27T18:17:44.674987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:44.675019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471058085547578:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11548) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:44.675032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11548) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:44.679386Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:44.679643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471059691667868:2049] [node 2] ICS01 socket: 317 reason# EndOfStream
2025-11-27T18:17:44.679693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471055396700482:2049] [node 2] ICP30 unregister session Session# [6:7577471059691667868:2049] VirtualId# [6:7577471059691667866:0]
2025-11-27T18:17:44.679725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471059691667868:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.679807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:44.679845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:44.679877Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:44.680077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471062931339683:2049] [node 2] ICS01 socket: 344 reason# EndOfStream
2025-11-27T18:17:44.680116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471058636372285:2049] [node 2] ICP30 unregister session Session# [5:7577471062931339683:2049] VirtualId# [5:7577471062931339676:0]
2025-11-27T18:17:44.680151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471062931339683:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.680209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:44.680247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:44.689437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471058085547578:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:44.693878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471056322534337:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11548) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:44.695856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471060617501737:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:44.695903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471056322534337:2049] [node 2] ICP30 unregister session Session# [3:7577471060617501737:2049] VirtualId# [3:7577471060617501729:0]
2025-11-27T18:17:44.695961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:44.696009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471056322534337:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11548) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:44.696025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11548) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:44.706401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471056322534337:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:44.738934Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:44.739178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471062380514985:2050] [node 3] ICS01 socket: 319 reason# EndOfStream
2025-11-27T18:17:44.739224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471058085547579:2050] [node 3] ICP30 unregister session Session# [4:7577471062380514985:2050] VirtualId# [4:7577471062380514980:0]
2025-11-27T18:17:44.739265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471062380514985:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.739341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:44.739384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:44.741609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:44.741824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471059691667876:2050] [node 3] ICS01 socket: 352 reason# EndOfStream
2025-11-27T18:17:44.741867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471055396700483:2050] [node 3] ICP30 unregister session Session# [6:7577471059691667876:2050] VirtualId# [6:7577471059691667873:0]
2025-11-27T18:17:44.741898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471059691667876:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.741977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:44.742026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:44.742145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:44.742315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471062931339681:2050] [node 3] ICS01 socket: 291 reason# EndOfStream
2025-11-27T18:17:44.742362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471058636372286:2050] [node 3] ICP30 unregister session Session# [5:7577471062931339681:2050] VirtualId# [5:7577471062931339671:0]
2025-11-27T18:17:44.742393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471062931339681:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.742441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:44.742471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:44.829458Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:44.829836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471062931339663:2109] [node 4] ICS07 socket disconnect 303 reason# EndOfStream
2025-11-27T18:17:44.829881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471062931339663:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.829974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:44.830050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471062931339663:2109] [node 4] ICS15 start handshake
2025-11-27T18:17:44.840112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471062931339644:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6065) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:44.830967Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:44.831289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471059691667880:2110] [node 4] ICS01 socket: 329 reason# EndOfStream
2025-11-27T18:17:44.831364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471059691667844:2110] [node 4] ICP30 unregister session Session# [6:7577471059691667880:2110] VirtualId# [6:7577471059691667878:0]
2025-11-27T18:17:44.831422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471059691667880:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.831551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:44.831633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:44.846813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471062931339663:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:44.847166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471062931339644:2109] [node 4] ICP30 unregister session Session# [5:7577471062931339663:2109] VirtualId# [5:7577471062931339660:0]
2025-11-27T18:17:44.847227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:44.847281Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471062931339644:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:44.847299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:44.893398Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:44.893653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471059691667870:2106] [node 5] ICS01 socket: 315 reason# EndOfStream
2025-11-27T18:17:44.893697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471055396700542:2106] [node 5] ICP30 unregister session Session# [6:7577471059691667870:2106] VirtualId# [6:7577471059691667858:0]
2025-11-27T18:17:44.893733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471059691667870:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:44.893814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:44.893872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:44.997588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471060408495513:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:45.001644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471060408495514:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:45.032577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471064583946193:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:45.032623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471064583946194:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:45.064381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471066563138333:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:45.064429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471066563138334:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:45.093168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471065800532416:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:45.093283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471065800532415:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:45.093402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471065800532417:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:45.114412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471066548937299:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:45.114445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471066548937300:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:45.114473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471066548937301:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:45.138908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471064057961645:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:45.138963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471064057961646:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:45.139076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471064057961647:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:45.555057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471066548937335:2084] [node 4] ICP01 ready to work
2025-11-27T18:17:45.655454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471065800532417:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(113672994ns) {Configure:104(113667497ns) {ProcessConfigured:135(97862ns) {StartInitialHandshake:157(94635ns) {DropHandshakes:470(3938ns) {DropIncomingHandshake:442(994ns) DropOutgoingHandshake:457(619ns)} GenerateSessionVirtualId:988(1363ns) PrepareNewSessionHandshake:477(2404ns) {SwitchToState:179(719ns)}}}}}
2025-11-27T18:17:45.588078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471064703462872:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:45.678574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471064583946260:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:45.677944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471064703462874:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:45.665985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471065800532481:2107] [node 6] ICP01 ready to work
2025-11-27T18:17:45.701675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471066563138397:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:45.731435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471066563138399:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:45.731918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471066563138398:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:45.721910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471065800532492:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:45.741727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471064057961646:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(107796870ns) {Configure:104(107790979ns) {ProcessConfigured:135(19740210ns) {StartInitialHandshake:157(19736866ns) {DropHandshakes:470(3089ns) {DropIncomingHandshake:442(650ns) DropOutgoingHandshake:457(637ns)} GenerateSessionVirtualId:988(972ns) PrepareNewSessionHandshake:477(6782ns) {SwitchToState:179(2841ns)}}}}}
2025-11-27T18:17:45.688880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471064583946261:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.781156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471066563138400:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49164
2025-11-27T18:17:45.713331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471066548937352:2088] [node 6] ICP01 ready to work
2025-11-27T18:17:45.681288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471064703462873:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.695070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471060408495513:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(166296780ns) {Configure:104(166288016ns) {ProcessConfigured:135(92262ns) {StartInitialHandshake:157(88546ns) {DropHandshakes:470(2587ns) {DropIncomingHandshake:442(587ns) DropOutgoingHandshake:457(511ns)} GenerateSessionVirtualId:988(1227ns) PrepareNewSessionHandshake:477(1532ns) {SwitchToState:179(527ns)}}}}}
2025-11-27T18:17:45.742684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471064057961663:2061] [node 5] ICP01 ready to work
2025-11-27T18:17:45.716051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471064583946262:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.802533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471064583946263:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4886
2025-11-27T18:17:45.846267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471060408495514:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(317505418ns) {Configure:104(317495396ns) {ProcessConfigured:135(55676ns) {StartInitialHandshake:157(52996ns) {DropHandshakes:470(2026ns) {DropIncomingHandshake:442(438ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(948ns) PrepareNewSessionHandshake:477(1476ns) {SwitchToState:179(505ns)}}}}}
2025-11-27T18:17:45.846694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471064703462883:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57128
2025-11-27T18:17:45.768583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471064057961666:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34642
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.873876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471066548937365:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.770281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471064057961676:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34642
2025-11-27T18:17:45.802598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471064583946264:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4886
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.872931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471066563138333:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(332369349ns) {Configure:104(332358100ns) {ProcessConfigured:135(71892ns) {StartInitialHandshake:157(67346ns) {DropHandshakes:470(2625ns) {DropIncomingHandshake:442(584ns) DropOutgoingHandshake:457(366ns)} GenerateSessionVirtualId:988(870ns) PrepareNewSessionHandshake:477(1861ns) {SwitchToState:179(500ns)}}}}}
2025-11-27T18:17:45.809658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471064057961707:2082] [node 4] ICP01 ready to work
2025-11-27T18:17:45.873914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471066548937365:2108] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.802662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471064583946265:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4886
2025-11-27T18:17:45.877867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471066548937365:2108] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:45.830705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471064057961676:2075] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.873835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471066563138406:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49164
2025-11-27T18:17:45.902093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471065800532494:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45550
2025-11-27T18:17:45.916123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471064703462883:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.862551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471064583946265:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.888148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471066548937299:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(330024519ns) {Configure:104(330017745ns) {ProcessConfigured:135(8347700ns) {StartInitialHandshake:157(8344964ns) {DropHandshakes:470(3840ns) {DropIncomingHandshake:442(1058ns) DropOutgoingHandshake:457(441ns)} GenerateSessionVirtualId:988(1520ns) PrepareNewSessionHandshake:477(4597ns) {SwitchToState:179(2276ns)}}}}}
2025-11-27T18:17:45.867038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471064057961663:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(121425434ns) {Configure:104(121419806ns) {ProcessConfigured:135(71101ns) {StartInitialHandshake:157(69071ns) {DropHandshakes:470(3599ns) {DropIncomingHandshake:442(889ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(930ns) PrepareNewSessionHandshake:477(2198ns) {SwitchToState:179(831ns)}}}}}
2025-11-27T18:17:45.906492Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471065800532481:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(216702677ns) {Configure:104(201874935ns) {ProcessConfigured:135(79165ns) {StartInitialHandshake:157(71770ns) {DropHandshakes:470(3383ns) {DropIncomingHandshake:442(819ns) DropOutgoingHandshake:457(481ns)} GenerateSessionVirtualId:988(980ns) PrepareNewSessionHandshake:477(1926ns) {SwitchToState:179(760ns)}}}}}
2025-11-27T18:17:45.862857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471064583946194:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(148241562ns) {Configure:104(148230935ns) {ProcessConfigured:135(15081144ns) {StartInitialHandshake:157(15078053ns) {DropHandshakes:470(3755ns) {DropIncomingHandshake:442(877ns) DropOutgoingHandshake:457(468ns)} GenerateSessionVirtualId:988(1324ns) PrepareNewSessionHandshake:477(6429ns) {SwitchToState:179(2329ns)}}}}}
2025-11-27T18:17:45.913800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471064583946264:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.938106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471064583946263:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.957817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471064583946271:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.957847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471064583946271:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:45.959538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471064583946277:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4886
2025-11-27T18:17:45.977806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471064583946193:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(275674446ns) {Configure:104(275665306ns) {ProcessConfigured:135(28613721ns) {StartInitialHandshake:157(16731099ns) {DropHandshakes:470(2348ns) {DropIncomingHandshake:442(1392ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(2172ns) PrepareNewSessionHandshake:477(4011ns) {SwitchToState:179(1813ns)}}}}}
2025-11-27T18:17:45.982632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471064583946271:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:45.987090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471064583946260:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(265240489ns) {Configure:104(265234209ns) {ProcessConfigured:135(26629776ns) {StartInitialHandshake:157(26626863ns) {DropHandshakes:470(2551ns) {DropIncomingHandshake:442(711ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(1025ns) PrepareNewSessionHandshake:477(3080ns) {SwitchToState:179(1651ns)}}}}}
2025-11-27T18:17:45.987296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471064583946269:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.987317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471064583946269:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:45.989888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471064583946193:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.879693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471066563138399:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(96752042ns) {Configure:104(96744127ns) {ProcessConfigured:135(98965ns) {StartInitialHandshake:157(96319ns) {DropHandshakes:470(6204ns) {DropIncomingHandshake:442(683ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(987ns) PrepareNewSessionHandshake:477(1625ns) {SwitchToState:179(562ns)}}}}}
2025-11-27T18:17:45.891576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471066563138411:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49164
2025-11-27T18:17:45.867277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471064057961663:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:45.982051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471064703462878:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.929925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471065800532497:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45550
2025-11-27T18:17:45.918344Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471066563138334:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(359153782ns) {Configure:104(359144368ns) {ProcessConfigured:135(16160130ns) {StartInitialHandshake:157(16158143ns) {DropHandshakes:470(2789ns) {DropIncomingHandshake:442(638ns) DropOutgoingHandshake:457(464ns)} GenerateSessionVirtualId:988(575ns) PrepareNewSessionHandshake:477(3006ns) {SwitchToState:179(129ns)}}}}}
2025-11-27T18:17:45.917709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471066548937300:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(357894762ns) {Configure:104(357885759ns) {ProcessConfigured:135(15667768ns) {StartInitialHandshake:157(15663111ns) {DropHandshakes:470(3601ns) {DropIncomingHandshake:442(824ns) DropOutgoingHandshake:457(448ns)} GenerateSessionVirtualId:988(1448ns) PrepareNewSessionHandshake:477(5412ns) {SwitchToState:179(2669ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.880467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471064057961725:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34642
2025-11-27T18:17:45.982123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471064703462878:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.919779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471066563138406:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.942118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471066548937352:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.012223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471064583946269:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:45.930080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471065800532498:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45550
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:46.002623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471064703462882:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.942184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.920113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471066563138333:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:45.946724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471064057961647:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(192386411ns) {Configure:104(192376540ns) {ProcessConfigured:135(74810ns) {StartInitialHandshake:157(72819ns) {DropHandshakes:470(1665ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(352ns) PrepareNewSessionHandshake:477(2057ns) {SwitchToState:179(736ns)}}}}}
2025-11-27T18:17:45.930159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471065800532499:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45550
2025-11-27T18:17:45.942249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471066548937352:2088] [node 6] ICP22 created new session: [5:7577471066548937376:2088]
2025-11-27T18:17:46.033467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471064583946194:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.002653Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471064703462882:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:45.946876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471064057961725:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.926719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471066563138398:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(134588350ns) {Configure:104(134583150ns) {ProcessConfigured:135(64892ns) {StartInitialHandshake:157(62447ns) {DropHandshakes:470(2313ns) {DropIncomingHandshake:442(485ns) DropOutgoingHandshake:457(383ns)} GenerateSessionVirtualId:988(426ns) PrepareNewSessionHandshake:477(2253ns) {SwitchToState:179(909ns)}}}}}
2025-11-27T18:17:45.938553Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471065800532415:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(376607917ns) {Configure:104(376602751ns) {ProcessConfigured:135(16455508ns) {StartInitialHandshake:157(16452486ns) {DropHandshakes:470(2473ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(311ns)} GenerateSessionVirtualId:988(984ns) PrepareNewSessionHandshake:477(2153ns) {SwitchToState:179(1236ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:46.041559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:45.942308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471066548937376:2088] [node 6] ICS09 handshake done sender: [5:7577471066548937365:2108] self: [5:7577471066548937364:0] peer: [6:7577471064057961722:0] socket: 303 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.948058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471065800532497:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.948357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471065800532498:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.962207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471065800532494:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.962277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471065800532417:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:45.977568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471065800532499:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.977747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471065800532416:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:45.997954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471065800532492:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(160128496ns) {Configure:104(160119843ns) {ProcessConfigured:135(40676873ns) {StartInitialHandshake:157(24982597ns) {DropHandshakes:470(4214ns) {DropIncomingHandshake:442(1191ns) DropOutgoingHandshake:457(415ns)} GenerateSessionVirtualId:988(3355ns) PrepareNewSessionHandshake:477(6786ns) {SwitchToState:179(3096ns)}}}}}
2025-11-27T18:17:46.003558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471064703462876:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.942344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471066548937376:2088] [node 6] ICS10 traffic start
2025-11-27T18:17:45.942381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:46.024391Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471066548937301:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(303171705ns) {Configure:104(292933878ns) {ProcessConfigured:135(36701637ns) {StartInitialHandshake:157(36697286ns) {DropHandshakes:470(3560ns) {DropIncomingHandshake:442(1301ns) DropOutgoingHandshake:457(475ns)} GenerateSessionVirtualId:988(787ns) PrepareNewSessionHandshake:477(3253ns) {SwitchToState:179(1855ns)}}}}}
2025-11-27T18:17:46.063663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471070843904677:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19765
2025-11-27T18:17:46.072776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471070843904678:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19765
2025-11-27T18:17:46.072836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471070843904679:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19765
2025-11-27T18:17:46.122550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471070843904679:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.127658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471066548937300:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:46.134500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471070843904677:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.140167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471066548937301:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:46.140543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471070843904682:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19765
2025-11-27T18:17:46.141997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471070843904678:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.946989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471064057961666:2066] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:46.041634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471064583946194:2049] [node 3] ICP22 created new session: [2:7577471068878913582:2049]
2025-11-27T18:17:45.953640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471066563138411:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.947630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471064057961647:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:46.005717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471065800532509:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45550
2025-11-27T18:17:45.953821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471066563138334:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:46.146743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471066548937299:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:46.003579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471064703462876:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:46.041695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471068878913582:2049] [node 3] ICS09 handshake done sender: [2:7577471064583946271:2119] self: [2:7577471064583946270:0] peer: [3:7577471066563138417:0] socket: 326 qp: -1
2025-11-27T18:17:46.004682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471064703462885:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57128
2025-11-27T18:17:45.947694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471064057961645:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:46.162069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471070843904682:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.023541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471070095499806:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45550
2025-11-27T18:17:45.954271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471066563138400:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.985583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471066563138409:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.005147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471068998430186:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57128
2025-11-27T18:17:46.032878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471068352929027:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34642
2025-11-27T18:17:46.041741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471068878913582:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:46.118570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471065800532417:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3264) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471065800532498:2123] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:45.985638Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471066563138409:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:45.987310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471066563138409:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:45.999521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471066563138402:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:45.999577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471066563138402:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:46.015535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471066563138402:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:46.015803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471066563138399:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.015918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471066563138397:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.016913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:46.022203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:46.022288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471066563138399:2109] [node 6] ICP22 created new session: [3:7577471070858105719:2109]
2025-11-27T18:17:46.022361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471070858105719:2109] [node 6] ICS09 handshake done sender: [3:7577471066563138409:2115] self: [3:7577471066563138408:0] peer: [6:7577471064057961729:0] socket: 322 qp: -1
2025-11-27T18:17:46.022409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471070858105719:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:46.022454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:46.025012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471066563138397:2107] [node 4] ICP22 created new session: [3:7577471070858105718:2107]
2025-11-27T18:17:46.025075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471070858105718:2107] [node 4] ICS09 handshake done sender: [3:7577471066563138402:2112] self: [3:7577471066563138401:0] peer: [4:7577471065800532506:0] socket: 316 qp: -1
2025-11-27T18:17:46.025139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471070858105718:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:46.025184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:46.046881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471070858105721:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49164
2025-11-27T18:17:46.048262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471066563138406:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:46.048530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471066563138333:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.048586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:46.048638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471066563138333:2048] [node 1] ICP22 created new session: [3:7577471070858105725:2048]
2025-11-27T18:17:46.048708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471070858105725:2048] [node 1] ICS09 handshake done sender: [3:7577471066563138406:2114] self: [3:7577471066563138414:0] peer: [1:7577471064703462881:0] socket: 317 qp: -1
2025-11-27T18:17:46.048751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471070858105725:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:46.048783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:46.157594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471066563138416:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.157637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471066563138416:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:46.170037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471070858105717:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49164
2025-11-27T18:17:46.185022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471066563138416:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:46.185882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471066563138398:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.185925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:46.185985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471066563138398:2108] [node 5] ICP22 created new session: [3:7577471070858105728:2108]
2025-11-27T18:17:46.186029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471070858105728:2108] [node 5] ICS09 handshake done sender: [3:7577471066563138416:2118] self: [3:7577471066563138415:0] peer: [5:7577471070843904681:0] socket: 302 qp: -1
2025-11-27T18:17:46.186068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471070858105728:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:46.186105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:46.190234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471066563138334:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5651) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471066563138411:2116] held: no
2025-11-27T18:17:46.184438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471070843904686:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19765
2025-11-27T18:17:46.185611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471070843904677:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:46.187979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471066548937301:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.188024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:46.188064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471066548937301:2050] [node 3] ICP22 created new session: [5:7577471070843904687:2050]
2025-11-27T18:17:46.188118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471070843904687:2050] [node 3] ICS09 handshake done sender: [5:7577471070843904677:2115] self: [5:7577471070843904681:0] peer: [3:7577471066563138415:0] socket: 354 qp: -1
2025-11-27T18:17:46.188142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471070843904687:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:46.188170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:46.193727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471070843904684:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19765
2025-11-27T18:17:46.197585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471066548937335:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:46.213830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471066548937335:2084] [node 4] ICP35 event processing took too much time PendingConnection:279(18828997ns) {IncomingHandshake:261(18827404ns) {DropIncomingHandshake:442(1142ns) PrepareNewSessionHandshake:477(1780ns) {SwitchToState:179(621ns)} GenerateSessionVirtualId:988(803ns) IssueIncomingHandshakeReply:193(61212ns) {SwitchToState:179(845ns)}}}
2025-11-27T18:17:46.253758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471066548937301:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(65536778ns) {HandleHandshakeStatus:329(65534645ns) {DropIncomingHandshake:442(407ns) DropOutgoingHandshake:457(4297ns) SwitchToState:179(189ns) ProcessPendingSessionEvents:495(43168651ns) {ForwardSessionEventToSession:612(43094652ns) ForwardSessionEventToSession:612(8904ns) ForwardSessionEventToSession:612(11624ns) ForwardSessionEventToSession:612(2887ns) ForwardSessionEventToSession:612(2531ns) ForwardSessionEventToSession:612(17475ns) ForwardSessionEventToSession:612(4163ns) ForwardSessionEventToSession:612(2493ns) ForwardSessionEventToSession:612(6907ns) ForwardSessionEventToSession:612(3717ns)}}}
2025-11-27T18:17:46.253989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471066548937299:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10463) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471070843904678:2116] held: no
2025-11-27T18:17:46.254895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471070843904690:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19765
2025-11-27T18:17:46.261586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471070843904691:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19765
2025-11-27T18:17:46.010175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471064703462872:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(288401097ns) {Configure:104(281792159ns) {ProcessConfigured:135(11773374ns) {StartInitialHandshake:157(11770130ns) {DropHandshakes:470(1851ns) {DropIncomingHandshake:442(106ns) DropOutgoingHandshake:457(437ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:477(4879ns) {SwitchToState:179(2548ns)}}}}}
2025-11-27T18:17:46.041798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:46.037806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471064057961724:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34642
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:46.010381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471064703462876:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:46.010729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471064703462878:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:46.016283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471064703462882:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:46.049999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471064703462885:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.050127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471060408495514:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.050174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:46.050219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471060408495514:2049] [node 3] ICP22 created new session: [1:7577471068998430192:2049]
2025-11-27T18:17:46.050269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471068998430192:2049] [node 3] ICS09 handshake done sender: [1:7577471064703462882:2112] self: [1:7577471064703462881:0] peer: [3:7577471066563138414:0] socket: 315 qp: -1
2025-11-27T18:17:46.050297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471068998430192:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:46.050333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:46.050987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471060408495513:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.051018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:46.051055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471060408495513:2048] [node 2] ICP22 created new session: [1:7577471068998430194:2048]
2025-11-27T18:17:46.051095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471068998430194:2048] [node 2] ICS09 handshake done sender: [1:7577471064703462876:2110] self: [1:7577471064703462875:0] peer: [2:7577471064583946282:0] socket: 298 qp: -1
2025-11-27T18:17:46.051122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471068998430194:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:46.051146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:46.051464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471064703462874:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.051508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:46.051546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471064703462874:2109] [node 6] ICP22 created new session: [1:7577471068998430196:2109]
2025-11-27T18:17:46.051582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471068998430196:2109] [node 6] ICS09 handshake done sender: [1:7577471064703462878:2111] self: [1:7577471064703462877:0] peer: [6:7577471064057961730:0] socket: 299 qp: -1
2025-11-27T18:17:46.051617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471068998430196:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:46.051640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:46.052051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471064703462886:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57128
2025-11-27T18:17:46.052925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471060408495514:2049] [node 3] ICP18 handshake (actor [1:7577471064703462885:2114]) is too old
2025-11-27T18:17:46.053648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471068998430186:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.056523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471064703462873:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(300667045ns) {Configure:104(300658954ns) {ProcessConfigured:135(34270191ns) {StartInitialHandshake:157(34267542ns) {DropHandshakes:470(2275ns) {DropIncomingHandshake:442(585ns) DropOutgoingHandshake:457(351ns)} GenerateSessionVirtualId:988(944ns) PrepareNewSessionHandshake:477(3505ns) {SwitchToState:179(1288ns)}}}}}
2025-11-27T18:17:46.062331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471064703462886:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.077851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471068998430198:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57128
2025-11-27T18:17:46.209433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471068998430191:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.209481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471068998430191:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:46.215942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471068998430191:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:46.216192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471064703462873:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.216250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:46.216305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471064703462873:2108] [node 5] ICP22 created new session: [1:7577471068998430204:2108]
2025-11-27T18:17:46.216349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471068998430204:2108] [node 5] ICS09 handshake done sender: [1:7577471068998430191:2118] self: [1:7577471064703462887:0] peer: [5:7577471070843904683:0] socket: 314 qp: -1
2025-11-27T18:17:46.216385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471068998430204:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:46.216420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:46.237845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471068998430190:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.237897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471068998430190:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:46.269373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471068998430190:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:46.273378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471066563138334:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(31134752ns) {HandleHandshakeStatus:395(31132321ns) {LogHandshakeFail:472(13363ns) {UpdateErrorStateLog:545(5733ns)} DropOutgoingHandshake:457(1358ns) ProcessPendingSessionEvents:495(19608242ns) {DropSessionEvent:512(18798ns) DropSessionEvent:512(5894ns) DropSessionEvent:512(26027ns) DropSessionEvent:512(5940ns) DropSessionEvent:512(5618ns) DropSessionEvent:512(5875ns) DropSessionEvent:512(5936ns) DropSessionEvent:512(7758ns) DropSessionEvent:512(15917490ns) DropSessionEvent:512(7125ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:46.118735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471065800532416:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5651) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471065800532499:2124] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:46.045490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471068352929028:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34642
2025-11-27T18:17:46.056768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471064057961726:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34642
2025-11-27T18:17:46.133755Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471064057961666:2066] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:46.140909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471064057961726:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.141038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471064057961725:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:46.141955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471064057961676:2075] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:46.149900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471064057961645:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10463) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471064057961666:2066] held: no
2025-11-27T18:17:46.150012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471064057961645:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.150059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:46.150107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471064057961645:2048] [node 1] ICP22 created new session: [6:7577471068352929030:2048]
2025-11-27T18:17:46.150158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471068352929030:2048] [node 1] ICS09 handshake done sender: [6:7577471064057961666:2066] self: [6:7577471064057961730:0] peer: [1:7577471064703462877:0] socket: 301 qp: -1
2025-11-27T18:17:46.150197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471068352929030:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:46.150230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:46.150637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471064057961647:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.150679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:46.150717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471064057961647:2050] [node 3] ICP22 created new session: [6:7577471068352929032:2050]
2025-11-27T18:17:46.150751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471068352929032:2050] [node 3] ICS09 handshake done sender: [6:7577471064057961725:2117] self: [6:7577471064057961729:0] peer: [3:7577471066563138408:0] socket: 323 qp: -1
2025-11-27T18:17:46.150784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471068352929032:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:46.150808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:46.151773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471064057961646:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:46.151875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471068352929029:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34642
2025-11-27T18:17:46.152700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471064057961663:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.152743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:46.152793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471064057961663:2061] [node 5] ICP22 created new session: [6:7577471068352929036:2061]
2025-11-27T18:17:46.152842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471068352929036:2061] [node 5] ICS09 handshake done sender: [6:7577471064057961676:2075] self: [6:7577471064057961722:0] peer: [5:7577471066548937364:0] socket: 305 qp: -1
2025-11-27T18:17:46.152872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471068352929036:2061] [node 5] ICS10 traffic start
2025-11-27T18:17:46.152908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:46.183307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471068352929029:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.183531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471064057961707:2082] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:46.184064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471068352929039:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34642
2025-11-27T18:17:46.184817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471064057961726:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:46.186874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471064057961646:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.186921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:46.186969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471064057961646:2049] [node 2] ICP22 created new session: [6:7577471068352929040:2049]
2025-11-27T18:17:46.187018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471068352929040:2049] [node 2] ICS09 handshake done sender: [6:7577471064057961726:2118] self: [6:7577471068352929034:0] peer: [2:7577471064583946266:0] socket: 330 qp: -1
2025-11-27T18:17:46.187046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471068352929040:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:46.187085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:46.193780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471064057961646:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:46.269690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471068352929042:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34642
2025-11-27T18:17:46.286223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471068352929029:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:46.119035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471065800532498:2123] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:46.275726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471070843904689:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19765
2025-11-27T18:17:46.283313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471066548937300:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5651) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471070843904679:2117] held: no
2025-11-27T18:17:46.285825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471070843904682:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:46.286874Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471070843904678:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:46.294156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471066548937299:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.294210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:46.294264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471066548937299:2048] [node 1] ICP22 created new session: [5:7577471070843904692:2048]
2025-11-27T18:17:46.294316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471070843904692:2048] [node 1] ICS09 handshake done sender: [5:7577471070843904678:2116] self: [5:7577471070843904683:0] peer: [1:7577471064703462887:0] socket: 359 qp: -1
2025-11-27T18:17:46.294369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471070843904692:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:46.294415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:46.294833Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471066548937335:2084] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61105) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471070843904682:2118] held: no
2025-11-27T18:17:46.294933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471066548937335:2084] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.294957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:46.294994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471066548937335:2084] [node 4] ICP22 created new session: [5:7577471070843904694:2084]
2025-11-27T18:17:46.295041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471070843904694:2084] [node 4] ICS09 handshake done sender: [5:7577471070843904682:2118] self: [5:7577471070843904685:0] peer: [4:7577471065800532501:0] socket: 357 qp: -1
2025-11-27T18:17:46.295067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471070843904694:2084] [node 4] ICS10 traffic start
2025-11-27T18:17:46.295099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:46.042337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471064583946262:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.042402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:46.042444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471064583946262:2109] [node 4] ICP22 created new session: [2:7577471068878913584:2109]
2025-11-27T18:17:46.042520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471068878913584:2109] [node 4] ICS09 handshake done sender: [2:7577471064583946269:2118] self: [2:7577471064583946268:0] peer: [4:7577471065800532507:0] socket: 325 qp: -1
2025-11-27T18:17:46.042560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471068878913584:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:46.042591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:46.046134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471068878913581:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4886
2025-11-27T18:17:46.047804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471064583946264:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:46.049629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471064583946193:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.054079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:46.057029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471068878913587:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4886
2025-11-27T18:17:46.064312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471064583946193:2048] [node 1] ICP22 created new session: [2:7577471068878913588:2048]
2025-11-27T18:17:46.065712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471064583946277:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.105516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471068878913588:2048] [node 1] ICS09 handshake done sender: [2:7577471064583946264:2115] self: [2:7577471064583946282:0] peer: [1:7577471064703462875:0] socket: 309 qp: -1
2025-11-27T18:17:46.113382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471068878913588:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:46.135478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471068878913587:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.141562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471064583946283:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.141604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471064583946283:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:46.142024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:46.145588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471064583946194:2049] [node 3] ICP18 handshake (actor [2:7577471068878913587:2124]) is too old
2025-11-27T18:17:46.158167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471064583946267:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.158189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471064583946267:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:46.159183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471064583946267:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:46.159474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471064583946261:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.159534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:46.159592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471064583946261:2108] [node 6] ICP22 created new session: [2:7577471068878913593:2108]
2025-11-27T18:17:46.159649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471068878913593:2108] [node 6] ICS09 handshake done sender: [2:7577471064583946267:2117] self: [2:7577471064583946266:0] peer: [6:7577471068352929034:0] socket: 324 qp: -1
2025-11-27T18:17:46.159686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471068878913593:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:46.159724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:46.188490Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471064583946193:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(138625348ns) {HandleHandshakeStatus:329(138622821ns) {DropIncomingHandshake:442(1021ns) DropOutgoingHandshake:457(4502ns) SwitchToState:179(309ns) ProcessPendingSessionEvents:495(15158039ns) {ForwardSessionEventToSession:612(15095862ns) ForwardSessionEventToSession:612(6796ns) ForwardSessionEventToSession:612(3930ns) ForwardSessionEventToSession:612(7335ns) ForwardSessionEventToSession:612(2899ns) ForwardSessionEventToSession:612(3788ns) ForwardSessionEventToSession:612(10429ns) ForwardSessionEventToSession:612(3126ns) ForwardSessionEventToSession:612(5957ns) ForwardSessionEventToSession:612(6036ns)}}}
2025-11-27T18:17:46.249509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471064583946283:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:46.249983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471064583946260:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.250056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:46.250109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471064583946260:2107] [node 5] ICP22 created new session: [2:7577471068878913596:2107]
2025-11-27T18:17:46.250172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471068878913596:2107] [node 5] ICS09 handshake done sender: [2:7577471064583946283:2122] self: [2:7577471064583946279:0] peer: [5:7577471070843904680:0] socket: 329 qp: -1
2025-11-27T18:17:46.250207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471068878913596:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:46.250245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:46.277340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471066563138411:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:46.282994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471066563138334:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.290067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:46.283139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471064703462872:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.290109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:46.299930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471066563138334:2049] [node 2] ICP22 created new session: [3:7577471070858105730:2049]
2025-11-27T18:17:46.300017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471070858105730:2049] [node 2] ICS09 handshake done sender: [3:7577471066563138411:2116] self: [3:7577471066563138417:0] peer: [2:7577471064583946270:0] socket: 331 qp: -1
2025-11-27T18:17:46.300062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471070858105730:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:46.300131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:46.300463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471066563138334:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17289638ns) {HandleHandshakeStatus:329(17286348ns) {DropIncomingHandshake:442(877ns) DropOutgoingHandshake:457(1404ns) SwitchToState:179(2940ns) ProcessPendingSessionEvents:495(1022ns)}}
2025-11-27T18:17:46.289667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471064057961707:2082] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.126051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471065800532417:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.300970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471064703462872:2107] [node 4] ICP22 created new session: [1:7577471068998430207:2107]
2025-11-27T18:17:46.301034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471068998430207:2107] [node 4] ICS09 handshake done sender: [1:7577471068998430190:2117] self: [1:7577471064703462888:0] peer: [4:7577471070095499813:0] socket: 358 qp: -1
2025-11-27T18:17:46.301066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471068998430207:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:46.301132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:46.313803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471064703462872:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(30440125ns) {HandleHandshakeStatus:329(30437766ns) {DropIncomingHandshake:442(16425ns) DropOutgoingHandshake:457(718ns) SwitchToState:179(759ns) ProcessPendingSessionEvents:495(12322832ns) {ForwardSessionEventToSession:612(8202903ns) ForwardSessionEventToSession:612(8515ns) ForwardSessionEventToSession:612(3512ns) ForwardSessionEventToSession:612(499258ns) ForwardSessionEventToSession:612(3551ns) ForwardSessionEventToSession:612(3566400ns) ForwardSessionEventToSession:612(4845ns) ForwardSessionEventToSession:612(3144ns) ForwardSessionEventToSession:612(12330ns) ForwardSessionEventToSession:612(2713ns)}}}
2025-11-27T18:17:46.289727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:46.290535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471064057961707:2082] [node 4] ICP22 created new session: [6:7577471068352929043:2082]
2025-11-27T18:17:46.314681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471068352929043:2082] [node 4] ICS09 handshake done sender: [6:7577471068352929029:2122] self: [6:7577471068352929038:0] peer: [4:7577471065800532495:0] socket: 360 qp: -1
2025-11-27T18:17:46.327204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471068352929043:2082] [node 4] ICS10 traffic start
2025-11-27T18:17:46.327266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:46.333972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471066548937300:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20039272ns) {HandleHandshakeStatus:395(20034020ns) {LogHandshakeFail:472(15315ns) {UpdateErrorStateLog:545(5765ns)} DropOutgoingHandshake:457(1663ns) ProcessPendingSessionEvents:495(18061356ns) {DropSessionEvent:512(11023ns) DropSessionEvent:512(4415ns) DropSessionEvent:512(4832ns) DropSessionEvent:512(4138ns) DropSessionEvent:512(6959475ns) DropSessionEvent:512(6210ns) DropSessionEvent:512(5789ns) DropSessionEvent:512(27630ns) DropSessionEvent:512(10809ns) DropSessionEvent:512(5868ns)}}}
2025-11-27T18:17:46.334125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471070843904679:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:46.126122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:46.126188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471065800532417:2050] [node 3] ICP22 created new session: [4:7577471070095499808:2050]
2025-11-27T18:17:46.126234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471070095499808:2050] [node 3] ICS09 handshake done sender: [4:7577471065800532498:2123] self: [4:7577471065800532506:0] peer: [3:7577471066563138401:0] socket: 334 qp: -1
2025-11-27T18:17:46.126271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471070095499808:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:46.126309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:46.126863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471070095499807:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45550
2025-11-27T18:17:46.134653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471065800532499:2124] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:46.173199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471070095499807:2129] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.173457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471065800532416:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.174150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:46.174201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471065800532416:2049] [node 2] ICP22 created new session: [4:7577471070095499812:2049]
2025-11-27T18:17:46.181735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471070095499812:2049] [node 2] ICS09 handshake done sender: [4:7577471065800532499:2124] self: [4:7577471065800532507:0] peer: [2:7577471064583946268:0] socket: 335 qp: -1
2025-11-27T18:17:46.181790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471070095499812:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:46.181843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:46.182072Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471065800532416:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16427366ns) {HandleHandshakeStatus:329(16425280ns) {DropIncomingHandshake:442(2952ns) DropOutgoingHandshake:457(487ns) SwitchToState:179(660ns) ProcessPendingSessionEvents:495(1234ns)}}
2025-11-27T18:17:46.193578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471065800532496:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.193611Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471065800532496:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:46.205962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471065800532415:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:46.225668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471065800532504:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:46.225703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471065800532504:2126] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:46.238267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471065800532504:2126] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:46.249460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471065800532415:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(35825521ns) {IncomingHandshake:261(34556280ns) {DropIncomingHandshake:442(271ns) PrepareNewSessionHandshake:477(673ns) {SwitchToState:179(199ns)} GenerateSessionVirtualId:988(258ns) IssueIncomingHandshakeReply:193(34534432ns) {SwitchToState:179(4199ns)}}}
2025-11-27T18:17:46.275189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471065800532496:2121] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:46.275878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471065800532492:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.275954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:46.276011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471065800532492:2108] [node 5] ICP22 created new session: [4:7577471070095499818:2108]
2025-11-27T18:17:46.276074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471070095499818:2108] [node 5] ICS09 handshake done sender: [4:7577471065800532504:2126] self: [4:7577471065800532501:0] peer: [5:7577471070843904685:0] socket: 297 qp: -1
2025-11-27T18:17:46.276118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471070095499818:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:46.276158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:46.277687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471070095499817:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45550
2025-11-27T18:17:46.286734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471070095499807:2129] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:46.287287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471065800532481:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:46.287348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:46.287399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471065800532481:2107] [node 6] ICP22 created new session: [4:7577471070095499820:2107]
2025-11-27T18:17:46.287446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471070095499820:2107] [node 6] ICS09 handshake done sender: [4:7577471065800532496:2121] self: [4:7577471065800532495:0] peer: [6:7577471068352929038:0] socket: 295 qp: -1
2025-11-27T18:17:46.287483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471070095499820:2107] [node 6] ICS10 traffic start
2025-11-27T18:17:46.287514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:46.289572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471065800532415:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.301586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:46.303215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471065800532415:2048] [node 1] ICP22 created new session: [4:7577471070095499822:2048]
2025-11-27T18:17:46.309870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471070095499822:2048] [node 1] ICS09 handshake done sender: [4:7577471070095499807:2129] self: [4:7577471070095499813:0] peer: [1:7577471064703462888:0] socket: 318 qp: -1
2025-11-27T18:17:46.309922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471070095499822:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:46.321716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:46.322212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471065800532415:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32424262ns) {HandleHandshakeStatus:329(32421386ns) {DropIncomingHandshake:442(881ns) DropOutgoingHandshake:457(14834ns) SwitchToState:179(159ns) ProcessPendingSessionEvents:495(120798ns) {ForwardSessionEventToSession:612(63421ns) ForwardSessionEventToSession:612(4047ns) ForwardSessionEventToSession:612(7081ns) ForwardSessionEventToSession:612(3265ns) ForwardSessionEventToSession:612(1936ns) ForwardSessionEventToSession:612(14938ns) ForwardSessionEventToSession:612(3778ns) ForwardSessionEventToSession:612(6575ns) ForwardSessionEventToSession:612(2511ns) ForwardSessionEventToSession:612(2299ns)}}}
2025-11-27T18:17:46.335049Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471064057961707:2082] [node 4] ICP35 event processing took too much time PendingConnection:279(45155348ns) {HandleHandshakeStatus:329(45153468ns) {DropIncomingHandshake:442(234ns) DropOutgoingHandshake:457(10829ns) SwitchToState:179(483ns) ProcessPendingSessionEvents:495(536403ns) {ForwardSessionEventToSession:612(478571ns) ForwardSessionEventToSession:612(8541ns) ForwardSessionEventToSession:612(6791ns) ForwardSessionEventToSession:612(2666ns) ForwardSessionEventToSession:612(5697ns) ForwardSessionEventToSession:612(2947ns) ForwardSessionEventToSession:612(5822ns) ForwardSessionEventToSession:612(3004ns) ForwardSessionEventToSession:612(3144ns) ForwardSessionEventToSession:612(5497ns)}}}
2025-11-27T18:17:46.343389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471066548937300:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:46.343455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:46.343608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471066548937300:2049] [node 2] ICP22 created new session: [5:7577471070843904696:2049]
2025-11-27T18:17:46.343674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471070843904696:2049] [node 2] ICS09 handshake done sender: [5:7577471070843904679:2117] self: [5:7577471070843904680:0] peer: [2:7577471064583946279:0] socket: 355 qp: -1
2025-11-27T18:17:46.343719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471070843904696:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:46.343762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:46.422762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:46.423015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471068878913588:2048] [node 1] ICS07 socket disconnect 309 reason# EndOfStream
2025-11-27T18:17:46.423205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471068878913588:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.423291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:46.423333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471068878913588:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:46.423117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:46.424097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471070858105725:2048] [node 1] ICS01 socket: 317 reason# EndOfStream
2025-11-27T18:17:46.424139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471066563138333:2048] [node 1] ICP30 unregister session Session# [3:7577471070858105725:2048] VirtualId# [3:7577471066563138414:0]
2025-11-27T18:17:46.424174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471070858105725:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.424236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:46.424280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:46.425724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471064583946193:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10463) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:46.425775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471068878913588:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:46.425808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471064583946193:2048] [node 1] ICP30 unregister session Session# [2:7577471068878913588:2048] VirtualId# [2:7577471064583946282:0]
2025-11-27T18:17:46.425846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:46.425876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471064583946193:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10463) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:46.425899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10463) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:46.437378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471064583946193:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:46.441436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:46.441687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471070843904692:2048] [node 1] ICS01 socket: 359 reason# EndOfStream
2025-11-27T18:17:46.441724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471066548937299:2048] [node 1] ICP30 unregister session Session# [5:7577471070843904692:2048] VirtualId# [5:7577471070843904683:0]
2025-11-27T18:17:46.441758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471070843904692:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.441835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:46.441877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:46.441981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:46.442163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471068352929030:2048] [node 1] ICS01 socket: 301 reason# EndOfStream
2025-11-27T18:17:46.442195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471064057961645:2048] [node 1] ICP30 unregister session Session# [6:7577471068352929030:2048] VirtualId# [6:7577471064057961730:0]
2025-11-27T18:17:46.442225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471068352929030:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.442273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:46.442306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:46.443931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:46.444135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471070095499822:2048] [node 1] ICS01 socket: 318 reason# EndOfStream
2025-11-27T18:17:46.444176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471065800532415:2048] [node 1] ICP30 unregister session Session# [4:7577471070095499822:2048] VirtualId# [4:7577471070095499813:0]
2025-11-27T18:17:46.444205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471070095499822:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.444288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:46.444327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:46.545425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:46.545711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471070858105730:2049] [node 2] ICS01 socket: 331 reason# EndOfStream
2025-11-27T18:17:46.545756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471066563138334:2049] [node 2] ICP30 unregister session Session# [3:7577471070858105730:2049] VirtualId# [3:7577471066563138417:0]
2025-11-27T18:17:46.545796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471070858105730:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.545886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:46.545946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:46.550895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:46.551130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471068352929040:2049] [node 2] ICS01 socket: 330 reason# EndOfStream
2025-11-27T18:17:46.551178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471064057961646:2049] [node 2] ICP30 unregister session Session# [6:7577471068352929040:2049] VirtualId# [6:7577471068352929034:0]
2025-11-27T18:17:46.551213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471068352929040:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.551274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:46.551309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:46.551399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:46.551597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471070843904696:2049] [node 2] ICS01 socket: 355 reason# EndOfStream
2025-11-27T18:17:46.551633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471066548937300:2049] [node 2] ICP30 unregister session Session# [5:7577471070843904696:2049] VirtualId# [5:7577471070843904680:0]
2025-11-27T18:17:46.551660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471070843904696:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.551713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:46.551751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:46.556685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:46.556916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471070095499812:2049] [node 2] ICS01 socket: 335 reason# EndOfStream
2025-11-27T18:17:46.556955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471065800532416:2049] [node 2] ICP30 unregister session Session# [4:7577471070095499812:2049] VirtualId# [4:7577471065800532507:0]
2025-11-27T18:17:46.556985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471070095499812:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.557050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:46.557104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:46.645419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:46.646206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471070095499808:2050] [node 3] ICS01 socket: 334 reason# EndOfStream
2025-11-27T18:17:46.645957Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:46.646497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471070843904687:2050] [node 3] ICS07 socket disconnect 354 reason# EndOfStream
2025-11-27T18:17:46.646536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471070843904687:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.646630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:46.646680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471070843904687:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:46.649064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471066548937301:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3264) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:46.649134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471070843904687:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:46.649172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471066548937301:2050] [node 3] ICP30 unregister session Session# [5:7577471070843904687:2050] VirtualId# [5:7577471070843904681:0]
2025-11-27T18:17:46.649215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:46.649245Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471066548937301:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:46.649280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:46.649997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471065800532417:2050] [node 3] ICP30 unregister session Session# [4:7577471070095499808:2050] VirtualId# [4:7577471065800532506:0]
2025-11-27T18:17:46.650056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471070095499808:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.650130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:46.650178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:46.652316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:46.652540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471068352929032:2050] [node 3] ICS07 socket disconnect 323 reason# EndOfStream
2025-11-27T18:17:46.652570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471068352929032:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.652648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:46.652702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471068352929032:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:46.656030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471066548937301:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:46.654231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471064057961647:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3264) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:46.654295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471068352929032:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:46.654327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471064057961647:2050] [node 3] ICP30 unregister session Session# [6:7577471068352929032:2050] VirtualId# [6:7577471064057961729:0]
2025-11-27T18:17:46.654364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:46.654393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471064057961647:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:46.654406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:46.666742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471064057961647:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:46.749531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:46.749747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471068352929043:2082] [node 4] ICS07 socket disconnect 360 reason# EndOfStream
2025-11-27T18:17:46.749785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471068352929043:2082] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.749871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:46.749908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471068352929043:2082] [node 4] ICS15 start handshake
2025-11-27T18:17:46.751332Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:46.761380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471070843904694:2084] [node 4] ICS01 socket: 357 reason# EndOfStream
2025-11-27T18:17:46.773978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471064057961707:2082] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61105) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:46.776623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471068352929043:2082] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:46.776672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471064057961707:2082] [node 4] ICP30 unregister session Session# [6:7577471068352929043:2082] VirtualId# [6:7577471068352929038:0]
2025-11-27T18:17:46.776736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:46.776794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471064057961707:2082] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61105) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:46.776811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61105) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:46.777068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [6:7577471064057961707:2082] [node 4] ICP35 event processing took too much time StateWork:294(25798649ns) {HandleHandshakeStatus:395(14464525ns) {LogHandshakeFail:472(943657ns) {UpdateErrorStateLog:545(2925ns)} DropOutgoingHandshake:457(2572ns) ProcessPendingSessionEvents:495(396ns) UnregisterSession:538(49575ns) {ProcessPendingSessionEvents:495(562ns) SwitchToInitialState:202(2347ns) {SwitchToState:179(1284ns)}} TransitToErrorState:752(75438ns) {SwitchToState:179(9997ns) ProcessPendingSessionEvents:495(53ns) DropHandshakes:470(1813ns) {DropIncomingHandshake:442(462ns) DropOutgoingHandshake:457(252ns)}}}}
2025-11-27T18:17:46.789712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471064057961707:2082] [node 4] ICP33 wake up from error state
2025-11-27T18:17:46.853915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:46.854159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471068352929036:2061] [node 5] ICS01 socket: 305 reason# EndOfStream
2025-11-27T18:17:46.854204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471064057961663:2061] [node 5] ICP30 unregister session Session# [6:7577471068352929036:2061] VirtualId# [6:7577471064057961722:0]
2025-11-27T18:17:46.854234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471068352929036:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:46.854322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:46.854367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:46.996193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471070650704562:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:46.996236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471070650704563:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:47.041609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471074918340427:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:47.041649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471074918340428:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:47.073456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471074529922477:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:47.073500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471074529922478:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:47.132961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471075441401104:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:47.133007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471075441401105:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:47.133037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471075441401106:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:47.159652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471075681521555:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:47.186467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471075677214126:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:47.186515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471075677214127:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:47.186548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471075677214128:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:47.159694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471075681521556:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:47.159723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471075681521557:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:47.820992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471075677214188:2105] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:47.827129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471075681521616:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13915
2025-11-27T18:17:47.856712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471074529922540:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:47.862088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471075441401170:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:47.837638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471075681521616:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:47.893975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471074529922542:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:47.841404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471074945671924:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:47.891370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471075681521618:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:47.928243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471074918340491:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:47.891303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471074945671925:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:47.976867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471075441401171:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:47.969961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471074918340492:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:47.948158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471075677214193:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8524
2025-11-27T18:17:47.928084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471074945671926:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:47.976067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471075677214197:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:47.976067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471074918340493:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:47.973799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471070650704563:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(147207896ns) {Configure:104(147203797ns) {ProcessConfigured:135(19039625ns) {StartInitialHandshake:157(19036535ns) {DropHandshakes:470(3154ns) {DropIncomingHandshake:442(738ns) DropOutgoingHandshake:457(769ns)} GenerateSessionVirtualId:988(991ns) PrepareNewSessionHandshake:477(4890ns) {SwitchToState:179(2508ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.065978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471079736368468:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-27T18:17:48.023325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471079976488916:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:48.008719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471079213307790:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19973
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.022948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471078824889842:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:47.981404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471074945671927:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33150
2025-11-27T18:17:48.103221Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471075681521555:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(209674091ns) {Configure:104(209666272ns) {ProcessConfigured:135(88003ns) {StartInitialHandshake:157(84983ns) {DropHandshakes:470(4087ns) {DropIncomingHandshake:442(1338ns) DropOutgoingHandshake:457(486ns)} GenerateSessionVirtualId:988(1375ns) PrepareNewSessionHandshake:477(5034ns) {SwitchToState:179(709ns)}}}}}
2025-11-27T18:17:48.048752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471078824889843:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10576
2025-11-27T18:17:48.035291Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471074918340493:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(33055451ns) {Configure:104(33050451ns) {ProcessConfigured:135(82545ns) {StartInitialHandshake:157(79171ns) {DropHandshakes:470(3280ns) {DropIncomingHandshake:442(924ns) DropOutgoingHandshake:457(432ns)} GenerateSessionVirtualId:988(620ns) PrepareNewSessionHandshake:477(2245ns) {SwitchToState:179(610ns)}}}}}
2025-11-27T18:17:48.027146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471074945671925:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(56941695ns) {Configure:104(50961829ns) {ProcessConfigured:135(96271ns) {StartInitialHandshake:157(93863ns) {DropHandshakes:470(3778ns) {DropIncomingHandshake:442(979ns) DropOutgoingHandshake:457(492ns)} GenerateSessionVirtualId:988(1016ns) PrepareNewSessionHandshake:477(1923ns) {SwitchToState:179(857ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.090547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471079972181494:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8524
2025-11-27T18:17:48.116583Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471075441401106:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(139485219ns) {Configure:104(139478531ns) {ProcessConfigured:135(49718ns) {StartInitialHandshake:157(48238ns) {DropHandshakes:470(2018ns) {DropIncomingHandshake:442(806ns) DropOutgoingHandshake:457(474ns)} GenerateSessionVirtualId:988(438ns) PrepareNewSessionHandshake:477(575ns) {SwitchToState:179(304ns)}}}}}
2025-11-27T18:17:48.116765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471079736368468:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.122164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471079736368472:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.126419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471079976488917:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13915
2025-11-27T18:17:48.093719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471075677214193:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.079779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471074945671927:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.055578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471074918340487:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19973
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.082232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471074529922478:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(127964577ns) {Configure:104(127955497ns) {ProcessConfigured:135(15364118ns) {StartInitialHandshake:157(15360500ns) {DropHandshakes:470(3304ns) {DropIncomingHandshake:442(894ns) DropOutgoingHandshake:457(435ns)} GenerateSessionVirtualId:988(910ns) PrepareNewSessionHandshake:477(4981ns) {SwitchToState:179(2511ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.134846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471075441401171:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(132782255ns) {Configure:104(132776037ns) {ProcessConfigured:135(79538ns) {StartInitialHandshake:157(76167ns) {DropHandshakes:470(3161ns) {DropIncomingHandshake:442(720ns) DropOutgoingHandshake:457(653ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:477(2325ns) {SwitchToState:179(792ns)}}}}}
2025-11-27T18:17:48.114891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471079240639230:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33150
2025-11-27T18:17:48.110256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471079972181494:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.083315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471079213307790:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.141208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471079976488924:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13915
2025-11-27T18:17:48.181837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471079240639230:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.082398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471078824889843:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.162003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471079736368472:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.083567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471079213307799:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19973
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.115731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471075677214127:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(157120378ns) {Configure:104(157108601ns) {ProcessConfigured:135(62537ns) {StartInitialHandshake:157(59765ns) {DropHandshakes:470(2596ns) {DropIncomingHandshake:442(492ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(812ns) PrepareNewSessionHandshake:477(1503ns) {SwitchToState:179(598ns)}}}}}
2025-11-27T18:17:48.141664Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471074529922540:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(253580238ns) {Configure:104(253574771ns) {ProcessConfigured:135(23207233ns) {StartInitialHandshake:157(23204790ns) {DropHandshakes:470(3785ns) {DropIncomingHandshake:442(1174ns) DropOutgoingHandshake:457(775ns)} GenerateSessionVirtualId:988(772ns) PrepareNewSessionHandshake:477(4518ns) {SwitchToState:179(2505ns)}}}}}
2025-11-27T18:17:48.145857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471075681521618:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(144275353ns) {Configure:104(144272411ns) {ProcessConfigured:135(112441ns) {StartInitialHandshake:157(64515ns) {DropHandshakes:470(3843ns) {DropIncomingHandshake:442(1104ns) DropOutgoingHandshake:457(451ns)} GenerateSessionVirtualId:988(5747ns) PrepareNewSessionHandshake:477(1812ns) {SwitchToState:179(725ns)}} IncomingHandshake:261(19137ns) {DropIncomingHandshake:442(566ns) PrepareNewSessionHandshake:477(324ns) {SwitchToState:179(236ns)} GenerateSessionVirtualId:988(109ns) IssueIncomingHandshakeReply:193(1028ns)}}}}
2025-11-27T18:17:48.205559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471075441401104:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(314217408ns) {Configure:104(314209162ns) {ProcessConfigured:135(19644971ns) {StartInitialHandshake:157(19642082ns) {DropHandshakes:470(1827ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(490ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(6190ns) {SwitchToState:179(3413ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.213335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471074945671929:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.101233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471074918340487:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.115810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471075677214127:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.171703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471078824889852:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10576
2025-11-27T18:17:48.146517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471079976488928:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13915
2025-11-27T18:17:48.109991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471079213307799:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.194744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471079972181501:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8524
2025-11-27T18:17:48.205337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471079972181500:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8524
2025-11-27T18:17:48.209587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471075677214128:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(197964645ns) {Configure:104(197957791ns) {ProcessConfigured:135(33874465ns) {StartInitialHandshake:157(33872085ns) {DropHandshakes:470(3136ns) {DropIncomingHandshake:442(894ns) DropOutgoingHandshake:457(532ns)} GenerateSessionVirtualId:988(879ns) PrepareNewSessionHandshake:477(1998ns) {SwitchToState:179(781ns)}}}}}
2025-11-27T18:17:48.209646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471075677214128:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.250023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471075677214197:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(247766799ns) {Configure:104(247762008ns) {ProcessConfigured:135(17003783ns) {StartInitialHandshake:157(17001146ns) {DropHandshakes:470(16921586ns) {DropIncomingHandshake:442(2592ns) DropOutgoingHandshake:457(879ns)} GenerateSessionVirtualId:988(1628ns) PrepareNewSessionHandshake:477(1591ns) {SwitchToState:179(112ns)}}}}}
2025-11-27T18:17:48.216760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471075441401104:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.213398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471074945671929:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.157507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471079976488917:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.182233Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471074529922477:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(332726083ns) {Configure:104(332714880ns) {ProcessConfigured:135(8487378ns) {StartInitialHandshake:157(8485694ns) {DropHandshakes:470(3014ns) {DropIncomingHandshake:442(706ns) DropOutgoingHandshake:457(469ns)} GenerateSessionVirtualId:988(1058ns) PrepareNewSessionHandshake:477(2026ns) {SwitchToState:179(95ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.129180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471079213307794:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.225758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471075441401170:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(229974930ns) {Configure:104(229961192ns) {ProcessConfigured:135(18158647ns) {StartInitialHandshake:157(18156101ns) {DropHandshakes:470(1812ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(399ns)} GenerateSessionVirtualId:988(543ns) PrepareNewSessionHandshake:477(4414ns) {SwitchToState:179(1774ns)}}}}}
2025-11-27T18:17:48.228770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471079736368482:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-27T18:17:48.259591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471079736368468:2111] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:48.259912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471075441401104:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.259972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:48.260023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471075441401104:2048] [node 1] ICP22 created new session: [4:7577471079736368483:2048]
2025-11-27T18:17:48.260065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471079736368483:2048] [node 1] ICS09 handshake done sender: [4:7577471079736368468:2111] self: [4:7577471079736368481:0] peer: [1:7577471079240639227:0] socket: 323 qp: -1
2025-11-27T18:17:48.260102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471079736368483:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:48.260143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:48.277696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471079736368485:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-27T18:17:48.282131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471079736368474:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.282169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471079736368474:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.182343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471074529922477:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.189324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471079976488924:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.227004Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471074945671929:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:48.129210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471079213307794:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.282058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471079972181501:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.227396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471079240639228:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.200469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471078824889852:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.189542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471075681521557:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(199495601ns) {Configure:104(199489745ns) {ProcessConfigured:135(83599ns) {StartInitialHandshake:157(81310ns) {DropHandshakes:470(714ns) {DropIncomingHandshake:442(175ns) DropOutgoingHandshake:457(166ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:477(1226ns) {SwitchToState:179(418ns)}}}}}
2025-11-27T18:17:48.286330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471079972181506:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8524
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.153907Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471074918340428:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(286094961ns) {Configure:104(286087824ns) {ProcessConfigured:135(21613847ns) {StartInitialHandshake:157(21611643ns) {DropHandshakes:470(3611ns) {DropIncomingHandshake:442(1061ns) DropOutgoingHandshake:457(487ns)} GenerateSessionVirtualId:988(720ns) PrepareNewSessionHandshake:477(2729ns) {SwitchToState:179(1207ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.341116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471079736368474:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.227420Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471079240639228:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.209474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471075681521557:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.158586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471079213307807:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19973
2025-11-27T18:17:48.226555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471078824889858:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10576
2025-11-27T18:17:48.288649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471075677214193:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:48.216020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471075681521556:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(233116404ns) {Configure:104(233110947ns) {ProcessConfigured:135(89898ns) {StartInitialHandshake:157(86875ns) {DropHandshakes:470(1889ns) {DropIncomingHandshake:442(329ns) DropOutgoingHandshake:457(333ns)} GenerateSessionVirtualId:988(501ns) PrepareNewSessionHandshake:477(1954ns) {SwitchToState:179(1237ns)}}}}}
2025-11-27T18:17:48.238573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471074945671919:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.238629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471074945671919:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.346842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471079736368485:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.166971Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471079213307794:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:48.229305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471078824889843:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:48.216242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471075681521556:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.288885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471075677214128:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.347201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471075441401106:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.244898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471079240639228:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:48.229375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471074529922544:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.281595Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471074945671924:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(320745457ns) {Configure:104(320735830ns) {ProcessConfigured:135(32396941ns) {StartInitialHandshake:157(32394368ns) {DropHandshakes:470(5563828ns) {DropIncomingHandshake:442(1608ns) DropOutgoingHandshake:457(424ns)} GenerateSessionVirtualId:988(1197ns) PrepareNewSessionHandshake:477(3777ns) {SwitchToState:179(1470ns)}}}}}
2025-11-27T18:17:48.258544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471079976488928:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.185419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471074918340427:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(281441243ns) {Configure:104(281437634ns) {ProcessConfigured:135(18837924ns) {StartInitialHandshake:157(18836024ns) {DropHandshakes:470(2702ns) {DropIncomingHandshake:442(725ns) DropOutgoingHandshake:457(481ns)} GenerateSessionVirtualId:988(699ns) PrepareNewSessionHandshake:477(1663ns) {SwitchToState:179(809ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.288940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.229405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471074529922544:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.293447Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471074945671926:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(269499723ns) {Configure:104(253755726ns) {ProcessConfigured:135(37973669ns) {StartInitialHandshake:157(37970717ns) {DropHandshakes:470(2955ns) {DropIncomingHandshake:442(442ns) DropOutgoingHandshake:457(769ns)} GenerateSessionVirtualId:988(653ns) PrepareNewSessionHandshake:477(6145ns) {SwitchToState:179(2692ns)}}}}}
2025-11-27T18:17:48.301590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471074945671919:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:48.302945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471070650704563:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.302994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:48.303039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471070650704563:2049] [node 3] ICP22 created new session: [1:7577471079240639240:2049]
2025-11-27T18:17:48.303084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471079240639240:2049] [node 3] ICS09 handshake done sender: [1:7577471074945671929:2112] self: [1:7577471074945671928:0] peer: [3:7577471078824889855:0] socket: 312 qp: -1
2025-11-27T18:17:48.303117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471079240639240:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:48.303165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:48.304080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471074945671925:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.304121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:48.304177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471074945671925:2108] [node 4] ICP22 created new session: [1:7577471079240639243:2108]
2025-11-27T18:17:48.304221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471079240639243:2108] [node 4] ICS09 handshake done sender: [1:7577471079240639228:2113] self: [1:7577471079240639227:0] peer: [4:7577471079736368481:0] socket: 320 qp: -1
2025-11-27T18:17:48.304254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471079240639243:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:48.304280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:48.329580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471070650704562:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.329621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:48.329656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471070650704562:2048] [node 2] ICP22 created new session: [1:7577471079240639246:2048]
2025-11-27T18:17:48.329700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471079240639246:2048] [node 2] ICS09 handshake done sender: [1:7577471074945671919:2107] self: [1:7577471074945671918:0] peer: [2:7577471079213307809:0] socket: 300 qp: -1
2025-11-27T18:17:48.329734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471079240639246:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:48.329761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:48.330408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471079240639245:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33150
2025-11-27T18:17:48.339225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471079240639245:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.339336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471070650704562:2048] [node 2] ICP18 handshake (actor [1:7577471079240639245:2117]) is too old
2025-11-27T18:17:48.417371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471079240639236:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.417417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471079240639236:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.441443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471079240639237:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.441491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471079240639237:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.449935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471079240639236:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:48.469741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471074945671926:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.217132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471074918340427:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.348602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471079736368487:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-27T18:17:48.258740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471079976488916:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.288989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471075677214128:2050] [node 3] ICP22 created new session: [6:7577471079972181507:2050]
2025-11-27T18:17:48.289030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471079972181507:2050] [node 3] ICS09 handshake done sender: [6:7577471075677214193:2111] self: [6:7577471079972181503:0] peer: [3:7577471074529922543:0] socket: 308 qp: -1
2025-11-27T18:17:48.289069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471079972181507:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:48.289123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:48.290663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471079972181494:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:48.333402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471075677214188:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.333433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471075677214127:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.340282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:48.355507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471075677214127:2049] [node 2] ICP22 created new session: [6:7577471079972181511:2049]
2025-11-27T18:17:48.361804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471079972181511:2049] [node 2] ICS09 handshake done sender: [6:7577471079972181494:2113] self: [6:7577471079972181498:0] peer: [2:7577471079213307793:0] socket: 316 qp: -1
2025-11-27T18:17:48.366229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471079972181512:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8524
2025-11-27T18:17:48.377905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471079972181511:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:48.389501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:48.393540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471075677214188:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(23107960ns) {IncomingHandshake:261(23104311ns) {DropIncomingHandshake:442(930ns) PrepareNewSessionHandshake:477(2497ns) {SwitchToState:179(794ns)} GenerateSessionVirtualId:988(791ns) IssueIncomingHandshakeReply:193(19626430ns) {SwitchToState:179(3359ns)}}}
2025-11-27T18:17:48.396396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471079972181512:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.405377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471075677214126:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.406590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471079972181515:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8524
2025-11-27T18:17:48.406725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471079972181516:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8524
2025-11-27T18:17:48.421777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471079972181515:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.422008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471075677214197:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.447301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471079972181501:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:48.451002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471075677214126:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25048546ns) {IncomingHandshake:261(25045960ns) {DropIncomingHandshake:442(1125ns) PrepareNewSessionHandshake:477(1900ns) {SwitchToState:179(672ns)} GenerateSessionVirtualId:988(695ns) IssueIncomingHandshakeReply:193(16257562ns) {SwitchToState:179(7382ns)}}}
2025-11-27T18:17:48.460376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471075677214188:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.463118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471079972181518:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8524
2025-11-27T18:17:48.465286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:48.469588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471075677214188:2105] [node 5] ICP22 created new session: [6:7577471079972181519:2105]
2025-11-27T18:17:48.251932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471074918340491:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.378799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471075441401171:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.251680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471078824889845:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.302637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471079976488937:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13915
2025-11-27T18:17:48.493479Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471075677214127:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(138386798ns) {HandleHandshakeStatus:329(138382348ns) {DropIncomingHandshake:442(3020ns) DropOutgoingHandshake:457(17225ns) SwitchToState:179(411ns) ProcessPendingSessionEvents:495(48347856ns) {ForwardSessionEventToSession:612(17677136ns) ForwardSessionEventToSession:612(8095ns) ForwardSessionEventToSession:612(14475514ns) ForwardSessionEventToSession:612(3369ns) ForwardSessionEventToSession:612(11840100ns) ForwardSessionEventToSession:612(20011ns) ForwardSessionEventToSession:612(4137ns) ForwardSessionEventToSession:612(4286559ns) ForwardSessionEventToSession:612(4042ns) ForwardSessionEventToSession:612(2204ns)}}}
2025-11-27T18:17:48.304743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471079976488938:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13915
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.251730Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471078824889845:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.493775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:48.501759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471074945671926:2110] [node 5] ICP22 created new session: [1:7577471079240639251:2110]
2025-11-27T18:17:48.513635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471079240639237:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:48.390702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:48.261544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471079213307792:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.531835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471079240639251:2110] [node 5] ICS09 handshake done sender: [1:7577471079240639236:2115] self: [1:7577471079240639231:0] peer: [5:7577471079976488946:0] socket: 342 qp: -1
2025-11-27T18:17:48.252732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471074529922477:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.390770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471075441401171:2109] [node 5] ICP22 created new session: [4:7577471079736368490:2109]
2025-11-27T18:17:48.493868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471079972181519:2105] [node 5] ICS09 handshake done sender: [6:7577471079972181501:2116] self: [6:7577471079972181510:0] peer: [5:7577471079976488925:0] socket: 343 qp: -1
2025-11-27T18:17:48.261585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471079213307792:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.318785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471079976488917:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:48.532968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471074945671924:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.533047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:48.533376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471079240639251:2110] [node 5] ICS10 traffic start
2025-11-27T18:17:48.543936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471074945671924:2109] [node 6] ICP22 created new session: [1:7577471079240639252:2109]
2025-11-27T18:17:48.544139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:48.552961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471079240639252:2109] [node 6] ICS09 handshake done sender: [1:7577471079240639237:2116] self: [1:7577471079240639233:0] peer: [6:7577471079972181514:0] socket: 362 qp: -1
2025-11-27T18:17:48.553021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471079240639252:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:48.553105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:48.556466Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471074945671924:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(25405160ns) {HandleHandshakeStatus:329(25402255ns) {DropIncomingHandshake:442(19874ns) DropOutgoingHandshake:457(576ns) SwitchToState:179(657ns) ProcessPendingSessionEvents:495(3052930ns) {ForwardSessionEventToSession:612(2996249ns) ForwardSessionEventToSession:612(11102ns) ForwardSessionEventToSession:612(3252ns) ForwardSessionEventToSession:612(6069ns) ForwardSessionEventToSession:612(3066ns) ForwardSessionEventToSession:612(2669ns) ForwardSessionEventToSession:612(4958ns) ForwardSessionEventToSession:612(5161ns) ForwardSessionEventToSession:612(2288ns) ForwardSessionEventToSession:612(7368ns)}}}
2025-11-27T18:17:48.261456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:48.281366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471074529922477:2048] [node 1] ICP22 created new session: [3:7577471078824889859:2048]
2025-11-27T18:17:48.281897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471074529922544:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:48.301470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471078824889859:2048] [node 1] ICS09 handshake done sender: [3:7577471078824889843:2112] self: [3:7577471078824889855:0] peer: [1:7577471074945671928:0] socket: 319 qp: -1
2025-11-27T18:17:48.306520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471078824889859:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:48.501825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471079972181519:2105] [node 5] ICS10 traffic start
2025-11-27T18:17:48.310488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471078824889845:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:48.311319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471074529922542:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.510374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471079972181515:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:48.311455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:48.512131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471075677214197:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19296) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471079972181515:2121] held: no
2025-11-27T18:17:48.314764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:48.514187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:48.320182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471078824889856:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10576
2025-11-27T18:17:48.333472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471074529922542:2107] [node 6] ICP22 created new session: [3:7577471078824889863:2107]
2025-11-27T18:17:48.529809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471075677214197:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(27464068ns) {HandleHandshakeStatus:395(27461142ns) {LogHandshakeFail:472(18491691ns) {UpdateErrorStateLog:545(7064ns)} DropOutgoingHandshake:457(561ns) ProcessPendingSessionEvents:495(8908311ns) {DropSessionEvent:512(14580ns) DropSessionEvent:512(4811ns) DropSessionEvent:512(5087ns) DropSessionEvent:512(8848225ns) DropSessionEvent:512(5260ns) DropSessionEvent:512(4630ns) DropSessionEvent:512(4365ns) DropSessionEvent:512(3759ns) DropSessionEvent:512(4346ns) DropSessionEvent:512(3548ns)}}}
2025-11-27T18:17:48.334147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471078824889842:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.529991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471075677214197:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.334179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:48.530034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:48.334230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471078824889842:2108] [node 5] ICP22 created new session: [3:7577471078824889865:2108]
2025-11-27T18:17:48.530084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471075677214197:2110] [node 4] ICP22 created new session: [6:7577471079972181521:2110]
2025-11-27T18:17:48.334281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471078824889865:2108] [node 5] ICS09 handshake done sender: [3:7577471078824889845:2113] self: [3:7577471078824889844:0] peer: [5:7577471079976488932:0] socket: 322 qp: -1
2025-11-27T18:17:48.530135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471079972181521:2110] [node 4] ICS09 handshake done sender: [6:7577471079972181515:2121] self: [6:7577471079972181517:0] peer: [4:7577471079736368477:0] socket: 368 qp: -1
2025-11-27T18:17:48.334322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471078824889865:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:48.530172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471079972181521:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:48.530203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:48.334358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:48.345513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471078824889863:2107] [node 6] ICS09 handshake done sender: [3:7577471074529922544:2111] self: [3:7577471074529922543:0] peer: [6:7577471079972181503:0] socket: 307 qp: -1
2025-11-27T18:17:48.539082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471075677214188:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(78549314ns) {HandleHandshakeStatus:329(78543874ns) {DropIncomingHandshake:442(1291ns) DropOutgoingHandshake:457(4816001ns) SwitchToState:179(1389ns) ProcessPendingSessionEvents:495(8206075ns) {ForwardSessionEventToSession:612(8157129ns) ForwardSessionEventToSession:612(8672ns) ForwardSessionEventToSession:612(5696ns) ForwardSessionEventToSession:612(2771ns) ForwardSessionEventToSession:612(5033ns) ForwardSessionEventToSession:612(2836ns) ForwardSessionEventToSession:612(5149ns) ForwardSessionEventToSession:612(2078ns) ForwardSessionEventToSession:612(1835ns) ForwardSessionEventToSession:612(4159ns)}}}
2025-11-27T18:17:48.543788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471075677214126:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471079972181512:2120] held: no
2025-11-27T18:17:48.358022Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471074529922477:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(105083114ns) {HandleHandshakeStatus:329(105079397ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(5326ns) SwitchToState:179(385ns) ProcessPendingSessionEvents:495(134867ns) {ForwardSessionEventToSession:612(74068ns) ForwardSessionEventToSession:612(3679ns) ForwardSessionEventToSession:612(2676ns) ForwardSessionEventToSession:612(6160ns) ForwardSessionEventToSession:612(3537ns) ForwardSessionEventToSession:612(3232ns) ForwardSessionEventToSession:612(9416ns) ForwardSessionEventToSession:612(12050ns) ForwardSessionEventToSession:612(5178ns) ForwardSessionEventToSession:612(1946ns)}}}
2025-11-27T18:17:48.360413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471078824889863:2107] [node 6] ICS10 traffic start
2025-11-27T18:17:48.361894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471078824889856:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.390850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471079736368490:2109] [node 5] ICS09 handshake done sender: [4:7577471079736368474:2114] self: [4:7577471079736368473:0] peer: [5:7577471079976488936:0] socket: 338 qp: -1
2025-11-27T18:17:48.369636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:48.377984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471078824889851:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.378025Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471078824889851:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.390890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471079736368490:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:48.400760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471078824889867:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10576
2025-11-27T18:17:48.390934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:48.392301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471079736368487:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.400844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471074529922478:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.426015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471079736368494:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-27T18:17:48.407302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471074529922542:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(95776182ns) {HandleHandshakeStatus:329(95772676ns) {DropIncomingHandshake:442(220ns) DropOutgoingHandshake:457(608ns) SwitchToState:179(528ns) ProcessPendingSessionEvents:495(6185085ns) {ForwardSessionEventToSession:612(6123235ns) ForwardSessionEventToSession:612(14335ns) ForwardSessionEventToSession:612(3964ns) ForwardSessionEventToSession:612(8875ns) ForwardSessionEventToSession:612(2094ns) ForwardSessionEventToSession:612(2993ns) ForwardSessionEventToSession:612(6920ns) ForwardSessionEventToSession:612(1685ns) ForwardSessionEventToSession:612(2629ns) ForwardSessionEventToSession:612(4188ns)}}}
2025-11-27T18:17:48.428963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471079736368485:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:48.407372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471078824889842:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:48.413541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471078824889851:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:48.429230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471075441401106:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.429303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:48.417565Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471074529922478:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(22442927ns) {IncomingHandshake:261(22440143ns) {DropIncomingHandshake:442(2097ns) PrepareNewSessionHandshake:477(3580ns) {SwitchToState:179(1625ns)} GenerateSessionVirtualId:988(907ns) IssueIncomingHandshakeReply:193(14404653ns) {SwitchToState:179(1428ns)}}}
2025-11-27T18:17:48.429351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471075441401106:2050] [node 3] ICP22 created new session: [4:7577471079736368495:2050]
2025-11-27T18:17:48.274042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:48.451866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471079736368479:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.451912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471079736368479:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.274104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471074918340491:2110] [node 6] ICP22 created new session: [2:7577471079213307810:2110]
2025-11-27T18:17:48.465659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471079736368495:2050] [node 3] ICS09 handshake done sender: [4:7577471079736368485:2118] self: [4:7577471079736368489:0] peer: [3:7577471078824889850:0] socket: 352 qp: -1
2025-11-27T18:17:48.285531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471079213307810:2110] [node 6] ICS09 handshake done sender: [2:7577471079213307794:2113] self: [2:7577471079213307793:0] peer: [6:7577471079972181498:0] socket: 315 qp: -1
2025-11-27T18:17:48.468117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471079736368495:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:48.285589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471079213307810:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:48.285638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:48.468160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:48.285924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471079213307813:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19973
2025-11-27T18:17:48.473508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471079736368479:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:48.474070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471075441401170:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.302082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471079213307792:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:48.317339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471079213307807:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.474168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:48.318644Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471074918340487:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:48.474226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471075441401170:2110] [node 6] ICP22 created new session: [4:7577471079736368498:2110]
2025-11-27T18:17:48.319312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471074918340491:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(74034654ns) {HandleHandshakeStatus:329(74029847ns) {DropIncomingHandshake:442(1852ns) DropOutgoingHandshake:457(617ns) SwitchToState:179(6863ns) ProcessPendingSessionEvents:495(13738193ns) {ForwardSessionEventToSession:612(13677823ns) ForwardSessionEventToSession:612(12171ns) ForwardSessionEventToSession:612(5164ns) ForwardSessionEventToSession:612(2548ns) ForwardSessionEventToSession:612(2490ns) ForwardSessionEventToSession:612(5630ns) ForwardSessionEventToSession:612(3043ns) ForwardSessionEventToSession:612(2766ns) ForwardSessionEventToSession:612(4255ns) ForwardSessionEventToSession:612(7606ns)}}}
2025-11-27T18:17:48.474279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471079736368498:2110] [node 6] ICS09 handshake done sender: [4:7577471079736368479:2115] self: [4:7577471079736368477:0] peer: [6:7577471079972181517:0] socket: 359 qp: -1
2025-11-27T18:17:48.474318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471079736368498:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:48.328128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471074918340492:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.474350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:48.328183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:48.532175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471075441401106:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(102716642ns) {HandleHandshakeStatus:329(102710582ns) {DropIncomingHandshake:442(990ns) DropOutgoingHandshake:457(5088ns) SwitchToState:179(684ns) ProcessPendingSessionEvents:495(43391507ns) {ForwardSessionEventToSession:612(23527753ns) ForwardSessionEventToSession:612(8891930ns) ForwardSessionEventToSession:612(3945ns) ForwardSessionEventToSession:612(1044093ns) ForwardSessionEventToSession:612(4461198ns) ForwardSessionEventToSession:612(6204ns) ForwardSessionEventToSession:612(5420904ns) ForwardSessionEventToSession:612(3582ns) ForwardSessionEventToSession:612(2877ns) ForwardSessionEventToSession:612(5630ns)}}}
2025-11-27T18:17:48.339129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471074918340427:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.540639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471079736368493:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-27T18:17:48.557288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471079736368493:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.557496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471075441401105:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.339185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:48.339237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471074918340427:2048] [node 1] ICP22 created new session: [2:7577471079213307816:2048]
2025-11-27T18:17:48.339278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471079213307816:2048] [node 1] ICS09 handshake done sender: [2:7577471074918340487:2107] self: [2:7577471079213307809:0] peer: [1:7577471074945671918:0] socket: 304 qp: -1
2025-11-27T18:17:48.339312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471079213307816:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:48.339349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:48.339943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471074918340491:2110] [node 6] ICP18 handshake (actor [2:7577471079213307807:2118]) is too old
2025-11-27T18:17:48.355222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471074918340492:2108] [node 5] ICP22 created new session: [2:7577471079213307815:2108]
2025-11-27T18:17:48.360221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471079213307815:2108] [node 5] ICS09 handshake done sender: [2:7577471079213307792:2112] self: [2:7577471079213307791:0] peer: [5:7577471079976488935:0] socket: 314 qp: -1
2025-11-27T18:17:48.386138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471079213307815:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:48.386213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:48.422660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471079213307803:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.422715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471079213307803:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.426635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471074918340492:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(98321950ns) {HandleHandshakeStatus:329(98319398ns) {DropIncomingHandshake:442(544ns) DropOutgoingHandshake:457(692ns) SwitchToState:179(925ns) ProcessPendingSessionEvents:495(10047297ns) {ForwardSessionEventToSession:612(9990139ns) ForwardSessionEventToSession:612(5491ns) ForwardSessionEventToSession:612(3437ns) ForwardSessionEventToSession:612(7906ns) ForwardSessionEventToSession:612(2365ns) ForwardSessionEventToSession:612(2864ns) ForwardSessionEventToSession:612(9325ns) ForwardSessionEventToSession:612(2379ns) ForwardSessionEventToSession:612(6919ns) ForwardSessionEventToSession:612(2605ns)}}}
2025-11-27T18:17:48.457758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471079213307803:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:48.468011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471074918340428:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.488734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:48.494108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471074918340428:2049] [node 3] ICP22 created new session: [2:7577471079213307821:2049]
2025-11-27T18:17:48.520829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471079213307821:2049] [node 3] ICS09 handshake done sender: [2:7577471079213307803:2116] self: [2:7577471079213307801:0] peer: [3:7577471078824889869:0] socket: 332 qp: -1
2025-11-27T18:17:48.520879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471079213307821:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:48.522123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:48.552078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471074918340428:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(83849386ns) {HandleHandshakeStatus:329(83846337ns) {DropIncomingHandshake:442(24635ns) DropOutgoingHandshake:457(624ns) SwitchToState:179(1107ns) ProcessPendingSessionEvents:495(8144528ns) {ForwardSessionEventToSession:612(8091162ns) ForwardSessionEventToSession:612(8640ns) ForwardSessionEventToSession:612(5317ns) ForwardSessionEventToSession:612(2583ns) ForwardSessionEventToSession:612(2267ns) ForwardSessionEventToSession:612(6120ns) ForwardSessionEventToSession:612(2529ns) ForwardSessionEventToSession:612(7370ns) ForwardSessionEventToSession:612(2067ns) ForwardSessionEventToSession:612(5525ns)}}}
2025-11-27T18:17:48.558793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471079213307798:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.558813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471079213307798:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.560487Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471074945671926:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(90536191ns) {HandleHandshakeStatus:329(90533529ns) {DropIncomingHandshake:442(11151ns) DropOutgoingHandshake:457(822ns) SwitchToState:179(702ns) ProcessPendingSessionEvents:495(1039400ns) {ForwardSessionEventToSession:612(980116ns) ForwardSessionEventToSession:612(14272ns) ForwardSessionEventToSession:612(3172ns) ForwardSessionEventToSession:612(6119ns) ForwardSessionEventToSession:612(2812ns) ForwardSessionEventToSession:612(2985ns) ForwardSessionEventToSession:612(5900ns) ForwardSessionEventToSession:612(2649ns) ForwardSessionEventToSession:612(2827ns) ForwardSessionEventToSession:612(9739ns)}}}
2025-11-27T18:17:48.335204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471075681521556:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.335252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:48.335295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471075681521556:2049] [node 2] ICP22 created new session: [5:7577471079976488940:2049]
2025-11-27T18:17:48.335346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471079976488940:2049] [node 2] ICS09 handshake done sender: [5:7577471079976488917:2111] self: [5:7577471079976488935:0] peer: [2:7577471079213307791:0] socket: 321 qp: -1
2025-11-27T18:17:48.335382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471079976488940:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:48.335419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:48.335945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471079976488924:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:48.336439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471079976488939:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13915
2025-11-27T18:17:48.359691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471079976488943:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13915
2025-11-27T18:17:48.361815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471075681521557:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.369413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471079976488926:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.369466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471079976488926:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:48.377600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:48.389689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471079976488928:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:48.393773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471079976488943:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:48.398456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471075681521555:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:48.398518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471079976488916:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.398565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:48.398612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471079976488916:2110] [node 4] ICP22 created new session: [5:7577471079976488947:2110]
2025-11-27T18:17:48.398678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471079976488947:2110] [node 4] ICS09 handshake done sender: [5:7577471079976488928:2116] self: [5:7577471079976488936:0] peer: [4:7577471079736368473:0] socket: 339 qp: -1
2025-11-27T18:17:48.398718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471079976488947:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:48.398756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:48.400678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471075681521557:2050] [node 3] ICP22 created new session: [5:7577471079976488944:2050]
2025-11-27T18:17:48.405639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471079976488916:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:48.405771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471079976488926:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:48.425447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471079976488944:2050] [node 3] ICS09 handshake done sender: [5:7577471079976488924:2114] self: [5:7577471079976488932:0] peer: [3:7577471078824889844:0] socket: 329 qp: -1
2025-11-27T18:17:48.434933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471079976488944:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:48.444865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471075681521618:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.444950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:48.445005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471075681521618:2109] [node 6] ICP22 created new session: [5:7577471079976488949:2109]
2025-11-27T18:17:48.445055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471079976488949:2109] [node 6] ICS09 handshake done sender: [5:7577471079976488926:2115] self: [5:7577471079976488925:0] peer: [6:7577471079972181510:0] socket: 340 qp: -1
2025-11-27T18:17:48.445108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471079976488949:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:48.445176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:48.449499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:48.461614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471079976488951:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13915
2025-11-27T18:17:48.481351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471079976488943:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:48.486754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471075681521557:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(145031926ns) {HandleHandshakeStatus:329(124757162ns) {DropIncomingHandshake:442(865ns) DropOutgoingHandshake:457(16183ns) SwitchToState:179(735ns) ProcessPendingSessionEvents:495(9037415ns) {ForwardSessionEventToSession:612(8966854ns) ForwardSessionEventToSession:612(6059ns) ForwardSessionEventToSession:612(3399ns) ForwardSessionEventToSession:612(5196ns) ForwardSessionEventToSession:612(2727ns) ForwardSessionEventToSession:612(2314ns) ForwardSessionEventToSession:612(5095ns) ForwardSessionEventToSession:612(2542ns) ForwardSessionEventToSession:612(29754ns) ForwardSessionEventToSession:612(2778ns)}}}
2025-11-27T18:17:48.494218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471075681521555:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.512615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:48.512676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471075681521555:2048] [node 1] ICP22 created new session: [5:7577471079976488953:2048]
2025-11-27T18:17:48.529692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471079976488953:2048] [node 1] ICS09 handshake done sender: [5:7577471079976488943:2122] self: [5:7577471079976488946:0] peer: [1:7577471079240639231:0] socket: 354 qp: -1
2025-11-27T18:17:48.530930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471079976488953:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:48.537759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:48.538167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471075681521555:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(49126600ns) {HandleHandshakeStatus:329(49123085ns) {DropIncomingHandshake:442(1321ns) DropOutgoingHandshake:457(12038ns) SwitchToState:179(1173ns) ProcessPendingSessionEvents:495(96578ns) {ForwardSessionEventToSession:612(51945ns) ForwardSessionEventToSession:612(3253ns) ForwardSessionEventToSession:612(2420ns) ForwardSessionEventToSession:612(5064ns) ForwardSessionEventToSession:612(1815ns) ForwardSessionEventToSession:612(12009ns) ForwardSessionEventToSession:612(2924ns) ForwardSessionEventToSession:612(2083ns) ForwardSessionEventToSession:612(4882ns) ForwardSessionEventToSession:612(1820ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:48.563003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471079972181523:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8524
2025-11-27T18:17:48.566735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471079972181512:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:48.450019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471074529922540:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.450083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:48.564747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471079213307798:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:48.565635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471074918340493:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:48.565718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:48.565769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471074918340493:2109] [node 4] ICP22 created new session: [2:7577471079213307824:2109]
2025-11-27T18:17:48.565809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471079213307824:2109] [node 4] ICS09 handshake done sender: [2:7577471079213307798:2114] self: [2:7577471079213307797:0] peer: [4:7577471079736368500:0] socket: 337 qp: -1
2025-11-27T18:17:48.565845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471079213307824:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:48.565884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:48.565070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471079736368501:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-27T18:17:48.567034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471079736368493:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:48.568357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471075441401105:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1358) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471079736368493:2120] held: no
2025-11-27T18:17:48.568905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471075441401105:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.568972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:48.569028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471075441401105:2049] [node 2] ICP22 created new session: [4:7577471079736368502:2049]
2025-11-27T18:17:48.569069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471079736368502:2049] [node 2] ICS09 handshake done sender: [4:7577471079736368493:2120] self: [4:7577471079736368500:0] peer: [2:7577471079213307797:0] socket: 366 qp: -1
2025-11-27T18:17:48.569115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471079736368502:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:48.569158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:48.450130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471074529922540:2109] [node 4] ICP22 created new session: [3:7577471078824889871:2109]
2025-11-27T18:17:48.573131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471075677214126:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.573210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:48.573282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471075677214126:2048] [node 1] ICP22 created new session: [6:7577471079972181524:2048]
2025-11-27T18:17:48.573331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471079972181524:2048] [node 1] ICS09 handshake done sender: [6:7577471079972181512:2120] self: [6:7577471079972181514:0] peer: [1:7577471079240639233:0] socket: 363 qp: -1
2025-11-27T18:17:48.573368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471079972181524:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:48.573408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:48.450189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471078824889871:2109] [node 4] ICS09 handshake done sender: [3:7577471078824889851:2115] self: [3:7577471078824889850:0] peer: [4:7577471079736368489:0] socket: 351 qp: -1
2025-11-27T18:17:48.450230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471078824889871:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:48.450271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:48.466220Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471074529922540:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(16028782ns) {HandleHandshakeStatus:329(16026182ns) {DropIncomingHandshake:442(16038ns) DropOutgoingHandshake:457(542ns) SwitchToState:179(729ns) ProcessPendingSessionEvents:495(93403ns) {ForwardSessionEventToSession:612(45969ns) ForwardSessionEventToSession:612(6121ns) ForwardSessionEventToSession:612(5019ns) ForwardSessionEventToSession:612(4636ns) ForwardSessionEventToSession:612(2262ns) ForwardSessionEventToSession:612(3166ns) ForwardSessionEventToSession:612(5314ns) ForwardSessionEventToSession:612(2743ns) ForwardSessionEventToSession:612(2222ns) ForwardSessionEventToSession:612(4892ns)}}}
2025-11-27T18:17:48.466839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471078824889873:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10576
2025-11-27T18:17:48.479436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471078824889856:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:48.497558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471074529922478:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:48.497634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:48.497745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471074529922478:2049] [node 2] ICP22 created new session: [3:7577471078824889874:2049]
2025-11-27T18:17:48.497810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471078824889874:2049] [node 2] ICS09 handshake done sender: [3:7577471078824889856:2118] self: [3:7577471078824889869:0] peer: [2:7577471079213307801:0] socket: 341 qp: -1
2025-11-27T18:17:48.497848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471078824889874:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:48.497885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:48.662942Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:48.663179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471079213307816:2048] [node 1] ICS07 socket disconnect 304 reason# EndOfStream
2025-11-27T18:17:48.663207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471079213307816:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.663283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:48.663323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471079213307816:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:48.668332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471074918340427:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32385) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:17:48.668379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471079213307816:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:48.669722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471074918340427:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32385) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:48.669775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471079213307816:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:48.669809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471074918340427:2048] [node 1] ICP30 unregister session Session# [2:7577471079213307816:2048] VirtualId# [2:7577471079213307809:0]
2025-11-27T18:17:48.669849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:48.669880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471074918340427:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.669894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.669736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:48.669928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471078824889859:2048] [node 1] ICS07 socket disconnect 319 reason# EndOfStream
2025-11-27T18:17:48.669954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471078824889859:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.670044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:48.670084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471078824889859:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:48.672559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471074529922477:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32385) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:48.672617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471078824889859:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:48.672649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471074529922477:2048] [node 1] ICP30 unregister session Session# [3:7577471078824889859:2048] VirtualId# [3:7577471078824889855:0]
2025-11-27T18:17:48.672688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:48.672717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471074529922477:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.672732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.677486Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:48.677731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471079972181524:2048] [node 1] ICS01 socket: 363 reason# EndOfStream
2025-11-27T18:17:48.677770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471075677214126:2048] [node 1] ICP30 unregister session Session# [6:7577471079972181524:2048] VirtualId# [6:7577471079972181514:0]
2025-11-27T18:17:48.677809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471079972181524:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.677841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:48.677893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:48.677492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:48.677731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471079736368483:2048] [node 1] ICS01 socket: 323 reason# EndOfStream
2025-11-27T18:17:48.677771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471075441401104:2048] [node 1] ICP30 unregister session Session# [4:7577471079736368483:2048] VirtualId# [4:7577471079736368481:0]
2025-11-27T18:17:48.677803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471079736368483:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.677886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:48.677923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:48.678004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:48.678190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471079976488953:2048] [node 1] ICS01 socket: 354 reason# EndOfStream
2025-11-27T18:17:48.678220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471075681521555:2048] [node 1] ICP30 unregister session Session# [5:7577471079976488953:2048] VirtualId# [5:7577471079976488946:0]
2025-11-27T18:17:48.678250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471079976488953:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.678300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:48.678334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:48.681530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471074918340427:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:48.685130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471074529922477:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:48.754914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:48.755143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471078824889874:2049] [node 2] ICS01 socket: 341 reason# EndOfStream
2025-11-27T18:17:48.755184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471074529922478:2049] [node 2] ICP30 unregister session Session# [3:7577471078824889874:2049] VirtualId# [3:7577471078824889869:0]
2025-11-27T18:17:48.755217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471078824889874:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.755293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:48.755333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:48.759074Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:48.759390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471079976488940:2049] [node 2] ICS01 socket: 321 reason# EndOfStream
2025-11-27T18:17:48.759458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471075681521556:2049] [node 2] ICP30 unregister session Session# [5:7577471079976488940:2049] VirtualId# [5:7577471079976488935:0]
2025-11-27T18:17:48.759495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471079976488940:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.759565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:48.759610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:48.760242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:48.760449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471079736368502:2049] [node 2] ICS01 socket: 366 reason# EndOfStream
2025-11-27T18:17:48.761319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471075441401105:2049] [node 2] ICP30 unregister session Session# [4:7577471079736368502:2049] VirtualId# [4:7577471079736368500:0]
2025-11-27T18:17:48.761366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471079736368502:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.761425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:48.764456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:48.764807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471079972181511:2049] [node 2] ICS07 socket disconnect 316 reason# EndOfStream
2025-11-27T18:17:48.761453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:48.764848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471079972181511:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.764950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:48.765031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471079972181511:2049] [node 2] ICS15 start handshake
2025-11-27T18:17:48.773246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471075677214127:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1358) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:48.773335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471079972181511:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:48.773377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471075677214127:2049] [node 2] ICP30 unregister session Session# [6:7577471079972181511:2049] VirtualId# [6:7577471079972181498:0]
2025-11-27T18:17:48.773435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:48.773474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471075677214127:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1358) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.773489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1358) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.785498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471075677214127:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:17:48.826219Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:48.826387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471079976488944:2050] [node 3] ICS07 socket disconnect 329 reason# EndOfStream
2025-11-27T18:17:48.826424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471079976488944:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.826481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:48.826525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471079976488944:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:48.825747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:48.825960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471079972181507:2050] [node 3] ICS01 socket: 308 reason# EndOfStream
2025-11-27T18:17:48.825993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471075677214128:2050] [node 3] ICP30 unregister session Session# [6:7577471079972181507:2050] VirtualId# [6:7577471079972181503:0]
2025-11-27T18:17:48.826022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471079972181507:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.826087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:48.826122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:48.834617Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:48.834842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471079736368495:2050] [node 3] ICS07 socket disconnect 352 reason# EndOfStream
2025-11-27T18:17:48.834870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471079736368495:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.834942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:48.834989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471079736368495:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:48.835816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471075441401106:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20521) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:48.835925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471079736368495:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:48.835963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471075441401106:2050] [node 3] ICP30 unregister session Session# [4:7577471079736368495:2050] VirtualId# [4:7577471079736368489:0]
2025-11-27T18:17:48.836003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:48.836045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471075441401106:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20521) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.836060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20521) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.836214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471075681521557:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20521) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:48.836267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471079976488944:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:48.836292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471075681521557:2050] [node 3] ICP30 unregister session Session# [5:7577471079976488944:2050] VirtualId# [5:7577471079976488932:0]
2025-11-27T18:17:48.836321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:48.836376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471075681521557:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20521) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.836388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20521) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.848675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471075681521557:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:48.901400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:48.901621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471079976488947:2110] [node 4] ICS01 socket: 339 reason# EndOfStream
2025-11-27T18:17:48.901664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471079976488916:2110] [node 4] ICP30 unregister session Session# [5:7577471079976488947:2110] VirtualId# [5:7577471079976488936:0]
2025-11-27T18:17:48.901689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471079976488947:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.901756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:48.901807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:48.901897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:48.902042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471079972181521:2110] [node 4] ICS01 socket: 368 reason# EndOfStream
2025-11-27T18:17:48.902077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471075677214197:2110] [node 4] ICP30 unregister session Session# [6:7577471079972181521:2110] VirtualId# [6:7577471079972181517:0]
2025-11-27T18:17:48.902098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471079972181521:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.902143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:48.902171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:48.945434Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:48.945656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471079972181519:2105] [node 5] ICS07 socket disconnect 343 reason# EndOfStream
2025-11-27T18:17:48.945685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471079972181519:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:48.945768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:48.945805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471079972181519:2105] [node 5] ICS15 start handshake
2025-11-27T18:17:48.946776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471075677214188:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23350) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:48.946832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471079972181519:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:48.946866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471075677214188:2105] [node 5] ICP30 unregister session Session# [6:7577471079972181519:2105] VirtualId# [6:7577471079972181510:0]
2025-11-27T18:17:48.946907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:17:48.946940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471075677214188:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23350) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.946953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23350) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:48.958032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471075677214188:2105] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:49.094570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471081728155510:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:49.094650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471081728155511:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:49.131697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471081178658923:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:49.131740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471081178658924:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:49.181653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471082453799030:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:49.181695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471082453799031:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:49.212020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471082001344897:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:49.212076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471082001344898:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:49.212109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471082001344899:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:49.243246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471080960323557:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:49.243299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471080960323558:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:49.243325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471080960323559:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:49.262423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471080999759876:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:49.262466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471080999759877:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:49.262497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471080999759878:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:49.469232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471080999759935:2104] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.510313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471082453799090:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28993
2025-11-27T18:17:49.535372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471081178658986:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:49.565248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471081728155591:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:49.565736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471081728155592:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:49.550498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471081178658989:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:49.550826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471081178658990:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:49.603421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471082453799094:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:49.566108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471081728155593:2117] [node 6] ICP01 ready to work
2025-11-27T18:17:49.666160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471081178658924:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(114985242ns) {Configure:104(114975581ns) {ProcessConfigured:135(20202380ns) {StartInitialHandshake:157(20199564ns) {DropHandshakes:470(3465ns) {DropIncomingHandshake:442(745ns) DropOutgoingHandshake:457(424ns)} GenerateSessionVirtualId:988(955ns) PrepareNewSessionHandshake:477(7397ns) {SwitchToState:179(3099ns)}}}}}
2025-11-27T18:17:49.612579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471081728155511:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(44209786ns) {Configure:104(44201902ns) {ProcessConfigured:135(18852213ns) {StartInitialHandshake:157(18849095ns) {DropHandshakes:470(3078ns) {DropIncomingHandshake:442(703ns) DropOutgoingHandshake:457(440ns)} GenerateSessionVirtualId:988(993ns) PrepareNewSessionHandshake:477(12588ns) {SwitchToState:179(2352ns)}}}}}
2025-11-27T18:17:49.668765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471082453799095:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:49.692927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471080999759876:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(79184354ns) {Configure:104(78111148ns) {ProcessConfigured:135(75037ns) {StartInitialHandshake:157(70382ns) {DropHandshakes:470(2938ns) {DropIncomingHandshake:442(812ns) DropOutgoingHandshake:457(594ns)} GenerateSessionVirtualId:988(1020ns) PrepareNewSessionHandshake:477(4907ns) {SwitchToState:179(651ns)}}}}}
2025-11-27T18:17:49.713410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471082453799096:2110] [node 5] ICP01 ready to work
2025-11-27T18:17:49.693479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471080999759948:2111] [node 4] ICP01 ready to work
2025-11-27T18:17:49.715097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471082001344897:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(151984598ns) {Configure:104(151976970ns) {ProcessConfigured:135(71113ns) {StartInitialHandshake:157(68062ns) {DropHandshakes:470(3709ns) {DropIncomingHandshake:442(803ns) DropOutgoingHandshake:457(479ns)} GenerateSessionVirtualId:988(1194ns) PrepareNewSessionHandshake:477(1718ns) {SwitchToState:179(612ns)}}}}}
2025-11-27T18:17:49.716583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471080960323621:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:49.754473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471081728155591:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(141538133ns) {Configure:104(141532577ns) {ProcessConfigured:135(65875ns) {StartInitialHandshake:157(64153ns) {DropHandshakes:470(4338ns) {DropIncomingHandshake:442(495ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(433ns) PrepareNewSessionHandshake:477(1295ns) {SwitchToState:179(644ns)}}}}}
2025-11-27T18:17:49.769866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471080960323622:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:49.715628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471082001344962:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.773891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471081728155593:2117] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17465197ns) {Configure:104(17462382ns) {ProcessConfigured:135(39925ns) {StartInitialHandshake:157(38551ns) {DropHandshakes:470(3379ns) {DropIncomingHandshake:442(368ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(491ns) {SwitchToState:179(169ns)}}}}}
2025-11-27T18:17:49.774791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471081728155604:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52536
2025-11-27T18:17:49.774849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471081728155605:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52536
2025-11-27T18:17:49.733598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471082001344963:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:49.751232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471082453799031:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34353669ns) {Configure:104(23994466ns) {ProcessConfigured:135(69618ns) {StartInitialHandshake:157(66749ns) {DropHandshakes:470(3315ns) {DropIncomingHandshake:442(732ns) DropOutgoingHandshake:457(634ns)} GenerateSessionVirtualId:988(827ns) PrepareNewSessionHandshake:477(1786ns) {SwitchToState:179(654ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.791548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471080960323618:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21316
2025-11-27T18:17:49.777827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471082453799090:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.805485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471080960323618:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.806353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471080999759954:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34378
2025-11-27T18:17:49.792898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471081728155605:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.795104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471081178659002:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36420
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.818264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471080960323633:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21316
2025-11-27T18:17:49.782065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471082453799105:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28993
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.798156Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471082001344899:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29175560ns) {Configure:104(29172197ns) {ProcessConfigured:135(46060ns) {StartInitialHandshake:157(44286ns) {DropHandshakes:470(3164ns) {DropIncomingHandshake:442(1010ns) DropOutgoingHandshake:457(382ns)} GenerateSessionVirtualId:988(575ns) PrepareNewSessionHandshake:477(1853ns) {SwitchToState:179(775ns)}}}}}
2025-11-27T18:17:49.795185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471081178659003:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36420
2025-11-27T18:17:49.826390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471081178659004:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36420
2025-11-27T18:17:49.826470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471081178659005:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36420
2025-11-27T18:17:49.800204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471082001344964:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27912
2025-11-27T18:17:49.782349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471082453799108:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28993
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.839220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471081178659003:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.839350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471081178658992:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.839371Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471081178658992:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:49.840694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471081178658992:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:49.800756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471082001344973:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27912
2025-11-27T18:17:49.825416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471082001344964:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.848064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471081178658924:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:49.848160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:49.848219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471081178658924:2049] [node 3] ICP22 created new session: [2:7577471081178659008:2049]
2025-11-27T18:17:49.848263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471081178659008:2049] [node 3] ICS09 handshake done sender: [2:7577471081178658992:2111] self: [2:7577471081178658991:0] peer: [3:7577471082453799113:0] socket: 312 qp: -1
2025-11-27T18:17:49.848297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471081178659008:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:49.848334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:49.846048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471081728155604:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.826158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471082453799105:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.836622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471082453799031:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:49.840507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471082453799114:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28993
2025-11-27T18:17:49.841297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471082453799105:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:49.849469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471082453799031:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:49.849515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:49.849560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471082453799031:2049] [node 2] ICP22 created new session: [3:7577471082453799115:2049]
2025-11-27T18:17:49.849599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471082453799115:2049] [node 2] ICS09 handshake done sender: [3:7577471082453799105:2115] self: [3:7577471082453799113:0] peer: [2:7577471081178658991:0] socket: 317 qp: -1
2025-11-27T18:17:49.849629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471082453799115:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:49.849660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.861027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471081728155609:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52536
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.863594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471080999759955:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34378
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.869360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471080960323638:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21316
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.870077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471081728155610:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52536
2025-11-27T18:17:49.870725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471081728155612:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52536
2025-11-27T18:17:49.871230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471081178659002:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.871355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471081178658923:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.871735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471082001344898:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:49.871940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471082001344973:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.872105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471082001344977:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27912
2025-11-27T18:17:49.872316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471082001344899:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:49.872785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471080999759954:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.873153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471081178659004:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.873287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471081178658994:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.873311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471081178658994:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:49.874706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471082453799108:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.874823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471082453799100:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.874842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471082453799100:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:49.875660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471082453799030:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:49.872911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471080999759948:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.876854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471082001344966:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.876881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471082001344966:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:49.878083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471082001344980:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27912
2025-11-27T18:17:49.878143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471082001344981:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27912
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.878902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471081178658994:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:49.879228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471081178658986:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:49.879312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:49.879366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471081178658986:2107] [node 4] ICP22 created new session: [2:7577471081178659014:2107]
2025-11-27T18:17:49.879409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471081178659014:2107] [node 4] ICS09 handshake done sender: [2:7577471081178658994:2112] self: [2:7577471081178658993:0] peer: [4:7577471082001344978:0] socket: 313 qp: -1
2025-11-27T18:17:49.879448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471081178659014:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:49.879482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:49.880536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471082453799100:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:49.876496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471080999759955:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.876618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471080999759878:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:49.878719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471080999759958:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34378
2025-11-27T18:17:49.880943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471080960323638:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.881048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471080960323633:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.873494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471081728155595:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.873532Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471081728155595:2128] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:49.886015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471080960323558:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:49.886102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471080960323557:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:49.887235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471081728155610:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.887322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471081728155609:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.887412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471081728155597:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.887433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471081728155597:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:49.888009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471081728155612:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.888103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471081728155599:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.888119Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471081728155599:2130] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.893838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471081178659005:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.893940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471081178658996:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.893959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471081178658996:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:49.895161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471081178659016:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36420
2025-11-27T18:17:49.895522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471082001344966:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:49.897515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471081728155595:2128] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:49.889895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471080999759954:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:49.896548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471082001344977:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.890057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471080960323641:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21316
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.917614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471081178658996:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:49.918119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471080960323642:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21316
2025-11-27T18:17:49.907156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471080999759948:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:49.919017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471081178658989:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:49.919105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:49.890259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471082453799104:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.919163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471081178658989:2108] [node 5] ICP22 created new session: [2:7577471081178659020:2108]
2025-11-27T18:17:49.925417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471081178659020:2108] [node 5] ICS09 handshake done sender: [2:7577471081178658996:2113] self: [2:7577471081178658995:0] peer: [5:7577471080960323639:0] socket: 350 qp: -1
2025-11-27T18:17:49.925481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471081178659020:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:49.925537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.935449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471081728155597:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:49.913938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:49.938749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471082001344963:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:49.938792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:49.938837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471082001344963:2109] [node 6] ICP22 created new session: [4:7577471082001344983:2109]
2025-11-27T18:17:49.938884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471082001344983:2109] [node 6] ICS09 handshake done sender: [4:7577471082001344966:2112] self: [4:7577471082001344965:0] peer: [6:7577471080999759956:0] socket: 333 qp: -1
2025-11-27T18:17:49.938913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471082001344983:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:49.938951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:49.939488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471082001344898:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17550) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471082001344964:2111] held: no
2025-11-27T18:17:49.939773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471082001344964:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:49.940347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471082001344973:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:49.940705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471082001344898:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:49.940746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:49.940787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471082001344898:2049] [node 2] ICP22 created new session: [4:7577471082001344987:2049]
2025-11-27T18:17:49.940827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471082001344987:2049] [node 2] ICS09 handshake done sender: [4:7577471082001344964:2111] self: [4:7577471082001344978:0] peer: [2:7577471081178658993:0] socket: 315 qp: -1
2025-11-27T18:17:49.940852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471082001344987:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:49.940884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:49.941223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471082001344899:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:49.941252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:49.941301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471082001344899:2050] [node 3] ICP22 created new session: [4:7577471082001344989:2050]
2025-11-27T18:17:49.941353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471082001344989:2050] [node 3] ICS09 handshake done sender: [4:7577471082001344973:2116] self: [4:7577471082001344979:0] peer: [3:7577471082453799099:0] socket: 331 qp: -1
2025-11-27T18:17:49.941390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471082001344989:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:49.941420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:49.890287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471082453799104:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.964026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471080960323643:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21316
2025-11-27T18:17:49.964294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471080960323638:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:49.925504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471080999759948:2111] [node 4] ICP22 created new session: [6:7577471080999759959:2111]
2025-11-27T18:17:49.933458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471080999759959:2111] [node 4] ICS09 handshake done sender: [6:7577471080999759954:2115] self: [6:7577471080999759956:0] peer: [4:7577471082001344965:0] socket: 336 qp: -1
2025-11-27T18:17:49.933524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471080999759959:2111] [node 4] ICS10 traffic start
2025-11-27T18:17:49.933588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:49.934096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471080999759948:2111] [node 4] ICP35 event processing took too much time PendingConnection:279(26735587ns) {HandleHandshakeStatus:329(26730548ns) {DropIncomingHandshake:442(1074ns) DropOutgoingHandshake:457(6372ns) SwitchToState:179(842ns) ProcessPendingSessionEvents:495(128333ns) {ForwardSessionEventToSession:612(76380ns) ForwardSessionEventToSession:612(5943ns) ForwardSessionEventToSession:612(2594ns) ForwardSessionEventToSession:612(2118ns) ForwardSessionEventToSession:612(5919ns) ForwardSessionEventToSession:612(2897ns) ForwardSessionEventToSession:612(9883ns) ForwardSessionEventToSession:612(2639ns) ForwardSessionEventToSession:612(5557ns) ForwardSessionEventToSession:612(5246ns)}}}
2025-11-27T18:17:49.935081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471080999759961:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34378
2025-11-27T18:17:49.935137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471080999759962:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34378
2025-11-27T18:17:49.936083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471080999759955:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:49.936571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471080999759878:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:49.936633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:49.936683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471080999759878:2050] [node 3] ICP22 created new session: [6:7577471080999759964:2050]
2025-11-27T18:17:49.936732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471080999759964:2050] [node 3] ICS09 handshake done sender: [6:7577471080999759955:2116] self: [6:7577471080999759957:0] peer: [3:7577471082453799103:0] socket: 329 qp: -1
2025-11-27T18:17:49.936769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471080999759964:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:49.936805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:49.892587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471082453799094:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:49.892655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:49.892717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471082453799094:2108] [node 4] ICP22 created new session: [3:7577471082453799121:2108]
2025-11-27T18:17:49.892761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471082453799121:2108] [node 4] ICS09 handshake done sender: [3:7577471082453799100:2112] self: [3:7577471082453799099:0] peer: [4:7577471082001344979:0] socket: 326 qp: -1
2025-11-27T18:17:49.892800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471082453799121:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:49.892836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:49.906368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471082453799120:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28993
2025-11-27T18:17:49.919413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471082453799104:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:49.937734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471082453799095:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:49.937800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:49.937851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471082453799095:2109] [node 6] ICP22 created new session: [3:7577471082453799125:2109]
2025-11-27T18:17:49.937910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471082453799125:2109] [node 6] ICS09 handshake done sender: [3:7577471082453799104:2114] self: [3:7577471082453799103:0] peer: [6:7577471080999759957:0] socket: 325 qp: -1
2025-11-27T18:17:49.937944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471082453799125:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:49.937972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:49.964899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471080960323558:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:49.964956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:49.965001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471080960323558:2049] [node 2] ICP22 created new session: [5:7577471080960323645:2049]
2025-11-27T18:17:49.965044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471080960323645:2049] [node 2] ICS09 handshake done sender: [5:7577471080960323638:2117] self: [5:7577471080960323639:0] peer: [2:7577471081178658995:0] socket: 351 qp: -1
2025-11-27T18:17:49.965088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471080960323645:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:49.965131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:49.969972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471081728155599:2130] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.983486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471080960323644:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21316
2025-11-27T18:17:49.982078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471081178658989:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(62822109ns) {HandleHandshakeStatus:329(62818358ns) {DropIncomingHandshake:442(15566ns) DropOutgoingHandshake:457(520ns) SwitchToState:179(667ns) ProcessPendingSessionEvents:495(22293518ns) {ForwardSessionEventToSession:612(11781562ns) ForwardSessionEventToSession:612(10886ns) ForwardSessionEventToSession:612(5116448ns) ForwardSessionEventToSession:612(3277ns) ForwardSessionEventToSession:612(1668246ns) ForwardSessionEventToSession:612(5769ns) ForwardSessionEventToSession:612(3512ns) ForwardSessionEventToSession:612(3673709ns) ForwardSessionEventToSession:612(5754ns) ForwardSessionEventToSession:612(2612ns)}}}
2025-11-27T18:17:49.984644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471081728155510:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:49.984722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:49.984790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471081728155510:2048] [node 2] ICP22 created new session: [1:7577471081728155620:2048]
2025-11-27T18:17:49.984854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471081728155620:2048] [node 2] ICS09 handshake done sender: [1:7577471081728155595:2128] self: [1:7577471081728155594:0] peer: [2:7577471081178659011:0] socket: 320 qp: -1
2025-11-27T18:17:49.984891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471081728155620:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:49.984930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:49.985382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471081728155511:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:49.985423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:49.985462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471081728155511:2049] [node 3] ICP22 created new session: [1:7577471081728155622:2049]
2025-11-27T18:17:49.985501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471081728155622:2049] [node 3] ICS09 handshake done sender: [1:7577471081728155597:2129] self: [1:7577471081728155596:0] peer: [3:7577471082453799119:0] socket: 321 qp: -1
2025-11-27T18:17:49.985523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471081728155622:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:49.985549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:49.985844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471081728155591:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:49.985882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:49.985923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471081728155591:2107] [node 5] ICP22 created new session: [1:7577471081728155624:2107]
2025-11-27T18:17:49.985991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471081728155624:2107] [node 5] ICS09 handshake done sender: [1:7577471081728155599:2130] self: [1:7577471081728155598:0] peer: [5:7577471080960323640:0] socket: 322 qp: -1
2025-11-27T18:17:49.986026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471081728155624:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:49.986052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:49.994179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471080960323644:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:49.994428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471080960323633:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:49.998272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471080960323643:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:50.007653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471082001344992:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27912
2025-11-27T18:17:49.998115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471082453799108:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:50.019032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471081178659002:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:49.999914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471080999759966:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34378
2025-11-27T18:17:50.019481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471080960323557:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:50.019548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:50.019607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471080960323557:2048] [node 1] ICP22 created new session: [5:7577471085255290944:2048]
2025-11-27T18:17:50.019654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471085255290944:2048] [node 1] ICS09 handshake done sender: [5:7577471080960323633:2116] self: [5:7577471080960323640:0] peer: [1:7577471081728155598:0] socket: 338 qp: -1
2025-11-27T18:17:50.019691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471085255290944:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:50.019724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:50.019962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471080960323557:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:50.020344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471080960323621:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:50.021059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471080960323559:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:50.028446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471080999759966:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:50.033548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471082001344992:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:50.041455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471082453799030:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14541) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471082453799108:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:50.041925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471082001344972:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:50.041965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471082001344972:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:50.042999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471082001344897:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:50.045407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471082001344972:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:50.043958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471082453799107:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:50.043990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471082453799107:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:50.044825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471082453799030:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:50.044879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:50.044936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471082453799030:2048] [node 1] ICP22 created new session: [3:7577471086748766425:2048]
2025-11-27T18:17:50.044986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471086748766425:2048] [node 1] ICS09 handshake done sender: [3:7577471082453799108:2117] self: [3:7577471082453799119:0] peer: [1:7577471081728155596:0] socket: 324 qp: -1
2025-11-27T18:17:50.045025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471086748766425:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:50.045065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:50.046240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471082453799127:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28993
2025-11-27T18:17:50.038171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471080999759876:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:50.050058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471081178658923:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:50.066833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471082001344962:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:50.066896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:50.066936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471082001344962:2108] [node 5] ICP22 created new session: [4:7577471086296312292:2108]
2025-11-27T18:17:50.066978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471086296312292:2108] [node 5] ICS09 handshake done sender: [4:7577471082001344972:2115] self: [4:7577471082001344971:0] peer: [5:7577471085255290946:0] socket: 372 qp: -1
2025-11-27T18:17:50.067006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471086296312292:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:50.067035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:50.038293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471085294727263:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34378
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:50.068973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471086748766424:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28993
2025-11-27T18:17:50.057712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471081728155603:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:50.060258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471080999759961:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:50.073522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471082453799107:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:50.050146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:50.057756Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471081728155603:2132] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:50.071997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471086296312291:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27912
2025-11-27T18:17:50.060551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471080999759877:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:50.062107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471085294727266:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34378
2025-11-27T18:17:50.062860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471080999759966:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:50.063046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471080999759876:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:50.063099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:50.063145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471080999759876:2048] [node 1] ICP22 created new session: [6:7577471085294727267:2048]
2025-11-27T18:17:50.063194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471085294727267:2048] [node 1] ICS09 handshake done sender: [6:7577471080999759966:2120] self: [6:7577471085294727264:0] peer: [1:7577471081728155602:0] socket: 327 qp: -1
2025-11-27T18:17:50.063233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471085294727267:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:50.063267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:50.084352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471085294727263:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:50.084804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471080999759935:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:50.084091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471082001344992:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:50.094565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471085294727270:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34378
2025-11-27T18:17:50.058551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471081728155601:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:50.050198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471081178658923:2048] [node 1] ICP22 created new session: [2:7577471085473626319:2048]
2025-11-27T18:17:50.050246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471085473626319:2048] [node 1] ICS09 handshake done sender: [2:7577471081178659002:2115] self: [2:7577471081178659011:0] peer: [1:7577471081728155594:0] socket: 323 qp: -1
2025-11-27T18:17:50.050278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471085473626319:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:50.050317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:50.050557Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471081178658923:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:50.051070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471085473626318:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36420
2025-11-27T18:17:50.061186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471085473626318:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:50.073678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471081178658998:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:50.073716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471081178658998:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:50.086312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471080960323621:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2157) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471080960323644:2121] held: no
2025-11-27T18:17:50.086519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471080960323632:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:50.086544Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471080960323632:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:50.087362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471085255290949:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21316
2025-11-27T18:17:50.102442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471085255290950:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21316
2025-11-27T18:17:50.058573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471081728155601:2131] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:50.062355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471081728155603:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:50.064880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471081728155601:2131] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:50.065109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471081728155593:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:50.065173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:50.065221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471081728155593:2117] [node 6] ICP22 created new session: [1:7577471086023122926:2117]
2025-11-27T18:17:50.065276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471086023122926:2117] [node 6] ICS09 handshake done sender: [1:7577471081728155603:2132] self: [1:7577471081728155602:0] peer: [6:7577471085294727264:0] socket: 318 qp: -1
2025-11-27T18:17:50.065474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471086023122926:2117] [node 6] ICS10 traffic start
2025-11-27T18:17:50.068282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:50.079376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471081728155592:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:50.079455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:50.079508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471081728155592:2108] [node 4] ICP22 created new session: [1:7577471086023122928:2108]
2025-11-27T18:17:50.079579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471086023122928:2108] [node 4] ICS09 handshake done sender: [1:7577471081728155601:2131] self: [1:7577471081728155600:0] peer: [4:7577471086296312290:0] socket: 309 qp: -1
2025-11-27T18:17:50.079624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471086023122928:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:50.079663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:50.105994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471085294727271:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34378
2025-11-27T18:17:50.108149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471082453799096:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:50.112183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:50.117832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471080960323644:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:50.105575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471081178658998:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:50.121491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471081178658990:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:50.121579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:50.121636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471081178658990:2109] [node 6] ICP22 created new session: [2:7577471085473626323:2109]
2025-11-27T18:17:50.121683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471085473626323:2109] [node 6] ICS09 handshake done sender: [2:7577471081178658998:2114] self: [2:7577471081178658997:0] peer: [6:7577471085294727265:0] socket: 363 qp: -1
2025-11-27T18:17:50.121720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471085473626323:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:50.121761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:50.124475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471082453799096:2110] [node 5] ICP22 created new session: [3:7577471086748766428:2110]
2025-11-27T18:17:50.122537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471082001344897:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:50.122618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:50.122668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471082001344897:2048] [node 1] ICP22 created new session: [4:7577471086296312294:2048]
2025-11-27T18:17:50.122721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471086296312294:2048] [node 1] ICS09 handshake done sender: [4:7577471082001344992:2120] self: [4:7577471086296312290:0] peer: [1:7577471081728155600:0] socket: 340 qp: -1
2025-11-27T18:17:50.122752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471086296312294:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:50.122780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:50.122979Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471082001344897:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:50.125281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471086748766428:2110] [node 5] ICS09 handshake done sender: [3:7577471082453799107:2116] self: [3:7577471082453799106:0] peer: [5:7577471085255290947:0] socket: 367 qp: -1
2025-11-27T18:17:50.125327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471086748766428:2110] [node 5] ICS10 traffic start
2025-11-27T18:17:50.125365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:50.125781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471082453799096:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(17452523ns) {HandleHandshakeStatus:329(17448949ns) {DropIncomingHandshake:442(2733ns) DropOutgoingHandshake:457(3390ns) SwitchToState:179(839ns) ProcessPendingSessionEvents:495(102354ns) {ForwardSessionEventToSession:612(50202ns) ForwardSessionEventToSession:612(6521ns) ForwardSessionEventToSession:612(3147ns) ForwardSessionEventToSession:612(3012ns) ForwardSessionEventToSession:612(5179ns) ForwardSessionEventToSession:612(3111ns) ForwardSessionEventToSession:612(8065ns) ForwardSessionEventToSession:612(2949ns) ForwardSessionEventToSession:612(5141ns) ForwardSessionEventToSession:612(4631ns)}}}
2025-11-27T18:17:50.125932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471086748766424:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:50.126290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471082453799096:2110] [node 5] ICP18 handshake (actor [3:7577471086748766424:2121]) is too old
2025-11-27T18:17:50.123598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471080960323621:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:50.123644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:50.123691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471080960323621:2109] [node 4] ICP22 created new session: [5:7577471085255290952:2109]
2025-11-27T18:17:50.123737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471085255290952:2109] [node 4] ICS09 handshake done sender: [5:7577471080960323644:2121] self: [5:7577471085255290946:0] peer: [4:7577471082001344971:0] socket: 371 qp: -1
2025-11-27T18:17:50.123768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471085255290952:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:50.123793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:50.124209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471080960323643:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:50.127450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471080999759961:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:50.127698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471080999759877:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:50.127743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:50.127788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471080999759877:2049] [node 2] ICP22 created new session: [6:7577471085294727272:2049]
2025-11-27T18:17:50.127839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471085294727272:2049] [node 2] ICS09 handshake done sender: [6:7577471080999759961:2118] self: [6:7577471085294727265:0] peer: [2:7577471081178658997:0] socket: 334 qp: -1
2025-11-27T18:17:50.127873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471085294727272:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:50.127903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:50.137422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471080960323632:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:50.138633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471085294727263:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:50.138951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471080960323559:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:50.139029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:50.139106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471080960323559:2050] [node 3] ICP22 created new session: [5:7577471085255290954:2050]
2025-11-27T18:17:50.139157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471085255290954:2050] [node 3] ICS09 handshake done sender: [5:7577471080960323643:2120] self: [5:7577471085255290947:0] peer: [3:7577471082453799106:0] socket: 368 qp: -1
2025-11-27T18:17:50.139195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471085255290954:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:50.139226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:50.141422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471080999759935:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:50.141482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:50.141524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471080999759935:2104] [node 5] ICP22 created new session: [6:7577471085294727274:2104]
2025-11-27T18:17:50.141571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471085294727274:2104] [node 5] ICS09 handshake done sender: [6:7577471085294727263:2121] self: [6:7577471085294727269:0] peer: [5:7577471080960323631:0] socket: 345 qp: -1
2025-11-27T18:17:50.141610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471085294727274:2104] [node 5] ICS10 traffic start
2025-11-27T18:17:50.141654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:50.143277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471080960323622:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:50.143336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:50.143380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471080960323622:2110] [node 6] ICP22 created new session: [5:7577471085255290956:2110]
2025-11-27T18:17:50.143423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471085255290956:2110] [node 6] ICS09 handshake done sender: [5:7577471080960323632:2115] self: [5:7577471080960323631:0] peer: [6:7577471085294727269:0] socket: 343 qp: -1
2025-11-27T18:17:50.143451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471085255290956:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:50.143480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:50.144135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471080960323559:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:50.164565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:50.164793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471085473626319:2048] [node 1] ICS01 socket: 323 reason# EndOfStream
2025-11-27T18:17:50.164836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471081178658923:2048] [node 1] ICP30 unregister session Session# [2:7577471085473626319:2048] VirtualId# [2:7577471081178659011:0]
2025-11-27T18:17:50.164871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471085473626319:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.164942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:50.164982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:50.166745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:50.166946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471086748766425:2048] [node 1] ICS01 socket: 324 reason# EndOfStream
2025-11-27T18:17:50.166975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471082453799030:2048] [node 1] ICP30 unregister session Session# [3:7577471086748766425:2048] VirtualId# [3:7577471082453799119:0]
2025-11-27T18:17:50.167002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471086748766425:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.167058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:50.167094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:50.169566Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:50.169760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471085255290944:2048] [node 1] ICS07 socket disconnect 338 reason# EndOfStream
2025-11-27T18:17:50.169787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471085255290944:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.169857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:50.169895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471085255290944:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:50.170903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471080960323557:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14541) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:50.170953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471085255290944:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:50.170986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471080960323557:2048] [node 1] ICP30 unregister session Session# [5:7577471085255290944:2048] VirtualId# [5:7577471080960323640:0]
2025-11-27T18:17:50.171033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:50.171063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471080960323557:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14541) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:50.171090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14541) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:50.173427Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:50.173617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471086296312294:2048] [node 1] ICS01 socket: 340 reason# EndOfStream
2025-11-27T18:17:50.173651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471082001344897:2048] [node 1] ICP30 unregister session Session# [4:7577471086296312294:2048] VirtualId# [4:7577471086296312290:0]
2025-11-27T18:17:50.173680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471086296312294:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.173735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:50.173769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:50.173851Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:50.174026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471085294727267:2048] [node 1] ICS01 socket: 327 reason# EndOfStream
2025-11-27T18:17:50.174052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471080999759876:2048] [node 1] ICP30 unregister session Session# [6:7577471085294727267:2048] VirtualId# [6:7577471085294727264:0]
2025-11-27T18:17:50.174075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471085294727267:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.174116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:50.174141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:50.190087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471080960323557:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:50.235886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:50.236101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471082453799115:2049] [node 2] ICS01 socket: 317 reason# EndOfStream
2025-11-27T18:17:50.236143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471082453799031:2049] [node 2] ICP30 unregister session Session# [3:7577471082453799115:2049] VirtualId# [3:7577471082453799113:0]
2025-11-27T18:17:50.236177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471082453799115:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.236248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:50.236285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:50.245460Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:50.245674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471080960323645:2049] [node 2] ICS01 socket: 351 reason# EndOfStream
2025-11-27T18:17:50.245728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471080960323558:2049] [node 2] ICP30 unregister session Session# [5:7577471080960323645:2049] VirtualId# [5:7577471080960323639:0]
2025-11-27T18:17:50.245788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:50.245766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471080960323645:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.245953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471082001344987:2049] [node 2] ICS01 socket: 315 reason# EndOfStream
2025-11-27T18:17:50.245847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:50.245983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471082001344898:2049] [node 2] ICP30 unregister session Session# [4:7577471082001344987:2049] VirtualId# [4:7577471082001344978:0]
2025-11-27T18:17:50.246005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471082001344987:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.245968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:50.246054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:50.246130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471085294727272:2049] [node 2] ICS01 socket: 334 reason# EndOfStream
2025-11-27T18:17:50.246099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:50.246162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471080999759877:2049] [node 2] ICP30 unregister session Session# [6:7577471085294727272:2049] VirtualId# [6:7577471085294727265:0]
2025-11-27T18:17:50.245887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:50.246185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471085294727272:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.246234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:50.246294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:50.301465Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:50.301693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471085255290954:2050] [node 3] ICS01 socket: 368 reason# EndOfStream
2025-11-27T18:17:50.301760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471080960323559:2050] [node 3] ICP30 unregister session Session# [5:7577471085255290954:2050] VirtualId# [5:7577471085255290947:0]
2025-11-27T18:17:50.301787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471085255290954:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.301838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:50.301878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:50.301446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:50.301694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471082001344989:2050] [node 3] ICS01 socket: 331 reason# EndOfStream
2025-11-27T18:17:50.301725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471082001344899:2050] [node 3] ICP30 unregister session Session# [4:7577471082001344989:2050] VirtualId# [4:7577471082001344979:0]
2025-11-27T18:17:50.301759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471082001344989:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.301840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:50.301878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:50.301964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:50.302122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471080999759964:2050] [node 3] ICS01 socket: 329 reason# EndOfStream
2025-11-27T18:17:50.302146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471080999759878:2050] [node 3] ICP30 unregister session Session# [6:7577471080999759964:2050] VirtualId# [6:7577471080999759957:0]
2025-11-27T18:17:50.302170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471080999759964:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.302222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:50.302255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:50.351380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:50.351674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:50.351846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471080999759959:2111] [node 4] ICS01 socket: 336 reason# EndOfStream
2025-11-27T18:17:50.351893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471080999759948:2111] [node 4] ICP30 unregister session Session# [6:7577471080999759959:2111] VirtualId# [6:7577471080999759956:0]
2025-11-27T18:17:50.351931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471080999759959:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.352015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:50.352073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:50.358552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471085255290952:2109] [node 4] ICS01 socket: 371 reason# EndOfStream
2025-11-27T18:17:50.358625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471080960323621:2109] [node 4] ICP30 unregister session Session# [5:7577471085255290952:2109] VirtualId# [5:7577471085255290946:0]
2025-11-27T18:17:50.358669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471085255290952:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.358762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:50.358807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:50.493426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:50.493698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471085294727274:2104] [node 5] ICS01 socket: 345 reason# EndOfStream
2025-11-27T18:17:50.493746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471080999759935:2104] [node 5] ICP30 unregister session Session# [6:7577471085294727274:2104] VirtualId# [6:7577471085294727269:0]
2025-11-27T18:17:50.493786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471085294727274:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:50.493889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:50.493937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:50.670324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471088135268288:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:50.670371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471088135268289:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:50.697109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471086315729069:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:50.697300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471086315729070:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:50.712465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471088883167388:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:50.712507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471088883167389:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:50.735944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471086108663349:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:50.735986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471086108663350:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:50.736013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471086108663351:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:50.758697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471085372411236:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:50.758742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471085372411237:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:50.759086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471085372411235:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:50.784362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471087861954518:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:50.784405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471087861954519:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:50.784436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471087861954520:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:51.280818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471092156921875:2105] [node 5] ICP01 ready to work
2025-11-27T18:17:51.323291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471090610696430:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:51.322286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471092430235647:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:51.373403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471092430235648:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:51.362904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471090610696432:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:51.453734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471088883167389:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(112281912ns) {Configure:104(112277382ns) {ProcessConfigured:135(31914148ns) {StartInitialHandshake:157(31911136ns) {DropHandshakes:470(3738ns) {DropIncomingHandshake:442(1043ns) DropOutgoingHandshake:457(578ns)} GenerateSessionVirtualId:988(1591ns) PrepareNewSessionHandshake:477(5796ns) {SwitchToState:179(3457ns)}}}}}
2025-11-27T18:17:51.465766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471093178134750:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:51.466085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471093178134751:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:51.466357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471093178134752:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:51.469855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471088883167388:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(144731735ns) {Configure:104(144724537ns) {ProcessConfigured:135(16261667ns) {StartInitialHandshake:157(16258623ns) {DropHandshakes:470(2907ns) {DropIncomingHandshake:442(616ns) DropOutgoingHandshake:457(561ns)} GenerateSessionVirtualId:988(1658ns) PrepareNewSessionHandshake:477(3107ns) {SwitchToState:179(1234ns)}}}}}
2025-11-27T18:17:51.462187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471092430235649:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:51.485305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471090403630711:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:51.510257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471090610696433:2112] [node 4] ICP01 ready to work
2025-11-27T18:17:51.521446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471089667378594:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:51.533370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471090403630712:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:51.549799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471089667378596:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:51.564544Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471088135268289:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(70418382ns) {Configure:104(70408181ns) {ProcessConfigured:135(77911ns) {StartInitialHandshake:157(76267ns) {DropHandshakes:470(3416ns) {DropIncomingHandshake:442(736ns) DropOutgoingHandshake:457(598ns)} GenerateSessionVirtualId:988(919ns) PrepareNewSessionHandshake:477(2263ns) {SwitchToState:179(721ns)}}}}}
2025-11-27T18:17:51.565527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471086315729070:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(241990992ns) {Configure:104(241979793ns) {ProcessConfigured:135(18824562ns) {StartInitialHandshake:157(18822127ns) {DropHandshakes:470(2886ns) {DropIncomingHandshake:442(721ns) DropOutgoingHandshake:457(481ns)} GenerateSessionVirtualId:988(942ns) PrepareNewSessionHandshake:477(3825ns) {SwitchToState:179(2476ns)}}}}}
2025-11-27T18:17:51.568423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471086108663351:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34681947ns) {Configure:104(34676784ns) {ProcessConfigured:135(59749ns) {StartInitialHandshake:157(57210ns) {DropHandshakes:470(1815ns) {DropIncomingHandshake:442(173ns) DropOutgoingHandshake:457(679ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(1528ns) {SwitchToState:179(462ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.594877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471088135268288:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(321532919ns) {Configure:104(321518577ns) {ProcessConfigured:135(16971123ns) {StartInitialHandshake:157(12803414ns) {DropHandshakes:470(2457ns) {DropIncomingHandshake:442(1000ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(1581ns) PrepareNewSessionHandshake:477(3425ns) {SwitchToState:179(740ns)}}}}}
2025-11-27T18:17:51.609963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471093178134763:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65309
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.626446Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471092430235648:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49251778ns) {Configure:104(49246802ns) {ProcessConfigured:135(103657ns) {StartInitialHandshake:157(100780ns) {DropHandshakes:470(3486ns) {DropIncomingHandshake:442(892ns) DropOutgoingHandshake:457(494ns)} GenerateSessionVirtualId:988(1084ns) PrepareNewSessionHandshake:477(2271ns) {SwitchToState:179(951ns)}}}}}
2025-11-27T18:17:51.627114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471092430235652:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63562
2025-11-27T18:17:51.627323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471092430235653:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63562
2025-11-27T18:17:51.627917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471092430235662:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63562
2025-11-27T18:17:51.591706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471086315729069:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(70301174ns) {Configure:104(70283038ns) {ProcessConfigured:135(70803ns) {StartInitialHandshake:157(68332ns) {DropHandshakes:470(3220ns) {DropIncomingHandshake:442(703ns) DropOutgoingHandshake:457(525ns)} GenerateSessionVirtualId:988(923ns) PrepareNewSessionHandshake:477(2214ns) {SwitchToState:179(759ns)}}}}}
2025-11-27T18:17:51.612994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471085372411235:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(200056096ns) {Configure:104(200050619ns) {ProcessConfigured:135(39760865ns) {StartInitialHandshake:157(39759130ns) {DropHandshakes:470(3211ns) {DropIncomingHandshake:442(909ns) DropOutgoingHandshake:457(481ns)} GenerateSessionVirtualId:988(1164ns) PrepareNewSessionHandshake:477(2167ns) {SwitchToState:179(212ns)}}}}}
2025-11-27T18:17:51.612214Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471092156921875:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(329716455ns) {Configure:104(329709708ns) {ProcessConfigured:135(6450787ns) {StartInitialHandshake:157(6448006ns) {DropHandshakes:470(3227ns) {DropIncomingHandshake:442(939ns) DropOutgoingHandshake:457(529ns)} GenerateSessionVirtualId:988(968ns) PrepareNewSessionHandshake:477(4024ns) {SwitchToState:179(1898ns)}}}}}
2025-11-27T18:17:51.594028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471090610696438:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4175
2025-11-27T18:17:51.628697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471093178134764:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65309
2025-11-27T18:17:51.646882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471092430235652:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.653879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471092430235653:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.612730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471092156921881:2110] [node 4] ICP01 ready to work
2025-11-27T18:17:51.640390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471086108663350:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(277363906ns) {Configure:104(277358135ns) {ProcessConfigured:135(9668591ns) {StartInitialHandshake:157(9666328ns) {DropHandshakes:470(3589ns) {DropIncomingHandshake:442(907ns) DropOutgoingHandshake:457(672ns)} GenerateSessionVirtualId:988(835ns) PrepareNewSessionHandshake:477(7347ns) {SwitchToState:179(4180ns)}}}}}
2025-11-27T18:17:51.670807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471087861954518:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(260368115ns) {Configure:104(260362216ns) {ProcessConfigured:135(8659932ns) {StartInitialHandshake:157(8656717ns) {DropHandshakes:470(4276ns) {DropIncomingHandshake:442(809ns) DropOutgoingHandshake:457(553ns)} GenerateSessionVirtualId:988(744ns) PrepareNewSessionHandshake:477(4835ns) {SwitchToState:179(2286ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.666886Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471090610696433:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(89191654ns) {Configure:104(89186807ns) {ProcessConfigured:135(81400ns) {StartInitialHandshake:157(78061ns) {DropHandshakes:470(3861ns) {DropIncomingHandshake:442(1181ns) DropOutgoingHandshake:457(572ns)} GenerateSessionVirtualId:988(1356ns) PrepareNewSessionHandshake:477(2436ns) {SwitchToState:179(878ns)}}}}}
2025-11-27T18:17:51.628787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471093178134765:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65309
2025-11-27T18:17:51.684903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471089667378594:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(103595006ns) {Configure:104(103590658ns) {ProcessConfigured:135(65858ns) {StartInitialHandshake:157(62927ns) {DropHandshakes:470(3836ns) {DropIncomingHandshake:442(938ns) DropOutgoingHandshake:457(433ns)} GenerateSessionVirtualId:988(949ns) PrepareNewSessionHandshake:477(5440ns) {SwitchToState:179(988ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.691444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471089667378601:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63282
2025-11-27T18:17:51.692744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471089667378596:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(79525272ns) {Configure:104(79518823ns) {ProcessConfigured:135(51663ns) {StartInitialHandshake:157(49363ns) {DropHandshakes:470(4743ns) {DropIncomingHandshake:442(308ns) DropOutgoingHandshake:457(478ns)} GenerateSessionVirtualId:988(629ns) PrepareNewSessionHandshake:477(638ns) {SwitchToState:179(137ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.679018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471090610696435:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.654261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471093178134763:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.693646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471092430235662:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.701202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471092430235657:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.701239Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471092430235657:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.709536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471092430235657:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:51.709936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471092430235670:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63562
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.672781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471093178134764:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.679047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471090610696435:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.695046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471089667378605:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63282
2025-11-27T18:17:51.713918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471087861954519:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(42577851ns) {Configure:104(42571083ns) {ProcessConfigured:135(10846779ns) {StartInitialHandshake:157(10843947ns) {DropHandshakes:470(1814ns) {DropIncomingHandshake:442(261ns) DropOutgoingHandshake:457(378ns)} GenerateSessionVirtualId:988(483ns) PrepareNewSessionHandshake:477(4399ns) {SwitchToState:179(1799ns)}}}}}
2025-11-27T18:17:51.741636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471092430235659:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.726304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471090403630716:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.710802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471089667378605:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.679866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471090610696448:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4175
2025-11-27T18:17:51.672932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471088883167389:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.741678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471092430235659:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.710943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471089667378594:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.716052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471092156921884:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.691266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471093178134768:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65309
2025-11-27T18:17:51.726357Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471090403630716:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.684062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471090610696438:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.689948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471090610696435:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:51.690482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471090610696448:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.690595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471086315729069:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.693871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471086315729070:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.693937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:51.693987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471086315729070:2049] [node 3] ICP22 created new session: [2:7577471090610696452:2049]
2025-11-27T18:17:51.694031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471090610696452:2049] [node 3] ICS09 handshake done sender: [2:7577471090610696435:2113] self: [2:7577471090610696434:0] peer: [3:7577471093178134767:0] socket: 325 qp: -1
2025-11-27T18:17:51.694065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471090610696452:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:51.694101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:51.717847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471090610696455:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4175
2025-11-27T18:17:51.719258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471090610696457:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4175
2025-11-27T18:17:51.743331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471090610696457:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.743449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471090610696437:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.743468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471090610696437:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.751933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471090610696437:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:51.768630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471090610696455:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.768742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471090610696442:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.768767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471090610696442:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.769463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471090610696430:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.769526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:51.769578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471090610696430:2111] [node 6] ICP22 created new session: [2:7577471090610696460:2111]
2025-11-27T18:17:51.769628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471090610696460:2111] [node 6] ICS09 handshake done sender: [2:7577471090610696437:2114] self: [2:7577471090610696436:0] peer: [6:7577471092156921895:0] socket: 326 qp: -1
2025-11-27T18:17:51.769663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471090610696460:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:51.769698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:51.770329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471086315729069:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19192) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471090610696448:2119] held: no
2025-11-27T18:17:51.716204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471092156921885:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:17:51.742283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471088135268288:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.795732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471090610696456:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4175
2025-11-27T18:17:51.753253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471093178134765:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.719531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471089667378601:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.762862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471093178134764:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:51.774975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471093178134754:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.775005Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471093178134754:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.775668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471088883167389:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20240) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471093178134764:2116] held: no
2025-11-27T18:17:51.776216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471088883167388:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.776305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471093178134769:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65309
2025-11-27T18:17:51.751891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471090403630716:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:51.742359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:51.719897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471089667378612:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63282
2025-11-27T18:17:51.727367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471092156921884:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.798694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471088883167389:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.783358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471090403630711:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.810277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471090610696442:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:51.727460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471092156921885:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.742417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471088135268288:2048] [node 2] ICP22 created new session: [1:7577471092430235672:2048]
2025-11-27T18:17:51.729592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471089667378613:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63282
2025-11-27T18:17:51.783405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:51.846662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471090610696448:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:51.847501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471090610696432:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.847584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:51.847646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471090610696432:2107] [node 5] ICP22 created new session: [2:7577471090610696465:2107]
2025-11-27T18:17:51.847700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471090610696465:2107] [node 5] ICS09 handshake done sender: [2:7577471090610696442:2117] self: [2:7577471090610696441:0] peer: [5:7577471089667378615:0] socket: 348 qp: -1
2025-11-27T18:17:51.847735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471090610696465:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:51.847777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:51.848162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471086315729069:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.848195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:51.848244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471086315729069:2048] [node 1] ICP22 created new session: [2:7577471090610696467:2048]
2025-11-27T18:17:51.848308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471090610696467:2048] [node 1] ICS09 handshake done sender: [2:7577471090610696448:2119] self: [2:7577471090610696451:0] peer: [1:7577471092430235656:0] socket: 336 qp: -1
2025-11-27T18:17:51.848346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471090610696467:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:51.848374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:51.798758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:51.742462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471092430235672:2048] [node 2] ICS09 handshake done sender: [1:7577471092430235657:2114] self: [1:7577471092430235656:0] peer: [2:7577471090610696451:0] socket: 332 qp: -1
2025-11-27T18:17:51.737710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471089667378612:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.731805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471087861954520:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42044946ns) {Configure:104(42036811ns) {ProcessConfigured:135(61284ns) {StartInitialHandshake:157(55751ns) {DropHandshakes:470(3212ns) {DropIncomingHandshake:442(863ns) DropOutgoingHandshake:457(520ns)} GenerateSessionVirtualId:988(900ns) PrepareNewSessionHandshake:477(1822ns) {SwitchToState:179(678ns)}}}}}
2025-11-27T18:17:51.783460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471090403630711:2109] [node 5] ICP22 created new session: [4:7577471090403630725:2109]
2025-11-27T18:17:51.783515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471090403630725:2109] [node 5] ICS09 handshake done sender: [4:7577471090403630716:2112] self: [4:7577471090403630715:0] peer: [5:7577471089667378611:0] socket: 323 qp: -1
2025-11-27T18:17:51.783549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471090403630725:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:51.783588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:51.793694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471090403630724:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5647
2025-11-27T18:17:51.818125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471090403630724:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.818505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471086108663349:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.818595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471090403630728:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5647
2025-11-27T18:17:51.821049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471090403630730:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5647
2025-11-27T18:17:51.825838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471090403630730:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.798809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471088883167389:2049] [node 2] ICP22 created new session: [3:7577471093178134773:2049]
2025-11-27T18:17:51.742500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471092430235672:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.733339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471087861954519:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.742541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:51.757021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471092430235659:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:51.765659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471092430235670:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.765744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471092430235649:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.765790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:51.765845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471092430235649:2109] [node 6] ICP22 created new session: [1:7577471092430235676:2109]
2025-11-27T18:17:51.765910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471092430235676:2109] [node 6] ICS09 handshake done sender: [1:7577471092430235659:2115] self: [1:7577471092430235658:0] peer: [6:7577471092156921896:0] socket: 333 qp: -1
2025-11-27T18:17:51.765944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471092430235676:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:51.765984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:51.773661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471092430235671:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63562
2025-11-27T18:17:51.785392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471092430235651:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.785433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471092430235651:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.800372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471092430235671:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.857895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471092430235655:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.857937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471092430235655:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.869761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471092430235661:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.869809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471092430235661:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.745801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471085372411237:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.798861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471093178134773:2049] [node 2] ICS09 handshake done sender: [3:7577471093178134764:2116] self: [3:7577471093178134767:0] peer: [2:7577471090610696434:0] socket: 329 qp: -1
2025-11-27T18:17:51.733402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471087861954518:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.752911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471092156921897:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:17:51.756566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471092156921884:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:51.761830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471087861954519:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.761899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:51.761960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471087861954519:2049] [node 2] ICP22 created new session: [6:7577471092156921899:2049]
2025-11-27T18:17:51.762014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471092156921899:2049] [node 2] ICS09 handshake done sender: [6:7577471092156921884:2113] self: [6:7577471092156921895:0] peer: [2:7577471090610696436:0] socket: 337 qp: -1
2025-11-27T18:17:51.762056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471092156921899:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:51.762096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:51.762607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471092156921898:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:17:51.796796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471092156921885:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:51.802261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471087861954518:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.802318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:51.802365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471087861954518:2048] [node 1] ICP22 created new session: [6:7577471092156921902:2048]
2025-11-27T18:17:51.802414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471092156921902:2048] [node 1] ICS09 handshake done sender: [6:7577471092156921885:2114] self: [6:7577471092156921896:0] peer: [1:7577471092430235658:0] socket: 338 qp: -1
2025-11-27T18:17:51.802453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471092156921902:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:51.802483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:51.819867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471092156921901:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:17:51.824452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471092156921904:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:17:51.853346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471092156921904:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.870901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471092156921881:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.887126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471092156921908:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:17:51.887306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471092156921901:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.871505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471090403630718:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.871556Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471090403630718:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.891189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471090403630728:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.752035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471085372411236:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.798894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471093178134773:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:51.798932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:51.799427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471093178134769:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.799988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471093178134754:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:51.850721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471093178134750:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.850755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:51.850806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471093178134750:2107] [node 5] ICP22 created new session: [3:7577471093178134775:2107]
2025-11-27T18:17:51.850857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471093178134775:2107] [node 5] ICS09 handshake done sender: [3:7577471093178134754:2111] self: [3:7577471093178134753:0] peer: [5:7577471089667378614:0] socket: 314 qp: -1
2025-11-27T18:17:51.850888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471093178134775:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:51.850918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:51.851131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471093178134750:2107] [node 5] ICP18 handshake (actor [3:7577471093178134769:2119]) is too old
2025-11-27T18:17:51.887360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471093178134777:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65309
2025-11-27T18:17:51.905780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471093178134777:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.897961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471090403630718:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:51.901698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471087861954520:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.784607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471089667378617:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63282
2025-11-27T18:17:51.911762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471092156921904:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:51.912198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471092156921881:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.912254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:51.912310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471092156921881:2110] [node 4] ICP22 created new session: [6:7577471092156921910:2110]
2025-11-27T18:17:51.912359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471092156921910:2110] [node 4] ICS09 handshake done sender: [6:7577471092156921904:2121] self: [6:7577471092156921907:0] peer: [4:7577471090403630717:0] socket: 371 qp: -1
2025-11-27T18:17:51.912395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471092156921910:2110] [node 4] ICS10 traffic start
2025-11-27T18:17:51.912425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:51.916654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471086108663350:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.916719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471090403630712:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.916765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:51.910093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471092430235649:2109] [node 6] ICP18 handshake (actor [1:7577471092430235671:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.784769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471089667378618:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63282
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.915665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471093178134757:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.915707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471093178134757:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.916778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471093178134757:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:51.912006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471092430235651:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:51.916809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471090403630712:2110] [node 6] ICP22 created new session: [4:7577471090403630735:2110]
2025-11-27T18:17:51.916869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471090403630735:2110] [node 6] ICS09 handshake done sender: [4:7577471090403630718:2113] self: [4:7577471090403630717:0] peer: [6:7577471092156921907:0] socket: 353 qp: -1
2025-11-27T18:17:51.916898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471090403630735:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:51.916930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.925474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471092156921913:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:17:51.914111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471092430235661:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:51.914378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471092430235655:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:51.914561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471088135268289:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.914623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:51.914671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471088135268289:2049] [node 3] ICP22 created new session: [1:7577471092430235683:2049]
2025-11-27T18:17:51.914709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471092430235683:2049] [node 3] ICS09 handshake done sender: [1:7577471092430235651:2110] self: [1:7577471092430235650:0] peer: [3:7577471093178134771:0] socket: 331 qp: -1
2025-11-27T18:17:51.914743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471092430235683:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:51.914778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:51.915107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471092430235648:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.915152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:51.915190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471092430235648:2108] [node 5] ICP22 created new session: [1:7577471092430235685:2108]
2025-11-27T18:17:51.915226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471092430235685:2108] [node 5] ICS09 handshake done sender: [1:7577471092430235661:2116] self: [1:7577471092430235660:0] peer: [5:7577471089667378620:0] socket: 363 qp: -1
2025-11-27T18:17:51.915255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471092430235685:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:51.915283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:51.921771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471092430235647:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.921820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:51.921862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471092430235647:2107] [node 4] ICP22 created new session: [1:7577471092430235687:2107]
2025-11-27T18:17:51.921899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471092430235687:2107] [node 4] ICS09 handshake done sender: [1:7577471092430235655:2113] self: [1:7577471092430235654:0] peer: [4:7577471090403630729:0] socket: 313 qp: -1
2025-11-27T18:17:51.921927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471092430235687:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:51.921954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:51.930249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471092156921912:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:17:51.931100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471092156921913:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.931241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471092156921875:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.934794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471088883167388:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19192) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471093178134765:2117] held: no
2025-11-27T18:17:51.934924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471093178134752:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.934983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:51.935028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471093178134752:2109] [node 6] ICP22 created new session: [3:7577471093178134782:2109]
2025-11-27T18:17:51.935083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471093178134782:2109] [node 6] ICS09 handshake done sender: [3:7577471093178134757:2113] self: [3:7577471093178134756:0] peer: [6:7577471092156921909:0] socket: 366 qp: -1
2025-11-27T18:17:51.935123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471093178134782:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:51.935156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:51.935585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471093178134781:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65309
2025-11-27T18:17:51.785036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471089667378616:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63282
2025-11-27T18:17:51.801916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471089667378617:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.802066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471089667378613:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.814539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471089667378605:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:51.814895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471085372411235:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.814957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471089667378601:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:51.826451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471089667378619:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63282
2025-11-27T18:17:51.834128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471089667378594:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.834218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:51.834292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471089667378594:2108] [node 4] ICP22 created new session: [5:7577471089667378622:2108]
2025-11-27T18:17:51.834359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471089667378622:2108] [node 4] ICS09 handshake done sender: [5:7577471089667378605:2115] self: [5:7577471089667378611:0] peer: [4:7577471090403630715:0] socket: 341 qp: -1
2025-11-27T18:17:51.834395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471089667378622:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:51.834447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:51.836451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471085372411237:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.836524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:51.836584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471085372411237:2050] [node 3] ICP22 created new session: [5:7577471089667378624:2050]
2025-11-27T18:17:51.941476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471093178134765:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:51.941720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471088883167388:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.941772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:51.941833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471088883167388:2048] [node 1] ICP22 created new session: [3:7577471093178134784:2048]
2025-11-27T18:17:51.941879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471093178134784:2048] [node 1] ICS09 handshake done sender: [3:7577471093178134765:2117] self: [3:7577471093178134771:0] peer: [1:7577471092430235650:0] socket: 334 qp: -1
2025-11-27T18:17:51.941914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471093178134784:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:51.941945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.949343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471090610696447:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.949399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471090610696447:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.952077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471090610696447:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:51.936138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471092156921901:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:51.836645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471089667378624:2050] [node 3] ICS09 handshake done sender: [5:7577471089667378601:2112] self: [5:7577471089667378614:0] peer: [3:7577471093178134753:0] socket: 321 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.962445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471090403630712:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(40678749ns) {HandleHandshakeStatus:329(40676756ns) {DropIncomingHandshake:442(8851ns) DropOutgoingHandshake:457(278ns) SwitchToState:179(99ns) ProcessPendingSessionEvents:495(22961298ns) {ForwardSessionEventToSession:612(3263352ns) ForwardSessionEventToSession:612(6057ns) ForwardSessionEventToSession:612(5717635ns) ForwardSessionEventToSession:612(4881ns) ForwardSessionEventToSession:612(4689783ns) ForwardSessionEventToSession:612(7027ns) ForwardSessionEventToSession:612(3352ns) ForwardSessionEventToSession:612(9218436ns) ForwardSessionEventToSession:612(20763ns) ForwardSessionEventToSession:612(4122ns)}}}
2025-11-27T18:17:51.962568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471090403630733:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5647
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.937803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471087861954520:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.952351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471090610696433:2112] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.836680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471089667378624:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:51.937856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:51.937905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471087861954520:2050] [node 3] ICP22 created new session: [6:7577471092156921915:2050]
2025-11-27T18:17:51.937948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471092156921915:2050] [node 3] ICS09 handshake done sender: [6:7577471092156921901:2120] self: [6:7577471092156921909:0] peer: [3:7577471093178134756:0] socket: 369 qp: -1
2025-11-27T18:17:51.937991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471092156921915:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:51.938019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:51.946187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471092156921917:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:17:51.958145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471092156921913:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:51.958448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471092156921875:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.958502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:51.958549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471092156921875:2105] [node 5] ICP22 created new session: [6:7577471092156921918:2105]
2025-11-27T18:17:51.958596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471092156921918:2105] [node 5] ICS09 handshake done sender: [6:7577471092156921913:2124] self: [6:7577471092156921914:0] peer: [5:7577471089667378606:0] socket: 384 qp: -1
2025-11-27T18:17:51.958631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471092156921918:2105] [node 5] ICS10 traffic start
2025-11-27T18:17:51.958668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:51.963021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471086108663349:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19192) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471090403630724:2115] held: no
2025-11-27T18:17:51.836739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:51.850201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471089667378612:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:51.862389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471085372411236:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.862443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:51.862493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471085372411236:2049] [node 2] ICP22 created new session: [5:7577471089667378626:2049]
2025-11-27T18:17:51.862547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471089667378626:2049] [node 2] ICS09 handshake done sender: [5:7577471089667378612:2117] self: [5:7577471089667378615:0] peer: [2:7577471090610696441:0] socket: 350 qp: -1
2025-11-27T18:17:51.862585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471089667378626:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:51.862615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:51.911205Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471085372411236:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:51.932154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471089667378607:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.932177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471089667378607:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.932913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471089667378629:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63282
2025-11-27T18:17:51.943105Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471085372411235:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19192) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471089667378617:2120] held: no
2025-11-27T18:17:51.943296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471089667378617:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:51.947865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471089667378607:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:51.961048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471085372411235:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.961123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:51.961180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471085372411235:2048] [node 1] ICP22 created new session: [5:7577471089667378631:2048]
2025-11-27T18:17:51.961226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471089667378631:2048] [node 1] ICS09 handshake done sender: [5:7577471089667378617:2120] self: [5:7577471089667378620:0] peer: [1:7577471092430235660:0] socket: 327 qp: -1
2025-11-27T18:17:51.961510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471089667378631:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:51.961551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:51.964101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471089667378596:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.969452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:51.970894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471089667378596:2109] [node 6] ICP22 created new session: [5:7577471089667378633:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.963756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471090403630737:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5647
2025-11-27T18:17:51.969035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471090403630739:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5647
2025-11-27T18:17:51.952400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:51.952466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471090610696433:2112] [node 4] ICP22 created new session: [2:7577471090610696470:2112]
2025-11-27T18:17:51.952509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471090610696470:2112] [node 4] ICS09 handshake done sender: [2:7577471090610696447:2118] self: [2:7577471090610696446:0] peer: [4:7577471090403630734:0] socket: 372 qp: -1
2025-11-27T18:17:51.952545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471090610696470:2112] [node 4] ICS10 traffic start
2025-11-27T18:17:51.952584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:51.971972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471090610696472:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4175
2025-11-27T18:17:51.973020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471090403630733:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.973124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471090403630724:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:51.973502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471086108663349:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.973554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:51.973615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471086108663349:2048] [node 1] ICP22 created new session: [4:7577471090403630740:2048]
2025-11-27T18:17:51.973653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471090403630740:2048] [node 1] ICS09 handshake done sender: [4:7577471090403630724:2115] self: [4:7577471090403630729:0] peer: [1:7577471092430235654:0] socket: 367 qp: -1
2025-11-27T18:17:51.973684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471090403630740:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:51.973714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:51.975592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471086108663351:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:51.977375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471093178134760:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.977418Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471093178134760:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:51.978651Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471093178134760:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:51.978917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471093178134751:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:51.978981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:51.979024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471093178134751:2108] [node 4] ICP22 created new session: [3:7577471093178134787:2108]
2025-11-27T18:17:51.979070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471093178134787:2108] [node 4] ICS09 handshake done sender: [3:7577471093178134760:2114] self: [3:7577471093178134759:0] peer: [4:7577471090403630742:0] socket: 342 qp: -1
2025-11-27T18:17:51.979102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471093178134787:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:51.979132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:51.986766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471090610696472:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:51.986964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471090610696433:2112] [node 4] ICP18 handshake (actor [2:7577471090610696472:2123]) is too old
2025-11-27T18:17:51.972694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471089667378633:2109] [node 6] ICS09 handshake done sender: [5:7577471089667378607:2116] self: [5:7577471089667378606:0] peer: [6:7577471092156921914:0] socket: 373 qp: -1
2025-11-27T18:17:51.972746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471089667378633:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:51.972827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:51.987621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471090403630728:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:51.988018Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471086108663351:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7679) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471090403630733:2118] held: no
2025-11-27T18:17:51.988134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471086108663350:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.988182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:51.988231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471086108663350:2049] [node 2] ICP22 created new session: [4:7577471090403630744:2049]
2025-11-27T18:17:51.988273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471090403630744:2049] [node 2] ICS09 handshake done sender: [4:7577471090403630728:2116] self: [4:7577471090403630734:0] peer: [2:7577471090610696446:0] socket: 351 qp: -1
2025-11-27T18:17:51.988306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471090403630744:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:51.988337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:51.990898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471090403630743:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5647
2025-11-27T18:17:51.991468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471090403630733:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:51.991689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471086108663351:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:51.991738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:51.991792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471086108663351:2050] [node 3] ICP22 created new session: [4:7577471090403630746:2050]
2025-11-27T18:17:51.991831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471090403630746:2050] [node 3] ICS09 handshake done sender: [4:7577471090403630733:2118] self: [4:7577471090403630742:0] peer: [3:7577471093178134759:0] socket: 376 qp: -1
2025-11-27T18:17:51.991867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471090403630746:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:51.991898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:52.054765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:52.055020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471090610696467:2048] [node 1] ICS01 socket: 336 reason# EndOfStream
2025-11-27T18:17:52.055074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471086315729069:2048] [node 1] ICP30 unregister session Session# [2:7577471090610696467:2048] VirtualId# [2:7577471090610696451:0]
2025-11-27T18:17:52.055110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471090610696467:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.055181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:52.055224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:52.059016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:52.059278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471093178134784:2048] [node 1] ICS01 socket: 334 reason# EndOfStream
2025-11-27T18:17:52.059320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471088883167388:2048] [node 1] ICP30 unregister session Session# [3:7577471093178134784:2048] VirtualId# [3:7577471093178134771:0]
2025-11-27T18:17:52.059356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471093178134784:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.059435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:52.059487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:52.062095Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:52.062342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471090403630740:2048] [node 1] ICS01 socket: 367 reason# EndOfStream
2025-11-27T18:17:52.062388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471086108663349:2048] [node 1] ICP30 unregister session Session# [4:7577471090403630740:2048] VirtualId# [4:7577471090403630729:0]
2025-11-27T18:17:52.062428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471090403630740:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.062499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:52.062537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:52.062865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:52.063084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471089667378631:2048] [node 1] ICS01 socket: 327 reason# EndOfStream
2025-11-27T18:17:52.063117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471085372411235:2048] [node 1] ICP30 unregister session Session# [5:7577471089667378631:2048] VirtualId# [5:7577471089667378620:0]
2025-11-27T18:17:52.063153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471089667378631:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.063232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:52.063271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:52.065532Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:52.065766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471092156921902:2048] [node 1] ICS07 socket disconnect 338 reason# EndOfStream
2025-11-27T18:17:52.065809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471092156921902:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.065880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:52.065916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471092156921902:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:52.067348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471087861954518:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19192) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:52.071911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471092156921902:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:52.073202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471087861954518:2048] [node 1] ICP30 unregister session Session# [6:7577471092156921902:2048] VirtualId# [6:7577471092156921896:0]
2025-11-27T18:17:52.073284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:52.073338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471087861954518:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:52.073358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:52.085620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471087861954518:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:52.121450Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:52.121714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471093178134773:2049] [node 2] ICS01 socket: 329 reason# EndOfStream
2025-11-27T18:17:52.121763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471088883167389:2049] [node 2] ICP30 unregister session Session# [3:7577471093178134773:2049] VirtualId# [3:7577471093178134767:0]
2025-11-27T18:17:52.121802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471093178134773:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.121878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:52.121916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:52.131274Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:52.131533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471092156921899:2049] [node 2] ICS01 socket: 337 reason# EndOfStream
2025-11-27T18:17:52.131574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471087861954519:2049] [node 2] ICP30 unregister session Session# [6:7577471092156921899:2049] VirtualId# [6:7577471092156921895:0]
2025-11-27T18:17:52.131607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471092156921899:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.131689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:52.131727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:52.131906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:52.132104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471090403630744:2049] [node 2] ICS01 socket: 351 reason# EndOfStream
2025-11-27T18:17:52.132150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471086108663350:2049] [node 2] ICP30 unregister session Session# [4:7577471090403630744:2049] VirtualId# [4:7577471090403630734:0]
2025-11-27T18:17:52.132176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471090403630744:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.132278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:52.132334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:52.133464Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:52.133683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471089667378626:2049] [node 2] ICS01 socket: 350 reason# EndOfStream
2025-11-27T18:17:52.133721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471085372411236:2049] [node 2] ICP30 unregister session Session# [5:7577471089667378626:2049] VirtualId# [5:7577471089667378615:0]
2025-11-27T18:17:52.133748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471089667378626:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.133820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:52.133858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:52.229333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:52.229575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471090403630746:2050] [node 3] ICS01 socket: 376 reason# EndOfStream
2025-11-27T18:17:52.229618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471086108663351:2050] [node 3] ICP30 unregister session Session# [4:7577471090403630746:2050] VirtualId# [4:7577471090403630742:0]
2025-11-27T18:17:52.229654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471090403630746:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.229727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:52.229768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:52.231812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:52.232028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471089667378624:2050] [node 3] ICS01 socket: 321 reason# EndOfStream
2025-11-27T18:17:52.232075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471085372411237:2050] [node 3] ICP30 unregister session Session# [5:7577471089667378624:2050] VirtualId# [5:7577471089667378614:0]
2025-11-27T18:17:52.232109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471089667378624:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.232191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:52.232242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:52.233828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:52.234052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471092156921915:2050] [node 3] ICS07 socket disconnect 369 reason# EndOfStream
2025-11-27T18:17:52.234082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471092156921915:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.234162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:52.234211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471092156921915:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:52.241369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471087861954520:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7679) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:52.241457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471092156921915:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:52.242335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471087861954520:2050] [node 3] ICP30 unregister session Session# [6:7577471092156921915:2050] VirtualId# [6:7577471092156921909:0]
2025-11-27T18:17:52.242407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:52.242445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471087861954520:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7679) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:52.242461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7679) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:52.253386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471087861954520:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:52.285466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:52.285734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471089667378622:2108] [node 4] ICS07 socket disconnect 341 reason# EndOfStream
2025-11-27T18:17:52.285767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471089667378622:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.285853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:52.285900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471089667378622:2108] [node 4] ICS15 start handshake
2025-11-27T18:17:52.286928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471089667378594:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3862) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:52.286990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471089667378622:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:52.287033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471089667378594:2108] [node 4] ICP30 unregister session Session# [5:7577471089667378622:2108] VirtualId# [5:7577471089667378611:0]
2025-11-27T18:17:52.287077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:52.287113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471089667378594:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3862) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:52.287129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3862) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:52.286718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:52.286977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471092156921910:2110] [node 4] ICS01 socket: 371 reason# EndOfStream
2025-11-27T18:17:52.287019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471092156921881:2110] [node 4] ICP30 unregister session Session# [6:7577471092156921910:2110] VirtualId# [6:7577471092156921907:0]
2025-11-27T18:17:52.287055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471092156921910:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.287135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:52.287178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:52.298669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471089667378594:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:17:52.373905Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:52.374331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471092156921918:2105] [node 5] ICS01 socket: 384 reason# EndOfStream
2025-11-27T18:17:52.374421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471092156921875:2105] [node 5] ICP30 unregister session Session# [6:7577471092156921918:2105] VirtualId# [6:7577471092156921914:0]
2025-11-27T18:17:52.374487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471092156921918:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:52.374620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:52.374708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:52.629532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471094954200856:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:52.629584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471094954200857:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:52.649339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471095313831258:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:52.649369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471095313831259:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:52.674478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471095928745019:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:52.674521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471095928745020:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:52.713572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471093714248452:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:52.713628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471093714248453:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:52.713659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471093714248454:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:52.744518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471096126156741:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:52.744558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471096126156742:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:52.744588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471096126156743:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:52.813595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471093651021266:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:52.813643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471093651021267:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:52.813671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471093651021268:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:53.346400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471097945988613:2097] [node 5] ICP01 ready to work
2025-11-27T18:17:53.352905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471097945988620:2100] [node 4] ICP01 ready to work
2025-11-27T18:17:53.383598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471099249168218:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:53.394772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471099249168219:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:53.481925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471099608798620:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:53.515936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471098009215811:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:53.521976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471100421124100:2109] [node 4] ICP01 ready to work
2025-11-27T18:17:53.418624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471099249168220:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:53.517552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471099608798624:2107] [node 6] ICP01 ready to work
2025-11-27T18:17:53.542227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471100421124101:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.577863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471097945988620:2100] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(224635750ns) {Configure:104(224629514ns) {ProcessConfigured:135(15734807ns) {StartInitialHandshake:157(15731016ns) {DropHandshakes:470(1929ns) {DropIncomingHandshake:442(402ns) DropOutgoingHandshake:457(341ns)} GenerateSessionVirtualId:988(379ns) PrepareNewSessionHandshake:477(3773ns) {SwitchToState:179(1956ns)}}}}}
2025-11-27T18:17:53.578802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471097945988635:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56126
2025-11-27T18:17:53.517619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471098009215812:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:53.584834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471100223712379:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:53.521714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471099608798623:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.583125Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471096126156741:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(45471829ns) {Configure:104(45460997ns) {ProcessConfigured:135(71943ns) {StartInitialHandshake:157(69973ns) {DropHandshakes:470(3084ns) {DropIncomingHandshake:442(909ns) DropOutgoingHandshake:457(371ns)} GenerateSessionVirtualId:988(960ns) PrepareNewSessionHandshake:477(2145ns) {SwitchToState:179(885ns)}}}}}
2025-11-27T18:17:53.586934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471100223712380:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:53.602741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471099249168220:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29586829ns) {Configure:104(29580735ns) {ProcessConfigured:135(82629ns) {StartInitialHandshake:157(79255ns) {DropHandshakes:470(3614ns) {DropIncomingHandshake:442(773ns) DropOutgoingHandshake:457(464ns)} GenerateSessionVirtualId:988(1085ns) PrepareNewSessionHandshake:477(2420ns) {SwitchToState:179(745ns)}}}}}
2025-11-27T18:17:53.557743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471099608798624:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28904216ns) {Configure:104(28899837ns) {ProcessConfigured:135(71136ns) {StartInitialHandshake:157(68300ns) {DropHandshakes:470(2876ns) {DropIncomingHandshake:442(640ns) DropOutgoingHandshake:457(485ns)} GenerateSessionVirtualId:988(918ns) PrepareNewSessionHandshake:477(1980ns) {SwitchToState:179(785ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.588615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471098009215819:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49956
2025-11-27T18:17:53.622910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471095928745019:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(231838524ns) {Configure:104(231828850ns) {ProcessConfigured:135(90210ns) {StartInitialHandshake:157(87184ns) {DropHandshakes:470(3503ns) {DropIncomingHandshake:442(1037ns) DropOutgoingHandshake:457(570ns)} GenerateSessionVirtualId:988(1142ns) PrepareNewSessionHandshake:477(2248ns) {SwitchToState:179(783ns)}}}}}
2025-11-27T18:17:53.638716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471099249168218:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(124938733ns) {Configure:104(124927979ns) {ProcessConfigured:135(91059ns) {StartInitialHandshake:157(88288ns) {DropHandshakes:470(3408ns) {DropIncomingHandshake:442(756ns) DropOutgoingHandshake:457(476ns)} GenerateSessionVirtualId:988(1547ns) PrepareNewSessionHandshake:477(2210ns) {SwitchToState:179(627ns)}}}}}
2025-11-27T18:17:53.639425Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471100421124101:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(53579937ns) {Configure:104(53571595ns) {ProcessConfigured:135(13117671ns) {StartInitialHandshake:157(13115671ns) {DropHandshakes:470(2121ns) {DropIncomingHandshake:442(732ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(533ns) PrepareNewSessionHandshake:477(2450ns) {SwitchToState:179(1226ns)}}}}}
2025-11-27T18:17:53.617633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471097945988640:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56126
2025-11-27T18:17:53.640589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471100223712386:2110] [node 5] ICP01 ready to work
2025-11-27T18:17:53.588687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471098009215820:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49956
2025-11-27T18:17:53.649839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471094954200856:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(268106305ns) {Configure:104(268097592ns) {ProcessConfigured:135(77336ns) {StartInitialHandshake:157(74220ns) {DropHandshakes:470(3258ns) {DropIncomingHandshake:442(707ns) DropOutgoingHandshake:457(421ns)} GenerateSessionVirtualId:988(950ns) PrepareNewSessionHandshake:477(2063ns) {SwitchToState:179(642ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.657238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471099608798636:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59675
2025-11-27T18:17:53.657466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471099608798637:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59675
2025-11-27T18:17:53.653380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471097945988640:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.653596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471093651021266:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.610412Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471098009215812:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(88988496ns) {Configure:104(88978899ns) {ProcessConfigured:135(76663ns) {StartInitialHandshake:157(70900ns) {DropHandshakes:470(2525ns) {DropIncomingHandshake:442(684ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(440ns) PrepareNewSessionHandshake:477(1902ns) {SwitchToState:179(699ns)}}}}}
2025-11-27T18:17:53.654212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471093651021268:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(77195937ns) {Configure:104(77192015ns) {ProcessConfigured:135(21651990ns) {StartInitialHandshake:157(21650331ns) {DropHandshakes:470(5511749ns) {DropIncomingHandshake:442(399ns) DropOutgoingHandshake:457(2618ns)} GenerateSessionVirtualId:988(4032ns) PrepareNewSessionHandshake:477(3070ns) {SwitchToState:179(390ns)}}}}}
2025-11-27T18:17:53.665683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471098009215820:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.672957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471099608798637:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.657774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471100223712375:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7442
2025-11-27T18:17:53.683577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471099249168224:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.683637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471099249168224:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:53.697288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471093714248452:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(187608338ns) {Configure:104(187600946ns) {ProcessConfigured:135(11312041ns) {StartInitialHandshake:157(11310162ns) {DropHandshakes:470(1915ns) {DropIncomingHandshake:442(578ns) DropOutgoingHandshake:457(362ns)} GenerateSessionVirtualId:988(351ns) PrepareNewSessionHandshake:477(2461ns) {SwitchToState:179(256ns)}}}}}
2025-11-27T18:17:53.702969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471097945988635:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.703169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471093651021267:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.717698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471099249168231:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38506
2025-11-27T18:17:53.673187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471099608798636:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.660033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471100223712385:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7442
2025-11-27T18:17:53.697640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471098009215827:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49956
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.735984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471099249168231:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.696827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471095313831258:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(278140851ns) {Configure:104(278130479ns) {ProcessConfigured:135(12624135ns) {StartInitialHandshake:157(12620823ns) {DropHandshakes:470(2795ns) {DropIncomingHandshake:442(726ns) DropOutgoingHandshake:457(471ns)} GenerateSessionVirtualId:988(870ns) PrepareNewSessionHandshake:477(3503ns) {SwitchToState:179(1506ns)}}}}}
2025-11-27T18:17:53.709552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471098009215819:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.746376Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471099249168224:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:53.663931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471095928745020:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(58459326ns) {Configure:104(58449572ns) {ProcessConfigured:135(213233ns) {StartInitialHandshake:157(210358ns) {DropHandshakes:470(2925ns) {DropIncomingHandshake:442(742ns) DropOutgoingHandshake:457(503ns)} GenerateSessionVirtualId:988(879ns) PrepareNewSessionHandshake:477(2456ns) {SwitchToState:179(640ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.758126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471097945988644:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56126
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.742711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471099608798630:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.769786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471097945988644:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.769972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471097945988613:2097] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.769567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471099249168222:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.769616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471099249168222:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.736541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471096126156742:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(194493539ns) {Configure:104(194486261ns) {ProcessConfigured:135(8576968ns) {StartInitialHandshake:157(8573619ns) {DropHandshakes:470(2644ns) {DropIncomingHandshake:442(716ns) DropOutgoingHandshake:457(494ns)} GenerateSessionVirtualId:988(1262ns) PrepareNewSessionHandshake:477(2750ns) {SwitchToState:179(1599ns)}}}}}
2025-11-27T18:17:53.715842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471093714248453:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.685956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471100223712385:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.785208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471097945988647:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56126
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.771702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471099249168213:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.719068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471098009215827:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.742760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471099608798630:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:53.705794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471100223712375:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.736760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471100421124108:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17461
2025-11-27T18:17:53.742959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471099608798628:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.788864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471097945988645:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56126
2025-11-27T18:17:53.800698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471097945988635:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:53.727610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471095928745019:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.771731Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471099249168213:2107] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.742296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471093714248454:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.736924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471100421124109:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17461
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.815546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471093651021267:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7145) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471097945988635:2114] held: no
2025-11-27T18:17:53.772456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471099249168220:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.742985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471099608798628:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.741783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471095928745019:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17787907ns) {IncomingHandshake:261(17785507ns) {DropIncomingHandshake:442(1502ns) PrepareNewSessionHandshake:477(4425ns) {SwitchToState:179(3527ns)} GenerateSessionVirtualId:988(841ns) IssueIncomingHandshakeReply:193(95492ns) {SwitchToState:179(1048ns)}}}
2025-11-27T18:17:53.773912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471098009215834:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49956
2025-11-27T18:17:53.781470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471098009215832:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49956
2025-11-27T18:17:53.786679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471093714248453:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7145) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471098009215819:2113] held: no
2025-11-27T18:17:53.787182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471098009215835:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49956
2025-11-27T18:17:53.797612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471098009215833:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49956
2025-11-27T18:17:53.737038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471100421124110:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17461
2025-11-27T18:17:53.746085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471099608798628:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:53.772497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:53.815742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471097945988640:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.744213Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471100223712380:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(93556320ns) {Configure:104(93550570ns) {ProcessConfigured:135(8712893ns) {StartInitialHandshake:157(8709118ns) {DropHandshakes:470(6845ns) {DropIncomingHandshake:442(4156ns) DropOutgoingHandshake:457(384ns)} GenerateSessionVirtualId:988(2255ns) PrepareNewSessionHandshake:477(3668ns) {SwitchToState:179(971ns)}}}}}
2025-11-27T18:17:53.800828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471098009215834:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.737164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471100421124111:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17461
2025-11-27T18:17:53.772541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471099249168220:2110] [node 6] ICP22 created new session: [1:7577471099249168235:2110]
2025-11-27T18:17:53.818413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471093651021267:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.818485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:53.818535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471093651021267:2049] [node 2] ICP22 created new session: [6:7577471097945988648:2049]
2025-11-27T18:17:53.818582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471097945988648:2049] [node 2] ICS09 handshake done sender: [6:7577471097945988635:2114] self: [6:7577471097945988643:0] peer: [2:7577471099608798629:0] socket: 324 qp: -1
2025-11-27T18:17:53.818618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471097945988648:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:53.818657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:53.818944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471093651021266:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.819000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:53.819041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471093651021266:2048] [node 1] ICP22 created new session: [6:7577471097945988650:2048]
2025-11-27T18:17:53.761737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471099608798630:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:53.819088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471097945988650:2048] [node 1] ICS09 handshake done sender: [6:7577471097945988640:2115] self: [6:7577471097945988642:0] peer: [1:7577471099249168223:0] socket: 337 qp: -1
2025-11-27T18:17:53.819117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471097945988650:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:53.819150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:53.857832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471097945988654:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56126
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.859184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471098009215818:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.764818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471100223712388:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.772585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471099249168235:2110] [node 6] ICS09 handshake done sender: [1:7577471099249168224:2112] self: [1:7577471099249168223:0] peer: [6:7577471097945988642:0] socket: 336 qp: -1
2025-11-27T18:17:53.770306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471099608798620:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.764846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471100223712388:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:53.905791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471097945988655:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56126
2025-11-27T18:17:53.746873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471100421124109:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.859210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471098009215818:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:53.772618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471099249168235:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:53.772651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:53.781960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471099249168213:2107] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:53.787576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471099249168222:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:53.787962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471094954200857:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.788020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:53.788069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471094954200857:2049] [node 3] ICP22 created new session: [1:7577471099249168240:2049]
2025-11-27T18:17:53.788114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471099249168240:2049] [node 3] ICS09 handshake done sender: [1:7577471099249168213:2107] self: [1:7577471099249168212:0] peer: [3:7577471100223712396:0] socket: 318 qp: -1
2025-11-27T18:17:53.788147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471099249168240:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:53.788180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:53.788594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471099249168219:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.788623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:53.797309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471099249168219:2109] [node 5] ICP22 created new session: [1:7577471099249168242:2109]
2025-11-27T18:17:53.825916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471099249168243:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38506
2025-11-27T18:17:53.828886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471099249168242:2109] [node 5] ICS09 handshake done sender: [1:7577471099249168222:2111] self: [1:7577471099249168221:0] peer: [5:7577471100421124119:0] socket: 335 qp: -1
2025-11-27T18:17:53.840933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471099249168242:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:53.840998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:53.864945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471099249168243:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.877995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471099249168219:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(89186455ns) {HandleHandshakeStatus:329(89184050ns) {DropIncomingHandshake:442(244ns) DropOutgoingHandshake:457(165ns) SwitchToState:179(278ns) ProcessPendingSessionEvents:495(13636041ns) {ForwardSessionEventToSession:612(6084511ns) ForwardSessionEventToSession:612(7478144ns) ForwardSessionEventToSession:612(6093ns) ForwardSessionEventToSession:612(2857ns) ForwardSessionEventToSession:612(21971ns) ForwardSessionEventToSession:612(3238ns) ForwardSessionEventToSession:612(8582ns) ForwardSessionEventToSession:612(3044ns) ForwardSessionEventToSession:612(3464ns) ForwardSessionEventToSession:612(5600ns)}}}
2025-11-27T18:17:53.878117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471099249168234:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38506
2025-11-27T18:17:53.886980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471099249168228:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.887026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471099249168228:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:53.914750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471099249168228:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.746997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471100421124111:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.928855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471097945988644:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:53.770368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:53.768765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471100223712388:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.860946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471098009215819:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.770414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471099608798620:2108] [node 4] ICP22 created new session: [2:7577471099608798644:2108]
2025-11-27T18:17:53.747083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471100421124112:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17461
2025-11-27T18:17:53.933597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471099249168230:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.933644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471099249168230:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:53.934419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471099249168234:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.934662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471099249168218:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.934708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:53.934769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471099249168218:2108] [node 4] ICP22 created new session: [1:7577471099249168249:2108]
2025-11-27T18:17:53.934826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471099249168249:2108] [node 4] ICS09 handshake done sender: [1:7577471099249168228:2113] self: [1:7577471099249168227:0] peer: [4:7577471098009215838:0] socket: 368 qp: -1
2025-11-27T18:17:53.934868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471099249168249:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:53.934904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:53.941631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471099249168230:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:53.942576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471094954200856:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.942634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:53.942684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471094954200856:2048] [node 2] ICP22 created new session: [1:7577471099249168252:2048]
2025-11-27T18:17:53.942751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471099249168252:2048] [node 2] ICS09 handshake done sender: [1:7577471099249168230:2114] self: [1:7577471099249168229:0] peer: [2:7577471099608798656:0] socket: 349 qp: -1
2025-11-27T18:17:53.942782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471099249168252:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:53.942828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:53.957124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471099249168219:2109] [node 5] ICP18 handshake (actor [1:7577471099249168234:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:53.861214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471098009215827:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:53.774227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471100223712384:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.747356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471096126156741:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.774260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471100223712384:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:53.776862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471100223712386:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.776910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:53.776954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471100223712386:2110] [node 5] ICP22 created new session: [3:7577471100223712400:2110]
2025-11-27T18:17:53.777000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471100223712400:2110] [node 5] ICS09 handshake done sender: [3:7577471100223712388:2114] self: [3:7577471100223712387:0] peer: [5:7577471100421124122:0] socket: 348 qp: -1
2025-11-27T18:17:53.777032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471100223712400:2110] [node 5] ICS10 traffic start
2025-11-27T18:17:53.777075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:53.797040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471100223712384:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:53.851154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471100223712379:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.851236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:53.851299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471100223712379:2108] [node 4] ICP22 created new session: [3:7577471100223712405:2108]
2025-11-27T18:17:53.851356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471100223712405:2108] [node 4] ICS09 handshake done sender: [3:7577471100223712384:2112] self: [3:7577471100223712383:0] peer: [4:7577471098009215831:0] socket: 347 qp: -1
2025-11-27T18:17:53.851392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471100223712405:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:53.851434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:53.852006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471095928745019:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27286) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471100223712375:2107] held: no
2025-11-27T18:17:53.858130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471100223712399:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7442
2025-11-27T18:17:53.858472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471100223712404:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7442
2025-11-27T18:17:53.858914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471100223712407:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7442
2025-11-27T18:17:53.874908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471100223712404:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.883403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471100223712375:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:53.893628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471095928745020:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.902758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471100223712407:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.902807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471095928745019:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.902857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:53.902909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471095928745019:2048] [node 1] ICP22 created new session: [3:7577471100223712410:2048]
2025-11-27T18:17:53.902955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471100223712410:2048] [node 1] ICS09 handshake done sender: [3:7577471100223712375:2107] self: [3:7577471100223712396:0] peer: [1:7577471099249168212:0] socket: 319 qp: -1
2025-11-27T18:17:53.902999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471100223712410:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:53.903063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:53.928222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471100223712413:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7442
2025-11-27T18:17:53.962752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471100223712404:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:53.982423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471095928745020:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.982508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:53.982564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471095928745020:2049] [node 2] ICP22 created new session: [3:7577471100223712414:2049]
2025-11-27T18:17:53.982613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471100223712414:2049] [node 2] ICS09 handshake done sender: [3:7577471100223712404:2118] self: [3:7577471100223712409:0] peer: [2:7577471099608798625:0] socket: 382 qp: -1
2025-11-27T18:17:53.982645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471100223712414:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:53.982673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:53.993351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471099249168247:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38506
2025-11-27T18:17:53.958695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471097945988613:2097] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.862135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471093714248454:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4637) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471098009215827:2116] held: no
2025-11-27T18:17:53.770464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471099608798644:2108] [node 4] ICS09 handshake done sender: [2:7577471099608798628:2116] self: [2:7577471099608798627:0] peer: [4:7577471098009215830:0] socket: 321 qp: -1
2025-11-27T18:17:53.999120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471099249168218:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:53.862236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471093714248454:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.862279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:53.862476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471093714248452:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.862700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471093714248453:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.862737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:53.862781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471093714248453:2049] [node 2] ICP22 created new session: [4:7577471098009215840:2049]
2025-11-27T18:17:53.862828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471098009215840:2049] [node 2] ICS09 handshake done sender: [4:7577471098009215819:2113] self: [4:7577471098009215830:0] peer: [2:7577471099608798627:0] socket: 329 qp: -1
2025-11-27T18:17:53.862864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471098009215840:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:53.862895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:53.864784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471098009215818:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:53.873633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471093714248454:2050] [node 3] ICP22 created new session: [4:7577471098009215837:2050]
2025-11-27T18:17:53.873683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471098009215837:2050] [node 3] ICS09 handshake done sender: [4:7577471098009215827:2116] self: [4:7577471098009215831:0] peer: [3:7577471100223712383:0] socket: 351 qp: -1
2025-11-27T18:17:53.873715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471098009215837:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:53.873759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:53.903894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471098009215811:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.903919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:53.903955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471098009215811:2108] [node 5] ICP22 created new session: [4:7577471098009215845:2108]
2025-11-27T18:17:53.904004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471098009215845:2108] [node 5] ICS09 handshake done sender: [4:7577471098009215818:2112] self: [4:7577471098009215817:0] peer: [5:7577471100421124125:0] socket: 333 qp: -1
2025-11-27T18:17:53.904042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471098009215845:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:53.904069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:53.904702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471098009215844:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49956
2025-11-27T18:17:53.904969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471098009215832:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.905088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471098009215811:2108] [node 5] ICP18 handshake (actor [4:7577471098009215832:2118]) is too old
2025-11-27T18:17:53.926202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471098009215834:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:53.936807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471093714248452:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.770500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471099608798644:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:53.965270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471097945988656:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56126
2025-11-27T18:17:53.978275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:53.992028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471097945988656:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.995045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471097945988613:2097] [node 5] ICP22 created new session: [6:7577471097945988657:2097]
2025-11-27T18:17:53.995175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471097945988655:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.756793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471100421124112:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.770534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:53.786267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471099608798632:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.786305Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471099608798632:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:53.788829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471099608798624:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.788875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:53.788921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471099608798624:2107] [node 6] ICP22 created new session: [2:7577471099608798648:2107]
2025-11-27T18:17:53.788957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471099608798648:2107] [node 6] ICS09 handshake done sender: [2:7577471099608798630:2117] self: [2:7577471099608798629:0] peer: [6:7577471097945988643:0] socket: 322 qp: -1
2025-11-27T18:17:53.788986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471099608798648:2107] [node 6] ICS10 traffic start
2025-11-27T18:17:53.789016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:53.796483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471099608798647:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59675
2025-11-27T18:17:53.797622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471099608798632:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:53.822132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471099608798624:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(33062432ns) {HandleHandshakeStatus:329(33060449ns) {DropIncomingHandshake:442(10234ns) DropOutgoingHandshake:457(192ns) SwitchToState:179(195ns) ProcessPendingSessionEvents:495(18012617ns) {ForwardSessionEventToSession:612(17956743ns) ForwardSessionEventToSession:612(5726ns) ForwardSessionEventToSession:612(3651ns) ForwardSessionEventToSession:612(6392ns) ForwardSessionEventToSession:612(2970ns) ForwardSessionEventToSession:612(3199ns) ForwardSessionEventToSession:612(9970ns) ForwardSessionEventToSession:612(3011ns) ForwardSessionEventToSession:612(5380ns) ForwardSessionEventToSession:612(2776ns)}}}
2025-11-27T18:17:53.829724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471099608798623:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.837857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471099608798652:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59675
2025-11-27T18:17:53.846669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:53.864959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471099608798623:2113] [node 5] ICP22 created new session: [2:7577471099608798653:2113]
2025-11-27T18:17:53.877361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471099608798653:2113] [node 5] ICS09 handshake done sender: [2:7577471099608798632:2118] self: [2:7577471099608798631:0] peer: [5:7577471100421124123:0] socket: 323 qp: -1
2025-11-27T18:17:53.877419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471099608798653:2113] [node 5] ICS10 traffic start
2025-11-27T18:17:53.879159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471099608798647:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.879531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:53.881391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471099608798652:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.905634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471095313831258:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.919647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471099608798623:2113] [node 5] ICP35 event processing took too much time PendingConnection:279(89661453ns) {HandleHandshakeStatus:329(89657576ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(544ns) SwitchToState:179(639ns) ProcessPendingSessionEvents:495(5855610ns) {ForwardSessionEventToSession:612(5795708ns) ForwardSessionEventToSession:612(6858ns) ForwardSessionEventToSession:612(2682ns) ForwardSessionEventToSession:612(6407ns) ForwardSessionEventToSession:612(2796ns) ForwardSessionEventToSession:612(2553ns) ForwardSessionEventToSession:612(11148ns) ForwardSessionEventToSession:612(3046ns) ForwardSessionEventToSession:612(4858ns) ForwardSessionEventToSession:612(2879ns)}}}
2025-11-27T18:17:53.925676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471099608798626:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.925720Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471099608798626:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:53.936038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471099608798658:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59675
2025-11-27T18:17:53.942121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471099608798652:2123] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:53.943599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471095313831258:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.943642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:53.943683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471095313831258:2048] [node 1] ICP22 created new session: [2:7577471099608798659:2048]
2025-11-27T18:17:53.943728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471099608798659:2048] [node 1] ICS09 handshake done sender: [2:7577471099608798652:2123] self: [2:7577471099608798656:0] peer: [1:7577471099249168229:0] socket: 381 qp: -1
2025-11-27T18:17:53.943769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471099608798659:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:53.943795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:53.947408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471099608798626:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:53.977935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471095313831259:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.999512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:54.019727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471095313831259:2049] [node 3] ICP22 created new session: [2:7577471099608798661:2049]
2025-11-27T18:17:53.939401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:53.954480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471093714248452:2048] [node 1] ICP22 created new session: [4:7577471098009215847:2048]
2025-11-27T18:17:53.974672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471098009215847:2048] [node 1] ICS09 handshake done sender: [4:7577471098009215834:2120] self: [4:7577471098009215838:0] peer: [1:7577471099249168227:0] socket: 369 qp: -1
2025-11-27T18:17:53.981603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471098009215847:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:53.992276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:54.023200Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471093714248452:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(92904608ns) {HandleHandshakeStatus:329(92901699ns) {DropIncomingHandshake:442(2395ns) DropOutgoingHandshake:457(14879ns) SwitchToState:179(693ns) ProcessPendingSessionEvents:495(16760734ns) {ForwardSessionEventToSession:612(16703431ns) ForwardSessionEventToSession:612(7032ns) ForwardSessionEventToSession:612(3129ns) ForwardSessionEventToSession:612(5651ns) ForwardSessionEventToSession:612(3615ns) ForwardSessionEventToSession:612(4922ns) ForwardSessionEventToSession:612(2856ns) ForwardSessionEventToSession:612(3568ns) ForwardSessionEventToSession:612(11695ns) ForwardSessionEventToSession:612(2725ns)}}}
2025-11-27T18:17:54.020343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471093651021268:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:54.021285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471097945988657:2097] [node 5] ICS09 handshake done sender: [6:7577471097945988644:2117] self: [6:7577471097945988646:0] peer: [5:7577471100421124106:0] socket: 360 qp: -1
2025-11-27T18:17:54.024495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471097945988657:2097] [node 5] ICS10 traffic start
2025-11-27T18:17:54.024543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:54.024324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471099608798661:2049] [node 3] ICS09 handshake done sender: [2:7577471099608798626:2115] self: [2:7577471099608798625:0] peer: [3:7577471100223712409:0] socket: 374 qp: -1
2025-11-27T18:17:54.024984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471099608798661:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:53.763461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471096126156743:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(201118541ns) {Configure:104(201111547ns) {ProcessConfigured:135(7131880ns) {StartInitialHandshake:157(7129071ns) {DropHandshakes:470(3655ns) {DropIncomingHandshake:442(719ns) DropOutgoingHandshake:457(515ns)} GenerateSessionVirtualId:988(762ns) PrepareNewSessionHandshake:477(3096ns) {SwitchToState:179(1391ns)}}}}}
2025-11-27T18:17:53.763907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471100421124108:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.764156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471096126156743:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.764375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471096126156742:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.769244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471100421124124:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17461
2025-11-27T18:17:53.769609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471100421124110:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.792335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471100421124100:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:53.817990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471100421124112:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:53.820784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471096126156743:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.820839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:53.820895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471096126156743:2050] [node 3] ICP22 created new session: [5:7577471100421124128:2050]
2025-11-27T18:17:53.820938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471100421124128:2050] [node 3] ICS09 handshake done sender: [5:7577471100421124112:2117] self: [5:7577471100421124122:0] peer: [3:7577471100223712387:0] socket: 352 qp: -1
2025-11-27T18:17:53.820967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471100421124128:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:53.820998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:53.838082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471100421124107:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:53.838139Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471100421124107:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:53.854531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471100421124126:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17461
2025-11-27T18:17:53.855017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471100421124127:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17461
2025-11-27T18:17:53.857419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471100421124108:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:53.861838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471100421124111:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:53.890387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471096126156742:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.890458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:53.890522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471096126156742:2049] [node 2] ICP22 created new session: [5:7577471100421124131:2049]
2025-11-27T18:17:53.890575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471100421124131:2049] [node 2] ICS09 handshake done sender: [5:7577471100421124108:2113] self: [5:7577471100421124123:0] peer: [2:7577471099608798631:0] socket: 339 qp: -1
2025-11-27T18:17:53.890617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471100421124131:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:53.890657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:53.891356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471100421124107:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:53.892936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471096126156741:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:53.892986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:53.893037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471096126156741:2048] [node 1] ICP22 created new session: [5:7577471100421124134:2048]
2025-11-27T18:17:53.893095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471100421124134:2048] [node 1] ICS09 handshake done sender: [5:7577471100421124111:2116] self: [5:7577471100421124119:0] peer: [1:7577471099249168221:0] socket: 342 qp: -1
2025-11-27T18:17:53.893132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471100421124134:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:53.893167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:53.937529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471100421124101:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:53.937622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:53.937675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471100421124101:2108] [node 6] ICP22 created new session: [5:7577471100421124137:2108]
2025-11-27T18:17:53.937725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471100421124137:2108] [node 6] ICS09 handshake done sender: [5:7577471100421124107:2112] self: [5:7577471100421124106:0] peer: [6:7577471097945988646:0] socket: 355 qp: -1
2025-11-27T18:17:53.937763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471100421124137:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:53.937805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:53.963157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471100421124100:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9411) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471100421124110:2115] held: no
2025-11-27T18:17:53.963461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471100421124133:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17461
2025-11-27T18:17:53.977608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471100421124110:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:54.000197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471096126156741:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27638093ns) {HandleHandshakeStatus:329(27634975ns) {DropIncomingHandshake:442(1022ns) DropOutgoingHandshake:457(5243ns) SwitchToState:179(926ns) ProcessPendingSessionEvents:495(27258562ns) {ForwardSessionEventToSession:612(8284962ns) ForwardSessionEventToSession:612(3787221ns) ForwardSessionEventToSession:612(7775ns) ForwardSessionEventToSession:612(5665ns) ForwardSessionEventToSession:612(2897ns) ForwardSessionEventToSession:612(4970ns) ForwardSessionEventToSession:612(2611ns) ForwardSessionEventToSession:612(1972ns) ForwardSessionEventToSession:612(10244851ns) ForwardSessionEventToSession:612(4889745ns)}}}
2025-11-27T18:17:54.000261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471100421124100:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:54.000309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:54.000363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471100421124100:2109] [node 4] ICP22 created new session: [5:7577471100421124139:2109]
2025-11-27T18:17:54.000409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471100421124139:2109] [node 4] ICS09 handshake done sender: [5:7577471100421124110:2115] self: [5:7577471100421124125:0] peer: [4:7577471098009215817:0] socket: 341 qp: -1
2025-11-27T18:17:54.000452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471100421124139:2109] [node 4] ICS10 traffic start
2025-11-27T18:17:54.000489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:54.025030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471097945988620:2100] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:54.033825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471102240955957:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56126
2025-11-27T18:17:54.041558Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471097945988613:2097] [node 5] ICP35 event processing took too much time PendingConnection:279(93765741ns) {HandleHandshakeStatus:329(93761743ns) {DropIncomingHandshake:442(3926ns) DropOutgoingHandshake:457(11887ns) SwitchToState:179(692ns) ProcessPendingSessionEvents:495(15789082ns) {ForwardSessionEventToSession:612(50980ns) ForwardSessionEventToSession:612(7755068ns) ForwardSessionEventToSession:612(19381ns) ForwardSessionEventToSession:612(7891209ns) ForwardSessionEventToSession:612(4492ns) ForwardSessionEventToSession:612(17538ns) ForwardSessionEventToSession:612(10491ns) ForwardSessionEventToSession:612(3123ns) ForwardSessionEventToSession:612(3288ns) ForwardSessionEventToSession:612(8755ns)}}}
2025-11-27T18:17:54.028095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:54.027920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471100223712397:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:54.027776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471098009215825:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:54.027814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471098009215825:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:54.037665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471098009215825:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:54.047823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471095313831259:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(69679378ns) {HandleHandshakeStatus:329(69675510ns) {DropIncomingHandshake:442(14622ns) DropOutgoingHandshake:457(549ns) SwitchToState:179(720ns) ProcessPendingSessionEvents:495(404773ns) {ForwardSessionEventToSession:612(348482ns) ForwardSessionEventToSession:612(10429ns) ForwardSessionEventToSession:612(3088ns) ForwardSessionEventToSession:612(3085ns) ForwardSessionEventToSession:612(5216ns) ForwardSessionEventToSession:612(2830ns) ForwardSessionEventToSession:612(3280ns) ForwardSessionEventToSession:612(4949ns) ForwardSessionEventToSession:612(3439ns) ForwardSessionEventToSession:612(9256ns)}}}
2025-11-27T18:17:54.027964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471100223712397:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:54.052046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471100223712397:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:54.046924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471097945988656:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:54.049544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471102240955958:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56126
2025-11-27T18:17:54.052550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471100223712380:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:54.052639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:54.052692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471100223712380:2109] [node 6] ICP22 created new session: [3:7577471104518679713:2109]
2025-11-27T18:17:54.052746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471104518679713:2109] [node 6] ICS09 handshake done sender: [3:7577471100223712397:2116] self: [3:7577471100223712395:0] peer: [6:7577471102240955954:0] socket: 383 qp: -1
2025-11-27T18:17:54.052783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471104518679713:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:54.052822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:54.049632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471098009215812:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:54.057106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471093651021268:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4637) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471097945988655:2121] held: no
2025-11-27T18:17:54.057548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471097945988620:2100] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:54.057629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:54.057684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471097945988620:2100] [node 4] ICP22 created new session: [6:7577471102240955959:2100]
2025-11-27T18:17:54.057736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471102240955959:2100] [node 4] ICS09 handshake done sender: [6:7577471097945988656:2122] self: [6:7577471102240955955:0] peer: [4:7577471098009215824:0] socket: 387 qp: -1
2025-11-27T18:17:54.057781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471102240955959:2100] [node 4] ICS10 traffic start
2025-11-27T18:17:54.057819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:54.058335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471097945988655:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:54.059054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:54.060165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471098009215812:2109] [node 6] ICP22 created new session: [4:7577471102304183146:2109]
2025-11-27T18:17:54.060215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471102304183146:2109] [node 6] ICS09 handshake done sender: [4:7577471098009215825:2115] self: [4:7577471098009215824:0] peer: [6:7577471102240955955:0] socket: 385 qp: -1
2025-11-27T18:17:54.060252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471102304183146:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:54.060297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:54.062425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471093651021268:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:54.062495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:54.062543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471093651021268:2050] [node 3] ICP22 created new session: [6:7577471102240955961:2050]
2025-11-27T18:17:54.062581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471102240955961:2050] [node 3] ICS09 handshake done sender: [6:7577471097945988655:2121] self: [6:7577471102240955954:0] peer: [3:7577471100223712395:0] socket: 384 qp: -1
2025-11-27T18:17:54.062612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471102240955961:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:54.062639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:54.136406Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:54.136637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471099608798659:2048] [node 1] ICS01 socket: 381 reason# EndOfStream
2025-11-27T18:17:54.136676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471095313831258:2048] [node 1] ICP30 unregister session Session# [2:7577471099608798659:2048] VirtualId# [2:7577471099608798656:0]
2025-11-27T18:17:54.136711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471099608798659:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.136786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:54.136827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:54.139472Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:54.139687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471100223712410:2048] [node 1] ICS01 socket: 319 reason# EndOfStream
2025-11-27T18:17:54.139722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471095928745019:2048] [node 1] ICP30 unregister session Session# [3:7577471100223712410:2048] VirtualId# [3:7577471100223712396:0]
2025-11-27T18:17:54.139761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471100223712410:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.139836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:54.139870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:54.145867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:54.146127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471100421124134:2048] [node 1] ICS07 socket disconnect 342 reason# EndOfStream
2025-11-27T18:17:54.146166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471100421124134:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.146235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:54.146275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471100421124134:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:54.149382Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:54.149625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471098009215847:2048] [node 1] ICS07 socket disconnect 369 reason# EndOfStream
2025-11-27T18:17:54.149653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471098009215847:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.149737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:54.149784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471098009215847:2048] [node 1] ICS15 start handshake
2025-11-27T18:17:54.149382Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:54.149602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471097945988650:2048] [node 1] ICS01 socket: 337 reason# EndOfStream
2025-11-27T18:17:54.149658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471093651021266:2048] [node 1] ICP30 unregister session Session# [6:7577471097945988650:2048] VirtualId# [6:7577471097945988642:0]
2025-11-27T18:17:54.149690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471097945988650:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.149760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:54.149800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:54.177641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471093714248452:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27286) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:54.177715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471098009215847:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:54.177759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471093714248452:2048] [node 1] ICP30 unregister session Session# [4:7577471098009215847:2048] VirtualId# [4:7577471098009215838:0]
2025-11-27T18:17:54.177819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:54.177868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471093714248452:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27286) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:54.177888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27286) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:54.177523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471096126156741:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27286) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:54.178353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471100421124134:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:54.178405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471096126156741:2048] [node 1] ICP30 unregister session Session# [5:7577471100421124134:2048] VirtualId# [5:7577471100421124119:0]
2025-11-27T18:17:54.178445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:54.178480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471096126156741:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27286) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:54.178494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27286) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:54.193585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471093714248452:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:54.194042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471096126156741:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:17:54.215275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:54.215555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471100223712414:2049] [node 2] ICS01 socket: 382 reason# EndOfStream
2025-11-27T18:17:54.215607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471095928745020:2049] [node 2] ICP30 unregister session Session# [3:7577471100223712414:2049] VirtualId# [3:7577471100223712409:0]
2025-11-27T18:17:54.215648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471100223712414:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.215766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:54.215821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:54.221456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:54.225338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471098009215840:2049] [node 2] ICS01 socket: 329 reason# EndOfStream
2025-11-27T18:17:54.225396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471093714248453:2049] [node 2] ICP30 unregister session Session# [4:7577471098009215840:2049] VirtualId# [4:7577471098009215830:0]
2025-11-27T18:17:54.225439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471098009215840:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.225536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:54.225600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:54.228854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:54.229112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471100421124131:2049] [node 2] ICS01 socket: 339 reason# EndOfStream
2025-11-27T18:17:54.229170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471096126156742:2049] [node 2] ICP30 unregister session Session# [5:7577471100421124131:2049] VirtualId# [5:7577471100421124123:0]
2025-11-27T18:17:54.229197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471100421124131:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.221464Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:54.229971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:54.221666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471097945988648:2049] [node 2] ICS01 socket: 324 reason# EndOfStream
2025-11-27T18:17:54.230070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:54.228764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471093651021267:2049] [node 2] ICP30 unregister session Session# [6:7577471097945988648:2049] VirtualId# [6:7577471097945988643:0]
2025-11-27T18:17:54.229621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471097945988648:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.229707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:54.229753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:54.274490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:54.274055Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:54.274235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471102240955961:2050] [node 3] ICS01 socket: 384 reason# EndOfStream
2025-11-27T18:17:54.273501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:54.273742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471098009215837:2050] [node 3] ICS01 socket: 351 reason# EndOfStream
2025-11-27T18:17:54.274270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471093651021268:2050] [node 3] ICP30 unregister session Session# [6:7577471102240955961:2050] VirtualId# [6:7577471102240955954:0]
2025-11-27T18:17:54.274292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471102240955961:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.274367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:54.274401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:54.274673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471100421124128:2050] [node 3] ICS07 socket disconnect 352 reason# EndOfStream
2025-11-27T18:17:54.274713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471100421124128:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.274776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:54.274815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471100421124128:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:54.273788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471093714248454:2050] [node 3] ICP30 unregister session Session# [4:7577471098009215837:2050] VirtualId# [4:7577471098009215831:0]
2025-11-27T18:17:54.273825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471098009215837:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.273906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:54.273953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:54.277617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471096126156743:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4637) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:54.277686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471100421124128:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:54.277717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471096126156743:2050] [node 3] ICP30 unregister session Session# [5:7577471100421124128:2050] VirtualId# [5:7577471100421124122:0]
2025-11-27T18:17:54.277766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:54.277800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471096126156743:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4637) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:54.277816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4637) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:54.287087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471096126156743:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:54.357468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:54.357717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471100421124139:2109] [node 4] ICS01 socket: 341 reason# EndOfStream
2025-11-27T18:17:54.357761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471100421124100:2109] [node 4] ICP30 unregister session Session# [5:7577471100421124139:2109] VirtualId# [5:7577471100421124125:0]
2025-11-27T18:17:54.357874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471100421124139:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.357962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:54.358009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:54.359445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:54.359676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471102240955959:2100] [node 4] ICS01 socket: 387 reason# EndOfStream
2025-11-27T18:17:54.359713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471097945988620:2100] [node 4] ICP30 unregister session Session# [6:7577471102240955959:2100] VirtualId# [6:7577471102240955955:0]
2025-11-27T18:17:54.359746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471102240955959:2100] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.359839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:54.359878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:54.405490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:54.405717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471097945988657:2097] [node 5] ICS07 socket disconnect 360 reason# EndOfStream
2025-11-27T18:17:54.405757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471097945988657:2097] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:54.405817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:54.405850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471097945988657:2097] [node 5] ICS15 start handshake
2025-11-27T18:17:54.407017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471097945988613:2097] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13636) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:54.407059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471097945988657:2097] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:54.407089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471097945988613:2097] [node 5] ICP30 unregister session Session# [6:7577471097945988657:2097] VirtualId# [6:7577471097945988646:0]
2025-11-27T18:17:54.407121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:17:54.407142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471097945988613:2097] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13636) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:54.407153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13636) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:54.417518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471097945988613:2097] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:54.548370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471105427692153:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:54.548412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471105427692154:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:54.633579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471103187952886:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:54.633641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471103187952887:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:54.693567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471104126858574:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:54.693628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471104126858575:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:54.709880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471103796657878:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:54.709925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471103796657879:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:54.709951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471103796657880:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:54.749924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471105497014714:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:54.749978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471105497014715:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:54.750006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471105497014716:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:54.791434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471103194601497:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:54.791478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471103194601498:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:54.791507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471103194601499:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:55.418151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471109722659511:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:55.455646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471107489568831:2076] [node 5] ICP01 ready to work
2025-11-27T18:17:55.557294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471109722659513:2108] [node 6] ICP01 ready to work
2025-11-27T18:17:55.627329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471105427692153:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41698645ns) {Configure:104(41693062ns) {ProcessConfigured:135(74611ns) {StartInitialHandshake:157(71384ns) {DropHandshakes:470(2502ns) {DropIncomingHandshake:442(605ns) DropOutgoingHandshake:457(317ns)} GenerateSessionVirtualId:988(1687ns) PrepareNewSessionHandshake:477(1328ns) {SwitchToState:179(486ns)}}}}}
2025-11-27T18:17:55.637343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471109722659514:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:55.641670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471107489568856:2097] [node 4] ICP01 ready to work
2025-11-27T18:17:55.621468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471109791982073:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:55.569667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471107482920247:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:55.605993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471108421825934:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.605856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471107482920248:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:55.641420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471108091625237:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:55.645949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471108091625238:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:55.606265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471107482920249:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:55.585663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471108421825933:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:55.701838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471103796657879:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(80111331ns) {Configure:104(80102880ns) {ProcessConfigured:135(85568ns) {StartInitialHandshake:157(81724ns) {DropHandshakes:470(3047ns) {DropIncomingHandshake:442(544ns) DropOutgoingHandshake:457(648ns)} GenerateSessionVirtualId:988(1206ns) PrepareNewSessionHandshake:477(1542ns) {SwitchToState:179(488ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.706078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471108091625245:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25664
2025-11-27T18:17:55.706175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471108091625246:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25664
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.669626Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471107482920247:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25492439ns) {Configure:104(25488902ns) {ProcessConfigured:135(23227765ns) {StartInitialHandshake:157(23223992ns) {DropHandshakes:470(1724ns) {DropIncomingHandshake:442(407ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(1103ns) PrepareNewSessionHandshake:477(4424ns) {SwitchToState:179(2182ns)}}}}}
2025-11-27T18:17:55.665708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471107489568859:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20005
2025-11-27T18:17:55.684747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471108421825935:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.730958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471109722659514:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(85072674ns) {Configure:104(85065913ns) {ProcessConfigured:135(71351ns) {StartInitialHandshake:157(66760ns) {DropHandshakes:470(2869ns) {DropIncomingHandshake:442(600ns) DropOutgoingHandshake:457(442ns)} GenerateSessionVirtualId:988(991ns) PrepareNewSessionHandshake:477(1045ns) {SwitchToState:179(264ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.732628Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471103796657878:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(279948211ns) {Configure:104(279941682ns) {ProcessConfigured:135(14805209ns) {StartInitialHandshake:157(14802819ns) {DropHandshakes:470(3118ns) {DropIncomingHandshake:442(545ns) DropOutgoingHandshake:457(568ns)} GenerateSessionVirtualId:988(1338ns) PrepareNewSessionHandshake:477(3430ns) {SwitchToState:179(1664ns)}}}}}
2025-11-27T18:17:55.734313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471108091625246:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.734605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471108091625251:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25664
2025-11-27T18:17:55.734686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471103796657879:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:55.693827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471109791982074:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.763204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471104126858574:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(85582078ns) {Configure:104(85568330ns) {ProcessConfigured:135(77473ns) {StartInitialHandshake:157(74258ns) {DropHandshakes:470(2928ns) {DropIncomingHandshake:442(500ns) DropOutgoingHandshake:457(427ns)} GenerateSessionVirtualId:988(1190ns) PrepareNewSessionHandshake:477(1932ns) {SwitchToState:179(525ns)}}}}}
2025-11-27T18:17:55.765593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471104126858575:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(340473598ns) {Configure:104(340468675ns) {ProcessConfigured:135(13044865ns) {StartInitialHandshake:157(13043298ns) {DropHandshakes:470(3249ns) {DropIncomingHandshake:442(569ns) DropOutgoingHandshake:457(877ns)} GenerateSessionVirtualId:988(905ns) PrepareNewSessionHandshake:477(2295ns) {SwitchToState:179(115ns)}}}}}
2025-11-27T18:17:55.711393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471107489568864:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20005
2025-11-27T18:17:55.670117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471107482920250:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57152
2025-11-27T18:17:55.757651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471108091625245:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.753424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471105427692154:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(305859625ns) {Configure:104(299002767ns) {ProcessConfigured:135(20597029ns) {StartInitialHandshake:157(20591205ns) {DropHandshakes:470(2665ns) {DropIncomingHandshake:442(531ns) DropOutgoingHandshake:457(312ns)} GenerateSessionVirtualId:988(817ns) PrepareNewSessionHandshake:477(2876ns) {SwitchToState:179(497ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.777973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471108421825941:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37665
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.714677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471103194601498:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(308253282ns) {Configure:104(308246271ns) {ProcessConfigured:135(56795ns) {StartInitialHandshake:157(53688ns) {DropHandshakes:470(3018ns) {DropIncomingHandshake:442(777ns) DropOutgoingHandshake:457(352ns)} GenerateSessionVirtualId:988(906ns) PrepareNewSessionHandshake:477(1447ns) {SwitchToState:179(513ns)}}}}}
2025-11-27T18:17:55.778969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471108421825941:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.779120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471104126858574:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:55.710467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471107482920257:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57152
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.776194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471109722659526:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.802155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471105497014714:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(192559721ns) {Configure:104(192553076ns) {ProcessConfigured:135(73224ns) {StartInitialHandshake:157(70783ns) {DropHandshakes:470(2443ns) {DropIncomingHandshake:442(499ns) DropOutgoingHandshake:457(291ns)} GenerateSessionVirtualId:988(1233ns) PrepareNewSessionHandshake:477(1500ns) {SwitchToState:179(455ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.778411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471108091625255:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25664
2025-11-27T18:17:55.738552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471107489568869:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20005
2025-11-27T18:17:55.732046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471107482920258:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57152
2025-11-27T18:17:55.809355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471109722659522:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.774011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471107482920254:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.774055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471107482920254:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:55.775275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471107482920259:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57152
2025-11-27T18:17:55.785614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471107482920254:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.766725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471107489568864:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.826499Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471108421825934:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(141498095ns) {Configure:104(141493082ns) {ProcessConfigured:135(4774062ns) {StartInitialHandshake:157(4771310ns) {DropHandshakes:470(2837ns) {DropIncomingHandshake:442(596ns) DropOutgoingHandshake:457(406ns)} GenerateSessionVirtualId:988(1184ns) PrepareNewSessionHandshake:477(2706ns) {SwitchToState:179(1545ns)}}}}}
2025-11-27T18:17:55.802662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471109791982081:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61728
2025-11-27T18:17:55.779665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471108091625244:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.835129Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471103187952886:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(356178395ns) {Configure:104(356173571ns) {ProcessConfigured:135(8018630ns) {StartInitialHandshake:157(8015869ns) {DropHandshakes:470(1293ns) {DropIncomingHandshake:442(241ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(340ns) PrepareNewSessionHandshake:477(2339ns) {SwitchToState:179(1266ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.809407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471109722659522:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:55.766858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471107489568856:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.810897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471109791982084:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61728
2025-11-27T18:17:55.779697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471108091625244:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:55.816105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471109722659526:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.835274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471107482920250:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.811024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471109791982085:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61728
2025-11-27T18:17:55.889448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471108421825940:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.779352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471107489568871:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20005
2025-11-27T18:17:55.835365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471107482920257:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.889502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471108421825940:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:55.901850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471108421825933:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(239644234ns) {Configure:104(239635701ns) {ProcessConfigured:135(34487389ns) {StartInitialHandshake:157(34483396ns) {DropHandshakes:470(1578ns) {DropIncomingHandshake:442(299ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(248ns) PrepareNewSessionHandshake:477(3354ns) {SwitchToState:179(1435ns)}}}}}
2025-11-27T18:17:55.926458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471108421825950:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37665
2025-11-27T18:17:55.926931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471108421825953:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37665
2025-11-27T18:17:55.927066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471108421825954:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37665
2025-11-27T18:17:55.927909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471108421825955:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37665
2025-11-27T18:17:55.801769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471108091625244:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.816615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471109722659527:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.936748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471108421825940:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:55.789939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471103194601497:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(402325627ns) {Configure:104(402320500ns) {ProcessConfigured:135(6312292ns) {StartInitialHandshake:157(6309195ns) {DropHandshakes:470(2734ns) {DropIncomingHandshake:442(799ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(1018ns) PrepareNewSessionHandshake:477(4576ns) {SwitchToState:179(2098ns)}}}}}
2025-11-27T18:17:55.833870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471109722659527:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.822130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471109791982081:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.835459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471107482920258:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.826099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471105497014716:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(217426034ns) {Configure:104(217417437ns) {ProcessConfigured:135(78646ns) {StartInitialHandshake:157(75300ns) {DropHandshakes:470(2757ns) {DropIncomingHandshake:442(1108ns) DropOutgoingHandshake:457(419ns)} GenerateSessionVirtualId:988(1291ns) PrepareNewSessionHandshake:477(1467ns) {SwitchToState:179(588ns)}}}}}
2025-11-27T18:17:55.841838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471103796657880:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(195579506ns) {Configure:104(195573940ns) {ProcessConfigured:135(88679ns) {StartInitialHandshake:157(85161ns) {DropHandshakes:470(2757ns) {DropIncomingHandshake:442(540ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(1518ns) PrepareNewSessionHandshake:477(1527ns) {SwitchToState:179(412ns)}}}}}
2025-11-27T18:17:55.938196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471108421825953:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.810766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471107489568871:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.853584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471109791982076:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.833994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471109722659518:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.835767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471103187952886:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.853626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471109791982076:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:55.866893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471109791982076:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:55.872199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471109791982084:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.873938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471105497014715:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(232339925ns) {Configure:104(232332170ns) {ProcessConfigured:135(8343786ns) {StartInitialHandshake:157(8339967ns) {DropHandshakes:470(2806ns) {DropIncomingHandshake:442(505ns) DropOutgoingHandshake:457(439ns)} GenerateSessionVirtualId:988(745ns) PrepareNewSessionHandshake:477(4037ns) {SwitchToState:179(1375ns)}}}}}
2025-11-27T18:17:55.874161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471105497014715:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:55.881776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471109791982085:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.909523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471109791982074:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:55.939402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471109791982095:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61728
2025-11-27T18:17:55.939526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471109791982096:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61728
2025-11-27T18:17:55.946140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471109791982084:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:55.950159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471109791982073:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:55.950248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:55.950305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471109791982073:2109] [node 6] ICP22 created new session: [5:7577471109791982097:2109]
2025-11-27T18:17:55.950350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471109791982097:2109] [node 6] ICS09 handshake done sender: [5:7577471109791982076:2110] self: [5:7577471109791982075:0] peer: [6:7577471107489568877:0] socket: 344 qp: -1
2025-11-27T18:17:55.950393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471109791982097:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:55.950463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:55.981794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471105497014715:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:55.981845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:55.981895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471105497014715:2049] [node 2] ICP22 created new session: [5:7577471109791982100:2049]
2025-11-27T18:17:55.981937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471109791982100:2049] [node 2] ICS09 handshake done sender: [5:7577471109791982084:2114] self: [5:7577471109791982093:0] peer: [2:7577471107482920251:0] socket: 361 qp: -1
2025-11-27T18:17:55.981970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471109791982100:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.834016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471109722659518:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:55.849537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471108091625246:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:55.827006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471107489568869:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.861898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471103187952887:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(227753162ns) {Configure:104(227747785ns) {ProcessConfigured:135(69361ns) {StartInitialHandshake:157(65918ns) {DropHandshakes:470(3097ns) {DropIncomingHandshake:442(1156ns) DropOutgoingHandshake:457(433ns)} GenerateSessionVirtualId:988(1072ns) PrepareNewSessionHandshake:477(1384ns) {SwitchToState:179(510ns)}}}}}
2025-11-27T18:17:55.877451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471103796657879:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:55.829721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471107489568859:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.982016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.842155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471109722659516:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.863838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471107482920268:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57152
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.829967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471107489568831:2076] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:56.001373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471108421825950:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.985700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471109791982102:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61728
2025-11-27T18:17:55.877513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:55.842179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471109722659516:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.830040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471103194601497:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:55.995026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471109791982095:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.011547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471108421825935:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:55.877559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471103796657879:2049] [node 2] ICP22 created new session: [4:7577471108091625259:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.881338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471107482920247:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:55.836575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471107489568879:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20005
2025-11-27T18:17:55.996074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471109791982085:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:55.845637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471109722659518:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:56.011597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:55.888963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:55.866296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471109722659511:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(244318395ns) {Configure:104(244311905ns) {ProcessConfigured:135(9562310ns) {StartInitialHandshake:157(9559731ns) {DropHandshakes:470(2841ns) {DropIncomingHandshake:442(577ns) DropOutgoingHandshake:457(317ns)} GenerateSessionVirtualId:988(539ns) PrepareNewSessionHandshake:477(3048ns) {SwitchToState:179(1682ns)}}}}}
2025-11-27T18:17:55.866466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471109722659516:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:55.874906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471105427692153:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:55.874985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:55.875037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471105427692153:2048] [node 2] ICP22 created new session: [1:7577471109722659534:2048]
2025-11-27T18:17:55.875078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471109722659534:2048] [node 2] ICS09 handshake done sender: [1:7577471109722659516:2110] self: [1:7577471109722659515:0] peer: [2:7577471107482920263:0] socket: 324 qp: -1
2025-11-27T18:17:55.875118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471109722659534:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:55.875152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:55.876261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471109722659513:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:55.876323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:55.876370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471109722659513:2108] [node 6] ICP22 created new session: [1:7577471109722659537:2108]
2025-11-27T18:17:55.876414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471109722659537:2108] [node 6] ICS09 handshake done sender: [1:7577471109722659518:2111] self: [1:7577471109722659517:0] peer: [6:7577471107489568878:0] socket: 325 qp: -1
2025-11-27T18:17:55.876444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471109722659537:2108] [node 6] ICS10 traffic start
2025-11-27T18:17:55.876480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:55.898888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471109722659542:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
2025-11-27T18:17:55.945897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471109722659522:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:55.946790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471109722659542:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.961874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471109722659533:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.961902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471109722659533:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:55.986376Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471109722659533:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:55.986454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471105427692154:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:55.998814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471109722659540:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62255
2025-11-27T18:17:56.004604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471105427692153:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:56.005369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:56.009861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471105427692154:2049] [node 3] ICP22 created new session: [1:7577471109722659545:2049]
2025-11-27T18:17:56.014852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471109722659545:2049] [node 3] ICS09 handshake done sender: [1:7577471109722659522:2112] self: [1:7577471109722659521:0] peer: [3:7577471108421825946:0] socket: 347 qp: -1
2025-11-27T18:17:56.015896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471109722659524:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.015931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471109722659524:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:56.016839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471109722659545:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:56.016883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:56.020016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471109722659524:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:56.020525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471109722659514:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:56.020617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:56.020693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471109722659514:2109] [node 5] ICP22 created new session: [1:7577471114017626844:2109]
2025-11-27T18:17:56.020737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471114017626844:2109] [node 5] ICS09 handshake done sender: [1:7577471109722659524:2113] self: [1:7577471109722659523:0] peer: [5:7577471114086949399:0] socket: 375 qp: -1
2025-11-27T18:17:56.020768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471114017626844:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:56.020822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:56.023479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471109722659511:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:56.037474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:56.037531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471109722659511:2107] [node 4] ICP22 created new session: [1:7577471114017626846:2107]
2025-11-27T18:17:56.052959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471105427692154:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(66281628ns) {HandleHandshakeStatus:329(66277764ns) {DropIncomingHandshake:442(16369ns) DropOutgoingHandshake:457(405ns) SwitchToState:179(586ns) ProcessPendingSessionEvents:495(16031102ns) {ForwardSessionEventToSession:612(15975873ns) ForwardSessionEventToSession:612(6393ns) ForwardSessionEventToSession:612(2953ns) ForwardSessionEventToSession:612(7609ns) ForwardSessionEventToSession:612(3282ns) ForwardSessionEventToSession:612(2681ns) ForwardSessionEventToSession:612(6241ns) ForwardSessionEventToSession:612(2286ns) ForwardSessionEventToSession:612(5320ns) ForwardSessionEventToSession:612(2785ns)}}}
2025-11-27T18:17:56.057342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471114017626846:2107] [node 4] ICS09 handshake done sender: [1:7577471109722659533:2116] self: [1:7577471109722659532:0] peer: [4:7577471108091625266:0] socket: 376 qp: -1
2025-11-27T18:17:56.061796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471114017626846:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:56.063646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:55.877597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471108091625259:2049] [node 2] ICS09 handshake done sender: [4:7577471108091625246:2114] self: [4:7577471108091625253:0] peer: [2:7577471107482920253:0] socket: 336 qp: -1
2025-11-27T18:17:55.865758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471103194601499:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(360069371ns) {Configure:104(360063093ns) {ProcessConfigured:135(19098128ns) {StartInitialHandshake:157(19093772ns) {DropHandshakes:470(3889ns) {DropIncomingHandshake:442(1111ns) DropOutgoingHandshake:457(485ns)} GenerateSessionVirtualId:988(1285ns) PrepareNewSessionHandshake:477(3585ns) {SwitchToState:179(1708ns)}}}}}
2025-11-27T18:17:56.005505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471105497014714:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:56.011642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471108421825935:2109] [node 6] ICP22 created new session: [3:7577471112716793254:2109]
2025-11-27T18:17:55.877632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471108091625259:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:55.877659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:55.878185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471108091625251:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.880440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471108091625261:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25664
2025-11-27T18:17:55.881499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471108091625237:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:17:55.881557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
2025-11-27T18:17:55.889446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471108091625237:2109] [node 6] ICP22 created new session: [4:7577471108091625265:2109]
2025-11-27T18:17:55.933884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471108091625265:2109] [node 6] ICS09 handshake done sender: [4:7577471108091625244:2112] self: [4:7577471108091625243:0] peer: [6:7577471107489568870:0] socket: 338 qp: -1
2025-11-27T18:17:55.937658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471108091625261:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.005745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471109791982074:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:55.889253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471107482920259:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.011683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471112716793254:2109] [node 6] ICS09 handshake done sender: [3:7577471108421825940:2111] self: [3:7577471108421825939:0] peer: [6:7577471107489568880:0] socket: 353 qp: -1
2025-11-27T18:17:55.865881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471103194601499:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:55.901369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471107482920247:2107] [node 4] ICP22 created new session: [2:7577471107482920269:2107]
2025-11-27T18:17:56.011720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471112716793254:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:55.937812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471103796657878:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:56.005787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:56.010571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471109791982074:2108] [node 4] ICP22 created new session: [5:7577471114086949400:2108]
2025-11-27T18:17:55.901534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471107489568859:2110] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:55.906486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471107482920252:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.011752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:55.945972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471108091625242:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.016698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471114086949400:2108] [node 4] ICS09 handshake done sender: [5:7577471109791982085:2115] self: [5:7577471109791982094:0] peer: [4:7577471108091625241:0] socket: 365 qp: -1
2025-11-27T18:17:55.906512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471107482920252:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:56.012296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471108421825955:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.902352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471103194601497:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:55.946003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471108091625242:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:56.023422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471114086949400:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:56.026131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471114086949401:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61728
2025-11-27T18:17:56.026390Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471105497014714:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12275) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471109791982095:2118] held: no
2025-11-27T18:17:56.027053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:56.045442Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471109791982095:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:56.053937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471114086949403:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61728
2025-11-27T18:17:56.065544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471105497014714:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:56.069883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471114086949403:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.069992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:56.070539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471105497014716:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:56.071002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471105497014714:2048] [node 1] ICP22 created new session: [5:7577471114086949404:2048]
2025-11-27T18:17:56.082324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471109722659511:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(62719259ns) {HandleHandshakeStatus:329(57580646ns) {DropIncomingHandshake:442(21514ns) DropOutgoingHandshake:457(427ns) SwitchToState:179(675ns) ProcessPendingSessionEvents:495(3991993ns) {ForwardSessionEventToSession:612(2251003ns) ForwardSessionEventToSession:612(525035ns) ForwardSessionEventToSession:612(3077ns) ForwardSessionEventToSession:612(448310ns) ForwardSessionEventToSession:612(711039ns) ForwardSessionEventToSession:612(5627ns) ForwardSessionEventToSession:612(13567ns) ForwardSessionEventToSession:612(2456ns) ForwardSessionEventToSession:612(2417ns) ForwardSessionEventToSession:612(15701ns)}}}
2025-11-27T18:17:56.071295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471114086949404:2048] [node 1] ICS09 handshake done sender: [5:7577471109791982095:2118] self: [5:7577471114086949399:0] peer: [1:7577471109722659523:0] socket: 389 qp: -1
2025-11-27T18:17:55.902403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:56.071361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471114086949404:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:55.902458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471103194601497:2048] [node 1] ICP22 created new session: [6:7577471107489568881:2048]
2025-11-27T18:17:56.071442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:55.902510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471107489568881:2048] [node 1] ICS09 handshake done sender: [6:7577471107489568859:2110] self: [6:7577471107489568878:0] peer: [1:7577471109722659517:0] socket: 327 qp: -1
2025-11-27T18:17:55.902549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471107489568881:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:55.902591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:55.938780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471107489568885:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20005
2025-11-27T18:17:55.945577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471107489568871:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:55.961442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471107489568874:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20005
2025-11-27T18:17:55.969555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471107489568856:2097] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16484) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471107489568864:2113] held: no
2025-11-27T18:17:56.076442Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471109791982074:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(68067552ns) {HandleHandshakeStatus:329(68065369ns) {DropIncomingHandshake:442(1309ns) DropOutgoingHandshake:457(9175ns) SwitchToState:179(212ns) ProcessPendingSessionEvents:495(8214178ns) {ForwardSessionEventToSession:612(5099780ns) ForwardSessionEventToSession:612(1446470ns) ForwardSessionEventToSession:612(13046ns) ForwardSessionEventToSession:612(545346ns) ForwardSessionEventToSession:612(164346ns) ForwardSessionEventToSession:612(3757ns) ForwardSessionEventToSession:612(2369ns) ForwardSessionEventToSession:612(509337ns) ForwardSessionEventToSession:612(4541ns) ForwardSessionEventToSession:612(407748ns)}}}
2025-11-27T18:17:55.974429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471103194601499:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:55.974510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:56.082337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471105497014716:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8595) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471114086949403:2122] held: no
2025-11-27T18:17:55.974567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471103194601499:2050] [node 3] ICP22 created new session: [6:7577471107489568886:2050]
2025-11-27T18:17:55.974614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471107489568886:2050] [node 3] ICS09 handshake done sender: [6:7577471107489568871:2116] self: [6:7577471107489568880:0] peer: [3:7577471108421825939:0] socket: 356 qp: -1
2025-11-27T18:17:55.974655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471107489568886:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:55.974689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:55.975272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471107489568831:2076] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8433) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471107489568869:2115] held: no
2025-11-27T18:17:55.976690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471107489568864:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:55.981598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471107489568888:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20005
2025-11-27T18:17:55.995671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471107489568856:2097] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:55.995737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:55.995780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471107489568856:2097] [node 4] ICP22 created new session: [6:7577471107489568889:2097]
2025-11-27T18:17:55.995844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471107489568889:2097] [node 4] ICS09 handshake done sender: [6:7577471107489568864:2113] self: [6:7577471107489568870:0] peer: [4:7577471108091625243:0] socket: 341 qp: -1
2025-11-27T18:17:55.995900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471107489568889:2097] [node 4] ICS10 traffic start
2025-11-27T18:17:55.996652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:56.005134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471107489568888:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.023418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471103194601498:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:56.023939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471107489568884:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20005
2025-11-27T18:17:55.949382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471108091625265:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:56.045396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471107489568869:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:55.949466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:56.070559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471107489568831:2076] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:55.966522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471108091625242:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:56.072390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:56.073681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471103194601498:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471107489568888:2123] held: no
2025-11-27T18:17:55.982977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471108091625237:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(101272812ns) {HandleHandshakeStatus:329(101270149ns) {DropIncomingHandshake:442(13218ns) DropOutgoingHandshake:457(249ns) SwitchToState:179(317ns) ProcessPendingSessionEvents:495(12278860ns) {ForwardSessionEventToSession:612(12224737ns) ForwardSessionEventToSession:612(7370ns) ForwardSessionEventToSession:612(2793ns) ForwardSessionEventToSession:612(4672ns) ForwardSessionEventToSession:612(7319ns) ForwardSessionEventToSession:612(2672ns) ForwardSessionEventToSession:612(6580ns) ForwardSessionEventToSession:612(5348ns) ForwardSessionEventToSession:612(2046ns) ForwardSessionEventToSession:612(5510ns)}}}
2025-11-27T18:17:56.074835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471107489568831:2076] [node 5] ICP22 created new session: [6:7577471111784536189:2076]
2025-11-27T18:17:55.994298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471108091625269:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25664
2025-11-27T18:17:56.076148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471111784536189:2076] [node 5] ICS09 handshake done sender: [6:7577471107489568869:2115] self: [6:7577471107489568877:0] peer: [5:7577471109791982075:0] socket: 346 qp: -1
2025-11-27T18:17:56.005284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471108091625238:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:56.076192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471111784536189:2076] [node 5] ICS10 traffic start
2025-11-27T18:17:56.005351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:56.076446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:56.007807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471108091625261:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:56.008081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471103796657878:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:56.078598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471111784536188:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20005
2025-11-27T18:17:56.008165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:56.008225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471103796657878:2048] [node 1] ICP22 created new session: [4:7577471112386592567:2048]
2025-11-27T18:17:56.008271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471112386592567:2048] [node 1] ICS09 handshake done sender: [4:7577471108091625261:2119] self: [4:7577471108091625266:0] peer: [1:7577471109722659532:0] socket: 377 qp: -1
2025-11-27T18:17:56.008302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471112386592567:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:56.008334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:56.011372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471108091625238:2108] [node 5] ICP22 created new session: [4:7577471112386592566:2108]
2025-11-27T18:17:55.914935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471107482920250:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:56.016034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471112386592566:2108] [node 5] ICS09 handshake done sender: [4:7577471108091625242:2111] self: [4:7577471108091625241:0] peer: [5:7577471109791982094:0] socket: 349 qp: -1
2025-11-27T18:17:56.017006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471112386592566:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:55.925362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471107482920269:2107] [node 4] ICS09 handshake done sender: [2:7577471107482920254:2114] self: [2:7577471107482920253:0] peer: [4:7577471108091625253:0] socket: 332 qp: -1
2025-11-27T18:17:56.025472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:56.071418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471112386592570:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25664
2025-11-27T18:17:55.933346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471107482920269:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:56.073136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471112386592570:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:55.933962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:56.074071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471103796657880:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:55.936973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471107482920252:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:56.076373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471108091625238:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(77648676ns) {HandleHandshakeStatus:329(76456929ns) {DropIncomingHandshake:442(17108ns) DropOutgoingHandshake:457(449ns) SwitchToState:179(462ns) ProcessPendingSessionEvents:495(13296853ns) {ForwardSessionEventToSession:612(8149321ns) ForwardSessionEventToSession:612(5726ns) ForwardSessionEventToSession:612(2809953ns) ForwardSessionEventToSession:612(1677ns) ForwardSessionEventToSession:612(974676ns) ForwardSessionEventToSession:612(4180ns) ForwardSessionEventToSession:612(515012ns) ForwardSessionEventToSession:612(810296ns) ForwardSessionEventToSession:612(3597ns) ForwardSessionEventToSession:612(3322ns)}}}
2025-11-27T18:17:55.939765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471103187952886:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:55.939822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:55.939866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471103187952886:2048] [node 1] ICP22 created new session: [2:7577471107482920275:2048]
2025-11-27T18:17:55.939912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471107482920275:2048] [node 1] ICS09 handshake done sender: [2:7577471107482920250:2112] self: [2:7577471107482920263:0] peer: [1:7577471109722659515:0] socket: 326 qp: -1
2025-11-27T18:17:55.939942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471107482920275:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:55.939979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:55.945742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471107482920248:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(303111781ns) {Configure:104(303104523ns) {ProcessConfigured:135(8869213ns) {StartInitialHandshake:157(8865568ns) {DropHandshakes:470(2683ns) {DropIncomingHandshake:442(627ns) DropOutgoingHandshake:457(378ns)} GenerateSessionVirtualId:988(943ns) PrepareNewSessionHandshake:477(1887ns) {SwitchToState:179(882ns)}}}}}
2025-11-27T18:17:55.966040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471107482920247:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(96463779ns) {HandleHandshakeStatus:329(96457191ns) {DropIncomingHandshake:442(24319ns) DropOutgoingHandshake:457(616ns) SwitchToState:179(270ns) ProcessPendingSessionEvents:495(12081412ns) {ForwardSessionEventToSession:612(12029478ns) ForwardSessionEventToSession:612(9287ns) ForwardSessionEventToSession:612(7573ns) ForwardSessionEventToSession:612(2712ns) ForwardSessionEventToSession:612(5222ns) ForwardSessionEventToSession:612(2643ns) ForwardSessionEventToSession:612(2921ns) ForwardSessionEventToSession:612(5019ns) ForwardSessionEventToSession:612(2365ns) ForwardSessionEventToSession:612(5224ns)}}}
2025-11-27T18:17:55.981053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471107482920249:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:55.995560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:55.995636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471107482920249:2108] [node 5] ICP22 created new session: [2:7577471107482920280:2108]
2025-11-27T18:17:55.995688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471107482920280:2108] [node 5] ICS09 handshake done sender: [2:7577471107482920252:2113] self: [2:7577471107482920251:0] peer: [5:7577471109791982093:0] socket: 331 qp: -1
2025-11-27T18:17:56.005844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471107482920280:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:56.005893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:56.006396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471107482920249:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(25134026ns) {HandleHandshakeStatus:329(25130622ns) {DropIncomingHandshake:442(677ns) DropOutgoingHandshake:457(436ns) SwitchToState:179(960ns) ProcessPendingSessionEvents:495(112251ns) {ForwardSessionEventToSession:612(55057ns) ForwardSessionEventToSession:612(6639ns) ForwardSessionEventToSession:612(2782ns) ForwardSessionEventToSession:612(2685ns) ForwardSessionEventToSession:612(13451ns) ForwardSessionEventToSession:612(2146ns) ForwardSessionEventToSession:612(5347ns) ForwardSessionEventToSession:612(3134ns) ForwardSessionEventToSession:612(5381ns) ForwardSessionEventToSession:612(5610ns)}}}
2025-11-27T18:17:56.036961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471107482920277:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.037002Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471107482920277:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:56.044482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471107482920277:2121] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:56.044721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471107482920248:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:56.016813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471104126858574:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12275) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471108421825941:2112] held: no
2025-11-27T18:17:56.086283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471107489568888:2123] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:56.044779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:56.016944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471108421825941:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:56.044825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471107482920248:2109] [node 6] ICP22 created new session: [2:7577471111777887579:2109]
2025-11-27T18:17:56.036002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471112716793253:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37665
2025-11-27T18:17:56.086593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471103194601498:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:56.089004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:56.036156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471104126858574:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:56.090822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471103796657880:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8595) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471112386592570:2121] held: no
2025-11-27T18:17:56.044865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471111777887579:2109] [node 6] ICS09 handshake done sender: [2:7577471107482920277:2121] self: [2:7577471107482920273:0] peer: [6:7577471111784536187:0] socket: 392 qp: -1
2025-11-27T18:17:56.093345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471114086949407:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61728
2025-11-27T18:17:56.090822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471103194601498:2049] [node 2] ICP22 created new session: [6:7577471111784536191:2049]
2025-11-27T18:17:56.092490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471111784536191:2049] [node 2] ICS09 handshake done sender: [6:7577471107489568888:2123] self: [6:7577471111784536187:0] peer: [2:7577471107482920273:0] socket: 328 qp: -1
2025-11-27T18:17:56.092741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471111784536191:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:56.093123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:56.036198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:56.036245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471104126858574:2048] [node 1] ICP22 created new session: [3:7577471112716793257:2048]
2025-11-27T18:17:56.036309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471112716793257:2048] [node 1] ICS09 handshake done sender: [3:7577471108421825941:2112] self: [3:7577471108421825946:0] peer: [1:7577471109722659521:0] socket: 351 qp: -1
2025-11-27T18:17:56.036343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471112716793257:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:56.036377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:56.044894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471111777887579:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:56.049463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471104126858575:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:56.105388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471114086949403:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:56.105560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471105497014716:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:56.044927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:56.114112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471112386592572:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25664
2025-11-27T18:17:56.069085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471107482920267:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.117699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:56.061987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471112716793253:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.121521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471112386592570:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:56.130542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471103796657880:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:56.125393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471105497014716:2050] [node 3] ICP22 created new session: [5:7577471114086949408:2050]
2025-11-27T18:17:56.069122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471107482920267:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:56.071079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471107482920267:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:56.072929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471103187952887:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:56.073721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:56.074473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471103187952887:2049] [node 3] ICP22 created new session: [2:7577471111777887582:2049]
2025-11-27T18:17:56.075919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471111777887582:2049] [node 3] ICS09 handshake done sender: [2:7577471107482920267:2119] self: [2:7577471107482920266:0] peer: [3:7577471112716793260:0] socket: 387 qp: -1
2025-11-27T18:17:56.076308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471111777887582:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:56.079216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:56.071359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471108421825951:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.129641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471114086949408:2050] [node 3] ICS09 handshake done sender: [5:7577471114086949403:2122] self: [5:7577471114086949405:0] peer: [3:7577471108421825949:0] socket: 367 qp: -1
2025-11-27T18:17:56.129700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471114086949408:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:56.134881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:56.145632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:56.145733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471103796657880:2050] [node 3] ICP22 created new session: [4:7577471112386592573:2050]
2025-11-27T18:17:56.145786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471112386592573:2050] [node 3] ICS09 handshake done sender: [4:7577471112386592570:2121] self: [4:7577471112386592571:0] peer: [3:7577471108421825947:0] socket: 352 qp: -1
2025-11-27T18:17:56.145831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471112386592573:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:56.145897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:56.071387Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471108421825951:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:56.075068Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471108421825951:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:56.076040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471112716793263:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37665
2025-11-27T18:17:56.076570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471108421825948:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:56.076604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471108421825948:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:56.077292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471104126858575:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471108421825955:2119] held: no
2025-11-27T18:17:56.077406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471108421825933:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:56.077500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:56.077539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471108421825933:2108] [node 5] ICP22 created new session: [3:7577471112716793265:2108]
2025-11-27T18:17:56.077589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471112716793265:2108] [node 5] ICS09 handshake done sender: [3:7577471108421825951:2116] self: [3:7577471108421825949:0] peer: [5:7577471114086949405:0] socket: 366 qp: -1
2025-11-27T18:17:56.077620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471112716793265:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:56.077655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:56.082574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471108421825948:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:56.086410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471108421825955:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:56.086535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471108421825934:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:56.086615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:56.087080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471108421825934:2107] [node 4] ICP22 created new session: [3:7577471112716793268:2107]
2025-11-27T18:17:56.087136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471112716793268:2107] [node 4] ICS09 handshake done sender: [3:7577471108421825948:2114] self: [3:7577471108421825947:0] peer: [4:7577471112386592571:0] socket: 345 qp: -1
2025-11-27T18:17:56.087177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471112716793268:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:56.087216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:56.089189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471104126858575:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:56.089228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:56.092558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471104126858575:2049] [node 2] ICP22 created new session: [3:7577471112716793270:2049]
2025-11-27T18:17:56.092792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471112716793270:2049] [node 2] ICS09 handshake done sender: [3:7577471108421825955:2119] self: [3:7577471112716793260:0] peer: [2:7577471107482920266:0] socket: 388 qp: -1
2025-11-27T18:17:56.093434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471112716793270:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:56.093491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:56.147788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471105497014716:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(42159701ns) {HandleHandshakeStatus:329(42156981ns) {DropIncomingHandshake:442(1217ns) DropOutgoingHandshake:457(481ns) SwitchToState:179(744ns) ProcessPendingSessionEvents:495(691ns)}}
2025-11-27T18:17:56.238925Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:56.239168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471107482920275:2048] [node 1] ICS01 socket: 326 reason# EndOfStream
2025-11-27T18:17:56.239218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471103187952886:2048] [node 1] ICP30 unregister session Session# [2:7577471107482920275:2048] VirtualId# [2:7577471107482920263:0]
2025-11-27T18:17:56.239252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471107482920275:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.239334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:56.239390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:56.239294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:56.239541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471112716793257:2048] [node 1] ICS01 socket: 351 reason# EndOfStream
2025-11-27T18:17:56.239582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471104126858574:2048] [node 1] ICP30 unregister session Session# [3:7577471112716793257:2048] VirtualId# [3:7577471108421825946:0]
2025-11-27T18:17:56.239618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471112716793257:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.239699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:56.239739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:56.242962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:56.243241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471112386592567:2048] [node 1] ICS01 socket: 377 reason# EndOfStream
2025-11-27T18:17:56.243296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471103796657878:2048] [node 1] ICP30 unregister session Session# [4:7577471112386592567:2048] VirtualId# [4:7577471108091625266:0]
2025-11-27T18:17:56.243329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471112386592567:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.243426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:56.243473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:56.244445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:56.243455Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:56.244656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471114086949404:2048] [node 1] ICS01 socket: 389 reason# EndOfStream
2025-11-27T18:17:56.244709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471105497014714:2048] [node 1] ICP30 unregister session Session# [5:7577471114086949404:2048] VirtualId# [5:7577471114086949399:0]
2025-11-27T18:17:56.243675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471107489568881:2048] [node 1] ICS01 socket: 327 reason# EndOfStream
2025-11-27T18:17:56.244734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471114086949404:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.243708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471103194601497:2048] [node 1] ICP30 unregister session Session# [6:7577471107489568881:2048] VirtualId# [6:7577471107489568878:0]
2025-11-27T18:17:56.243737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471107489568881:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.244796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:56.243797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:56.244841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:56.243851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:56.280625Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:56.280872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471112716793270:2049] [node 2] ICS01 socket: 388 reason# EndOfStream
2025-11-27T18:17:56.280914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471104126858575:2049] [node 2] ICP30 unregister session Session# [3:7577471112716793270:2049] VirtualId# [3:7577471112716793260:0]
2025-11-27T18:17:56.280948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471112716793270:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.281019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:56.281072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:56.284960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:56.285227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471111784536191:2049] [node 2] ICS01 socket: 328 reason# EndOfStream
2025-11-27T18:17:56.285287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471103194601498:2049] [node 2] ICP30 unregister session Session# [6:7577471111784536191:2049] VirtualId# [6:7577471111784536187:0]
2025-11-27T18:17:56.285319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471111784536191:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.285403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:56.285463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:56.285648Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:56.285886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471108091625259:2049] [node 2] ICS01 socket: 336 reason# EndOfStream
2025-11-27T18:17:56.285932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471103796657879:2049] [node 2] ICP30 unregister session Session# [4:7577471108091625259:2049] VirtualId# [4:7577471108091625253:0]
2025-11-27T18:17:56.285964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471108091625259:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.286052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:56.286101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:56.287649Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:56.291273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471109791982100:2049] [node 2] ICS01 socket: 361 reason# EndOfStream
2025-11-27T18:17:56.291320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471105497014715:2049] [node 2] ICP30 unregister session Session# [5:7577471109791982100:2049] VirtualId# [5:7577471109791982093:0]
2025-11-27T18:17:56.291354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471109791982100:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.291439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:56.291490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:56.348866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:56.349154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471112386592573:2050] [node 3] ICS01 socket: 352 reason# EndOfStream
2025-11-27T18:17:56.349228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471103796657880:2050] [node 3] ICP30 unregister session Session# [4:7577471112386592573:2050] VirtualId# [4:7577471112386592571:0]
2025-11-27T18:17:56.349308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471112386592573:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.349406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:56.349462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:56.349862Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:56.350086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471114086949408:2050] [node 3] ICS01 socket: 367 reason# EndOfStream
2025-11-27T18:17:56.350126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471105497014716:2050] [node 3] ICP30 unregister session Session# [5:7577471114086949408:2050] VirtualId# [5:7577471114086949405:0]
2025-11-27T18:17:56.350155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471114086949408:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.350227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:56.350284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:56.350758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:56.350930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471107489568886:2050] [node 3] ICS01 socket: 356 reason# EndOfStream
2025-11-27T18:17:56.350958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471103194601499:2050] [node 3] ICP30 unregister session Session# [6:7577471107489568886:2050] VirtualId# [6:7577471107489568880:0]
2025-11-27T18:17:56.350976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471107489568886:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.351038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:56.351117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:56.405730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:56.405984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471114086949400:2108] [node 4] ICS01 socket: 365 reason# EndOfStream
2025-11-27T18:17:56.406039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471109791982074:2108] [node 4] ICP30 unregister session Session# [5:7577471114086949400:2108] VirtualId# [5:7577471109791982094:0]
2025-11-27T18:17:56.406100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471114086949400:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.406181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:56.406234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:56.408993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:56.409213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471107489568889:2097] [node 4] ICS01 socket: 341 reason# EndOfStream
2025-11-27T18:17:56.409274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471107489568856:2097] [node 4] ICP30 unregister session Session# [6:7577471107489568889:2097] VirtualId# [6:7577471107489568870:0]
2025-11-27T18:17:56.409309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471107489568889:2097] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.409391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:56.409457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:56.486877Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:56.487239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471111784536189:2076] [node 5] ICS01 socket: 346 reason# EndOfStream
2025-11-27T18:17:56.487287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471107489568831:2076] [node 5] ICP30 unregister session Session# [6:7577471111784536189:2076] VirtualId# [6:7577471107489568877:0]
2025-11-27T18:17:56.487331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471111784536189:2076] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:56.487437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:56.487494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:56.610278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471113636730730:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:56.610298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471113636730731:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:56.631671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471112587479070:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:56.631965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471112587479069:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:56.656305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471114226805941:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:56.656720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471114226805942:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:56.677934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471113780315726:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:56.679161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471113780315728:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:56.679181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471113780315727:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:56.703074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471114201967174:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:56.703211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471114201967173:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:56.703300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471114201967175:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:56.728211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471111601892960:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:56.728347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471111601892961:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:56.728483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471111601892962:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:56.808989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471114201967235:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:56.827802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471112587479132:2109] [node 5] ICP01 ready to work
2025-11-27T18:17:56.812972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471111601893019:2104] [node 5] ICP01 ready to work
2025-11-27T18:17:56.810760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471113636730791:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:56.838333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471113636730794:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:56.857088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471114226806022:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:56.851131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471111601893021:2103] [node 4] ICP01 ready to work
2025-11-27T18:17:56.884658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471112587479137:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:56.905465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471113636730795:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:56.921758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471114226806023:2116] [node 6] ICP01 ready to work
2025-11-27T18:17:56.946037Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471111601893019:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(132823024ns) {Configure:104(130607769ns) {ProcessConfigured:135(58064ns) {StartInitialHandshake:157(56114ns) {DropHandshakes:470(1677ns) {DropIncomingHandshake:442(361ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(1039ns) PrepareNewSessionHandshake:477(1558ns) {SwitchToState:179(719ns)}}}}}
2025-11-27T18:17:56.941772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471114201967237:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:56.905288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471112587479136:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:56.935892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471113780315799:2110] [node 6] ICP01 ready to work
2025-11-27T18:17:56.946323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471112587479132:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(101695553ns) {Configure:104(101688882ns) {ProcessConfigured:135(24170638ns) {StartInitialHandshake:157(24162532ns) {DropHandshakes:470(3246ns) {DropIncomingHandshake:442(843ns) DropOutgoingHandshake:457(487ns)} GenerateSessionVirtualId:988(1241ns) PrepareNewSessionHandshake:477(8226854ns) {SwitchToState:179(7542ns)}}}}}
2025-11-27T18:17:56.943784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471113780315800:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:56.941751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471114226806024:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:56.960932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471112587479069:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(147497862ns) {Configure:104(147493766ns) {ProcessConfigured:135(89031ns) {StartInitialHandshake:157(86217ns) {DropHandshakes:470(5756ns) {DropIncomingHandshake:442(902ns) DropOutgoingHandshake:457(480ns)} GenerateSessionVirtualId:988(1126ns) PrepareNewSessionHandshake:477(2038ns) {SwitchToState:179(759ns)}}}}}
2025-11-27T18:17:56.985837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471113780315727:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(147760974ns) {Configure:104(147756037ns) {ProcessConfigured:135(78174ns) {StartInitialHandshake:157(75870ns) {DropHandshakes:470(3119ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(666ns)} GenerateSessionVirtualId:988(883ns) PrepareNewSessionHandshake:477(1733ns) {SwitchToState:179(546ns)}}}}}
2025-11-27T18:17:56.986376Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471114201967235:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(177036758ns) {Configure:104(177022212ns) {ProcessConfigured:135(5060105ns) {StartInitialHandshake:157(5056780ns) {DropHandshakes:470(3136ns) {DropIncomingHandshake:442(646ns) DropOutgoingHandshake:457(699ns)} GenerateSessionVirtualId:988(1076ns) PrepareNewSessionHandshake:477(3065ns) {SwitchToState:179(1892ns)}}}}}
2025-11-27T18:17:56.978785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471113636730730:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(164732076ns) {Configure:104(164724770ns) {ProcessConfigured:135(7739923ns) {StartInitialHandshake:157(7737403ns) {DropHandshakes:470(1859ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:477(3930ns) {SwitchToState:179(1647ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:56.988678Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471114201967174:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(120351433ns) {Configure:104(120338202ns) {ProcessConfigured:135(38672621ns) {StartInitialHandshake:157(38669098ns) {DropHandshakes:470(2646ns) {DropIncomingHandshake:442(612ns) DropOutgoingHandshake:457(392ns)} GenerateSessionVirtualId:988(2497ns) PrepareNewSessionHandshake:477(5883ns) {SwitchToState:179(2643ns)}}}}}
2025-11-27T18:17:57.006256Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471112587479136:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(88264999ns) {Configure:104(88254584ns) {ProcessConfigured:135(61861ns) {StartInitialHandshake:157(59405ns) {DropHandshakes:470(2097ns) {DropIncomingHandshake:442(538ns) DropOutgoingHandshake:457(266ns)} GenerateSessionVirtualId:988(1111ns) PrepareNewSessionHandshake:477(1416ns) {SwitchToState:179(459ns)}}}}}
2025-11-27T18:17:57.007043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471114201967240:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57875
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.000167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471113636730801:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16426
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.045737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471114226805941:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(215974853ns) {Configure:104(215970452ns) {ProcessConfigured:135(15380876ns) {StartInitialHandshake:157(15377734ns) {DropHandshakes:470(2755ns) {DropIncomingHandshake:442(778ns) DropOutgoingHandshake:457(551ns)} GenerateSessionVirtualId:988(1184ns) PrepareNewSessionHandshake:477(3996ns) {SwitchToState:179(1964ns)}}}}}
2025-11-27T18:17:57.038370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471111601892961:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(170190520ns) {Configure:104(170181341ns) {ProcessConfigured:135(50810ns) {StartInitialHandshake:157(48508ns) {DropHandshakes:470(1483ns) {DropIncomingHandshake:442(626ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(686ns) PrepareNewSessionHandshake:477(1661ns) {SwitchToState:179(537ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.058972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471113636730791:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(219422712ns) {Configure:104(219417288ns) {ProcessConfigured:135(4829502ns) {StartInitialHandshake:157(4826484ns) {DropHandshakes:470(2879ns) {DropIncomingHandshake:442(607ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(1191ns) PrepareNewSessionHandshake:477(3579ns) {SwitchToState:179(1411ns)}}}}}
2025-11-27T18:17:57.022839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471114201967245:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57875
2025-11-27T18:17:57.054354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471116882446443:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54635
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.059192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471113780315728:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(185147729ns) {Configure:104(183065771ns) {ProcessConfigured:135(12421855ns) {StartInitialHandshake:157(12417591ns) {DropHandshakes:470(3267ns) {DropIncomingHandshake:442(1075ns) DropOutgoingHandshake:457(425ns)} GenerateSessionVirtualId:988(1104ns) PrepareNewSessionHandshake:477(3342ns) {SwitchToState:179(1308ns)}}}}}
2025-11-27T18:17:57.038755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471115896860324:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12027
2025-11-27T18:17:57.066326Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471114226806022:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(156734351ns) {Configure:104(156726145ns) {ProcessConfigured:135(99082ns) {StartInitialHandshake:157(95270ns) {DropHandshakes:470(3496ns) {DropIncomingHandshake:442(894ns) DropOutgoingHandshake:457(624ns)} GenerateSessionVirtualId:988(889ns) PrepareNewSessionHandshake:477(1419ns) {SwitchToState:179(764ns)}}}}}
2025-11-27T18:17:57.068192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471113636730731:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(233071042ns) {Configure:104(233063857ns) {ProcessConfigured:135(14233443ns) {StartInitialHandshake:157(14230614ns) {DropHandshakes:470(1531ns) {DropIncomingHandshake:442(405ns) DropOutgoingHandshake:457(487ns)} GenerateSessionVirtualId:988(449ns) PrepareNewSessionHandshake:477(3223ns) {SwitchToState:179(1328ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.080058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471116882446444:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54635
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.034345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471114201967240:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.077505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471113780315726:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(210449540ns) {Configure:104(210443469ns) {ProcessConfigured:135(16165454ns) {StartInitialHandshake:157(6979054ns) {DropHandshakes:470(2727ns) {DropIncomingHandshake:442(329ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(2788ns) PrepareNewSessionHandshake:477(2688ns) {SwitchToState:179(1422ns)}}}}}
2025-11-27T18:17:57.053536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471111601893021:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(202115626ns) {Configure:104(202111310ns) {ProcessConfigured:135(11545561ns) {StartInitialHandshake:157(11541373ns) {DropHandshakes:470(1970ns) {DropIncomingHandshake:442(500ns) DropOutgoingHandshake:457(231ns)} GenerateSessionVirtualId:988(1693ns) PrepareNewSessionHandshake:477(4054ns) {SwitchToState:179(2202ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.097091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471114226806024:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(139429744ns) {Configure:104(139424055ns) {ProcessConfigured:135(86104ns) {StartInitialHandshake:157(84201ns) {DropHandshakes:470(3224ns) {DropIncomingHandshake:442(1092ns) DropOutgoingHandshake:457(430ns)} GenerateSessionVirtualId:988(2020ns) PrepareNewSessionHandshake:477(2701ns) {SwitchToState:179(1125ns)}}}}}
2025-11-27T18:17:57.092791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471113636730801:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.079573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471118496934549:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57875
2025-11-27T18:17:57.080356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471112587479070:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(237049066ns) {Configure:104(237036103ns) {ProcessConfigured:135(11848508ns) {StartInitialHandshake:157(11844110ns) {DropHandshakes:470(1411ns) {DropIncomingHandshake:442(392ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(425ns) PrepareNewSessionHandshake:477(2040ns) {SwitchToState:179(542ns)}}}}}
2025-11-27T18:17:57.078390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471118075283103:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5375
2025-11-27T18:17:57.100082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471113636730797:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.100122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471113636730797:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.107305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471113636730794:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(223709376ns) {Configure:104(223703836ns) {ProcessConfigured:135(52931618ns) {StartInitialHandshake:157(52929784ns) {DropHandshakes:470(1053ns) {DropIncomingHandshake:442(298ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(329ns) PrepareNewSessionHandshake:477(2980ns) {SwitchToState:179(1912ns)}}}}}
2025-11-27T18:17:57.121430Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471113636730797:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:57.089505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471114201967173:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(189064904ns) {Configure:104(189059467ns) {ProcessConfigured:135(8197836ns) {StartInitialHandshake:157(8194415ns) {DropHandshakes:470(2397ns) {DropIncomingHandshake:442(572ns) DropOutgoingHandshake:457(695ns)} GenerateSessionVirtualId:988(601ns) PrepareNewSessionHandshake:477(3505ns) {SwitchToState:179(1416ns)}}}}}
2025-11-27T18:17:57.089962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471115896860324:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.110459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471118521773332:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41546
2025-11-27T18:17:57.116135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471118521773332:2132] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.092010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471116882446449:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54635
2025-11-27T18:17:57.092232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471118075283106:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5375
2025-11-27T18:17:57.091362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471111601892961:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.129674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471114226805942:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(174259678ns) {Configure:104(174250153ns) {ProcessConfigured:135(14702965ns) {StartInitialHandshake:157(14686227ns) {DropHandshakes:470(3019ns) {DropIncomingHandshake:442(852ns) DropOutgoingHandshake:457(375ns)} GenerateSessionVirtualId:988(1237ns) PrepareNewSessionHandshake:477(2836ns) {SwitchToState:179(1698ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.090654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471114201967173:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.116272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471111601892962:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(172702052ns) {Configure:104(172695301ns) {ProcessConfigured:135(23486276ns) {StartInitialHandshake:157(23482650ns) {DropHandshakes:470(2156ns) {DropIncomingHandshake:442(467ns) DropOutgoingHandshake:457(294ns)} GenerateSessionVirtualId:988(799ns) PrepareNewSessionHandshake:477(1311ns) {SwitchToState:179(290ns)}}}}}
2025-11-27T18:17:57.092535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471113780315799:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(117942196ns) {Configure:104(117935389ns) {ProcessConfigured:135(14437496ns) {StartInitialHandshake:157(14435199ns) {DropHandshakes:470(1747ns) {DropIncomingHandshake:442(386ns) DropOutgoingHandshake:457(365ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:477(3965ns) {SwitchToState:179(1178ns)}}}}}
2025-11-27T18:17:57.091006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471118496934550:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57875
2025-11-27T18:17:57.092722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471118496934549:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.094019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471114201967245:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.100040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471116882446440:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.106358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471118496934553:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57875
2025-11-27T18:17:57.100080Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471116882446440:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.129540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471115896860336:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12027
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.100058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471118075283103:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:17:57.131792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471111601892960:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(258543170ns) {Configure:104(248355017ns) {ProcessConfigured:135(16421591ns) {StartInitialHandshake:157(16418962ns) {DropHandshakes:470(2503ns) {DropIncomingHandshake:442(666ns) DropOutgoingHandshake:457(437ns)} GenerateSessionVirtualId:988(655ns) PrepareNewSessionHandshake:477(2938ns) {SwitchToState:179(1163ns)}}}}}
2025-11-27T18:17:57.107218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471118075283107:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5375
2025-11-27T18:17:57.133220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471115896860337:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12027
2025-11-27T18:17:57.116142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471118075283106:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.117605Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471114201967175:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(157197123ns) {Configure:104(157191839ns) {ProcessConfigured:135(4536259ns) {StartInitialHandshake:157(4532867ns) {DropHandshakes:470(3310ns) {DropIncomingHandshake:442(1026ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(671ns) PrepareNewSessionHandshake:477(4208ns) {SwitchToState:179(2023ns)}}}}}
2025-11-27T18:17:57.101428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471116882446443:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.130828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471114226805941:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.107268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471116882446447:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54635
2025-11-27T18:17:57.129996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471118075283107:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.133804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471117931698106:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16426
2025-11-27T18:17:57.116108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471114201967174:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.131401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471114226806023:2116] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(163519490ns) {Configure:104(163504269ns) {ProcessConfigured:135(6936505ns) {StartInitialHandshake:157(6932283ns) {DropHandshakes:470(2175ns) {DropIncomingHandshake:442(455ns) DropOutgoingHandshake:457(492ns)} GenerateSessionVirtualId:988(930ns) PrepareNewSessionHandshake:477(2029ns) {SwitchToState:179(726ns)}}}}}
2025-11-27T18:17:57.116227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471116882446444:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.120720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471112587479137:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(175926852ns) {Configure:104(175921965ns) {ProcessConfigured:135(22518730ns) {StartInitialHandshake:157(22516650ns) {DropHandshakes:470(2760ns) {DropIncomingHandshake:442(631ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(896ns) PrepareNewSessionHandshake:477(2649ns) {SwitchToState:179(1492ns)}}}}}
2025-11-27T18:17:57.130217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471112587479139:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.130257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471112587479139:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.130758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471116882446440:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:57.130761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471116882446449:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.131387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471112587479069:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.133651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471112587479136:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.137389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:57.137585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471112587479136:2110] [node 6] ICP22 created new session: [2:7577471116882446456:2110]
2025-11-27T18:17:57.140910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471116882446456:2110] [node 6] ICS09 handshake done sender: [2:7577471116882446440:2115] self: [2:7577471116882446439:0] peer: [6:7577471115896860332:0] socket: 339 qp: -1
2025-11-27T18:17:57.141720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471116882446456:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:57.130513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471113780315800:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(162265991ns) {Configure:104(162257294ns) {ProcessConfigured:135(15547370ns) {StartInitialHandshake:157(15544555ns) {DropHandshakes:470(1525ns) {DropIncomingHandshake:442(447ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(592ns) PrepareNewSessionHandshake:477(3697ns) {SwitchToState:179(2093ns)}}}}}
2025-11-27T18:17:57.137665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471118521773325:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.131082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471114201967174:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(21251623ns) {IncomingHandshake:261(19799434ns) {DropIncomingHandshake:442(3946ns) PrepareNewSessionHandshake:477(4431ns) {SwitchToState:179(3466ns)} GenerateSessionVirtualId:988(1047ns) IssueIncomingHandshakeReply:193(6851719ns) {SwitchToState:179(3062ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.133983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471113636730800:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.132656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471113780315728:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.134014Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471113636730800:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.133985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471114201967240:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.151756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471118075283117:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5375
2025-11-27T18:17:57.143130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471112587479139:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.148426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471115896860324:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:57.137695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471118521773325:2129] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.140742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471114201967173:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.144756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471118521773325:2129] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:57.149837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471115896860336:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.144464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471116882446458:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54635
2025-11-27T18:17:57.135021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471113636730795:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.152690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471118075283105:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.149978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471111601893021:2103] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.135107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471117931698107:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16426
2025-11-27T18:17:57.139797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471117931698102:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.139809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:57.139830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471117931698102:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.139838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471117931698106:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.140994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471113636730800:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:57.143776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471117931698107:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.145069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471113636730730:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.147279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471113636730795:2108] [node 5] ICP22 created new session: [1:7577471117931698112:2108]
2025-11-27T18:17:57.151133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471117931698112:2108] [node 5] ICS09 handshake done sender: [1:7577471113636730797:2110] self: [1:7577471113636730796:0] peer: [5:7577471118496934551:0] socket: 330 qp: -1
2025-11-27T18:17:57.151168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:57.152658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471117931698112:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:57.152814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471117931698102:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:57.153167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471113636730730:2048] [node 2] ICP22 created new session: [1:7577471117931698115:2048]
2025-11-27T18:17:57.153934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471117931698115:2048] [node 2] ICS09 handshake done sender: [1:7577471113636730800:2111] self: [1:7577471113636730799:0] peer: [2:7577471116882446453:0] socket: 359 qp: -1
2025-11-27T18:17:57.156336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:57.156371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471117931698115:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:57.171514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:57.167583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471118521773337:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41546
2025-11-27T18:17:57.152727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471118075283105:2124] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.147417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:57.152224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471112587479132:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.152278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:57.152411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471116882446447:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.141429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471118496934550:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.151156Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471111601892961:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471115896860324:2111] held: no
2025-11-27T18:17:57.171298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471118521773329:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.171339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471118521773329:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.153202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471112587479132:2109] [node 5] ICP22 created new session: [2:7577471116882446460:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.152926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471111601892961:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.144867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:57.175630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471114226806022:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.175696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:57.175745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471114226806022:2107] [node 4] ICP22 created new session: [3:7577471118521773340:2107]
2025-11-27T18:17:57.175786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471118521773340:2107] [node 4] ICS09 handshake done sender: [3:7577471118521773325:2129] self: [3:7577471118521773324:0] peer: [4:7577471118075283115:0] socket: 353 qp: -1
2025-11-27T18:17:57.175818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471118521773340:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:57.175852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:57.196152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471118521773336:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41546
2025-11-27T18:17:57.198532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471118521773337:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.154841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471118075283116:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5375
2025-11-27T18:17:57.177185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471113636730731:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.155749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471116882446460:2109] [node 5] ICS09 handshake done sender: [2:7577471112587479139:2113] self: [2:7577471112587479138:0] peer: [5:7577471118496934554:0] socket: 350 qp: -1
2025-11-27T18:17:57.156054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471116882446449:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:57.162259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471116882446460:2109] [node 5] ICS10 traffic start
2025-11-27T18:17:57.169834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471112587479069:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471116882446449:2120] held: no
2025-11-27T18:17:57.169953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471112587479069:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.170078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:57.170126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471112587479069:2048] [node 1] ICP22 created new session: [2:7577471116882446463:2048]
2025-11-27T18:17:57.170172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471116882446463:2048] [node 1] ICS09 handshake done sender: [2:7577471116882446449:2120] self: [2:7577471116882446453:0] peer: [1:7577471113636730799:0] socket: 360 qp: -1
2025-11-27T18:17:57.170208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471116882446463:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:57.170239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:57.174857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:57.202190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471118521773329:2130] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:57.174647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471118075283105:2124] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:57.198498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.225493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471114226805941:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10816) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471118521773332:2132] held: no
2025-11-27T18:17:57.225651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471114226806024:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.148546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471114201967173:2048] [node 1] ICP22 created new session: [5:7577471118496934555:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.156321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.150739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471114201967175:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.171460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471111601892961:2049] [node 2] ICP22 created new session: [6:7577471115896860340:2049]
2025-11-27T18:17:57.202700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471113636730731:2049] [node 3] ICP22 created new session: [1:7577471117931698119:2049]
2025-11-27T18:17:57.225710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:57.174710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471118075283116:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.215645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471112587479136:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(43732219ns) {HandleHandshakeStatus:329(43730046ns) {DropIncomingHandshake:442(22741ns) DropOutgoingHandshake:457(552ns) SwitchToState:179(298ns) ProcessPendingSessionEvents:495(23930208ns) {ForwardSessionEventToSession:612(3227916ns) ForwardSessionEventToSession:612(6319ns) ForwardSessionEventToSession:612(5204532ns) ForwardSessionEventToSession:612(4842ns) ForwardSessionEventToSession:612(2871ns) ForwardSessionEventToSession:612(4954005ns) ForwardSessionEventToSession:612(4790877ns) ForwardSessionEventToSession:612(3305032ns) ForwardSessionEventToSession:612(3712ns) ForwardSessionEventToSession:612(28353ns)}}}
2025-11-27T18:17:57.213570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471117931698099:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.186935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471115896860340:2049] [node 2] ICS09 handshake done sender: [6:7577471115896860324:2111] self: [6:7577471115896860332:0] peer: [2:7577471116882446439:0] socket: 344 qp: -1
2025-11-27T18:17:57.182146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471113780315726:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.225761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471114226806024:2108] [node 5] ICP22 created new session: [3:7577471118521773346:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.229710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471116882446466:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54635
2025-11-27T18:17:57.151169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471118496934555:2048] [node 1] ICS09 handshake done sender: [5:7577471114201967240:2111] self: [5:7577471118496934551:0] peer: [1:7577471113636730796:0] socket: 331 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.186991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471115896860340:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:57.233408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471116882446448:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.213604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471117931698099:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.193227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471118075283107:2126] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:57.225806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471118521773346:2108] [node 5] ICS09 handshake done sender: [3:7577471118521773329:2130] self: [3:7577471118521773328:0] peer: [5:7577471118496934557:0] socket: 361 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.233439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471116882446448:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.234726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471116882446448:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:57.238527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471112587479132:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(54080992ns) {HandleHandshakeStatus:329(54073922ns) {DropIncomingHandshake:442(13542ns) DropOutgoingHandshake:457(256ns) SwitchToState:179(279ns) ProcessPendingSessionEvents:495(31263824ns) {ForwardSessionEventToSession:612(11895565ns) ForwardSessionEventToSession:612(7667ns) ForwardSessionEventToSession:612(6257980ns) ForwardSessionEventToSession:612(3297ns) ForwardSessionEventToSession:612(2943214ns) ForwardSessionEventToSession:612(4972ns) ForwardSessionEventToSession:612(2484986ns) ForwardSessionEventToSession:612(4123496ns) ForwardSessionEventToSession:612(3510782ns) ForwardSessionEventToSession:612(4357ns)}}}
2025-11-27T18:17:57.243073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471112587479070:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.254607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:57.262114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471112587479070:2049] [node 3] ICP22 created new session: [2:7577471116882446468:2049]
2025-11-27T18:17:57.271013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471116882446468:2049] [node 3] ICS09 handshake done sender: [2:7577471116882446448:2119] self: [2:7577471116882446445:0] peer: [3:7577471118521773348:0] socket: 378 qp: -1
2025-11-27T18:17:57.271050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471116882446468:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:57.271091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:57.152221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471118496934555:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:57.223682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471117931698119:2049] [node 3] ICS09 handshake done sender: [1:7577471117931698102:2114] self: [1:7577471117931698100:0] peer: [3:7577471118521773335:0] socket: 363 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.225844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471118521773346:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:57.193898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471115896860339:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12027
2025-11-27T18:17:57.194983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471113780315799:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.153113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:57.225890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:57.224417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471117931698099:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:57.195058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:57.226495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471114226805942:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.174735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471114201967174:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471118496934549:2117] held: no
2025-11-27T18:17:57.227691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471117931698119:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:57.195119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471113780315799:2110] [node 6] ICP22 created new session: [4:7577471118075283121:2110]
2025-11-27T18:17:57.303963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471116882446466:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.205958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.305238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471116882446450:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.194721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471118496934560:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57875
2025-11-27T18:17:57.226668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471118521773345:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41546
2025-11-27T18:17:57.232278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:57.234540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471117931698122:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16426
2025-11-27T18:17:57.235304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471113636730791:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.235340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:57.240615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471113636730791:2107] [node 4] ICP22 created new session: [1:7577471117931698123:2107]
2025-11-27T18:17:57.245404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471117931698105:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.245434Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471117931698105:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.210044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471111601893021:2103] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65300) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471115896860336:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.242873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471118521773336:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.215165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471118496934556:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57875
2025-11-27T18:17:57.195172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471118075283121:2110] [node 6] ICS09 handshake done sender: [4:7577471118075283105:2124] self: [4:7577471118075283104:0] peer: [6:7577471115896860338:0] socket: 369 qp: -1
2025-11-27T18:17:57.252864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471117931698105:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:57.235826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471115896860339:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.305290Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471116882446450:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.195214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471118075283121:2110] [node 6] ICS10 traffic start
2025-11-27T18:17:57.224180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471118496934550:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:57.238711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471114201967173:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(55509432ns) {HandleHandshakeStatus:329(55266370ns) {DropIncomingHandshake:442(2684ns) DropOutgoingHandshake:457(7546ns) SwitchToState:179(409ns) ProcessPendingSessionEvents:495(21867354ns) {ForwardSessionEventToSession:612(3840747ns) ForwardSessionEventToSession:612(5484ns) ForwardSessionEventToSession:612(2479ns) ForwardSessionEventToSession:612(2486652ns) ForwardSessionEventToSession:612(3280ns) ForwardSessionEventToSession:612(4215149ns) ForwardSessionEventToSession:612(5175962ns) ForwardSessionEventToSession:612(4129ns) ForwardSessionEventToSession:612(6105612ns) ForwardSessionEventToSession:612(6292ns)}}}
2025-11-27T18:17:57.243404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471114201967175:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.249149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471118496934549:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:57.249498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471118521773339:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41546
2025-11-27T18:17:57.330917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471116882446450:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:57.244443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471111601892960:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.257111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471117931698123:2107] [node 4] ICS09 handshake done sender: [1:7577471117931698099:2113] self: [1:7577471117931698098:0] peer: [4:7577471118075283120:0] socket: 377 qp: -1
2025-11-27T18:17:57.249893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:57.249695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471118521773345:2138] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.257166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471117931698123:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:57.195248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:57.254910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471115896860342:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12027
2025-11-27T18:17:57.333347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471112587479070:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(66168372ns) {HandleHandshakeStatus:329(66164615ns) {DropIncomingHandshake:442(604ns) DropOutgoingHandshake:457(491ns) SwitchToState:179(805ns) ProcessPendingSessionEvents:495(9081520ns) {ForwardSessionEventToSession:612(7674604ns) ForwardSessionEventToSession:612(5067ns) ForwardSessionEventToSession:612(7421ns) ForwardSessionEventToSession:612(5836ns) ForwardSessionEventToSession:612(2823ns) ForwardSessionEventToSession:612(5520ns) ForwardSessionEventToSession:612(2858ns) ForwardSessionEventToSession:612(2810ns) ForwardSessionEventToSession:612(1356172ns) ForwardSessionEventToSession:612(3554ns)}}}
2025-11-27T18:17:57.249935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471114201967175:2050] [node 3] ICP22 created new session: [5:7577471118496934562:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.225164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471118075283123:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5375
2025-11-27T18:17:57.262775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471118521773343:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41546
2025-11-27T18:17:57.337223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471112587479137:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.258723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:57.263206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471118496934562:2050] [node 3] ICS09 handshake done sender: [5:7577471118496934550:2118] self: [5:7577471118496934557:0] peer: [3:7577471118521773328:0] socket: 362 qp: -1
2025-11-27T18:17:57.255082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471111601892961:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(79197340ns) {HandleHandshakeStatus:329(79195101ns) {DropIncomingHandshake:442(392ns) DropOutgoingHandshake:457(113ns) SwitchToState:179(565ns) ProcessPendingSessionEvents:495(1136ns)}}
2025-11-27T18:17:57.227831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471113780315728:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.258794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471113636730730:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(85187183ns) {HandleHandshakeStatus:329(85184248ns) {DropIncomingHandshake:442(11922ns) DropOutgoingHandshake:457(357ns) SwitchToState:179(486ns) ProcessPendingSessionEvents:495(36918130ns) {ForwardSessionEventToSession:612(9294627ns) ForwardSessionEventToSession:612(3432652ns) ForwardSessionEventToSession:612(3323ns) ForwardSessionEventToSession:612(11098139ns) ForwardSessionEventToSession:612(3743ns) ForwardSessionEventToSession:612(2559ns) ForwardSessionEventToSession:612(10462641ns) ForwardSessionEventToSession:612(7466ns) ForwardSessionEventToSession:612(3334ns) ForwardSessionEventToSession:612(2586502ns)}}}
2025-11-27T18:17:57.276511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471113636730794:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.289589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471117931698126:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16426
2025-11-27T18:17:57.290067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471113636730795:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(100234150ns) {HandleHandshakeStatus:329(100231593ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(395ns) SwitchToState:179(197ns) ProcessPendingSessionEvents:495(48234815ns) {ForwardSessionEventToSession:612(23326523ns) ForwardSessionEventToSession:612(6486ns) ForwardSessionEventToSession:612(5001512ns) ForwardSessionEventToSession:612(8303971ns) ForwardSessionEventToSession:612(4107267ns) ForwardSessionEventToSession:612(7626ns) ForwardSessionEventToSession:612(4550938ns) ForwardSessionEventToSession:612(3457ns) ForwardSessionEventToSession:612(2886313ns) ForwardSessionEventToSession:612(7554ns)}}}
2025-11-27T18:17:57.290588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:57.298532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471113636730795:2108] [node 5] ICP18 handshake (actor [1:7577471117931698107:2117]) is too old
2025-11-27T18:17:57.303860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471113636730794:2109] [node 6] ICP22 created new session: [1:7577471117931698127:2109]
2025-11-27T18:17:57.306406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471113636730791:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(49128873ns) {HandleHandshakeStatus:329(49126558ns) {DropIncomingHandshake:442(1013ns) DropOutgoingHandshake:457(1130ns) SwitchToState:179(421ns) ProcessPendingSessionEvents:495(22103108ns) {ForwardSessionEventToSession:612(76366ns) ForwardSessionEventToSession:612(573717ns) ForwardSessionEventToSession:612(19855ns) ForwardSessionEventToSession:612(4096ns) ForwardSessionEventToSession:612(5600580ns) ForwardSessionEventToSession:612(3030ns) ForwardSessionEventToSession:612(7767703ns) ForwardSessionEventToSession:612(7842ns) ForwardSessionEventToSession:612(2533ns) ForwardSessionEventToSession:612(4561394ns)}}}
2025-11-27T18:17:57.306966Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471113636730791:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:57.316997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471117931698127:2109] [node 6] ICS09 handshake done sender: [1:7577471117931698105:2115] self: [1:7577471117931698101:0] peer: [6:7577471115896860344:0] socket: 386 qp: -1
2025-11-27T18:17:57.317674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471113636730731:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(123086213ns) {HandleHandshakeStatus:329(123083973ns) {DropIncomingHandshake:442(11019ns) DropOutgoingHandshake:457(2739ns) SwitchToState:179(576ns) ProcessPendingSessionEvents:495(33645016ns) {ForwardSessionEventToSession:612(9543611ns) ForwardSessionEventToSession:612(8181497ns) ForwardSessionEventToSession:612(2685928ns) ForwardSessionEventToSession:612(3730ns) ForwardSessionEventToSession:612(2729ns) ForwardSessionEventToSession:612(3180771ns) ForwardSessionEventToSession:612(3439ns) ForwardSessionEventToSession:612(576785ns) ForwardSessionEventToSession:612(5033ns) ForwardSessionEventToSession:612(9438058ns)}}}
2025-11-27T18:17:57.318798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471117931698127:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:57.318862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:57.319273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471117931698126:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.326701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471113636730794:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(56600748ns) {HandleHandshakeStatus:329(55174119ns) {DropIncomingHandshake:442(1534ns) DropOutgoingHandshake:457(2406ns) SwitchToState:179(284ns) ProcessPendingSessionEvents:495(116868ns) {ForwardSessionEventToSession:612(64924ns) ForwardSessionEventToSession:612(3642ns) ForwardSessionEventToSession:612(6274ns) ForwardSessionEventToSession:612(3162ns) ForwardSessionEventToSession:612(5242ns) ForwardSessionEventToSession:612(8061ns) ForwardSessionEventToSession:612(2718ns) ForwardSessionEventToSession:612(6285ns) ForwardSessionEventToSession:612(2803ns) ForwardSessionEventToSession:612(2299ns)}}}
2025-11-27T18:17:57.327034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471113636730794:2109] [node 6] ICP18 handshake (actor [1:7577471117931698126:2119]) is too old
2025-11-27T18:17:57.349122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:57.269118Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471114226806024:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:57.273789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471115896860345:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12027
2025-11-27T18:17:57.266278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471118496934563:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57875
2025-11-27T18:17:57.270549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471114226806022:2107] [node 4] ICP18 handshake (actor [3:7577471118521773336:2134]) is too old
2025-11-27T18:17:57.278469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471118521773332:2132] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:57.279175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471114226805941:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.231794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471118075283116:2128] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:57.351148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471112587479137:2108] [node 4] ICP22 created new session: [2:7577471116882446471:2108]
2025-11-27T18:17:57.273892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471115896860348:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12027
2025-11-27T18:17:57.279542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471115896860336:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:57.266746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471118496934562:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:57.279261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:57.355571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471116882446471:2108] [node 4] ICS09 handshake done sender: [2:7577471116882446450:2121] self: [2:7577471116882446446:0] peer: [4:7577471118075283129:0] socket: 342 qp: -1
2025-11-27T18:17:57.235558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:57.288977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471115896860339:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:57.268529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471114201967174:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.279313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471114226805941:2048] [node 1] ICP22 created new session: [3:7577471118521773352:2048]
2025-11-27T18:17:57.356819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471116882446471:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:57.300850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471111601893021:2103] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.268583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:57.246692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471113780315726:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.279372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471118521773352:2048] [node 1] ICS09 handshake done sender: [3:7577471118521773332:2132] self: [3:7577471118521773335:0] peer: [1:7577471117931698100:0] socket: 365 qp: -1
2025-11-27T18:17:57.268614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471114201967174:2049] [node 2] ICP22 created new session: [5:7577471118496934564:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:57.276242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:57.283826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471118496934564:2049] [node 2] ICS09 handshake done sender: [5:7577471118496934549:2117] self: [5:7577471118496934554:0] peer: [2:7577471112587479138:0] socket: 352 qp: -1
2025-11-27T18:17:57.284294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471118496934563:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.286813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471118496934564:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:57.298501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:57.305008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471114201967235:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.359002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:57.300930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:57.301070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471111601892960:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.301148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:57.301199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471111601892960:2048] [node 1] ICP22 created new session: [6:7577471115896860350:2048]
2025-11-27T18:17:57.301253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471115896860350:2048] [node 1] ICS09 handshake done sender: [6:7577471115896860339:2118] self: [6:7577471115896860344:0] peer: [1:7577471117931698101:0] socket: 337 qp: -1
2025-11-27T18:17:57.301311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471115896860350:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:57.301343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:57.301833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471115896860345:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.302071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471111601892962:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.303892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471111601893021:2103] [node 4] ICP22 created new session: [6:7577471115896860349:2103]
2025-11-27T18:17:57.318747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471115896860349:2103] [node 4] ICS09 handshake done sender: [6:7577471115896860336:2116] self: [6:7577471115896860338:0] peer: [4:7577471118075283104:0] socket: 370 qp: -1
2025-11-27T18:17:57.319417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471115896860349:2103] [node 4] ICS10 traffic start
2025-11-27T18:17:57.321866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:57.331233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471115896860347:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12027
2025-11-27T18:17:57.333798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471115896860347:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.334581Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471111601893021:2103] [node 4] ICP35 event processing took too much time PendingConnection:279(30288713ns) {HandleHandshakeStatus:329(30285964ns) {DropIncomingHandshake:442(766ns) DropOutgoingHandshake:457(289ns) SwitchToState:179(1002ns) ProcessPendingSessionEvents:495(1038ns)}}
2025-11-27T18:17:57.338877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471111601893019:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.352489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471115896860354:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12027
2025-11-27T18:17:57.356625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471111601892962:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19106) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471115896860345:2120] held: no
2025-11-27T18:17:57.358293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471115896860345:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:57.359678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471111601892962:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.361050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:57.361099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471111601892962:2050] [node 3] ICP22 created new session: [6:7577471115896860357:2050]
2025-11-27T18:17:57.361358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471115896860357:2050] [node 3] ICS09 handshake done sender: [6:7577471115896860345:2120] self: [6:7577471115896860352:0] peer: [3:7577471118521773333:0] socket: 399 qp: -1
2025-11-27T18:17:57.361396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471115896860357:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:57.361434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:57.318200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471118496934568:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57875
2025-11-27T18:17:57.279408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471118521773352:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:57.246758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:57.361961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471112587479070:2049] [node 3] ICP18 handshake (actor [2:7577471116882446466:2123]) is too old
2025-11-27T18:17:57.323040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471118496934563:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:57.279682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:57.283848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471118521773350:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41546
2025-11-27T18:17:57.294173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471118521773337:2135] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:57.294420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471114226805942:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.294470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:57.294517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471114226805942:2049] [node 2] ICP22 created new session: [3:7577471118521773354:2049]
2025-11-27T18:17:57.294557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471118521773354:2049] [node 2] ICS09 handshake done sender: [3:7577471118521773337:2135] self: [3:7577471118521773348:0] peer: [2:7577471116882446445:0] socket: 380 qp: -1
2025-11-27T18:17:57.294587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471118521773354:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:57.294623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:57.316966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471118521773334:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.317000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471118521773334:2133] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.329566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471118521773334:2133] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:57.337342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471114226806023:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.338381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:57.339424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471114226806023:2116] [node 6] ICP22 created new session: [3:7577471118521773357:2116]
2025-11-27T18:17:57.349006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471118521773357:2116] [node 6] ICS09 handshake done sender: [3:7577471118521773334:2133] self: [3:7577471118521773333:0] peer: [6:7577471115896860352:0] socket: 396 qp: -1
2025-11-27T18:17:57.351270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471118521773357:2116] [node 6] ICS10 traffic start
2025-11-27T18:17:57.356058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:57.357553Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471114226806023:2116] [node 6] ICP35 event processing took too much time PendingConnection:279(22556099ns) {HandleHandshakeStatus:329(22552709ns) {DropIncomingHandshake:442(25991ns) DropOutgoingHandshake:457(616ns) SwitchToState:179(2657ns) ProcessPendingSessionEvents:495(121181ns) {ForwardSessionEventToSession:612(58181ns) ForwardSessionEventToSession:612(2816ns) ForwardSessionEventToSession:612(6358ns) ForwardSessionEventToSession:612(2622ns) ForwardSessionEventToSession:612(23376ns) ForwardSessionEventToSession:612(5911ns) ForwardSessionEventToSession:612(3314ns) ForwardSessionEventToSession:612(3188ns) ForwardSessionEventToSession:612(5616ns) ForwardSessionEventToSession:612(2274ns)}}}
2025-11-27T18:17:57.361687Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471111601893019:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5090) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471115896860347:2121] held: no
2025-11-27T18:17:57.246821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471113780315726:2048] [node 1] ICP22 created new session: [4:7577471118075283126:2048]
2025-11-27T18:17:57.246888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471118075283126:2048] [node 1] ICS09 handshake done sender: [4:7577471118075283116:2128] self: [4:7577471118075283120:0] peer: [1:7577471117931698098:0] socket: 379 qp: -1
2025-11-27T18:17:57.246921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471118075283126:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:57.246959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:57.249712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471118075283119:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5375
2025-11-27T18:17:57.243152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471113780315728:2050] [node 3] ICP22 created new session: [4:7577471118075283125:2050]
2025-11-27T18:17:57.270011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471118075283125:2050] [node 3] ICS09 handshake done sender: [4:7577471118075283107:2126] self: [4:7577471118075283115:0] peer: [3:7577471118521773324:0] socket: 356 qp: -1
2025-11-27T18:17:57.270719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471118075283119:2130] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.276229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471118075283125:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:57.279799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471113780315727:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:57.280114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:57.308956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471118075283111:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.309010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471118075283111:2127] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.316476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471118075283111:2127] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:57.318611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471113780315800:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.329369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:57.331007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471113780315800:2108] [node 5] ICP22 created new session: [4:7577471118075283132:2108]
2025-11-27T18:17:57.333380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471118075283132:2108] [node 5] ICS09 handshake done sender: [4:7577471118075283111:2127] self: [4:7577471118075283110:0] peer: [5:7577471118496934566:0] socket: 395 qp: -1
2025-11-27T18:17:57.378114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471115896860356:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12027
2025-11-27T18:17:57.333426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471118075283132:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:57.336942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:57.384794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471115896860347:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:57.356960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471113780315728:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(110148088ns) {HandleHandshakeStatus:329(110144569ns) {DropIncomingHandshake:442(1089ns) DropOutgoingHandshake:457(5164ns) SwitchToState:179(350ns) ProcessPendingSessionEvents:495(14985375ns) {ForwardSessionEventToSession:612(7068325ns) ForwardSessionEventToSession:612(8951ns) ForwardSessionEventToSession:612(3657ns) ForwardSessionEventToSession:612(3679715ns) ForwardSessionEventToSession:612(4078ns) ForwardSessionEventToSession:612(3794ns) ForwardSessionEventToSession:612(3681443ns) ForwardSessionEventToSession:612(10570ns) ForwardSessionEventToSession:612(497953ns) ForwardSessionEventToSession:612(4640ns)}}}
2025-11-27T18:17:57.359538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471113780315727:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471118075283119:2130] held: no
2025-11-27T18:17:57.360938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471118075283133:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5375
2025-11-27T18:17:57.362352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471118075283119:2130] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:57.363449Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471113780315800:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(39948259ns) {HandleHandshakeStatus:329(39944966ns) {DropIncomingHandshake:442(17783ns) DropOutgoingHandshake:457(663ns) SwitchToState:179(1004ns) ProcessPendingSessionEvents:495(11185724ns) {ForwardSessionEventToSession:612(5634697ns) ForwardSessionEventToSession:612(6222ns) ForwardSessionEventToSession:612(2860ns) ForwardSessionEventToSession:612(2981299ns) ForwardSessionEventToSession:612(5655ns) ForwardSessionEventToSession:612(1052124ns) ForwardSessionEventToSession:612(17115ns) ForwardSessionEventToSession:612(2572ns) ForwardSessionEventToSession:612(1464215ns) ForwardSessionEventToSession:612(3214ns)}}}
2025-11-27T18:17:57.363679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471113780315727:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.377716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:57.384069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471113780315727:2049] [node 2] ICP22 created new session: [4:7577471118075283135:2049]
2025-11-27T18:17:57.384125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471118075283135:2049] [node 2] ICS09 handshake done sender: [4:7577471118075283119:2130] self: [4:7577471118075283129:0] peer: [2:7577471116882446446:0] socket: 389 qp: -1
2025-11-27T18:17:57.384161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471118075283135:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:57.384210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:57.384466Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471113780315727:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20684067ns) {HandleHandshakeStatus:329(20682380ns) {DropIncomingHandshake:442(1643ns) DropOutgoingHandshake:457(306ns) SwitchToState:179(342ns) ProcessPendingSessionEvents:495(1225ns)}}
2025-11-27T18:17:57.331185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471114201967174:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(51485908ns) {HandleHandshakeStatus:329(49450377ns) {DropIncomingHandshake:442(537ns) DropOutgoingHandshake:457(365ns) SwitchToState:179(655ns) ProcessPendingSessionEvents:495(767ns)}}
2025-11-27T18:17:57.331704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471114201967235:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.331779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:57.331819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471114201967235:2108] [node 4] ICP22 created new session: [5:7577471118496934569:2108]
2025-11-27T18:17:57.331863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471118496934569:2108] [node 4] ICS09 handshake done sender: [5:7577471118496934563:2122] self: [5:7577471118496934566:0] peer: [4:7577471118075283110:0] socket: 401 qp: -1
2025-11-27T18:17:57.331896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471118496934569:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:57.331933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:57.332153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471114201967235:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:57.345227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471114201967244:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:57.345275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471114201967244:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:57.348542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471114201967175:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(84406392ns) {HandleHandshakeStatus:329(84403152ns) {DropIncomingHandshake:442(1406ns) DropOutgoingHandshake:457(6238ns) SwitchToState:179(317ns) ProcessPendingSessionEvents:495(33158705ns) {ForwardSessionEventToSession:612(19814818ns) ForwardSessionEventToSession:612(7001371ns) ForwardSessionEventToSession:612(8823ns) ForwardSessionEventToSession:612(760633ns) ForwardSessionEventToSession:612(3880ns) ForwardSessionEventToSession:612(4219ns) ForwardSessionEventToSession:612(1665245ns) ForwardSessionEventToSession:612(3056ns) ForwardSessionEventToSession:612(2382784ns) ForwardSessionEventToSession:612(1491671ns)}}}
2025-11-27T18:17:57.353763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471114201967244:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:57.359630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471114201967237:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:57.361002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:57.361070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471114201967237:2109] [node 6] ICP22 created new session: [5:7577471118496934572:2109]
2025-11-27T18:17:57.361381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471118496934572:2109] [node 6] ICS09 handshake done sender: [5:7577471114201967244:2113] self: [5:7577471114201967243:0] peer: [6:7577471115896860355:0] socket: 397 qp: -1
2025-11-27T18:17:57.361418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471118496934572:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:57.361456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:57.393541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471112587479137:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(36923201ns) {HandleHandshakeStatus:329(36920100ns) {DropIncomingHandshake:442(15627ns) DropOutgoingHandshake:457(439ns) SwitchToState:179(464ns) ProcessPendingSessionEvents:495(12909919ns) {ForwardSessionEventToSession:612(1745234ns) ForwardSessionEventToSession:612(5081ns) ForwardSessionEventToSession:612(1042978ns) ForwardSessionEventToSession:612(4716455ns) ForwardSessionEventToSession:612(3369ns) ForwardSessionEventToSession:612(961774ns) ForwardSessionEventToSession:612(3549ns) ForwardSessionEventToSession:612(12910ns) ForwardSessionEventToSession:612(1349404ns) ForwardSessionEventToSession:612(3745ns)}}}
2025-11-27T18:17:57.394550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471111601893019:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:57.394630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:57.394691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471111601893019:2104] [node 5] ICP22 created new session: [6:7577471115896860359:2104]
2025-11-27T18:17:57.394741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471115896860359:2104] [node 5] ICS09 handshake done sender: [6:7577471115896860347:2121] self: [6:7577471115896860355:0] peer: [5:7577471114201967243:0] socket: 400 qp: -1
2025-11-27T18:17:57.394777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471115896860359:2104] [node 5] ICS10 traffic start
2025-11-27T18:17:57.394816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:57.464770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:57.465005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471116882446463:2048] [node 1] ICS01 socket: 360 reason# EndOfStream
2025-11-27T18:17:57.465060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471112587479069:2048] [node 1] ICP30 unregister session Session# [2:7577471116882446463:2048] VirtualId# [2:7577471116882446453:0]
2025-11-27T18:17:57.465097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471116882446463:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.465167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:57.465214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:57.468448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:57.468691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471118521773352:2048] [node 1] ICS01 socket: 365 reason# EndOfStream
2025-11-27T18:17:57.468737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471114226805941:2048] [node 1] ICP30 unregister session Session# [3:7577471118521773352:2048] VirtualId# [3:7577471118521773335:0]
2025-11-27T18:17:57.468768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471118521773352:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.468850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:57.468906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:57.471853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:57.472375Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:57.472585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471118496934555:2048] [node 1] ICS01 socket: 331 reason# EndOfStream
2025-11-27T18:17:57.472613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471114201967173:2048] [node 1] ICP30 unregister session Session# [5:7577471118496934555:2048] VirtualId# [5:7577471118496934551:0]
2025-11-27T18:17:57.472635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471118496934555:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.472680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:57.472714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:57.472095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471118075283126:2048] [node 1] ICS01 socket: 379 reason# EndOfStream
2025-11-27T18:17:57.472141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471113780315726:2048] [node 1] ICP30 unregister session Session# [4:7577471118075283126:2048] VirtualId# [4:7577471118075283120:0]
2025-11-27T18:17:57.472172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471118075283126:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.472245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:57.472288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:57.477415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:57.477926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471115896860350:2048] [node 1] ICS01 socket: 337 reason# EndOfStream
2025-11-27T18:17:57.477975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471111601892960:2048] [node 1] ICP30 unregister session Session# [6:7577471115896860350:2048] VirtualId# [6:7577471115896860344:0]
2025-11-27T18:17:57.478009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471115896860350:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.478078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:57.478121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:57.528836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:57.529097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471118521773354:2049] [node 2] ICS01 socket: 380 reason# EndOfStream
2025-11-27T18:17:57.529136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471114226805942:2049] [node 2] ICP30 unregister session Session# [3:7577471118521773354:2049] VirtualId# [3:7577471118521773348:0]
2025-11-27T18:17:57.529174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471118521773354:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.529253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:57.529310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:57.537578Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:57.537818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471118075283135:2049] [node 2] ICS01 socket: 389 reason# EndOfStream
2025-11-27T18:17:57.537858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471113780315727:2049] [node 2] ICP30 unregister session Session# [4:7577471118075283135:2049] VirtualId# [4:7577471118075283129:0]
2025-11-27T18:17:57.537889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471118075283135:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.537987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:57.538026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:57.538117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:57.538311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471118496934564:2049] [node 2] ICS01 socket: 352 reason# EndOfStream
2025-11-27T18:17:57.538348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471114201967174:2049] [node 2] ICP30 unregister session Session# [5:7577471118496934564:2049] VirtualId# [5:7577471118496934554:0]
2025-11-27T18:17:57.538378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471118496934564:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.538419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:57.538458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:57.546787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:57.547042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471115896860340:2049] [node 2] ICS01 socket: 344 reason# EndOfStream
2025-11-27T18:17:57.547110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471111601892961:2049] [node 2] ICP30 unregister session Session# [6:7577471115896860340:2049] VirtualId# [6:7577471115896860332:0]
2025-11-27T18:17:57.547145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471115896860340:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.547220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:57.547269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:57.625154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:57.625409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471118075283125:2050] [node 3] ICS07 socket disconnect 356 reason# EndOfStream
2025-11-27T18:17:57.625437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471118075283125:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.625518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:57.625564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471118075283125:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:57.625960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:57.626131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471118496934562:2050] [node 3] ICS07 socket disconnect 362 reason# EndOfStream
2025-11-27T18:17:57.626156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471118496934562:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.626211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:57.626249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471118496934562:2050] [node 3] ICS15 start handshake
2025-11-27T18:17:57.626286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:57.629531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471114201967175:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19106) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:57.626490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471115896860357:2050] [node 3] ICS01 socket: 399 reason# EndOfStream
2025-11-27T18:17:57.629596Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471113780315728:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19106) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:17:57.631399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471118496934562:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:57.631631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471114201967175:2050] [node 3] ICP30 unregister session Session# [5:7577471118496934562:2050] VirtualId# [5:7577471118496934557:0]
2025-11-27T18:17:57.629725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471118075283125:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:17:57.631666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:57.630767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471113780315728:2050] [node 3] ICP30 unregister session Session# [4:7577471118075283125:2050] VirtualId# [4:7577471118075283115:0]
2025-11-27T18:17:57.630823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:57.630867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471113780315728:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:57.630882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:57.631683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471114201967175:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:57.626550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471111601892962:2050] [node 3] ICP30 unregister session Session# [6:7577471115896860357:2050] VirtualId# [6:7577471115896860352:0]
2025-11-27T18:17:57.632076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:17:57.626589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471115896860357:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.626716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:57.626764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:57.645386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471114201967175:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:17:57.693512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:57.693726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471118496934569:2108] [node 4] ICS01 socket: 401 reason# EndOfStream
2025-11-27T18:17:57.693767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471114201967235:2108] [node 4] ICP30 unregister session Session# [5:7577471118496934569:2108] VirtualId# [5:7577471118496934566:0]
2025-11-27T18:17:57.693801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471118496934569:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.693883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:57.693926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:57.701388Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:57.701601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471115896860349:2103] [node 4] ICS01 socket: 370 reason# EndOfStream
2025-11-27T18:17:57.701648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471111601893021:2103] [node 4] ICP30 unregister session Session# [6:7577471115896860349:2103] VirtualId# [6:7577471115896860338:0]
2025-11-27T18:17:57.701681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471115896860349:2103] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:57.701762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:57.701805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:57.861500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471118477169854:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:57.861548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471118477169855:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:57.877547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471116591389325:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:57.877574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471116591389326:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:57.898588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471115273032590:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:57.898625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471115273032591:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:57.937484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471115111647526:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:57.937540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471115111647527:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:57.937586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471115111647528:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:57.949405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471115596045721:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:57.949455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471115596045722:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:57.949492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471115596045723:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:57.986686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471116109317606:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:57.986799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471116109317607:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:57.991436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471116109317608:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:58.122906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471120404284963:2107] [node 5] ICP01 ready to work
2025-11-27T18:17:58.208158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471119567999949:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:58.210413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471119567999952:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:58.213186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471119567999953:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:58.217645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471115596045721:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(72827532ns) {Configure:104(72823533ns) {ProcessConfigured:135(4198376ns) {StartInitialHandshake:157(4194791ns) {DropHandshakes:470(2014ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(584ns)} GenerateSessionVirtualId:988(861ns) PrepareNewSessionHandshake:477(3893ns) {SwitchToState:179(1966ns)}}}}}
2025-11-27T18:17:58.218182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471119891013082:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:58.218479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471119891013083:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:58.210785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471120886356702:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:58.222019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471120404284963:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(83785363ns) {Configure:104(83775569ns) {ProcessConfigured:135(47681ns) {StartInitialHandshake:157(46375ns) {DropHandshakes:470(1464ns) {DropIncomingHandshake:442(340ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(1308ns) PrepareNewSessionHandshake:477(857ns) {SwitchToState:179(333ns)}}}}}
2025-11-27T18:17:58.222284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471120404284968:2108] [node 4] ICP01 ready to work
2025-11-27T18:17:58.220256Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471115111647526:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(85566889ns) {Configure:104(85561433ns) {ProcessConfigured:135(223529ns) {StartInitialHandshake:157(221706ns) {DropHandshakes:470(2140ns) {DropIncomingHandshake:442(434ns) DropOutgoingHandshake:457(329ns)} GenerateSessionVirtualId:988(827ns) PrepareNewSessionHandshake:477(1818ns) {SwitchToState:179(897ns)}}}}}
2025-11-27T18:17:58.220687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471119406614887:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:58.220935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471119406614888:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:58.226727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471118477169854:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(115497029ns) {Configure:104(115482699ns) {ProcessConfigured:135(52646ns) {StartInitialHandshake:157(49832ns) {DropHandshakes:470(1524ns) {DropIncomingHandshake:442(361ns) DropOutgoingHandshake:457(174ns)} GenerateSessionVirtualId:988(750ns) PrepareNewSessionHandshake:477(1118ns) {SwitchToState:179(276ns)}}}}}
2025-11-27T18:17:58.235282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471115596045722:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16144636ns) {Configure:104(16140564ns) {ProcessConfigured:135(147634ns) {StartInitialHandshake:157(146153ns) {DropHandshakes:470(1253ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(443ns) PrepareNewSessionHandshake:477(1321ns) {SwitchToState:179(269ns)}}}}}
2025-11-27T18:17:58.236686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471116591389325:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17799211ns) {Configure:104(16528316ns) {ProcessConfigured:135(43139ns) {StartInitialHandshake:157(40981ns) {DropHandshakes:470(963ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(599ns) PrepareNewSessionHandshake:477(603ns) {SwitchToState:179(138ns)}}}}}
2025-11-27T18:17:58.237089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471120886356705:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:58.237483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471120886356706:2114] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.246688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471119567999965:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38675
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.256851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471119406614900:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:17:58.258196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471122772137217:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:58.259278Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471119891013082:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17255536ns) {Configure:104(17251223ns) {ProcessConfigured:135(74172ns) {StartInitialHandshake:157(71939ns) {DropHandshakes:470(2715ns) {DropIncomingHandshake:442(488ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(1067ns) PrepareNewSessionHandshake:477(1505ns) {SwitchToState:179(258ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.262536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471119891013092:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19814
2025-11-27T18:17:58.263325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471122772137218:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:58.263654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471122772137219:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:58.266906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471119891013092:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.269942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471120886356702:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25699688ns) {Configure:104(25696153ns) {ProcessConfigured:135(58090ns) {StartInitialHandshake:157(54829ns) {DropHandshakes:470(9887ns) {DropIncomingHandshake:442(294ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(1118ns) PrepareNewSessionHandshake:477(906ns) {SwitchToState:179(171ns)}}}}}
2025-11-27T18:17:58.285755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471119406614901:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.297951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471122772137220:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22281
2025-11-27T18:17:58.298109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471122772137221:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22281
2025-11-27T18:17:58.298301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471122772137222:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22281
2025-11-27T18:17:58.298819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471122772137223:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22281
2025-11-27T18:17:58.299129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471122772137232:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22281
2025-11-27T18:17:58.274185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471120886356711:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33786
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.274337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471120886356712:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33786
2025-11-27T18:17:58.274451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471120886356713:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33786
2025-11-27T18:17:58.274508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471120886356714:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33786
2025-11-27T18:17:58.281665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471120886356711:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.281843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471120886356712:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.297093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471120886356714:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.301279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471120886356710:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.301301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471120886356710:2130] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.297570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471119406614901:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.300277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471115111647527:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.300798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471119406614903:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:17:58.310520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471122772137223:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.309838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471120886356725:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33786
2025-11-27T18:17:58.319438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471122772137221:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.319606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471122772137232:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.319756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471122772137220:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.338423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471119567999966:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38675
2025-11-27T18:17:58.338562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471119567999967:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38675
2025-11-27T18:17:58.338671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471119567999968:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38675
2025-11-27T18:17:58.325669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471120886356710:2130] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.339666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471119406614904:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:17:58.340610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471119406614901:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:58.340863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471119567999965:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.340956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471119567999966:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.341276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471122772137222:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.341736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471120886356713:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.341920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471120886356702:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.341976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:58.342018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471120886356702:2107] [node 4] ICP22 created new session: [2:7577471120886356727:2107]
2025-11-27T18:17:58.341973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471119406614900:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.342017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471115111647527:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.342050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:58.342099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471115111647527:2049] [node 2] ICP22 created new session: [4:7577471119406614905:2049]
2025-11-27T18:17:58.342144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471119406614905:2049] [node 2] ICS09 handshake done sender: [4:7577471119406614901:2116] self: [4:7577471119406614902:0] peer: [2:7577471120886356709:0] socket: 367 qp: -1
2025-11-27T18:17:58.342171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471119406614905:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:58.342221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:58.342826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471119406614903:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.343524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471119567999968:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.343604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471119567999967:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.343715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471115273032590:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.343758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471115273032591:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.342090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471120886356727:2107] [node 4] ICS09 handshake done sender: [2:7577471120886356710:2130] self: [2:7577471120886356709:0] peer: [4:7577471119406614902:0] socket: 365 qp: -1
2025-11-27T18:17:58.342131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471120886356727:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:58.342165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:58.349050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471120886356708:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.349079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471120886356708:2129] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.351617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471119891013097:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19814
2025-11-27T18:17:58.352854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471115111647528:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.352931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471115111647526:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.350172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471122772137216:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.353449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471119567999973:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38675
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.350199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471122772137216:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.354587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471120886356725:2137] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.359134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471119891013098:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19814
2025-11-27T18:17:58.350840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471122772137216:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.353596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471119567999974:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38675
2025-11-27T18:17:58.354755Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471120886356708:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:58.351152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471118477169855:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.357507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471119567999957:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.357530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471119567999957:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.365515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471115273032590:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2391) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471119567999968:2118] held: no
2025-11-27T18:17:58.367511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471119567999968:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:58.367837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471115273032590:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.367909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:58.367959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471115273032590:2048] [node 1] ICP22 created new session: [3:7577471119567999976:2048]
2025-11-27T18:17:58.368006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471119567999976:2048] [node 1] ICS09 handshake done sender: [3:7577471119567999968:2118] self: [3:7577471119567999971:0] peer: [1:7577471122772137215:0] socket: 379 qp: -1
2025-11-27T18:17:58.368041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471119567999976:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:58.368073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:58.368447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471119567999967:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:58.368687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471115273032591:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.368733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:58.368774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471115273032591:2049] [node 2] ICP22 created new session: [3:7577471119567999979:2049]
2025-11-27T18:17:58.368812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471119567999979:2049] [node 2] ICS09 handshake done sender: [3:7577471119567999967:2117] self: [3:7577471119567999972:0] peer: [2:7577471120886356707:0] socket: 378 qp: -1
2025-11-27T18:17:58.368837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471119567999979:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:58.368866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:58.367283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471119891013099:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19814
2025-11-27T18:17:58.351230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:58.351705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471118477169855:2049] [node 3] ICP22 created new session: [1:7577471122772137240:2049]
2025-11-27T18:17:58.351748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471122772137240:2049] [node 3] ICS09 handshake done sender: [1:7577471122772137216:2111] self: [1:7577471122772137215:0] peer: [3:7577471119567999971:0] socket: 370 qp: -1
2025-11-27T18:17:58.351776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471122772137240:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:58.351808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:58.366041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471122772137225:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.366065Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471122772137225:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.377707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471122772137225:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:58.381633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471119891013100:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19814
2025-11-27T18:17:58.382319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471116591389326:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.389553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471122772137217:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.389648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:58.389700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471122772137217:2107] [node 4] ICP22 created new session: [1:7577471122772137244:2107]
2025-11-27T18:17:58.389746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471122772137244:2107] [node 4] ICS09 handshake done sender: [1:7577471122772137225:2116] self: [1:7577471122772137224:0] peer: [4:7577471119406614908:0] socket: 371 qp: -1
2025-11-27T18:17:58.389782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471122772137244:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:58.389834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.398347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471119891013099:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.398428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471120404284980:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57364
2025-11-27T18:17:58.382379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:58.384049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471116591389326:2049] [node 3] ICP22 created new session: [2:7577471120886356732:2049]
2025-11-27T18:17:58.385043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471120886356732:2049] [node 3] ICS09 handshake done sender: [2:7577471120886356708:2129] self: [2:7577471120886356707:0] peer: [3:7577471119567999972:0] socket: 364 qp: -1
2025-11-27T18:17:58.385087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471120886356732:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:58.385134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:58.403576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471115596045721:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.403334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471119567999957:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:58.406971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471119891013098:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.407109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471119891013082:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.408045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471116591389325:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.411137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471120404284981:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57364
2025-11-27T18:17:58.408928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471122772137227:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.408960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471122772137227:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.411744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471122772137214:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.411765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471122772137214:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.408643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471119567999949:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.408696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:58.408739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471119567999949:2107] [node 4] ICP22 created new session: [3:7577471119567999981:2107]
2025-11-27T18:17:58.408780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471119567999981:2107] [node 4] ICS09 handshake done sender: [3:7577471119567999957:2112] self: [3:7577471119567999956:0] peer: [4:7577471119406614907:0] socket: 347 qp: -1
2025-11-27T18:17:58.408810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471119567999981:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:58.408844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:58.413875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471120404284980:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.414556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471116109317606:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.413778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471122772137227:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:58.412931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471119891013097:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.415176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471122772137214:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:58.414250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471115596045722:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.415489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471122772137229:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.415509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471122772137229:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.414330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471119891013103:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19814
2025-11-27T18:17:58.416254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471122772137218:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.416336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:58.416378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471122772137218:2108] [node 5] ICP22 created new session: [1:7577471122772137249:2108]
2025-11-27T18:17:58.416435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471122772137249:2108] [node 5] ICS09 handshake done sender: [1:7577471122772137227:2117] self: [1:7577471122772137226:0] peer: [5:7577471119891013101:0] socket: 345 qp: -1
2025-11-27T18:17:58.416475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471122772137249:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:58.416505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:58.418859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471119891013099:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:58.419101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471115596045721:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.419157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:58.419196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471115596045721:2048] [node 1] ICP22 created new session: [5:7577471119891013105:2048]
2025-11-27T18:17:58.419234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471119891013105:2048] [node 1] ICS09 handshake done sender: [5:7577471119891013099:2118] self: [5:7577471119891013101:0] peer: [1:7577471122772137226:0] socket: 386 qp: -1
2025-11-27T18:17:58.419256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471119891013105:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:58.419284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.421133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471120886356716:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.421164Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471120886356716:2135] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.421968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471120886356736:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33786
2025-11-27T18:17:58.422354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471120404284981:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.422484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471116109317607:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.423822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471119891013107:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19814
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.424883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471120404284984:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57364
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.425557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471119406614910:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:17:58.417762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471118477169854:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.426419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471119406614911:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:17:58.417832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:58.417880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471118477169854:2048] [node 2] ICP22 created new session: [1:7577471122772137251:2048]
2025-11-27T18:17:58.417920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471122772137251:2048] [node 2] ICS09 handshake done sender: [1:7577471122772137214:2110] self: [1:7577471122772137213:0] peer: [2:7577471120886356734:0] socket: 369 qp: -1
2025-11-27T18:17:58.417951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471122772137251:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:58.417977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:58.428154Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471115111647526:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2391) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471119406614903:2117] held: no
2025-11-27T18:17:58.428349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471119406614894:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.428373Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471119406614894:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.430167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471120404284980:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:58.429705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471122772137229:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:58.431195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471119891013100:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.431322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471115596045723:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.431335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471115111647528:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5015) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471119406614900:2115] held: no
2025-11-27T18:17:58.431504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471119406614903:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:58.430498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471116109317606:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.431751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471120886356718:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.431786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471120886356718:2136] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.432103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471119567999959:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.432130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471119567999959:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.433419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:58.433989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471116109317606:2048] [node 1] ICP22 created new session: [6:7577471120404284985:2048]
2025-11-27T18:17:58.435173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471119567999959:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:58.441662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471116591389325:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2391) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471120886356725:2137] held: no
2025-11-27T18:17:58.441939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471120886356716:2135] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:58.435495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471122772137219:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.434863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471119406614900:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:58.435556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471120404284985:2048] [node 1] ICS09 handshake done sender: [6:7577471120404284980:2115] self: [6:7577471120404284982:0] peer: [1:7577471122772137228:0] socket: 392 qp: -1
2025-11-27T18:17:58.435763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471120404284985:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:58.436673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:58.436850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471119567999952:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.445435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:58.449206Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471116109317606:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18495115ns) {HandleHandshakeStatus:329(18493204ns) {DropIncomingHandshake:442(674ns) DropOutgoingHandshake:457(5931ns) SwitchToState:179(907ns) ProcessPendingSessionEvents:495(2992326ns) {ForwardSessionEventToSession:612(2931699ns) ForwardSessionEventToSession:612(5612ns) ForwardSessionEventToSession:612(2555ns) ForwardSessionEventToSession:612(13423ns) ForwardSessionEventToSession:612(2983ns) ForwardSessionEventToSession:612(5938ns) ForwardSessionEventToSession:612(2312ns) ForwardSessionEventToSession:612(2129ns) ForwardSessionEventToSession:612(5989ns) ForwardSessionEventToSession:612(1866ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.454333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471120404284987:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57364
2025-11-27T18:17:58.436910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.447156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471115111647526:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.454467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471120404284988:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57364
2025-11-27T18:17:58.445916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471122772137219:2109] [node 6] ICP22 created new session: [1:7577471122772137254:2109]
2025-11-27T18:17:58.460843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471120886356725:2137] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:58.447202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:58.447229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471115111647526:2048] [node 1] ICP22 created new session: [4:7577471119406614914:2048]
2025-11-27T18:17:58.447258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471119406614914:2048] [node 1] ICS09 handshake done sender: [4:7577471119406614903:2117] self: [4:7577471119406614908:0] peer: [1:7577471122772137224:0] socket: 373 qp: -1
2025-11-27T18:17:58.447284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471119406614914:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:58.447334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:58.436946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471119567999952:2108] [node 5] ICP22 created new session: [3:7577471119567999985:2108]
2025-11-27T18:17:58.456966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471122772137254:2109] [node 6] ICS09 handshake done sender: [1:7577471122772137229:2118] self: [1:7577471122772137228:0] peer: [6:7577471120404284982:0] socket: 355 qp: -1
2025-11-27T18:17:58.437003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471119567999985:2108] [node 5] ICS09 handshake done sender: [3:7577471119567999959:2113] self: [3:7577471119567999958:0] peer: [5:7577471119891013109:0] socket: 388 qp: -1
2025-11-27T18:17:58.447796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471115111647528:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.457017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471122772137254:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:58.447819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:58.447844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471115111647528:2050] [node 3] ICP22 created new session: [4:7577471119406614917:2050]
2025-11-27T18:17:58.447868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471119406614917:2050] [node 3] ICS09 handshake done sender: [4:7577471119406614900:2115] self: [4:7577471119406614907:0] peer: [3:7577471119567999956:0] socket: 349 qp: -1
2025-11-27T18:17:58.447886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471119406614917:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:58.447903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:58.448321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471119406614912:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:17:58.465372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471119406614894:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:58.437042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471119567999985:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:58.437074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:58.469695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471120404284987:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.471335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471116109317608:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.471421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471120404284990:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57364
2025-11-27T18:17:58.457082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:58.457561Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471122772137219:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(21849073ns) {HandleHandshakeStatus:329(21846031ns) {DropIncomingHandshake:442(14651ns) DropOutgoingHandshake:457(218ns) SwitchToState:179(794ns) ProcessPendingSessionEvents:495(123018ns) {ForwardSessionEventToSession:612(51286ns) ForwardSessionEventToSession:612(3451ns) ForwardSessionEventToSession:612(27220ns) ForwardSessionEventToSession:612(2595ns) ForwardSessionEventToSession:612(2974ns) ForwardSessionEventToSession:612(5207ns) ForwardSessionEventToSession:612(6764ns) ForwardSessionEventToSession:612(5755ns) ForwardSessionEventToSession:612(2528ns) ForwardSessionEventToSession:612(2932ns)}}}
2025-11-27T18:17:58.459260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471119567999987:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38675
2025-11-27T18:17:58.465840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471120886356705:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.473253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471115596045723:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5015) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471119891013100:2119] held: no
2025-11-27T18:17:58.465921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:58.465988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471120886356705:2108] [node 5] ICP22 created new session: [2:7577471120886356739:2108]
2025-11-27T18:17:58.466036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471120886356739:2108] [node 5] ICS09 handshake done sender: [2:7577471120886356716:2135] self: [2:7577471120886356715:0] peer: [5:7577471119891013104:0] socket: 360 qp: -1
2025-11-27T18:17:58.466090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471120886356739:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:58.466135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:58.466657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471116591389325:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.466686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:58.466725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471116591389325:2048] [node 1] ICP22 created new session: [2:7577471120886356741:2048]
2025-11-27T18:17:58.466770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471120886356741:2048] [node 1] ICS09 handshake done sender: [2:7577471120886356725:2137] self: [2:7577471120886356734:0] peer: [1:7577471122772137213:0] socket: 372 qp: -1
2025-11-27T18:17:58.466798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471120886356741:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:58.466827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:58.475149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471120886356718:2136] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:58.474782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471119406614887:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.474802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471119567999961:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.474841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471119567999961:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.474833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:58.474898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471119406614887:2108] [node 5] ICP22 created new session: [4:7577471119406614919:2108]
2025-11-27T18:17:58.475290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471119406614919:2108] [node 5] ICS09 handshake done sender: [4:7577471119406614894:2113] self: [4:7577471119406614893:0] peer: [5:7577471119891013102:0] socket: 384 qp: -1
2025-11-27T18:17:58.475393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471119406614919:2108] [node 5] ICS10 traffic start
2025-11-27T18:17:58.475426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:58.475873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471119406614912:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.477443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471119406614887:2108] [node 5] ICP18 handshake (actor [4:7577471119406614912:2121]) is too old
2025-11-27T18:17:58.479510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471120886356706:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.479571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:58.479616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471120886356706:2114] [node 6] ICP22 created new session: [2:7577471120886356743:2114]
2025-11-27T18:17:58.479656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471120886356743:2114] [node 6] ICS09 handshake done sender: [2:7577471120886356718:2136] self: [2:7577471120886356717:0] peer: [6:7577471120404284983:0] socket: 346 qp: -1
2025-11-27T18:17:58.479688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471120886356743:2114] [node 6] ICS10 traffic start
2025-11-27T18:17:58.479720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:58.481421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471119567999961:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:58.481841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471119567999953:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.481879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:58.481914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471119567999953:2109] [node 6] ICP22 created new session: [3:7577471119567999989:2109]
2025-11-27T18:17:58.481958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471119567999989:2109] [node 6] ICS09 handshake done sender: [3:7577471119567999961:2114] self: [3:7577471119567999960:0] peer: [6:7577471120404284991:0] socket: 397 qp: -1
2025-11-27T18:17:58.481989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471119567999989:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:58.482031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.483501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471119891013110:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19814
2025-11-27T18:17:58.484449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471119891013097:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:58.484690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471119567999987:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.484789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471119567999953:2109] [node 6] ICP18 handshake (actor [3:7577471119567999987:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.486444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471115596045722:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64131) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471119891013097:2116] held: no
2025-11-27T18:17:58.486535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471115596045722:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.486588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:58.486632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471115596045722:2049] [node 2] ICP22 created new session: [5:7577471119891013112:2049]
2025-11-27T18:17:58.486675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471119891013112:2049] [node 2] ICS09 handshake done sender: [5:7577471119891013097:2116] self: [5:7577471119891013104:0] peer: [2:7577471120886356715:0] socket: 382 qp: -1
2025-11-27T18:17:58.486726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471119891013112:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:58.486753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:58.487299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471119891013111:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19814
2025-11-27T18:17:58.488716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471116109317607:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64131) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471120404284981:2116] held: no
2025-11-27T18:17:58.488879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471119891013100:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:58.490824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471120404284988:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.490901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471120404284981:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.491255Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471116109317608:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5015) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471120404284987:2118] held: no
2025-11-27T18:17:58.491465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471116109317607:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.491527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:58.491571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471116109317607:2049] [node 2] ICP22 created new session: [6:7577471120404284994:2049]
2025-11-27T18:17:58.491608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471120404284994:2049] [node 2] ICS09 handshake done sender: [6:7577471120404284981:2116] self: [6:7577471120404284983:0] peer: [2:7577471120886356717:0] socket: 361 qp: -1
2025-11-27T18:17:58.491637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471120404284994:2049] [node 2] ICS10 traffic start
2025-11-27T18:17:58.491667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:58.492034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471120404284992:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57364
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.493279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471120404284968:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.493613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471120404284993:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57364
2025-11-27T18:17:58.493661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471120404284987:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:58.492862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471119891013082:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12886) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471119891013098:2117] held: no
2025-11-27T18:17:58.492984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471115596045723:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.493057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:58.493100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471115596045723:2050] [node 3] ICP22 created new session: [5:7577471119891013115:2050]
2025-11-27T18:17:58.493159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471119891013115:2050] [node 3] ICS09 handshake done sender: [5:7577471119891013100:2119] self: [5:7577471119891013109:0] peer: [3:7577471119567999958:0] socket: 389 qp: -1
2025-11-27T18:17:58.493193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471119891013115:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:58.493219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:58.493708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471119891013098:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.494314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471119406614921:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:17:58.494697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471116109317608:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.494749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:17:58.494785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471116109317608:2050] [node 3] ICP22 created new session: [6:7577471120404284998:2050]
2025-11-27T18:17:58.494825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471120404284998:2050] [node 3] ICS09 handshake done sender: [6:7577471120404284987:2118] self: [6:7577471120404284991:0] peer: [3:7577471119567999960:0] socket: 337 qp: -1
2025-11-27T18:17:58.494854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471120404284998:2050] [node 3] ICS10 traffic start
2025-11-27T18:17:58.494880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:58.495659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471119406614896:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.495685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471119406614896:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:58.496909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471119406614896:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:58.497374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471119406614888:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.497413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:58.497458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471119406614888:2109] [node 6] ICP22 created new session: [4:7577471119406614923:2109]
2025-11-27T18:17:58.497503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471119406614923:2109] [node 6] ICS09 handshake done sender: [4:7577471119406614896:2114] self: [4:7577471119406614895:0] peer: [6:7577471120404284997:0] socket: 377 qp: -1
2025-11-27T18:17:58.497532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471119406614923:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:58.497564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.500726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471120404285000:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57364
2025-11-27T18:17:58.501444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471120404284988:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:58.501628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471119406614921:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.501736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471119406614888:2109] [node 6] ICP18 handshake (actor [4:7577471119406614921:2122]) is too old
2025-11-27T18:17:58.505021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471120404284968:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.505120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:58.505165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471120404284968:2108] [node 4] ICP22 created new session: [6:7577471120404285001:2108]
2025-11-27T18:17:58.505209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471120404285001:2108] [node 4] ICS09 handshake done sender: [6:7577471120404284988:2119] self: [6:7577471120404284997:0] peer: [4:7577471119406614895:0] socket: 400 qp: -1
2025-11-27T18:17:58.505246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471120404285001:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:58.505292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:58.505522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471120404284968:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:17:58.507230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471119891013082:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.507308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:58.507350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471119891013082:2108] [node 4] ICP22 created new session: [5:7577471119891013117:2108]
2025-11-27T18:17:58.507394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471119891013117:2108] [node 4] ICS09 handshake done sender: [5:7577471119891013098:2117] self: [5:7577471119891013102:0] peer: [4:7577471119406614893:0] socket: 356 qp: -1
2025-11-27T18:17:58.507425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471119891013117:2108] [node 4] ICS10 traffic start
2025-11-27T18:17:58.507456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:58.512460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471120404284993:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.512613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471120404284963:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:58.513010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471119891013091:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:58.513055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471119891013091:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:58.514521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471120404285004:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57364
2025-11-27T18:17:58.514727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471119891013091:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:58.515171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471120404284993:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:58.515468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471119891013083:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:58.515544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:58.515589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471119891013083:2109] [node 6] ICP22 created new session: [5:7577471119891013120:2109]
2025-11-27T18:17:58.515632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471119891013120:2109] [node 6] ICS09 handshake done sender: [5:7577471119891013091:2114] self: [5:7577471119891013090:0] peer: [6:7577471120404285003:0] socket: 338 qp: -1
2025-11-27T18:17:58.515660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471119891013120:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:58.515690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:58.516551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471120404284963:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:58.516605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:58.516641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471120404284963:2107] [node 5] ICP22 created new session: [6:7577471120404285005:2107]
2025-11-27T18:17:58.516679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471120404285005:2107] [node 5] ICS09 handshake done sender: [6:7577471120404284993:2122] self: [6:7577471120404285003:0] peer: [5:7577471119891013090:0] socket: 342 qp: -1
2025-11-27T18:17:58.516707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471120404285005:2107] [node 5] ICS10 traffic start
2025-11-27T18:17:58.516731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:58.534564Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:17:58.534801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471120886356741:2048] [node 1] ICS01 socket: 372 reason# EndOfStream
2025-11-27T18:17:58.534864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471116591389325:2048] [node 1] ICP30 unregister session Session# [2:7577471120886356741:2048] VirtualId# [2:7577471120886356734:0]
2025-11-27T18:17:58.534899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471120886356741:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.534964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:58.535005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:58.535806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:17:58.536030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471119567999976:2048] [node 1] ICS01 socket: 379 reason# EndOfStream
2025-11-27T18:17:58.536086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471115273032590:2048] [node 1] ICP30 unregister session Session# [3:7577471119567999976:2048] VirtualId# [3:7577471119567999971:0]
2025-11-27T18:17:58.536111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471119567999976:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.536159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:58.536194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:58.538632Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:17:58.538843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471119406614914:2048] [node 1] ICS01 socket: 373 reason# EndOfStream
2025-11-27T18:17:58.538886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471115111647526:2048] [node 1] ICP30 unregister session Session# [4:7577471119406614914:2048] VirtualId# [4:7577471119406614908:0]
2025-11-27T18:17:58.538916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471119406614914:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.538976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:58.539007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:58.539773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:17:58.539990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471119891013105:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-11-27T18:17:58.540023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471115596045721:2048] [node 1] ICP30 unregister session Session# [5:7577471119891013105:2048] VirtualId# [5:7577471119891013101:0]
2025-11-27T18:17:58.540047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471119891013105:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.540094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:58.540136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:58.540763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:17:58.541144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471120404284985:2048] [node 1] ICS01 socket: 392 reason# EndOfStream
2025-11-27T18:17:58.541282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471116109317606:2048] [node 1] ICP30 unregister session Session# [6:7577471120404284985:2048] VirtualId# [6:7577471120404284982:0]
2025-11-27T18:17:58.541308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471120404284985:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.541479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:17:58.541523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:17:58.580202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:17:58.580425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471119567999979:2049] [node 2] ICS01 socket: 378 reason# EndOfStream
2025-11-27T18:17:58.580557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471115273032591:2049] [node 2] ICP30 unregister session Session# [3:7577471119567999979:2049] VirtualId# [3:7577471119567999972:0]
2025-11-27T18:17:58.581313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471119567999979:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.581415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:58.581521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:58.583570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:17:58.583798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471119406614905:2049] [node 2] ICS01 socket: 367 reason# EndOfStream
2025-11-27T18:17:58.583846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471115111647527:2049] [node 2] ICP30 unregister session Session# [4:7577471119406614905:2049] VirtualId# [4:7577471119406614902:0]
2025-11-27T18:17:58.583880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471119406614905:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.583958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:58.583999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:58.584889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:17:58.585106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471119891013112:2049] [node 2] ICS01 socket: 382 reason# EndOfStream
2025-11-27T18:17:58.585145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471115596045722:2049] [node 2] ICP30 unregister session Session# [5:7577471119891013112:2049] VirtualId# [5:7577471119891013104:0]
2025-11-27T18:17:58.585173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471119891013112:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.585269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:58.585345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:58.587961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:17:58.588361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471120404284994:2049] [node 2] ICS01 socket: 361 reason# EndOfStream
2025-11-27T18:17:58.588429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471116109317607:2049] [node 2] ICP30 unregister session Session# [6:7577471120404284994:2049] VirtualId# [6:7577471120404284983:0]
2025-11-27T18:17:58.588455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471120404284994:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.588528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:17:58.588571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:17:58.633678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:17:58.633891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471119406614917:2050] [node 3] ICS01 socket: 349 reason# EndOfStream
2025-11-27T18:17:58.633931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471115111647528:2050] [node 3] ICP30 unregister session Session# [4:7577471119406614917:2050] VirtualId# [4:7577471119406614907:0]
2025-11-27T18:17:58.633963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471119406614917:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.634053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:58.634099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:58.634206Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:17:58.634380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471119891013115:2050] [node 3] ICS01 socket: 389 reason# EndOfStream
2025-11-27T18:17:58.634412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471115596045723:2050] [node 3] ICP30 unregister session Session# [5:7577471119891013115:2050] VirtualId# [5:7577471119891013109:0]
2025-11-27T18:17:58.634441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471119891013115:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.634507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:58.634542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:58.634629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:17:58.634784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471120404284998:2050] [node 3] ICS01 socket: 337 reason# EndOfStream
2025-11-27T18:17:58.634826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471116109317608:2050] [node 3] ICP30 unregister session Session# [6:7577471120404284998:2050] VirtualId# [6:7577471120404284991:0]
2025-11-27T18:17:58.634852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471120404284998:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.634912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:17:58.634944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:17:58.713207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:17:58.713440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471119891013117:2108] [node 4] ICS01 socket: 356 reason# EndOfStream
2025-11-27T18:17:58.713501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471119891013082:2108] [node 4] ICP30 unregister session Session# [5:7577471119891013117:2108] VirtualId# [5:7577471119891013102:0]
2025-11-27T18:17:58.713533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471119891013117:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.713612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:58.713656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:58.713745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:17:58.713905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471120404285001:2108] [node 4] ICS01 socket: 400 reason# EndOfStream
2025-11-27T18:17:58.713938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471120404284968:2108] [node 4] ICP30 unregister session Session# [6:7577471120404285001:2108] VirtualId# [6:7577471120404284997:0]
2025-11-27T18:17:58.713962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471120404285001:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.714006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:17:58.714052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:17:58.742373Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:17:58.742624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471120404285005:2107] [node 5] ICS01 socket: 342 reason# EndOfStream
2025-11-27T18:17:58.742668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471120404284963:2107] [node 5] ICP30 unregister session Session# [6:7577471120404285005:2107] VirtualId# [6:7577471120404285003:0]
2025-11-27T18:17:58.742700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471120404285005:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:17:58.742788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:17:58.742835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:17:58.869241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471119745528901:2048] [node 2] ICP01 ready to work
2025-11-27T18:17:58.869296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471119745528902:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:58.901751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471123272728321:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:58.901808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471123272728322:2049] [node 3] ICP01 ready to work
2025-11-27T18:17:58.921614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471121868164521:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:58.921655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471121868164522:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:58.950763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471120352653628:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:58.950804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471120352653629:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:58.950829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471120352653630:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:58.985842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471123019382471:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:58.985892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471123019382472:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:58.985919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471123019382473:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:59.009990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471123839581415:2048] [node 1] ICP01 ready to work
2025-11-27T18:17:59.010049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471123839581416:2049] [node 2] ICP01 ready to work
2025-11-27T18:17:59.010095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471123839581417:2050] [node 3] ICP01 ready to work
2025-11-27T18:17:59.486938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471127567695696:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:59.489712Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471123019382472:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16895530ns) {Configure:104(16892373ns) {ProcessConfigured:135(5678286ns) {StartInitialHandshake:157(5676103ns) {DropHandshakes:470(2106ns) {DropIncomingHandshake:442(514ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(1476ns) PrepareNewSessionHandshake:477(1526ns) {SwitchToState:179(698ns)}}}}}
2025-11-27T18:17:59.493546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471127314349803:2076] [node 6] ICP01 ready to work
2025-11-27T18:17:59.493898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471127314349804:2078] [node 4] ICP01 ready to work
2025-11-27T18:17:59.487281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471127567695697:2111] [node 6] ICP01 ready to work
2025-11-27T18:17:59.481736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471124040496260:2107] [node 4] ICP01 ready to work
2025-11-27T18:17:59.490481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471124647620990:2102] [node 6] ICP01 ready to work
2025-11-27T18:17:59.561318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471127567695700:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:59.494698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471126163131896:2104] [node 6] ICP01 ready to work
2025-11-27T18:17:59.482224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471124040496261:2109] [node 6] ICP01 ready to work
2025-11-27T18:17:59.521503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471126163131897:2124] [node 5] ICP01 ready to work
2025-11-27T18:17:59.558464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471124647620991:2094] [node 5] ICP01 ready to work
2025-11-27T18:17:59.499832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471124040496262:2108] [node 5] ICP01 ready to work
2025-11-27T18:17:59.721996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471123839581417:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(249315910ns) {Configure:104(249311366ns) {ProcessConfigured:135(54988ns) {StartInitialHandshake:157(53057ns) {DropHandshakes:470(1368ns) {DropIncomingHandshake:442(287ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(838ns) PrepareNewSessionHandshake:477(1116ns) {SwitchToState:179(359ns)}}}}}
2025-11-27T18:17:59.562142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471119745528902:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(62028848ns) {Configure:104(62023839ns) {ProcessConfigured:135(80565ns) {StartInitialHandshake:157(78268ns) {DropHandshakes:470(2254ns) {DropIncomingHandshake:442(413ns) DropOutgoingHandshake:457(352ns)} GenerateSessionVirtualId:988(1413ns) PrepareNewSessionHandshake:477(1849ns) {SwitchToState:179(949ns)}}}}}
2025-11-27T18:17:59.685763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471126163131902:2125] [node 4] ICP01 ready to work
2025-11-27T18:17:59.723804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471123839581481:2069] [node 4] ICP01 ready to work
2025-11-27T18:17:59.724101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471123839581482:2071] [node 5] ICP01 ready to work
2025-11-27T18:17:59.742956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471123839581415:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(236711085ns) {Configure:104(236705928ns) {ProcessConfigured:135(84567387ns) {StartInitialHandshake:157(84563921ns) {DropHandshakes:470(2312ns) {DropIncomingHandshake:442(553ns) DropOutgoingHandshake:457(347ns)} GenerateSessionVirtualId:988(1102ns) PrepareNewSessionHandshake:477(6605716ns) {SwitchToState:179(7572ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.734063Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471123019382471:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(244109261ns) {Configure:104(244104668ns) {ProcessConfigured:135(77331ns) {StartInitialHandshake:157(74182ns) {DropHandshakes:470(2437ns) {DropIncomingHandshake:442(580ns) DropOutgoingHandshake:457(471ns)} GenerateSessionVirtualId:988(1005ns) PrepareNewSessionHandshake:477(5361ns) {SwitchToState:179(518ns)}}}}}
2025-11-27T18:17:59.754460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471123019382473:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(260332467ns) {Configure:104(260327901ns) {ProcessConfigured:135(61714ns) {StartInitialHandshake:157(59483ns) {DropHandshakes:470(2382ns) {DropIncomingHandshake:442(513ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(658ns) PrepareNewSessionHandshake:477(1822ns) {SwitchToState:179(762ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.602547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471119745528901:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(120042706ns) {Configure:104(120037568ns) {ProcessConfigured:135(44445159ns) {StartInitialHandshake:157(44441760ns) {DropHandshakes:470(2526ns) {DropIncomingHandshake:442(562ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(953ns) PrepareNewSessionHandshake:477(4300ns) {SwitchToState:179(2483ns)}}}}}
2025-11-27T18:17:59.750741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471126163131908:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45842
2025-11-27T18:17:59.774781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471123272728321:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(268821972ns) {Configure:104(268816643ns) {ProcessConfigured:135(70476ns) {StartInitialHandshake:157(67653ns) {DropHandshakes:470(2210ns) {DropIncomingHandshake:442(633ns) DropOutgoingHandshake:457(295ns)} GenerateSessionVirtualId:988(1277ns) PrepareNewSessionHandshake:477(1522ns) {SwitchToState:179(554ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.781542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471127314349803:2076] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(35176582ns) {Configure:104(35171251ns) {ProcessConfigured:135(28027158ns) {StartInitialHandshake:157(28023449ns) {DropHandshakes:470(1322ns) {DropIncomingHandshake:442(409ns) DropOutgoingHandshake:457(265ns)} GenerateSessionVirtualId:988(891ns) PrepareNewSessionHandshake:477(1067278ns) {SwitchToState:179(3209ns)}}}}}
2025-11-27T18:17:59.729289Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471124040496262:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(165020711ns) {Configure:104(165000921ns) {ProcessConfigured:135(8486357ns) {StartInitialHandshake:157(8481366ns) {DropHandshakes:470(2804ns) {DropIncomingHandshake:442(621ns) DropOutgoingHandshake:457(421ns)} GenerateSessionVirtualId:988(1071ns) PrepareNewSessionHandshake:477(3482ns) {SwitchToState:179(1425ns)}}}}}
2025-11-27T18:17:59.766116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471124040496276:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42057
2025-11-27T18:17:59.766189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471124040496277:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42057
2025-11-27T18:17:59.787837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471124040496265:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.787866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471124040496265:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:59.793469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471124040496265:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:59.767300Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471126163131896:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(145073234ns) {Configure:104(145069500ns) {ProcessConfigured:135(32146831ns) {StartInitialHandshake:157(32144812ns) {DropHandshakes:470(4389987ns) {DropIncomingHandshake:442(637ns) DropOutgoingHandshake:457(1633ns)} GenerateSessionVirtualId:988(2008ns) PrepareNewSessionHandshake:477(2096ns) {SwitchToState:179(403ns)}}}}}
2025-11-27T18:17:59.808508Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471127567695697:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(246843824ns) {Configure:104(246838068ns) {ProcessConfigured:135(165223ns) {StartInitialHandshake:157(71944ns) {DropHandshakes:470(3204ns) {DropIncomingHandshake:442(794ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(1367ns) PrepareNewSessionHandshake:477(2225ns) {SwitchToState:179(627ns)}}}}}
2025-11-27T18:17:59.792846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471120352653629:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(123315766ns) {Configure:104(123303310ns) {ProcessConfigured:135(66214ns) {StartInitialHandshake:157(63171ns) {DropHandshakes:470(2860ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(1179ns) PrepareNewSessionHandshake:477(1475ns) {SwitchToState:179(478ns)}}}}}
2025-11-27T18:17:59.777883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471126163131908:2130] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.834341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471120352653628:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(128843938ns) {Configure:104(128833718ns) {ProcessConfigured:135(71351ns) {StartInitialHandshake:157(68637ns) {DropHandshakes:470(2556ns) {DropIncomingHandshake:442(447ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(1286ns) PrepareNewSessionHandshake:477(1573ns) {SwitchToState:179(457ns)}}}}}
2025-11-27T18:17:59.835084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471124647621000:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44040
2025-11-27T18:17:59.835240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471124647621001:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44040
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.836534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471127314349845:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20835
2025-11-27T18:17:59.837268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471119745528902:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:17:59.837331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.841413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471119745528902:2049] [node 3] ICP22 created new session: [1:7577471124040496279:2049]
2025-11-27T18:17:59.841479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471124040496279:2049] [node 3] ICS09 handshake done sender: [1:7577471124040496265:2110] self: [1:7577471124040496264:0] peer: [3:7577471126163131914:0] socket: 342 qp: -1
2025-11-27T18:17:59.841527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471124040496279:2049] [node 3] ICS10 traffic start
2025-11-27T18:17:59.841574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:17:59.842086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471124040496277:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.842179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471124040496276:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.842444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471119745528902:2049] [node 3] ICP18 handshake (actor [1:7577471124040496276:2115]) is too old
2025-11-27T18:17:59.843149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471124647621000:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.843365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471124647621001:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.843529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471120352653628:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:59.843578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471120352653630:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:59.782260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471121868164521:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.853954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471123839581481:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(105202944ns) {Configure:104(105196255ns) {ProcessConfigured:135(20212658ns) {StartInitialHandshake:157(20209785ns) {DropHandshakes:470(1299ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(573ns) PrepareNewSessionHandshake:477(3973ns) {SwitchToState:179(1990ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.782363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471126163131913:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45842
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.865520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471124647621009:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44040
2025-11-27T18:17:59.864856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471127314349845:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.854776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471123839581482:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(111606736ns) {Configure:104(111601708ns) {ProcessConfigured:135(52919ns) {StartInitialHandshake:157(50737ns) {DropHandshakes:470(1389ns) {DropIncomingHandshake:442(329ns) DropOutgoingHandshake:457(172ns)} GenerateSessionVirtualId:988(621ns) PrepareNewSessionHandshake:477(2287ns) {SwitchToState:179(851ns)}}}}}
2025-11-27T18:17:59.861901Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471127567695700:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(84838957ns) {Configure:104(84834469ns) {ProcessConfigured:135(64376ns) {StartInitialHandshake:157(62123ns) {DropHandshakes:470(2691ns) {DropIncomingHandshake:442(494ns) DropOutgoingHandshake:457(357ns)} GenerateSessionVirtualId:988(995ns) PrepareNewSessionHandshake:477(1902ns) {SwitchToState:179(826ns)}}}}}
2025-11-27T18:17:59.838096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471126163131916:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45842
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.878429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471124647621010:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44040
2025-11-27T18:17:59.845488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471126163131913:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.867732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471124040496267:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.862859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471127567695707:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55311
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.879383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471127314349804:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:59.888409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471124647620997:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.888439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471124647620997:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:59.889077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471124647621009:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.889185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471124647621000:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.890261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471127314349847:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20835
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.891692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471123839581493:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15422
2025-11-27T18:17:59.891811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471123839581494:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15422
2025-11-27T18:17:59.863301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471127567695708:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55311
2025-11-27T18:17:59.864174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471126163131905:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.867775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471124040496267:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:59.889792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471120352653629:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:59.894826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471123839581493:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.898198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471123839581415:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:59.871355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471124040496283:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42057
2025-11-27T18:17:59.878917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471124040496267:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:59.892511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471124040496268:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.892530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471124040496268:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:17:59.893039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471124040496260:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:59.893082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:59.893121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471124040496260:2107] [node 4] ICP22 created new session: [1:7577471124040496285:2107]
2025-11-27T18:17:59.893163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471124040496285:2107] [node 4] ICS09 handshake done sender: [1:7577471124040496267:2111] self: [1:7577471124040496266:0] peer: [4:7577471124647621007:0] socket: 343 qp: -1
2025-11-27T18:17:59.893207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471124040496285:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:59.893239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:59.899173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471124040496268:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:17:59.864209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471126163131905:2128] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:59.864325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471126163131915:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45842
2025-11-27T18:17:59.900119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471121868164521:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18852) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471126163131908:2130] held: no
2025-11-27T18:17:59.900729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471126163131908:2130] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:59.863437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471127567695709:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55311
2025-11-27T18:17:59.863541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471127567695710:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55311
2025-11-27T18:17:59.871597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471127567695709:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.871719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471127567695710:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.873636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471127567695707:2130] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.886655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471127567695716:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55311
2025-11-27T18:17:59.886859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471123272728321:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:59.890460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471127567695704:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.890478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471127567695704:2128] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:17:59.898955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471127567695704:2128] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:59.902645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471127314349845:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:17:59.901918Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471124647620997:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:59.902235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471121868164521:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.903158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471124040496270:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.902294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:59.903035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471123839581494:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.905544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471123839581416:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:59.906138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471123839581497:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15422
2025-11-27T18:17:59.906557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471123839581493:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:59.906709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471123839581415:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:59.906786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:59.906825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471123839581415:2048] [node 1] ICP22 created new session: [6:7577471123839581498:2048]
2025-11-27T18:17:59.906882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471123839581498:2048] [node 1] ICS09 handshake done sender: [6:7577471123839581493:2115] self: [6:7577471123839581495:0] peer: [1:7577471124040496269:0] socket: 377 qp: -1
2025-11-27T18:17:59.906912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471123839581498:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:59.906940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:59.901909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471127567695696:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:59.905334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471127314349804:2078] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:17:59.902340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471121868164521:2048] [node 1] ICP22 created new session: [3:7577471126163131921:2048]
2025-11-27T18:17:59.902378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471126163131921:2048] [node 1] ICS09 handshake done sender: [3:7577471126163131908:2130] self: [3:7577471126163131914:0] peer: [1:7577471124040496264:0] socket: 346 qp: -1
2025-11-27T18:17:59.902405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471126163131921:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:59.902439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:59.904797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471126163131918:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45842
2025-11-27T18:17:59.904927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471126163131919:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45842
2025-11-27T18:17:59.905096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471126163131905:2128] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.902015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:59.903191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471124040496270:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:59.903715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471119745528901:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:17:59.903752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:17:59.903789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471119745528901:2048] [node 2] ICP22 created new session: [1:7577471124040496289:2048]
2025-11-27T18:17:59.903829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471124040496289:2048] [node 2] ICS09 handshake done sender: [1:7577471124040496268:2112] self: [1:7577471124040496263:0] peer: [2:7577471127567695719:0] socket: 344 qp: -1
2025-11-27T18:17:59.903858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471124040496289:2048] [node 2] ICS10 traffic start
2025-11-27T18:17:59.903884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:17:59.909642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471124040496270:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:59.928002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471124040496283:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.905381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.938560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471123839581500:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15422
2025-11-27T18:17:59.913964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471126163131918:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.934156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471120352653628:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:17:59.945336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:59.945409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471120352653628:2048] [node 1] ICP22 created new session: [4:7577471124647621015:2048]
2025-11-27T18:17:59.909529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471127567695696:2107] [node 4] ICP22 created new session: [2:7577471127567695722:2107]
2025-11-27T18:17:59.909454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471127314349804:2078] [node 4] ICP22 created new session: [5:7577471127314349849:2078]
2025-11-27T18:17:59.913975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471126163131902:2125] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:17:59.949778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471124040496261:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:59.913439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471127567695708:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.957376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471124647621015:2048] [node 1] ICS09 handshake done sender: [4:7577471124647621000:2117] self: [4:7577471124647621007:0] peer: [1:7577471124040496266:0] socket: 359 qp: -1
2025-11-27T18:17:59.912973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471127314349848:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20835
2025-11-27T18:17:59.957446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471124647621015:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:59.957501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:59.957954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471120352653628:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23592273ns) {HandleHandshakeStatus:329(23589203ns) {DropIncomingHandshake:442(896ns) DropOutgoingHandshake:457(140774ns) SwitchToState:179(652ns) ProcessPendingSessionEvents:495(100118ns) {ForwardSessionEventToSession:612(56176ns) ForwardSessionEventToSession:612(3462ns) ForwardSessionEventToSession:612(2417ns) ForwardSessionEventToSession:612(6783ns) ForwardSessionEventToSession:612(2214ns) ForwardSessionEventToSession:612(7397ns) ForwardSessionEventToSession:612(3213ns) ForwardSessionEventToSession:612(2716ns) ForwardSessionEventToSession:612(5326ns) ForwardSessionEventToSession:612(2842ns)}}}
2025-11-27T18:17:59.958002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471124647620991:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:59.958040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:59.958085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471124647620991:2094] [node 5] ICP22 created new session: [4:7577471124647621017:2094]
2025-11-27T18:17:59.958139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471124647621017:2094] [node 5] ICS09 handshake done sender: [4:7577471124647620997:2115] self: [4:7577471124647620996:0] peer: [5:7577471127314349846:0] socket: 365 qp: -1
2025-11-27T18:17:59.958171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471124647621017:2094] [node 5] ICS10 traffic start
2025-11-27T18:17:59.958197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:59.949843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:59.961900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471123839581500:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.914022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:17:59.915341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471127567695722:2107] [node 4] ICS09 handshake done sender: [2:7577471127567695704:2128] self: [2:7577471127567695703:0] peer: [4:7577471124647621012:0] socket: 374 qp: -1
2025-11-27T18:17:59.922418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471127314349849:2078] [node 4] ICS09 handshake done sender: [5:7577471127314349845:2115] self: [5:7577471127314349846:0] peer: [4:7577471124647620996:0] socket: 367 qp: -1
2025-11-27T18:17:59.949888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471124040496261:2109] [node 6] ICP22 created new session: [1:7577471124040496292:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.914071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471126163131902:2125] [node 4] ICP22 created new session: [3:7577471126163131924:2125]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.962039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471123839581482:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.915386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471127567695722:2107] [node 4] ICS10 traffic start
2025-11-27T18:17:59.928457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471127314349849:2078] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.949937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471124040496292:2109] [node 6] ICS09 handshake done sender: [1:7577471124040496270:2113] self: [1:7577471124040496269:0] peer: [6:7577471123839581495:0] socket: 372 qp: -1
2025-11-27T18:17:59.975772Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471120352653629:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4056) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471124647621009:2120] held: no
2025-11-27T18:17:59.949974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471124040496292:2109] [node 6] ICS10 traffic start
2025-11-27T18:17:59.918883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:59.922504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471126163131924:2125] [node 4] ICS09 handshake done sender: [3:7577471126163131905:2128] self: [3:7577471126163131904:0] peer: [4:7577471124647621008:0] socket: 349 qp: -1
2025-11-27T18:17:59.980711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471123839581501:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15422
2025-11-27T18:17:59.930277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471127314349851:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20835
2025-11-27T18:17:59.976282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471124647621014:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44040
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.969487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471127567695716:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.972626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471127567695706:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.972665Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471127567695706:2129] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:17:59.976745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471127567695696:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(70558426ns) {HandleHandshakeStatus:329(70555572ns) {DropIncomingHandshake:442(16309ns) DropOutgoingHandshake:457(1346ns) SwitchToState:179(345ns) ProcessPendingSessionEvents:495(30798259ns) {ForwardSessionEventToSession:612(18794204ns) ForwardSessionEventToSession:612(4155ns) ForwardSessionEventToSession:612(4361731ns) ForwardSessionEventToSession:612(15878ns) ForwardSessionEventToSession:612(2863ns) ForwardSessionEventToSession:612(1575304ns) ForwardSessionEventToSession:612(2862ns) ForwardSessionEventToSession:612(1438571ns) ForwardSessionEventToSession:612(4932ns) ForwardSessionEventToSession:612(2220ns)}}}
2025-11-27T18:17:59.976896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471127567695699:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.976914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471127567695699:2126] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:17:59.978350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471127567695721:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55311
2025-11-27T18:17:59.988890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471127567695707:2130] [node 1] ICH04 handshake succeeded
2025-11-27T18:17:59.995661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471127567695706:2129] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:59.996172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471127567695697:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:59.996247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:59.996294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471127567695697:2111] [node 6] ICP22 created new session: [2:7577471127567695729:2111]
2025-11-27T18:17:59.996336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471127567695729:2111] [node 6] ICS09 handshake done sender: [2:7577471127567695706:2129] self: [2:7577471127567695705:0] peer: [6:7577471123839581496:0] socket: 375 qp: -1
2025-11-27T18:17:59.996370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471127567695729:2111] [node 6] ICS10 traffic start
2025-11-27T18:17:59.996419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:59.998816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471127567695699:2126] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:59.950008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:59.927930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471126163131924:2125] [node 4] ICS10 traffic start
2025-11-27T18:17:59.932047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:17:59.986712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471123839581504:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15422
2025-11-27T18:17:59.994515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471120352653630:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4787) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471124647621001:2118] held: no
2025-11-27T18:17:59.968290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471124040496261:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18337991ns) {HandleHandshakeStatus:329(18335350ns) {DropIncomingHandshake:442(439ns) DropOutgoingHandshake:457(499ns) SwitchToState:179(525ns) ProcessPendingSessionEvents:495(17974293ns) {ForwardSessionEventToSession:612(49340ns) ForwardSessionEventToSession:612(2996ns) ForwardSessionEventToSession:612(10239512ns) ForwardSessionEventToSession:612(7673ns) ForwardSessionEventToSession:612(3040ns) ForwardSessionEventToSession:612(7635823ns) ForwardSessionEventToSession:612(3570ns) ForwardSessionEventToSession:612(5874ns) ForwardSessionEventToSession:612(2599ns) ForwardSessionEventToSession:612(2957ns)}}}
2025-11-27T18:17:59.927985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:00.009597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471123272728321:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:00.002102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471124647621019:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44040
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.941829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471127314349851:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.986765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471123839581505:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15422
2025-11-27T18:17:59.968727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471124040496261:2109] [node 6] ICP18 handshake (actor [1:7577471124040496283:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.953940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471126163131916:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:00.009666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:17:59.969878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471127314349804:2078] [node 4] ICP35 event processing took too much time PendingConnection:279(64375579ns) {HandleHandshakeStatus:329(64373586ns) {DropIncomingHandshake:442(465ns) DropOutgoingHandshake:457(4381ns) SwitchToState:179(486ns) ProcessPendingSessionEvents:495(9381890ns) {ForwardSessionEventToSession:612(9330076ns) ForwardSessionEventToSession:612(12366ns) ForwardSessionEventToSession:612(2555ns) ForwardSessionEventToSession:612(7984ns) ForwardSessionEventToSession:612(2461ns) ForwardSessionEventToSession:612(1951ns) ForwardSessionEventToSession:612(4719ns) ForwardSessionEventToSession:612(1711ns) ForwardSessionEventToSession:612(5259ns) ForwardSessionEventToSession:612(4725ns)}}}
2025-11-27T18:17:59.988519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471123839581500:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:59.979271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471124040496275:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:00.009710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471123272728321:2048] [node 1] ICP22 created new session: [2:7577471131862663027:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:00.002242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471124647621013:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44040
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:17:59.979291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471124040496275:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:17:59.970290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471126163131919:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.969950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471123019382473:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:00.009762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471131862663027:2048] [node 1] ICS09 handshake done sender: [2:7577471127567695707:2130] self: [2:7577471127567695719:0] peer: [1:7577471124040496263:0] socket: 361 qp: -1
2025-11-27T18:17:59.989965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471123839581482:2071] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:17:59.970379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471121868164522:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:17:59.983457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471124040496275:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:00.023060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471124647621001:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:17:59.971787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471127314349844:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.994761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:00.009793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471131862663027:2048] [node 1] ICS10 traffic start
2025-11-27T18:17:59.973737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471126163131902:2125] [node 4] ICP35 event processing took too much time PendingConnection:279(58759300ns) {HandleHandshakeStatus:329(58756077ns) {DropIncomingHandshake:442(436ns) DropOutgoingHandshake:457(384ns) SwitchToState:179(644ns) ProcessPendingSessionEvents:495(26958211ns) {ForwardSessionEventToSession:612(20375647ns) ForwardSessionEventToSession:612(3407813ns) ForwardSessionEventToSession:612(5247ns) ForwardSessionEventToSession:612(1604685ns) ForwardSessionEventToSession:612(4030ns) ForwardSessionEventToSession:612(2356ns) ForwardSessionEventToSession:612(773366ns) ForwardSessionEventToSession:612(3723ns) ForwardSessionEventToSession:612(511417ns) ForwardSessionEventToSession:612(244350ns)}}}
2025-11-27T18:17:59.998724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471124040496262:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:59.971817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471127314349844:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:00.028858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471120352653630:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(18455613ns) {HandleHandshakeStatus:395(18453829ns) {LogHandshakeFail:472(11313ns) {UpdateErrorStateLog:545(4247ns)} DropOutgoingHandshake:457(1192ns) ProcessPendingSessionEvents:495(12979569ns) {DropSessionEvent:512(3898218ns) DropSessionEvent:512(18488ns) DropSessionEvent:512(5595ns) DropSessionEvent:512(5866ns) DropSessionEvent:512(4793ns) DropSessionEvent:512(3822ns) DropSessionEvent:512(4580ns) DropSessionEvent:512(3503ns) DropSessionEvent:512(4078ns) DropSessionEvent:512(15103ns)}}}
2025-11-27T18:18:00.002458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471123839581482:2071] [node 5] ICP22 created new session: [6:7577471123839581506:2071]
2025-11-27T18:18:00.009825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:17:59.973863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471126163131907:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.978376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471127314349848:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:17:59.998766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:00.048814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471124647621019:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:00.006609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471123839581501:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:00.017688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471123272728322:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:00.013034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471124040496262:2108] [node 5] ICP22 created new session: [1:7577471124040496295:2108]
2025-11-27T18:18:00.024082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471124040496295:2108] [node 5] ICS09 handshake done sender: [1:7577471124040496275:2114] self: [1:7577471124040496274:0] peer: [5:7577471127314349854:0] socket: 390 qp: -1
2025-11-27T18:18:00.024127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471124040496295:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:00.029615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:00.037816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471124040496262:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(38907597ns) {HandleHandshakeStatus:329(38904430ns) {DropIncomingHandshake:442(9634ns) DropOutgoingHandshake:457(285ns) SwitchToState:179(126ns) ProcessPendingSessionEvents:495(135252ns) {ForwardSessionEventToSession:612(78514ns) ForwardSessionEventToSession:612(3399ns) ForwardSessionEventToSession:612(2933ns) ForwardSessionEventToSession:612(8220ns) ForwardSessionEventToSession:612(12948ns) ForwardSessionEventToSession:612(2497ns) ForwardSessionEventToSession:612(6169ns) ForwardSessionEventToSession:612(2050ns) ForwardSessionEventToSession:612(7917ns) ForwardSessionEventToSession:612(2178ns)}}}
2025-11-27T18:17:59.973883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471126163131907:2129] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:00.048906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471120352653630:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:17:59.979012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471123019382471:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:00.017792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:00.017841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471123272728322:2049] [node 3] ICP22 created new session: [2:7577471131862663030:2049]
2025-11-27T18:18:00.017881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471131862663030:2049] [node 3] ICS09 handshake done sender: [2:7577471127567695699:2126] self: [2:7577471127567695698:0] peer: [3:7577471126163131928:0] socket: 373 qp: -1
2025-11-27T18:18:00.017911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471131862663030:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:00.017945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:00.053508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471127567695712:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:00.053530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471127567695712:2134] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:00.060898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471127567695712:2134] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:00.064703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471127567695700:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:00.065445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:00.065497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471127567695700:2108] [node 5] ICP22 created new session: [2:7577471131862663033:2108]
2025-11-27T18:18:00.065541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471131862663033:2108] [node 5] ICS09 handshake done sender: [2:7577471127567695712:2134] self: [2:7577471127567695711:0] peer: [5:7577471131609317161:0] socket: 411 qp: -1
2025-11-27T18:18:00.065578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471131862663033:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:00.065604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:17:59.985849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471127314349844:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:17:59.986261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471127314349856:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20835
2025-11-27T18:18:00.012944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471123839581506:2071] [node 5] ICS09 handshake done sender: [6:7577471123839581500:2118] self: [6:7577471123839581503:0] peer: [5:7577471127314349835:0] socket: 402 qp: -1
2025-11-27T18:18:00.017039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471123839581506:2071] [node 5] ICS10 traffic start
2025-11-27T18:18:00.018755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:00.030103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471123839581482:2071] [node 5] ICP35 event processing took too much time PendingConnection:279(40492154ns) {HandleHandshakeStatus:329(40489761ns) {DropIncomingHandshake:442(3433ns) DropOutgoingHandshake:457(9222ns) SwitchToState:179(458ns) ProcessPendingSessionEvents:495(111535ns) {ForwardSessionEventToSession:612(61355ns) ForwardSessionEventToSession:612(3340ns) ForwardSessionEventToSession:612(6188ns) ForwardSessionEventToSession:612(2967ns) ForwardSessionEventToSession:612(8269ns) ForwardSessionEventToSession:612(5679ns) ForwardSessionEventToSession:612(3290ns) ForwardSessionEventToSession:612(7523ns) ForwardSessionEventToSession:612(2662ns) ForwardSessionEventToSession:612(2688ns)}}}
2025-11-27T18:18:00.030384Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471123839581416:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4056) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471123839581494:2116] held: no
2025-11-27T18:18:00.031092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471123839581494:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:00.033760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471123839581417:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:00.033988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471128134548804:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15422
2025-11-27T18:18:00.038868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471123839581416:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:00.053813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:00.055123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471128134548807:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15422
2025-11-27T18:18:00.055905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471123839581501:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:00.056089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471123839581417:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:00.056147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:00.056184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471123839581417:2050] [node 3] ICP22 created new session: [6:7577471128134548808:2050]
2025-11-27T18:18:00.056233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471128134548808:2050] [node 3] ICS09 handshake done sender: [6:7577471123839581501:2119] self: [6:7577471128134548805:0] peer: [3:7577471126163131903:0] socket: 404 qp: -1
2025-11-27T18:18:00.056271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471128134548808:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:00.056296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:00.053853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471123839581416:2049] [node 2] ICP22 created new session: [6:7577471128134548806:2049]
2025-11-27T18:18:00.061799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471128134548804:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:00.062430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471128134548806:2049] [node 2] ICS09 handshake done sender: [6:7577471123839581494:2116] self: [6:7577471123839581496:0] peer: [2:7577471127567695705:0] socket: 382 qp: -1
2025-11-27T18:18:00.062620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471123839581481:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:00.063940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471128134548806:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:00.064644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:00.066838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471128134548812:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15422
2025-11-27T18:18:00.069894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471123839581416:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(30903998ns) {HandleHandshakeStatus:329(30901683ns) {DropIncomingHandshake:442(1017ns) DropOutgoingHandshake:457(286ns) SwitchToState:179(411ns) ProcessPendingSessionEvents:495(803ns)}}
2025-11-27T18:18:00.073231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471128134548804:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:00.078975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471123839581481:2069] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2220) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471128134548804:2122] held: no
2025-11-27T18:18:00.079074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471123839581481:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:00.079141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:00.079192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471123839581481:2069] [node 4] ICP22 created new session: [6:7577471128134548813:2069]
2025-11-27T18:18:00.079237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471128134548813:2069] [node 4] ICS09 handshake done sender: [6:7577471128134548804:2122] self: [6:7577471128134548810:0] peer: [4:7577471124647620994:0] socket: 363 qp: -1
2025-11-27T18:18:00.079274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471128134548813:2069] [node 4] ICS10 traffic start
2025-11-27T18:18:00.079319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:00.048954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:00.048999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471120352653630:2050] [node 3] ICP22 created new session: [4:7577471128942588317:2050]
2025-11-27T18:18:00.049059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471128942588317:2050] [node 3] ICS09 handshake done sender: [4:7577471124647621001:2118] self: [4:7577471124647621008:0] peer: [3:7577471126163131904:0] socket: 360 qp: -1
2025-11-27T18:18:00.049100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471128942588317:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:00.049131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:00.049528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471124647621009:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:00.051516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471120352653629:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:00.051555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:00.051598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471120352653629:2049] [node 2] ICP22 created new session: [4:7577471128942588319:2049]
2025-11-27T18:18:00.051650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471128942588319:2049] [node 2] ICS09 handshake done sender: [4:7577471124647621009:2120] self: [4:7577471124647621012:0] peer: [2:7577471127567695703:0] socket: 378 qp: -1
2025-11-27T18:18:00.051676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471128942588319:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:00.051708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:00.062884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471124647620995:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:00.062908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471124647620995:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:00.064858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471124647620995:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:00.067048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471124647620990:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:00.067093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:00.067132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471124647620990:2102] [node 6] ICP22 created new session: [4:7577471128942588323:2102]
2025-11-27T18:18:00.067171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471128942588323:2102] [node 6] ICS09 handshake done sender: [4:7577471124647620995:2114] self: [4:7577471124647620994:0] peer: [6:7577471128134548810:0] socket: 354 qp: -1
2025-11-27T18:18:00.067199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471128942588323:2102] [node 6] ICS10 traffic start
2025-11-27T18:18:00.067224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:59.974392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471126163131902:2125] [node 4] ICP18 handshake (actor [3:7577471126163131916:2134]) is too old
2025-11-27T18:17:59.981067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471126163131907:2129] [node 5] ICH04 handshake succeeded
2025-11-27T18:17:59.997582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471126163131897:2124] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:17:59.997629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:17:59.997667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471126163131897:2124] [node 5] ICP22 created new session: [3:7577471126163131931:2124]
2025-11-27T18:17:59.997706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471126163131931:2124] [node 5] ICS09 handshake done sender: [3:7577471126163131907:2129] self: [3:7577471126163131906:0] peer: [5:7577471127314349853:0] socket: 401 qp: -1
2025-11-27T18:17:59.997737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471126163131931:2124] [node 5] ICS10 traffic start
2025-11-27T18:17:59.997762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:00.043969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471126163131909:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:00.043999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471126163131909:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:00.044896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471126163131930:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45842
2025-11-27T18:18:00.054439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471121868164522:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4056) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471126163131919:2136] held: no
2025-11-27T18:18:00.054728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471126163131919:2136] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:00.055527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471126163131909:2131] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:00.057519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471121868164522:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:00.057552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:00.057599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471121868164522:2049] [node 2] ICP22 created new session: [3:7577471130458099230:2049]
2025-11-27T18:18:00.062361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471130458099230:2049] [node 2] ICS09 handshake done sender: [3:7577471126163131919:2136] self: [3:7577471126163131928:0] peer: [2:7577471127567695698:0] socket: 392 qp: -1
2025-11-27T18:18:00.063967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471130458099230:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:00.064008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:00.067927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471126163131896:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:00.067969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:00.068000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471126163131896:2104] [node 6] ICP22 created new session: [3:7577471130458099232:2104]
2025-11-27T18:18:00.068040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471130458099232:2104] [node 6] ICS09 handshake done sender: [3:7577471126163131909:2131] self: [3:7577471126163131903:0] peer: [6:7577471128134548805:0] socket: 403 qp: -1
2025-11-27T18:18:00.068063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471130458099232:2104] [node 6] ICS10 traffic start
2025-11-27T18:18:00.068089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:59.987363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471127314349857:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20835
2025-11-27T18:17:59.987702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471127314349803:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:17:59.987747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:17:59.987789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471127314349803:2076] [node 6] ICP22 created new session: [5:7577471127314349858:2076]
2025-11-27T18:17:59.987842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471127314349858:2076] [node 6] ICS09 handshake done sender: [5:7577471127314349844:2114] self: [5:7577471127314349835:0] peer: [6:7577471123839581503:0] socket: 400 qp: -1
2025-11-27T18:17:59.987875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471127314349858:2076] [node 6] ICS10 traffic start
2025-11-27T18:17:59.987903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:17:59.993797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471127314349851:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:00.012558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471127314349848:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:00.012850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471123019382473:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:00.012898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:00.021390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471123019382473:2050] [node 3] ICP22 created new session: [5:7577471131609317156:2050]
2025-11-27T18:18:00.021446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471131609317156:2050] [node 3] ICS09 handshake done sender: [5:7577471127314349851:2118] self: [5:7577471127314349853:0] peer: [3:7577471126163131906:0] socket: 405 qp: -1
2025-11-27T18:18:00.021476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471131609317156:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:00.021519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:00.045453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471123019382471:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:00.045506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:00.045553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471123019382471:2048] [node 1] ICP22 created new session: [5:7577471131609317159:2048]
2025-11-27T18:18:00.045595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471131609317159:2048] [node 1] ICS09 handshake done sender: [5:7577471127314349848:2117] self: [5:7577471127314349854:0] peer: [1:7577471124040496274:0] socket: 394 qp: -1
2025-11-27T18:18:00.045631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471131609317159:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:00.045659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:00.045879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471123019382471:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:00.046733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471131609317158:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20835
2025-11-27T18:18:00.050415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471131609317158:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:00.051089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471123019382472:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:00.061270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471131609317162:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20835
2025-11-27T18:18:00.062168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471131609317158:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:00.069725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471123019382472:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:00.070049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:00.071323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471123019382472:2049] [node 2] ICP22 created new session: [5:7577471131609317163:2049]
2025-11-27T18:18:00.073702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471131609317163:2049] [node 2] ICS09 handshake done sender: [5:7577471131609317158:2121] self: [5:7577471131609317161:0] peer: [2:7577471127567695711:0] socket: 355 qp: -1
2025-11-27T18:18:00.073738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471131609317163:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:00.073771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:00.104807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:00.105102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471131862663027:2048] [node 1] ICS01 socket: 361 reason# EndOfStream
2025-11-27T18:18:00.105146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471123272728321:2048] [node 1] ICP30 unregister session Session# [2:7577471131862663027:2048] VirtualId# [2:7577471127567695719:0]
2025-11-27T18:18:00.105179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471131862663027:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.105289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:00.105349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:00.108348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:00.108563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471126163131921:2048] [node 1] ICS01 socket: 346 reason# EndOfStream
2025-11-27T18:18:00.108595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471121868164521:2048] [node 1] ICP30 unregister session Session# [3:7577471126163131921:2048] VirtualId# [3:7577471126163131914:0]
2025-11-27T18:18:00.108627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471126163131921:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.108701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:00.108755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:00.112672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:00.112931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471123839581498:2048] [node 1] ICS01 socket: 377 reason# EndOfStream
2025-11-27T18:18:00.112969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471123839581415:2048] [node 1] ICP30 unregister session Session# [6:7577471123839581498:2048] VirtualId# [6:7577471123839581495:0]
2025-11-27T18:18:00.113003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471123839581498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.113076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:00.113132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:00.113438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:00.113655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471131609317159:2048] [node 1] ICS01 socket: 394 reason# EndOfStream
2025-11-27T18:18:00.113481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:00.113679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471124647621015:2048] [node 1] ICS07 socket disconnect 359 reason# EndOfStream
2025-11-27T18:18:00.113704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471124647621015:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.113767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:00.113803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471124647621015:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:00.113684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471123019382471:2048] [node 1] ICP30 unregister session Session# [5:7577471131609317159:2048] VirtualId# [5:7577471127314349854:0]
2025-11-27T18:18:00.113705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471131609317159:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.113766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:00.113804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:00.115378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471120352653628:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18852) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:00.115431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471124647621015:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:00.115467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471120352653628:2048] [node 1] ICP30 unregister session Session# [4:7577471124647621015:2048] VirtualId# [4:7577471124647621007:0]
2025-11-27T18:18:00.115504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:00.115533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471120352653628:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18852) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:00.115546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18852) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:00.121863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471120352653628:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:00.148322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:00.148589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471130458099230:2049] [node 2] ICS01 socket: 392 reason# EndOfStream
2025-11-27T18:18:00.148629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471121868164522:2049] [node 2] ICP30 unregister session Session# [3:7577471130458099230:2049] VirtualId# [3:7577471126163131928:0]
2025-11-27T18:18:00.148662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471130458099230:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.148748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:00.149061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:00.150189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:00.150404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471128942588319:2049] [node 2] ICS01 socket: 378 reason# EndOfStream
2025-11-27T18:18:00.150448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471120352653629:2049] [node 2] ICP30 unregister session Session# [4:7577471128942588319:2049] VirtualId# [4:7577471124647621012:0]
2025-11-27T18:18:00.150478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471128942588319:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.150539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:00.150573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:00.152328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:00.152533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471128134548806:2049] [node 2] ICS01 socket: 382 reason# EndOfStream
2025-11-27T18:18:00.152569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471123839581416:2049] [node 2] ICP30 unregister session Session# [6:7577471128134548806:2049] VirtualId# [6:7577471123839581496:0]
2025-11-27T18:18:00.152595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471128134548806:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.152648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:00.152679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:00.153138Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:00.153632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471131609317163:2049] [node 2] ICS01 socket: 355 reason# EndOfStream
2025-11-27T18:18:00.153670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471123019382472:2049] [node 2] ICP30 unregister session Session# [5:7577471131609317163:2049] VirtualId# [5:7577471131609317161:0]
2025-11-27T18:18:00.153698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471131609317163:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.153770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:00.153812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:00.208683Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:00.208906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471128134548808:2050] [node 3] ICS01 socket: 404 reason# EndOfStream
2025-11-27T18:18:00.208947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471123839581417:2050] [node 3] ICP30 unregister session Session# [6:7577471128134548808:2050] VirtualId# [6:7577471128134548805:0]
2025-11-27T18:18:00.208978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471128134548808:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.209078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:00.209123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:00.209047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:00.209220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471131609317156:2050] [node 3] ICS07 socket disconnect 405 reason# EndOfStream
2025-11-27T18:18:00.209246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471131609317156:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.209321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:00.209364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471131609317156:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:00.222888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471123019382473:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4787) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:00.222951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471131609317156:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:00.222983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471123019382473:2050] [node 3] ICP30 unregister session Session# [5:7577471131609317156:2050] VirtualId# [5:7577471127314349853:0]
2025-11-27T18:18:00.223025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:00.223052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471123019382473:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4787) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:00.223064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4787) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:00.234420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471123019382473:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:00.259832Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:00.260038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471127314349849:2078] [node 4] ICS01 socket: 367 reason# EndOfStream
2025-11-27T18:18:00.260075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471127314349804:2078] [node 4] ICP30 unregister session Session# [5:7577471127314349849:2078] VirtualId# [5:7577471127314349846:0]
2025-11-27T18:18:00.260106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471127314349849:2078] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.260193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:00.260240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:00.260337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:00.260489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471128134548813:2069] [node 4] ICS01 socket: 363 reason# EndOfStream
2025-11-27T18:18:00.260519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471123839581481:2069] [node 4] ICP30 unregister session Session# [6:7577471128134548813:2069] VirtualId# [6:7577471128134548810:0]
2025-11-27T18:18:00.260542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471128134548813:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:00.260591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:00.260626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:00.491271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471130614370401:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:00.491320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471130614370402:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:00.519953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471132250004989:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:00.520414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471132250004990:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:00.545972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471128901231997:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:00.546023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471128901231998:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:00.577885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471131983971295:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:00.577927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471131983971296:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:00.578909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471131983971294:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:00.605981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471130032566324:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:00.605982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471130032566325:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:00.606023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471130032566326:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:00.629581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471130956204829:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:00.629631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471130956204830:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:00.629894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471130956204831:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:01.064941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471134909337759:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:01.117498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471136544972348:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:01.137434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471134327533682:2108] [node 6] ICP01 ready to work
2025-11-27T18:18:01.093881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471134909337760:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:01.182505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471133196199357:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:01.187696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471136544972349:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:01.173898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471134909337765:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:01.150938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471134327533687:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:01.201843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471136278938656:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:01.199665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471130614370402:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32742160ns) {Configure:104(32735205ns) {ProcessConfigured:135(72244ns) {StartInitialHandshake:157(69925ns) {DropHandshakes:470(3371ns) {DropIncomingHandshake:442(808ns) DropOutgoingHandshake:457(458ns)} GenerateSessionVirtualId:988(1073ns) PrepareNewSessionHandshake:477(1959ns) {SwitchToState:179(751ns)}}}}}
2025-11-27T18:18:01.187291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471133196199356:2108] [node 6] ICP01 ready to work
2025-11-27T18:18:01.208241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471136544972350:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:01.226355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471135251172175:2092] [node 5] ICP01 ready to work
2025-11-27T18:18:01.226671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471136278938657:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:01.239148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471132250004989:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(121347059ns) {Configure:104(121341264ns) {ProcessConfigured:135(71624ns) {StartInitialHandshake:157(68564ns) {DropHandshakes:470(2945ns) {DropIncomingHandshake:442(780ns) DropOutgoingHandshake:457(454ns)} GenerateSessionVirtualId:988(1013ns) PrepareNewSessionHandshake:477(1832ns) {SwitchToState:179(577ns)}}}}}
2025-11-27T18:18:01.229415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471131983971295:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56272993ns) {Configure:104(56258366ns) {ProcessConfigured:135(80629ns) {StartInitialHandshake:157(77990ns) {DropHandshakes:470(5616ns) {DropIncomingHandshake:442(797ns) DropOutgoingHandshake:457(465ns)} GenerateSessionVirtualId:988(736ns) PrepareNewSessionHandshake:477(2518ns) {SwitchToState:179(1379ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.242076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471133196199358:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:01.287686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471130614370401:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(74695626ns) {Configure:104(74691046ns) {ProcessConfigured:135(18151647ns) {StartInitialHandshake:157(18146010ns) {DropHandshakes:470(1572ns) {DropIncomingHandshake:442(622ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(1416ns) PrepareNewSessionHandshake:477(4466ns) {SwitchToState:179(2653ns)}}}}}
2025-11-27T18:18:01.296589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471134909337759:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(180775450ns) {Configure:104(180765487ns) {ProcessConfigured:135(80561ns) {StartInitialHandshake:157(77359ns) {DropHandshakes:470(3732ns) {DropIncomingHandshake:442(978ns) DropOutgoingHandshake:457(727ns)} GenerateSessionVirtualId:988(1242ns) PrepareNewSessionHandshake:477(2063ns) {SwitchToState:179(818ns)}}}}}
2025-11-27T18:18:01.266370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471136544972348:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25578118ns) {Configure:104(25573718ns) {ProcessConfigured:135(48433ns) {StartInitialHandshake:157(46371ns) {DropHandshakes:470(2140ns) {DropIncomingHandshake:442(615ns) DropOutgoingHandshake:457(519ns)} GenerateSessionVirtualId:988(671ns) PrepareNewSessionHandshake:477(1170ns) {SwitchToState:179(235ns)}}}}}
2025-11-27T18:18:01.258324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471128901231997:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(56294548ns) {Configure:104(56286490ns) {ProcessConfigured:135(71536ns) {StartInitialHandshake:157(68018ns) {DropHandshakes:470(6136ns) {DropIncomingHandshake:442(657ns) DropOutgoingHandshake:457(403ns)} GenerateSessionVirtualId:988(1085ns) PrepareNewSessionHandshake:477(1748ns) {SwitchToState:179(771ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.278865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471136278938664:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55361
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.311086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471133196199368:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32550
2025-11-27T18:18:01.311166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471133196199369:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32550
2025-11-27T18:18:01.311779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471134327533687:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(123369160ns) {Configure:104(123363977ns) {ProcessConfigured:135(3839363ns) {StartInitialHandshake:157(3837847ns) {DropHandshakes:470(1066ns) {DropIncomingHandshake:442(258ns) DropOutgoingHandshake:457(107ns)} GenerateSessionVirtualId:988(304ns) PrepareNewSessionHandshake:477(2529ns) {SwitchToState:179(1638ns)}}}}}
2025-11-27T18:18:01.313678Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471134327533682:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(131533639ns) {Configure:104(131524534ns) {ProcessConfigured:135(36020ns) {StartInitialHandshake:157(33563ns) {DropHandshakes:470(1139ns) {DropIncomingHandshake:442(404ns) DropOutgoingHandshake:457(364ns)} GenerateSessionVirtualId:988(490ns) PrepareNewSessionHandshake:477(709ns) {SwitchToState:179(368ns)}}}}}
2025-11-27T18:18:01.303563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471136278938665:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55361
2025-11-27T18:18:01.307353Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471136278938656:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(55316480ns) {Configure:104(55307616ns) {ProcessConfigured:135(68968ns) {StartInitialHandshake:157(66875ns) {DropHandshakes:470(2833ns) {DropIncomingHandshake:442(591ns) DropOutgoingHandshake:457(407ns)} GenerateSessionVirtualId:988(733ns) PrepareNewSessionHandshake:477(1897ns) {SwitchToState:179(786ns)}}}}}
2025-11-27T18:18:01.309621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471130956204830:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(163360541ns) {Configure:104(148049151ns) {ProcessConfigured:135(44184ns) {StartInitialHandshake:157(42564ns) {DropHandshakes:470(1269ns) {DropIncomingHandshake:442(608ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(456ns) PrepareNewSessionHandshake:477(1782ns) {SwitchToState:179(717ns)}}}}}
2025-11-27T18:18:01.308164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471134909337764:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27144
2025-11-27T18:18:01.310026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471135251172191:2094] [node 4] ICP01 ready to work
2025-11-27T18:18:01.312951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471134909337763:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27144
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.326389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471134327533692:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45933
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.320306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471133196199369:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.326741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471130032566326:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(168138308ns) {Configure:104(168130450ns) {ProcessConfigured:135(6010873ns) {StartInitialHandshake:157(6008589ns) {DropHandshakes:470(418393ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(1002ns)} GenerateSessionVirtualId:988(965ns) PrepareNewSessionHandshake:477(1960ns) {SwitchToState:179(712ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.332574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471136278938668:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55361
2025-11-27T18:18:01.330793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471136544972350:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(83935876ns) {Configure:104(75059857ns) {ProcessConfigured:135(4150626ns) {StartInitialHandshake:157(4148038ns) {DropHandshakes:470(1214059ns) {DropIncomingHandshake:442(1078ns) DropOutgoingHandshake:457(308ns)} GenerateSessionVirtualId:988(697ns) PrepareNewSessionHandshake:477(2166ns) {SwitchToState:179(1084ns)}}}}}
No RDMA context found for GID: 2025-11-27T18:18:01.347665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471136278938665:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:01.326817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471134327533695:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45933
2025-11-27T18:18:01.336419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471130956204831:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(236977686ns) {Configure:104(226567574ns) {ProcessConfigured:135(5671373ns) {StartInitialHandshake:157(5667226ns) {DropHandshakes:470(1160ns) {DropIncomingHandshake:442(359ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(344ns) PrepareNewSessionHandshake:477(2579ns) {SwitchToState:179(1204ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.354567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471133196199373:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32550
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.342058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471134909337764:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.349120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471136278938664:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.363021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471135251172201:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57159
2025-11-27T18:18:01.332118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471134327533694:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45933
2025-11-27T18:18:01.355859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471133196199362:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.369637Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471130956204829:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(136219483ns) {Configure:104(136213156ns) {ProcessConfigured:135(3855704ns) {StartInitialHandshake:157(3853193ns) {DropHandshakes:470(2962ns) {DropIncomingHandshake:442(746ns) DropOutgoingHandshake:457(520ns)} GenerateSessionVirtualId:988(938ns) PrepareNewSessionHandshake:477(2025ns) {SwitchToState:179(160ns)}}}}}
2025-11-27T18:18:01.361484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471136544972362:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51272
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.377700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471135251172204:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57159
2025-11-27T18:18:01.365311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471136278938669:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55361
2025-11-27T18:18:01.346165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471134909337763:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.354517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471134327533694:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.378613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471136544972362:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.366894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471131983971296:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.381617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471135251172204:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.381753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471130956204829:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:01.355884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471133196199362:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.354650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471130032566326:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:01.379522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471134909337776:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27144
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.375752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471136278938657:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(119194184ns) {Configure:104(119188731ns) {ProcessConfigured:135(3417934ns) {StartInitialHandshake:157(3415798ns) {DropHandshakes:470(3099ns) {DropIncomingHandshake:442(559ns) DropOutgoingHandshake:457(402ns)} GenerateSessionVirtualId:988(660ns) PrepareNewSessionHandshake:477(3045ns) {SwitchToState:179(924ns)}}}}}
2025-11-27T18:18:01.382481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471136544972356:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.365328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471134327533692:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.362167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471133196199368:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.382276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471134909337771:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.366766Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471128901231998:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(144884764ns) {Configure:104(144874464ns) {ProcessConfigured:135(15618097ns) {StartInitialHandshake:157(15615102ns) {DropHandshakes:470(2922ns) {DropIncomingHandshake:442(519ns) DropOutgoingHandshake:457(528ns)} GenerateSessionVirtualId:988(413ns) PrepareNewSessionHandshake:477(5925ns) {SwitchToState:179(2679ns)}}}}}
2025-11-27T18:18:01.375864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471131983971295:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:01.394495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471135251172201:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.382514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471136544972356:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.369716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471134327533695:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.382301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471134909337771:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.418184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471135251172175:2092] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:01.377363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471128901231998:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.427414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471135251172208:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57159
2025-11-27T18:18:01.427524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471135251172209:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57159
2025-11-27T18:18:01.379700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471136278938668:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.375656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471130032566325:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(54129071ns) {Configure:104(54118658ns) {ProcessConfigured:135(20419630ns) {StartInitialHandshake:157(20416699ns) {DropHandshakes:470(2825ns) {DropIncomingHandshake:442(403ns) DropOutgoingHandshake:457(381ns)} GenerateSessionVirtualId:988(502ns) PrepareNewSessionHandshake:477(2732ns) {SwitchToState:179(1152ns)}}}}}
2025-11-27T18:18:01.378202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471133196199373:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.378704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471133196199364:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.378728Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471133196199364:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.379404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471133196199362:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:01.380211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471133196199356:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(106072260ns) {Configure:104(106061807ns) {ProcessConfigured:135(4040723ns) {StartInitialHandshake:157(4038568ns) {DropHandshakes:470(2318588ns) {DropIncomingHandshake:442(1356ns) DropOutgoingHandshake:457(836ns)} GenerateSessionVirtualId:988(737ns) PrepareNewSessionHandshake:477(1542ns) {SwitchToState:179(74ns)}}}}}
2025-11-27T18:18:01.382618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471128901231997:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:01.383177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471133196199364:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:01.385321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471133196199357:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.385381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:01.385429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471133196199357:2109] [node 5] ICP22 created new session: [3:7577471133196199380:2109]
2025-11-27T18:18:01.385479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471133196199380:2109] [node 5] ICS09 handshake done sender: [3:7577471133196199362:2111] self: [3:7577471133196199361:0] peer: [5:7577471134327533700:0] socket: 361 qp: -1
2025-11-27T18:18:01.385510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471133196199380:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:01.385551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:01.385987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471133196199358:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.386027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:01.386066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471133196199358:2107] [node 4] ICP22 created new session: [3:7577471133196199382:2107]
2025-11-27T18:18:01.386103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471133196199382:2107] [node 4] ICS09 handshake done sender: [3:7577471133196199364:2112] self: [3:7577471133196199363:0] peer: [4:7577471136278938670:0] socket: 362 qp: -1
2025-11-27T18:18:01.386129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471133196199382:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:01.386157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:01.402278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471133196199386:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32550
2025-11-27T18:18:01.402335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471133196199387:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32550
2025-11-27T18:18:01.403917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471133196199368:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:01.404079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471128901231998:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.404128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:01.404175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471128901231998:2049] [node 2] ICP22 created new session: [3:7577471133196199388:2049]
2025-11-27T18:18:01.404230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471133196199388:2049] [node 2] ICS09 handshake done sender: [3:7577471133196199368:2113] self: [3:7577471133196199375:0] peer: [2:7577471136544972353:0] socket: 366 qp: -1
2025-11-27T18:18:01.404261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471133196199388:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:01.404291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:01.428639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471135251172175:2092] [node 5] ICP35 event processing took too much time PendingConnection:279(22147977ns) {IncomingHandshake:261(22146532ns) {DropIncomingHandshake:442(1925ns) PrepareNewSessionHandshake:477(2914ns) {SwitchToState:179(2246ns)} GenerateSessionVirtualId:988(597ns) IssueIncomingHandshakeReply:193(12541348ns) {SwitchToState:179(3062ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.383986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471136544972358:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.393872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471134909337767:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.433804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471133196199373:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:01.384005Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471136544972358:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.393782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471136278938669:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.393902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471134909337767:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.381804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471130032566325:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:01.432549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471135251172210:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57159
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.440209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471128901231997:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.440278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:01.440331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471128901231997:2048] [node 1] ICP22 created new session: [3:7577471133196199390:2048]
2025-11-27T18:18:01.440373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471133196199390:2048] [node 1] ICS09 handshake done sender: [3:7577471133196199373:2116] self: [3:7577471133196199379:0] peer: [1:7577471134909337766:0] socket: 387 qp: -1
2025-11-27T18:18:01.440416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471133196199390:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:01.440448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:01.394988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471136278938674:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55361
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.407191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471134909337771:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:01.440105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471135251172204:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:01.381942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471134327533701:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45933
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.395916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471136278938665:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:01.387150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471136544972356:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:01.413564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471134909337767:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:01.393686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471134327533701:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.442014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471135251172211:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57159
2025-11-27T18:18:01.450836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471135251172210:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.452896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471130956204829:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.452946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:01.452997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471130956204829:2048] [node 1] ICP22 created new session: [6:7577471135251172212:2048]
2025-11-27T18:18:01.453055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471135251172212:2048] [node 1] ICS09 handshake done sender: [6:7577471135251172204:2116] self: [6:7577471135251172205:0] peer: [1:7577471134909337770:0] socket: 389 qp: -1
2025-11-27T18:18:01.453092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471135251172212:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:01.453136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:01.453559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471135251172208:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.396401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471131983971296:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9855) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471136278938665:2114] held: no
2025-11-27T18:18:01.396500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471131983971296:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.396540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:01.396595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471131983971296:2050] [node 3] ICP22 created new session: [4:7577471136278938678:2050]
2025-11-27T18:18:01.396637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471136278938678:2050] [node 3] ICS09 handshake done sender: [4:7577471136278938665:2114] self: [4:7577471136278938670:0] peer: [3:7577471133196199363:0] socket: 365 qp: -1
2025-11-27T18:18:01.396672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471136278938678:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:01.396700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:01.414048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471131983971295:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18632) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471136278938664:2113] held: no
2025-11-27T18:18:01.387510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471136544972358:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.413952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471134909337765:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.395452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471130032566324:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.485752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471133196199377:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.485789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471133196199377:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.387739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471136544972348:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.426889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471136278938680:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55361
2025-11-27T18:18:01.459210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471135251172191:2094] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:01.490567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471133196199392:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32550
2025-11-27T18:18:01.490784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471133196199377:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.450395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471136278938675:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55361
2025-11-27T18:18:01.414023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:01.395533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471134327533706:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45933
2025-11-27T18:18:01.387791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:01.459283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471130956204830:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.518680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471133196199356:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.432822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471134327533691:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.414069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471134909337765:2109] [node 6] ICP22 created new session: [1:7577471134909337785:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.432845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471134327533691:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.387844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471136544972348:2107] [node 4] ICP22 created new session: [2:7577471136544972369:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:01.540889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471133196199392:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.541591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:01.543361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471133196199356:2108] [node 6] ICP22 created new session: [3:7577471133196199394:2108]
2025-11-27T18:18:01.433432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471134327533692:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:01.450741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471136278938680:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.457788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471131983971294:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:01.440496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471130032566325:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.440569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:01.464497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471136278938664:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:01.441068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471130032566325:2049] [node 2] ICP22 created new session: [5:7577471134327533711:2049]
2025-11-27T18:18:01.482565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471136278938672:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.441131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471134327533711:2049] [node 2] ICS09 handshake done sender: [5:7577471134327533692:2113] self: [5:7577471134327533703:0] peer: [2:7577471136544972357:0] socket: 369 qp: -1
2025-11-27T18:18:01.482602Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471136278938672:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.441156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471134327533711:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:01.498230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471131983971295:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.441187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:01.498286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:01.441566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471134327533705:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45933
2025-11-27T18:18:01.498354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471131983971295:2049] [node 2] ICP22 created new session: [4:7577471136278938685:2049]
2025-11-27T18:18:01.452555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471134327533709:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45933
2025-11-27T18:18:01.498408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471136278938685:2049] [node 2] ICS09 handshake done sender: [4:7577471136278938664:2113] self: [4:7577471136278938673:0] peer: [2:7577471136544972355:0] socket: 359 qp: -1
2025-11-27T18:18:01.452738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471134327533691:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:01.498447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471136278938685:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:01.462724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471134327533709:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.462770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471134327533682:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.498490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:01.499417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471136278938684:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55361
2025-11-27T18:18:01.462820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:01.504801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471136278938672:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:01.462874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471134327533682:2108] [node 6] ICP22 created new session: [5:7577471134327533713:2108]
2025-11-27T18:18:01.462916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471134327533713:2108] [node 6] ICS09 handshake done sender: [5:7577471134327533691:2112] self: [5:7577471134327533690:0] peer: [6:7577471135251172207:0] socket: 378 qp: -1
2025-11-27T18:18:01.459371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471135251172206:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57159
2025-11-27T18:18:01.505202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471136278938667:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.417980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471134909337776:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.505270Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471136278938667:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.517518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471136278938680:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:01.530313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471136278938657:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.530370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:01.530424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471136278938657:2110] [node 6] ICP22 created new session: [4:7577471136278938688:2110]
2025-11-27T18:18:01.530482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471136278938688:2110] [node 6] ICS09 handshake done sender: [4:7577471136278938672:2118] self: [4:7577471136278938671:0] peer: [6:7577471135251172214:0] socket: 413 qp: -1
2025-11-27T18:18:01.530515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471136278938688:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:01.530560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:01.531091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471131983971294:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.531126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:01.531168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471131983971294:2048] [node 1] ICP22 created new session: [4:7577471136278938690:2048]
2025-11-27T18:18:01.531209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471136278938690:2048] [node 1] ICS09 handshake done sender: [4:7577471136278938680:2121] self: [4:7577471136278938683:0] peer: [1:7577471134909337774:0] socket: 410 qp: -1
2025-11-27T18:18:01.531242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471136278938690:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:01.531270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:01.531455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471131983971294:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:01.533108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471136278938667:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:01.545043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471136278938656:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.549907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:01.549966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471136278938656:2109] [node 5] ICP22 created new session: [4:7577471136278938693:2109]
2025-11-27T18:18:01.466171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471135251172206:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.550021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471136278938693:2109] [node 5] ICS09 handshake done sender: [4:7577471136278938667:2115] self: [4:7577471136278938666:0] peer: [5:7577471134327533718:0] socket: 411 qp: -1
2025-11-27T18:18:01.550074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471136278938693:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:01.481121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471135251172201:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:01.550121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:01.481439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471135251172175:2092] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28083) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471135251172201:2114] held: no
2025-11-27T18:18:01.481970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471130956204831:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:01.462954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471134327533713:2108] [node 6] ICS10 traffic start
2025-11-27T18:18:01.482339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471135251172216:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57159
2025-11-27T18:18:01.499893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471130956204830:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18632) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471135251172208:2118] held: no
2025-11-27T18:18:01.462986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:01.500021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471135251172175:2092] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.500067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:01.463444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471134327533694:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:01.463761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471130032566324:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2154) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471134327533701:2117] held: no
2025-11-27T18:18:01.500113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471135251172175:2092] [node 5] ICP22 created new session: [6:7577471135251172221:2092]
2025-11-27T18:18:01.470047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471130032566326:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.500164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471135251172221:2092] [node 5] ICS09 handshake done sender: [6:7577471135251172201:2114] self: [6:7577471135251172207:0] peer: [5:7577471134327533690:0] socket: 381 qp: -1
2025-11-27T18:18:01.470128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:01.500199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471135251172221:2092] [node 5] ICS10 traffic start
2025-11-27T18:18:01.470184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471130032566326:2050] [node 3] ICP22 created new session: [5:7577471134327533715:2050]
2025-11-27T18:18:01.500231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:01.470233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471134327533715:2050] [node 3] ICS09 handshake done sender: [5:7577471134327533694:2114] self: [5:7577471134327533700:0] peer: [3:7577471133196199361:0] socket: 375 qp: -1
2025-11-27T18:18:01.501091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471135251172208:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:01.470284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471134327533715:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:01.504572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471130956204830:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.470321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:01.504624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:01.471479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471134327533687:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:01.504674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471130956204830:2049] [node 2] ICP22 created new session: [6:7577471135251172223:2049]
2025-11-27T18:18:01.485193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471134327533708:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45933
2025-11-27T18:18:01.504720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471135251172223:2049] [node 2] ICS09 handshake done sender: [6:7577471135251172208:2118] self: [6:7577471135251172215:0] peer: [2:7577471136544972363:0] socket: 406 qp: -1
2025-11-27T18:18:01.511655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471134327533701:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:01.504754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471135251172223:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:01.541973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471130032566324:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.504786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:01.542029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:01.523164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471135251172220:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57159
2025-11-27T18:18:01.542083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471130032566324:2048] [node 1] ICP22 created new session: [5:7577471134327533720:2048]
2025-11-27T18:18:01.523830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471135251172210:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:01.542141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471134327533720:2048] [node 1] ICS09 handshake done sender: [5:7577471134327533701:2117] self: [5:7577471134327533707:0] peer: [1:7577471134909337768:0] socket: 386 qp: -1
2025-11-27T18:18:01.542180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471134327533720:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:01.524029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471135251172191:2094] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.542220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:01.524099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:01.542810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471134327533719:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45933
2025-11-27T18:18:01.524147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471135251172191:2094] [node 4] ICP22 created new session: [6:7577471135251172225:2094]
2025-11-27T18:18:01.524198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471135251172225:2094] [node 4] ICS09 handshake done sender: [6:7577471135251172210:2120] self: [6:7577471135251172214:0] peer: [4:7577471136278938671:0] socket: 414 qp: -1
2025-11-27T18:18:01.524238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471135251172225:2094] [node 4] ICS10 traffic start
2025-11-27T18:18:01.524273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:01.551105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471135251172219:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57159
2025-11-27T18:18:01.387895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471136544972369:2107] [node 4] ICS09 handshake done sender: [2:7577471136544972356:2112] self: [2:7577471136544972355:0] peer: [4:7577471136278938673:0] socket: 355 qp: -1
2025-11-27T18:18:01.393907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471136544972369:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:01.393949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:01.395694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471136544972354:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.395724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471136544972354:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.395800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471136544972349:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.400830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471136544972365:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51272
2025-11-27T18:18:01.401216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471136544972371:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51272
2025-11-27T18:18:01.403355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471136544972371:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.403484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471136544972354:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:01.406739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:01.414267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471132250004990:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.414290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:01.414323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471132250004990:2049] [node 3] ICP22 created new session: [2:7577471136544972376:2049]
2025-11-27T18:18:01.414354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471136544972376:2049] [node 3] ICS09 handshake done sender: [2:7577471136544972354:2111] self: [2:7577471136544972353:0] peer: [3:7577471133196199375:0] socket: 354 qp: -1
2025-11-27T18:18:01.414377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471136544972376:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:01.414410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:01.421488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471136544972349:2108] [node 5] ICP22 created new session: [2:7577471136544972375:2108]
2025-11-27T18:18:01.421597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471132250004990:2049] [node 3] ICP18 handshake (actor [2:7577471136544972371:2117]) is too old
2025-11-27T18:18:01.426231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471136544972375:2108] [node 5] ICS09 handshake done sender: [2:7577471136544972358:2113] self: [2:7577471136544972357:0] peer: [5:7577471134327533703:0] socket: 356 qp: -1
2025-11-27T18:18:01.427922Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471136544972348:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(39956587ns) {HandleHandshakeStatus:329(39953733ns) {DropIncomingHandshake:442(10222ns) DropOutgoingHandshake:457(231ns) SwitchToState:179(144ns) ProcessPendingSessionEvents:495(3260361ns) {ForwardSessionEventToSession:612(3208960ns) ForwardSessionEventToSession:612(5621ns) ForwardSessionEventToSession:612(7449ns) ForwardSessionEventToSession:612(2783ns) ForwardSessionEventToSession:612(3087ns) ForwardSessionEventToSession:612(4738ns) ForwardSessionEventToSession:612(3286ns) ForwardSessionEventToSession:612(7679ns) ForwardSessionEventToSession:612(2326ns) ForwardSessionEventToSession:612(2828ns)}}}
2025-11-27T18:18:01.433926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471136544972375:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:01.435820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:01.437606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471136544972365:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.446581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471136544972379:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51272
2025-11-27T18:18:01.457357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471136544972379:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.460429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471132250004989:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:01.460596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471136544972364:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.460622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471136544972364:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.470892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471136544972364:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:01.472310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471136544972350:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.472366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:01.472413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471136544972350:2109] [node 6] ICP22 created new session: [2:7577471136544972383:2109]
2025-11-27T18:18:01.472464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471136544972383:2109] [node 6] ICS09 handshake done sender: [2:7577471136544972364:2115] self: [2:7577471136544972363:0] peer: [6:7577471135251172215:0] socket: 404 qp: -1
2025-11-27T18:18:01.472507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471136544972383:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:01.472561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:01.483306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471136544972349:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(70894477ns) {HandleHandshakeStatus:329(69767073ns) {DropIncomingHandshake:442(221ns) DropOutgoingHandshake:457(805ns) SwitchToState:179(377ns) ProcessPendingSessionEvents:495(19832254ns) {ForwardSessionEventToSession:612(4656334ns) ForwardSessionEventToSession:612(1774092ns) ForwardSessionEventToSession:612(3777ns) ForwardSessionEventToSession:612(2958ns) ForwardSessionEventToSession:612(1621682ns) ForwardSessionEventToSession:612(3424ns) ForwardSessionEventToSession:612(3834162ns) ForwardSessionEventToSession:612(2522305ns) ForwardSessionEventToSession:612(3254ns) ForwardSessionEventToSession:612(5386294ns)}}}
2025-11-27T18:18:01.484105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471136544972385:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51272
2025-11-27T18:18:01.497664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471136544972379:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:01.501632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471132250004989:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.501682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:01.501731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471132250004989:2048] [node 1] ICP22 created new session: [2:7577471136544972386:2048]
2025-11-27T18:18:01.501788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471136544972386:2048] [node 1] ICS09 handshake done sender: [2:7577471136544972379:2118] self: [2:7577471136544972381:0] peer: [1:7577471134909337772:0] socket: 363 qp: -1
2025-11-27T18:18:01.501829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471136544972386:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:01.501874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:01.550477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471133196199394:2108] [node 6] ICS09 handshake done sender: [3:7577471133196199377:2117] self: [3:7577471133196199376:0] peer: [6:7577471135251172218:0] socket: 397 qp: -1
2025-11-27T18:18:01.554834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471134327533709:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:01.419392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471134909337785:2109] [node 6] ICS09 handshake done sender: [1:7577471134909337771:2115] self: [1:7577471134909337770:0] peer: [6:7577471135251172205:0] socket: 384 qp: -1
2025-11-27T18:18:01.553920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471133196199394:2108] [node 6] ICS10 traffic start
2025-11-27T18:18:01.553978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:01.556509Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471133196199356:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(44426560ns) {HandleHandshakeStatus:329(37621680ns) {DropIncomingHandshake:442(2015ns) DropOutgoingHandshake:457(1849ns) SwitchToState:179(680ns) ProcessPendingSessionEvents:495(142637ns) {ForwardSessionEventToSession:612(72453ns) ForwardSessionEventToSession:612(3867ns) ForwardSessionEventToSession:612(22284ns) ForwardSessionEventToSession:612(3098ns) ForwardSessionEventToSession:612(2368ns) ForwardSessionEventToSession:612(6176ns) ForwardSessionEventToSession:612(2653ns) ForwardSessionEventToSession:612(8630ns) ForwardSessionEventToSession:612(4882ns) ForwardSessionEventToSession:612(3221ns)}}}
2025-11-27T18:18:01.556593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471133196199356:2108] [node 6] ICP18 handshake (actor [3:7577471133196199392:2120]) is too old
2025-11-27T18:18:01.419434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471134909337785:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:01.419479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:01.420243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471130614370402:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.420282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:01.421737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471130614370402:2049] [node 3] ICP22 created new session: [1:7577471134909337789:2049]
2025-11-27T18:18:01.421853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471134909337783:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27144
2025-11-27T18:18:01.422862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471134909337769:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.422885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471134909337769:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.425800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471134909337769:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:01.428532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471134909337789:2049] [node 3] ICS09 handshake done sender: [1:7577471134909337767:2113] self: [1:7577471134909337766:0] peer: [3:7577471133196199379:0] socket: 382 qp: -1
2025-11-27T18:18:01.428722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471134909337760:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.435748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471134909337789:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:01.437893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:01.438427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:01.438482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471134909337760:2108] [node 5] ICP22 created new session: [1:7577471134909337791:2108]
2025-11-27T18:18:01.442444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471134909337791:2108] [node 5] ICS09 handshake done sender: [1:7577471134909337769:2114] self: [1:7577471134909337768:0] peer: [5:7577471134327533707:0] socket: 383 qp: -1
2025-11-27T18:18:01.445798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471134909337791:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:01.447009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471134909337783:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.447489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:01.456285Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471130614370402:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(37948212ns) {HandleHandshakeStatus:329(37945834ns) {DropIncomingHandshake:442(1592ns) DropOutgoingHandshake:457(521ns) SwitchToState:179(301ns) ProcessPendingSessionEvents:495(10240992ns) {ForwardSessionEventToSession:612(5718137ns) ForwardSessionEventToSession:612(4453890ns) ForwardSessionEventToSession:612(4122ns) ForwardSessionEventToSession:612(2690ns) ForwardSessionEventToSession:612(8780ns) ForwardSessionEventToSession:612(3202ns) ForwardSessionEventToSession:612(6452ns) ForwardSessionEventToSession:612(2924ns) ForwardSessionEventToSession:612(3032ns) ForwardSessionEventToSession:612(23583ns)}}}
2025-11-27T18:18:01.456362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471130614370402:2049] [node 3] ICP18 handshake (actor [1:7577471134909337776:2118]) is too old
2025-11-27T18:18:01.467180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471134909337775:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.467209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471134909337775:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.467848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471134909337773:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:01.467869Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471134909337773:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:01.484648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471134909337775:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:01.487801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471134909337773:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:01.509939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471134909337759:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.510022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:01.510082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471134909337759:2107] [node 4] ICP22 created new session: [1:7577471134909337797:2107]
2025-11-27T18:18:01.510137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471134909337797:2107] [node 4] ICS09 handshake done sender: [1:7577471134909337775:2117] self: [1:7577471134909337774:0] peer: [4:7577471136278938683:0] socket: 367 qp: -1
2025-11-27T18:18:01.510181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471134909337797:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:01.510219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:01.515238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471130614370401:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:01.515304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:01.515356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471130614370401:2048] [node 2] ICP22 created new session: [1:7577471134909337799:2048]
2025-11-27T18:18:01.515396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471134909337799:2048] [node 2] ICS09 handshake done sender: [1:7577471134909337773:2116] self: [1:7577471134909337772:0] peer: [2:7577471136544972381:0] socket: 358 qp: -1
2025-11-27T18:18:01.515432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471134909337799:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:01.515489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:01.554395Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471134909337760:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(103011220ns) {HandleHandshakeStatus:329(103008422ns) {DropIncomingHandshake:442(12870ns) DropOutgoingHandshake:457(575ns) SwitchToState:179(1136ns) ProcessPendingSessionEvents:495(63981565ns) {ForwardSessionEventToSession:612(25882915ns) ForwardSessionEventToSession:612(7711ns) ForwardSessionEventToSession:612(3585ns) ForwardSessionEventToSession:612(2353698ns) ForwardSessionEventToSession:612(4355ns) ForwardSessionEventToSession:612(17250864ns) ForwardSessionEventToSession:612(7242048ns) ForwardSessionEventToSession:612(3071ns) ForwardSessionEventToSession:612(11195046ns) ForwardSessionEventToSession:612(5885ns)}}}
2025-11-27T18:18:01.554564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471135251172206:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:01.559693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471130956204831:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.559765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:01.559818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471130956204831:2050] [node 3] ICP22 created new session: [6:7577471135251172227:2050]
2025-11-27T18:18:01.559864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471135251172227:2050] [node 3] ICS09 handshake done sender: [6:7577471135251172206:2117] self: [6:7577471135251172218:0] peer: [3:7577471133196199376:0] socket: 400 qp: -1
2025-11-27T18:18:01.560118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471135251172227:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:01.559062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471134327533687:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:01.560101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:01.560839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471134327533687:2110] [node 4] ICP22 created new session: [5:7577471134327533722:2110]
2025-11-27T18:18:01.563809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471134327533722:2110] [node 4] ICS09 handshake done sender: [5:7577471134327533709:2122] self: [5:7577471134327533718:0] peer: [4:7577471136278938666:0] socket: 415 qp: -1
2025-11-27T18:18:01.563857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471134327533722:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:01.563910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:01.562486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:01.610277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:01.610537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471136544972386:2048] [node 1] ICS01 socket: 363 reason# EndOfStream
2025-11-27T18:18:01.610581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471132250004989:2048] [node 1] ICP30 unregister session Session# [2:7577471136544972386:2048] VirtualId# [2:7577471136544972381:0]
2025-11-27T18:18:01.610618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471136544972386:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.610698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:01.610784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:01.612938Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:01.613193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471133196199390:2048] [node 1] ICS01 socket: 387 reason# EndOfStream
2025-11-27T18:18:01.613232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471128901231997:2048] [node 1] ICP30 unregister session Session# [3:7577471133196199390:2048] VirtualId# [3:7577471133196199379:0]
2025-11-27T18:18:01.613284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471133196199390:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.613357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:01.613399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:01.617951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:01.618193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471136278938690:2048] [node 1] ICS01 socket: 410 reason# EndOfStream
2025-11-27T18:18:01.618233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471131983971294:2048] [node 1] ICP30 unregister session Session# [4:7577471136278938690:2048] VirtualId# [4:7577471136278938683:0]
2025-11-27T18:18:01.618268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471136278938690:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.618338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:01.618395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:01.621022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:01.621284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471134327533720:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-11-27T18:18:01.621323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471130032566324:2048] [node 1] ICP30 unregister session Session# [5:7577471134327533720:2048] VirtualId# [5:7577471134327533707:0]
2025-11-27T18:18:01.621197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:01.621361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471134327533720:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.621431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:01.621421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471135251172212:2048] [node 1] ICS01 socket: 389 reason# EndOfStream
2025-11-27T18:18:01.621481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:01.621458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471130956204829:2048] [node 1] ICP30 unregister session Session# [6:7577471135251172212:2048] VirtualId# [6:7577471135251172205:0]
2025-11-27T18:18:01.621486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471135251172212:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.621565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:01.621606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:01.682423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:01.682706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471133196199388:2049] [node 2] ICS01 socket: 366 reason# EndOfStream
2025-11-27T18:18:01.682809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471128901231998:2049] [node 2] ICP30 unregister session Session# [3:7577471133196199388:2049] VirtualId# [3:7577471133196199375:0]
2025-11-27T18:18:01.682856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471133196199388:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.682941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:01.682985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:01.685149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:01.685414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471136278938685:2049] [node 2] ICS01 socket: 359 reason# EndOfStream
2025-11-27T18:18:01.685478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471131983971295:2049] [node 2] ICP30 unregister session Session# [4:7577471136278938685:2049] VirtualId# [4:7577471136278938673:0]
2025-11-27T18:18:01.685511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471136278938685:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.685570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:01.685616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:01.689412Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:01.689407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:01.689630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471135251172223:2049] [node 2] ICS01 socket: 406 reason# EndOfStream
2025-11-27T18:18:01.689693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471130956204830:2049] [node 2] ICP30 unregister session Session# [6:7577471135251172223:2049] VirtualId# [6:7577471135251172215:0]
2025-11-27T18:18:01.689728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471135251172223:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.689834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:01.689918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:01.691211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471134327533711:2049] [node 2] ICS01 socket: 369 reason# EndOfStream
2025-11-27T18:18:01.691299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471130032566325:2049] [node 2] ICP30 unregister session Session# [5:7577471134327533711:2049] VirtualId# [5:7577471134327533703:0]
2025-11-27T18:18:01.691368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471134327533711:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.691477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:01.691573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:01.781589Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:01.781828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471135251172227:2050] [node 3] ICS01 socket: 400 reason# EndOfStream
2025-11-27T18:18:01.781909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471130956204831:2050] [node 3] ICP30 unregister session Session# [6:7577471135251172227:2050] VirtualId# [6:7577471135251172218:0]
2025-11-27T18:18:01.781950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471135251172227:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.782040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:01.782095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:01.782328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:01.782522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471136278938678:2050] [node 3] ICS01 socket: 365 reason# EndOfStream
2025-11-27T18:18:01.782556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471131983971296:2050] [node 3] ICP30 unregister session Session# [4:7577471136278938678:2050] VirtualId# [4:7577471136278938670:0]
2025-11-27T18:18:01.782574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471136278938678:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.782633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:01.782663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:01.784609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:01.784809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471134327533715:2050] [node 3] ICS01 socket: 375 reason# EndOfStream
2025-11-27T18:18:01.784843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471130032566326:2050] [node 3] ICP30 unregister session Session# [5:7577471134327533715:2050] VirtualId# [5:7577471134327533700:0]
2025-11-27T18:18:01.784871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471134327533715:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.785034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:01.785179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:01.919008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:01.919256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471134327533722:2110] [node 4] ICS01 socket: 415 reason# EndOfStream
2025-11-27T18:18:01.919294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471134327533687:2110] [node 4] ICP30 unregister session Session# [5:7577471134327533722:2110] VirtualId# [5:7577471134327533718:0]
2025-11-27T18:18:01.919332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471134327533722:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.919440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:01.919487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:01.924775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:01.925046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471135251172225:2094] [node 4] ICS01 socket: 414 reason# EndOfStream
2025-11-27T18:18:01.925103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471135251172191:2094] [node 4] ICP30 unregister session Session# [6:7577471135251172225:2094] VirtualId# [6:7577471135251172214:0]
2025-11-27T18:18:01.925147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471135251172225:2094] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:01.925237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:01.926401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:02.021638Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:02.021895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471135251172221:2092] [node 5] ICS01 socket: 381 reason# EndOfStream
2025-11-27T18:18:02.021945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471135251172175:2092] [node 5] ICP30 unregister session Session# [6:7577471135251172221:2092] VirtualId# [6:7577471135251172207:0]
2025-11-27T18:18:02.021987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471135251172221:2092] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:02.022077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:02.022127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:02.213510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471140011090921:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:02.213574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471140011090922:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:02.261557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471137457167559:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:02.261609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471137457167560:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:02.284923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471136706394245:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:02.284964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471136706394246:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:02.313383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471137519948920:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:02.313441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471137519948921:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:02.313474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471137519948922:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:02.349533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471137005081698:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:02.349589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471137005081699:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:02.349624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471137005081700:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:02.365906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471140435047333:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:02.365955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471140435047334:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:02.365989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471140435047335:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:03.034402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471140435047382:2085] [node 5] ICP01 ready to work
2025-11-27T18:18:03.083367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471144306058284:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:03.125596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471141001361604:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:03.101983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471144306058285:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:03.107961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471141814916279:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:03.113923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471141752134918:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:03.106953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471144730014683:2091] [node 4] ICP01 ready to work
2025-11-27T18:18:03.113777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471141300049058:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.147816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471140435047333:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(64280718ns) {Configure:104(64274715ns) {ProcessConfigured:135(65715ns) {StartInitialHandshake:157(63533ns) {DropHandshakes:470(2632ns) {DropIncomingHandshake:442(684ns) DropOutgoingHandshake:457(399ns)} GenerateSessionVirtualId:988(846ns) PrepareNewSessionHandshake:477(2355ns) {SwitchToState:179(810ns)}}}}}
2025-11-27T18:18:03.130008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471141001361605:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:03.117619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471144306058286:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:03.125423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471141752134919:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:03.145492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471141814916280:2108] [node 6] ICP01 ready to work
2025-11-27T18:18:03.122097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471141300049059:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.198741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471137519948920:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(115434623ns) {Configure:104(115422281ns) {ProcessConfigured:135(77886ns) {StartInitialHandshake:157(75469ns) {DropHandshakes:470(3125ns) {DropIncomingHandshake:442(729ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(732ns) PrepareNewSessionHandshake:477(1638ns) {SwitchToState:179(621ns)}}}}}
2025-11-27T18:18:03.200915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471141814916285:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14379
2025-11-27T18:18:03.141709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471141001361609:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:03.129794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471141752134920:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.216101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471140435047334:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(101833338ns) {Configure:104(101826558ns) {ProcessConfigured:135(80689ns) {StartInitialHandshake:157(78355ns) {DropHandshakes:470(2780ns) {DropIncomingHandshake:442(708ns) DropOutgoingHandshake:457(536ns)} GenerateSessionVirtualId:988(1517ns) PrepareNewSessionHandshake:477(1834ns) {SwitchToState:179(683ns)}}}}}
2025-11-27T18:18:03.216761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471144730014698:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1099
2025-11-27T18:18:03.170194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471141001361606:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59199
2025-11-27T18:18:03.229882Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471137457167560:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(229455219ns) {Configure:104(229445785ns) {ProcessConfigured:135(7871034ns) {StartInitialHandshake:157(7868410ns) {DropHandshakes:470(1113ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(158ns)} GenerateSessionVirtualId:988(432ns) PrepareNewSessionHandshake:477(3958ns) {SwitchToState:179(2156ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.241336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471137005081698:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(134936923ns) {Configure:104(134922681ns) {ProcessConfigured:135(53571ns) {StartInitialHandshake:157(51482ns) {DropHandshakes:470(3176ns) {DropIncomingHandshake:442(677ns) DropOutgoingHandshake:457(458ns)} GenerateSessionVirtualId:988(1408ns) PrepareNewSessionHandshake:477(1399ns) {SwitchToState:179(720ns)}}}}}
2025-11-27T18:18:03.242770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471141300049059:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(79574851ns) {Configure:104(79566906ns) {ProcessConfigured:135(14153693ns) {StartInitialHandshake:157(14151760ns) {DropHandshakes:470(1132ns) {DropIncomingHandshake:442(451ns) DropOutgoingHandshake:457(155ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(2920ns) {SwitchToState:179(1327ns)}}}}}
2025-11-27T18:18:03.213641Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471137519948922:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(69439198ns) {Configure:104(69434630ns) {ProcessConfigured:135(35732436ns) {StartInitialHandshake:157(35729283ns) {DropHandshakes:470(2883ns) {DropIncomingHandshake:442(802ns) DropOutgoingHandshake:457(593ns)} GenerateSessionVirtualId:988(531ns) PrepareNewSessionHandshake:477(3471ns) {SwitchToState:179(1658ns)}}}}}
2025-11-27T18:18:03.201622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471141001361617:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59199
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.266319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471144306058285:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(136378644ns) {Configure:104(136368311ns) {ProcessConfigured:135(68257ns) {StartInitialHandshake:157(61603ns) {DropHandshakes:470(858ns) {DropIncomingHandshake:442(183ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(270ns) PrepareNewSessionHandshake:477(1737ns) {SwitchToState:179(619ns)}}}}}
2025-11-27T18:18:03.268664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471144306058289:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56353
2025-11-27T18:18:03.268843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471144306058290:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56353
2025-11-27T18:18:03.268918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471144306058291:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56353
2025-11-27T18:18:03.269060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471144306058292:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56353
2025-11-27T18:18:03.243638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471141300049065:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43330
2025-11-27T18:18:03.243708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471141300049066:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43330
2025-11-27T18:18:03.243767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471141300049067:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43330
2025-11-27T18:18:03.274316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471137457167559:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(136945259ns) {Configure:104(136938552ns) {ProcessConfigured:135(8694303ns) {StartInitialHandshake:157(8690313ns) {DropHandshakes:470(2922ns) {DropIncomingHandshake:442(678ns) DropOutgoingHandshake:457(407ns)} GenerateSessionVirtualId:988(795ns) PrepareNewSessionHandshake:477(5477ns) {SwitchToState:179(3424ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.271102Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471144730014683:2091] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(90206391ns) {Configure:104(90200963ns) {ProcessConfigured:135(12604528ns) {StartInitialHandshake:157(12601437ns) {DropHandshakes:470(2692ns) {DropIncomingHandshake:442(258ns) DropOutgoingHandshake:457(574ns)} GenerateSessionVirtualId:988(508ns) PrepareNewSessionHandshake:477(3508ns) {SwitchToState:179(1855ns)}}}}}
2025-11-27T18:18:03.273763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471144730014698:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.283254Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471140011090921:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(302998520ns) {Configure:104(294446547ns) {ProcessConfigured:135(12377119ns) {StartInitialHandshake:157(6021627ns) {DropHandshakes:470(4745ns) {DropIncomingHandshake:442(999ns) DropOutgoingHandshake:457(623ns)} GenerateSessionVirtualId:988(1656ns) PrepareNewSessionHandshake:477(4409ns) {SwitchToState:179(2477ns)}}}}}
2025-11-27T18:18:03.282853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471141814916293:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14379
2025-11-27T18:18:03.282923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471141814916294:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14379
2025-11-27T18:18:03.237193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471141001361618:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59199
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.278241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471141752134932:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
2025-11-27T18:18:03.284021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471141001361617:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.294822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471137519948921:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(130715081ns) {Configure:104(130708895ns) {ProcessConfigured:135(16924084ns) {StartInitialHandshake:157(16920999ns) {DropHandshakes:470(3453ns) {DropIncomingHandshake:442(1262ns) DropOutgoingHandshake:457(411ns)} GenerateSessionVirtualId:988(621ns) PrepareNewSessionHandshake:477(1413ns) {SwitchToState:179(136ns)}}}}}
2025-11-27T18:18:03.302348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471141814916294:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.326480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471140435047382:2085] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(189005927ns) {Configure:104(189000629ns) {ProcessConfigured:135(100112ns) {StartInitialHandshake:157(97129ns) {DropHandshakes:470(3638ns) {DropIncomingHandshake:442(898ns) DropOutgoingHandshake:457(734ns)} GenerateSessionVirtualId:988(1547ns) PrepareNewSessionHandshake:477(2199ns) {SwitchToState:179(704ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.333738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471140435047335:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(341029145ns) {Configure:104(341022283ns) {ProcessConfigured:135(11573919ns) {StartInitialHandshake:157(11571400ns) {DropHandshakes:470(3057ns) {DropIncomingHandshake:442(737ns) DropOutgoingHandshake:457(438ns)} GenerateSessionVirtualId:988(878ns) PrepareNewSessionHandshake:477(4114ns) {SwitchToState:179(2120ns)}}}}}
2025-11-27T18:18:03.305484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471140011090982:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.284116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471141001361606:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.335174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471141814916285:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.297807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471141752134932:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.307065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471141300049058:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(147749374ns) {Configure:104(147741845ns) {ProcessConfigured:135(87346ns) {StartInitialHandshake:157(85243ns) {DropHandshakes:470(3109ns) {DropIncomingHandshake:442(744ns) DropOutgoingHandshake:457(440ns)} GenerateSessionVirtualId:988(1525ns) PrepareNewSessionHandshake:477(1881ns) {SwitchToState:179(675ns)}}}}}
2025-11-27T18:18:03.284285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471136706394245:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.305548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471141001361618:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.339249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471141001361613:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.339277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471141001361613:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.353378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471141001361611:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.353429Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471141001361611:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.302967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471141752134933:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
2025-11-27T18:18:03.333982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471140435047335:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.361895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471137519948921:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.312651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471137005081700:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(163540384ns) {Configure:104(163536273ns) {ProcessConfigured:135(42573041ns) {StartInitialHandshake:157(42569051ns) {DropHandshakes:470(1510ns) {DropIncomingHandshake:442(435ns) DropOutgoingHandshake:457(447ns)} GenerateSessionVirtualId:988(406ns) PrepareNewSessionHandshake:477(6234ns) {SwitchToState:179(3101ns)}}}}}
2025-11-27T18:18:03.356706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471141001361613:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:03.366007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471136706394246:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(255072639ns) {Configure:104(255064743ns) {ProcessConfigured:135(9396702ns) {StartInitialHandshake:157(9394067ns) {DropHandshakes:470(3891ns) {DropIncomingHandshake:442(1242ns) DropOutgoingHandshake:457(560ns)} GenerateSessionVirtualId:988(1053ns) PrepareNewSessionHandshake:477(3691ns) {SwitchToState:179(1804ns)}}}}}
2025-11-27T18:18:03.366283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471136706394246:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.366703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471141001361623:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59199
2025-11-27T18:18:03.305541Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471140011090982:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.369364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471144730014708:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1099
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.369804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471141814916293:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.369954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471141814916287:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.369994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471141814916287:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.379380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471141814916287:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:03.381920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471141001361611:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:03.321468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471144306058286:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(148057187ns) {Configure:104(148052752ns) {ProcessConfigured:135(18463914ns) {StartInitialHandshake:157(18461350ns) {DropHandshakes:470(2462ns) {DropIncomingHandshake:442(438ns) DropOutgoingHandshake:457(499ns)} GenerateSessionVirtualId:988(507ns) PrepareNewSessionHandshake:477(3813ns) {SwitchToState:179(2290ns)}}}}}
2025-11-27T18:18:03.389874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471141814916279:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.389953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:03.390001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471141814916279:2109] [node 5] ICP22 created new session: [4:7577471141814916298:2109]
2025-11-27T18:18:03.390046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471141814916298:2109] [node 5] ICS09 handshake done sender: [4:7577471141814916287:2113] self: [4:7577471141814916286:0] peer: [5:7577471141300049076:0] socket: 366 qp: -1
2025-11-27T18:18:03.390077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471141814916298:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:03.390105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.360919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471141752134933:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.379019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471144730014710:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1099
2025-11-27T18:18:03.382323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471144730014708:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.382646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471140435047382:2085] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.330447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471141300049065:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.383122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471141001361605:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.330536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471141300049066:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.327461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471144306058289:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.361153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471137457167559:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.400500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471141814916302:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14379
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.383184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:03.434658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471141814916302:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.444414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471141814916304:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14379
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.330923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471141300049073:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43330
2025-11-27T18:18:03.328783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471144306058290:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.385645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471141752134928:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.465416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471144730014698:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.383239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471141001361605:2109] [node 6] ICP22 created new session: [3:7577471141001361628:2109]
2025-11-27T18:18:03.385698Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471141752134928:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.451543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471141814916280:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(196033698ns) {Configure:104(187438870ns) {ProcessConfigured:135(28030466ns) {StartInitialHandshake:157(28029281ns) {DropHandshakes:470(2180ns) {DropIncomingHandshake:442(550ns) DropOutgoingHandshake:457(507ns)} GenerateSessionVirtualId:988(382ns) PrepareNewSessionHandshake:477(1874ns) {SwitchToState:179(92ns)}}}}}
2025-11-27T18:18:03.331103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471137005081700:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.350575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471144306058292:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.383283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471141001361628:2109] [node 6] ICS09 handshake done sender: [3:7577471141001361613:2113] self: [3:7577471141001361612:0] peer: [6:7577471144730014709:0] socket: 358 qp: -1
2025-11-27T18:18:03.481896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471140435047335:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.385916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471141752134922:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.460062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471137519948922:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.353475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471144306058291:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.340240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471141300049067:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.383312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471141001361628:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.356611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471140011090982:2107] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:03.487868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:03.471243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471141814916285:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:03.385941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471141752134922:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.340369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471141300049059:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.498075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471140435047335:2050] [node 3] ICP22 created new session: [6:7577471144730014714:2050]
2025-11-27T18:18:03.532674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471144730014714:2050] [node 3] ICS09 handshake done sender: [6:7577471144730014698:2112] self: [6:7577471144730014709:0] peer: [3:7577471141001361612:0] socket: 367 qp: -1
2025-11-27T18:18:03.533923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471144730014713:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1099
2025-11-27T18:18:03.537577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471144730014714:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:03.537625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:03.545908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471144730014715:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1099
2025-11-27T18:18:03.546234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471144730014717:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1099
2025-11-27T18:18:03.550587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471140435047335:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(68471185ns) {HandleHandshakeStatus:329(68468578ns) {DropIncomingHandshake:442(2707ns) DropOutgoingHandshake:457(9633ns) SwitchToState:179(150ns) ProcessPendingSessionEvents:495(115227ns) {ForwardSessionEventToSession:612(67703ns) ForwardSessionEventToSession:612(4062ns) ForwardSessionEventToSession:612(8276ns) ForwardSessionEventToSession:612(3047ns) ForwardSessionEventToSession:612(2692ns) ForwardSessionEventToSession:612(6397ns) ForwardSessionEventToSession:612(3179ns) ForwardSessionEventToSession:612(4778ns) ForwardSessionEventToSession:612(3075ns) ForwardSessionEventToSession:612(3282ns)}}}
2025-11-27T18:18:03.382250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471140011090922:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.383345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.400153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471141752134935:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
2025-11-27T18:18:03.379426Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471137005081699:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(137817888ns) {Configure:104(137812265ns) {ProcessConfigured:135(11499071ns) {StartInitialHandshake:157(11496615ns) {DropHandshakes:470(2497ns) {DropIncomingHandshake:442(299ns) DropOutgoingHandshake:457(472ns)} GenerateSessionVirtualId:988(476ns) PrepareNewSessionHandshake:477(2679ns) {SwitchToState:179(1223ns)}}}}}
2025-11-27T18:18:03.472132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471137519948921:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.385487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471141300049073:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.393733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471137005081699:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.410148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471137005081698:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.421326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471141300049072:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.421385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471141300049072:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.444707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471141300049059:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11064) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471141300049067:2115] held: no
2025-11-27T18:18:03.450786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471141300049081:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43330
2025-11-27T18:18:03.457404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471141300049080:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43330
2025-11-27T18:18:03.466045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471141300049067:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:03.466294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471141300049065:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:03.466531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471141300049059:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.466582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:03.466630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471141300049059:2109] [node 4] ICP22 created new session: [5:7577471141300049085:2109]
2025-11-27T18:18:03.466677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471141300049085:2109] [node 4] ICS09 handshake done sender: [5:7577471141300049067:2115] self: [5:7577471141300049076:0] peer: [4:7577471141814916286:0] socket: 376 qp: -1
2025-11-27T18:18:03.466713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471141300049085:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:03.466754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:03.467202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471137005081700:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.467240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:03.467288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471137005081700:2050] [node 3] ICP22 created new session: [5:7577471141300049087:2050]
2025-11-27T18:18:03.467330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471141300049087:2050] [node 3] ICS09 handshake done sender: [5:7577471141300049065:2113] self: [5:7577471141300049075:0] peer: [3:7577471141001361610:0] socket: 374 qp: -1
2025-11-27T18:18:03.467365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471141300049087:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:03.467392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:03.468012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471141300049084:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43330
2025-11-27T18:18:03.471791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471141300049072:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:03.481352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471141300049066:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:03.489810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471141300049058:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.489869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:03.489924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471141300049058:2108] [node 6] ICP22 created new session: [5:7577471141300049092:2108]
2025-11-27T18:18:03.489983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471141300049092:2108] [node 6] ICS09 handshake done sender: [5:7577471141300049072:2117] self: [5:7577471141300049071:0] peer: [6:7577471144730014711:0] socket: 394 qp: -1
2025-11-27T18:18:03.490017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471141300049092:2108] [node 6] ICS10 traffic start
2025-11-27T18:18:03.490047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:03.490862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471141300049091:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43330
2025-11-27T18:18:03.493510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471137005081699:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.493557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:03.493610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471137005081699:2049] [node 2] ICP22 created new session: [5:7577471141300049094:2049]
2025-11-27T18:18:03.493657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471141300049094:2049] [node 2] ICS09 handshake done sender: [5:7577471141300049066:2114] self: [5:7577471141300049079:0] peer: [2:7577471141752134923:0] socket: 375 qp: -1
2025-11-27T18:18:03.493689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471141300049094:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:03.493726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:03.514596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471141300049073:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:03.524829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471137005081698:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.534322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:03.548032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471137005081698:2048] [node 1] ICP22 created new session: [5:7577471141300049096:2048]
2025-11-27T18:18:03.559389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471141300049096:2048] [node 1] ICS09 handshake done sender: [5:7577471141300049073:2118] self: [5:7577471141300049082:0] peer: [1:7577471144306058287:0] socket: 393 qp: -1
2025-11-27T18:18:03.573541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471141300049096:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:03.573618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:03.574113Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471137005081698:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(49060503ns) {HandleHandshakeStatus:329(49058356ns) {DropIncomingHandshake:442(223ns) DropOutgoingHandshake:457(4622ns) SwitchToState:179(182ns) ProcessPendingSessionEvents:495(133898ns) {ForwardSessionEventToSession:612(66559ns) ForwardSessionEventToSession:612(24757ns) ForwardSessionEventToSession:612(3604ns) ForwardSessionEventToSession:612(2172ns) ForwardSessionEventToSession:612(4699ns) ForwardSessionEventToSession:612(5166ns) ForwardSessionEventToSession:612(2663ns) ForwardSessionEventToSession:612(5198ns) ForwardSessionEventToSession:612(2215ns) ForwardSessionEventToSession:612(6596ns)}}}
2025-11-27T18:18:03.383842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471141001361606:2110] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:03.472190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.570087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471144730014717:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.413446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471141752134924:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.382327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:03.382379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471140011090922:2049] [node 3] ICP22 created new session: [1:7577471144306058304:2049]
2025-11-27T18:18:03.382427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471144306058304:2049] [node 3] ICS09 handshake done sender: [1:7577471140011090982:2107] self: [1:7577471140011090981:0] peer: [3:7577471141001361619:0] socket: 354 qp: -1
2025-11-27T18:18:03.382466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471144306058304:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:03.382502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:03.400161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471144306058296:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.400204Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471144306058296:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.449831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471144306058296:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:03.459445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471144306058309:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56353
2025-11-27T18:18:03.461963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471144306058288:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.462001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471144306058288:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.469878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471144306058284:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(303989811ns) {Configure:104(303981683ns) {ProcessConfigured:135(51189665ns) {StartInitialHandshake:157(40149270ns) {DropHandshakes:470(2694ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(2931ns) PrepareNewSessionHandshake:477(4047ns) {SwitchToState:179(633ns)}}}}}
2025-11-27T18:18:03.480462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471140011090921:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.487703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471144306058288:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:03.494456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:03.494496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471144306058309:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.506438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471140011090921:2048] [node 2] ICP22 created new session: [1:7577471144306058313:2048]
2025-11-27T18:18:03.522566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471144306058285:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.522626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:03.522679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471144306058285:2109] [node 5] ICP22 created new session: [1:7577471144306058314:2109]
2025-11-27T18:18:03.522737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471144306058314:2109] [node 5] ICS09 handshake done sender: [1:7577471144306058288:2112] self: [1:7577471144306058287:0] peer: [5:7577471141300049082:0] socket: 387 qp: -1
2025-11-27T18:18:03.522774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471144306058314:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:03.522812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:03.529575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471144306058313:2048] [node 2] ICS09 handshake done sender: [1:7577471144306058296:2117] self: [1:7577471144306058295:0] peer: [2:7577471141752134934:0] socket: 391 qp: -1
2025-11-27T18:18:03.532819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471144306058306:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.532854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471144306058306:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.537987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471144306058306:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:03.538238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471144306058284:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.538297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:03.538345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471144306058284:2108] [node 4] ICP22 created new session: [1:7577471144306058317:2108]
2025-11-27T18:18:03.538400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471144306058317:2108] [node 4] ICS09 handshake done sender: [1:7577471144306058306:2119] self: [1:7577471144306058303:0] peer: [4:7577471141814916311:0] socket: 403 qp: -1
2025-11-27T18:18:03.538438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471144306058317:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:03.538468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:03.554738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471144306058313:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:03.554796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:03.384613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471141001361604:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.384645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:03.384694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471141001361604:2108] [node 5] ICP22 created new session: [3:7577471141001361630:2108]
2025-11-27T18:18:03.384738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471141001361630:2108] [node 5] ICS09 handshake done sender: [3:7577471141001361611:2112] self: [3:7577471141001361610:0] peer: [5:7577471141300049075:0] socket: 357 qp: -1
2025-11-27T18:18:03.384761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471141001361630:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:03.384785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:03.393707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471141001361609:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(177610137ns) {Configure:104(177602756ns) {ProcessConfigured:135(3187288ns) {StartInitialHandshake:157(3183088ns) {DropHandshakes:470(2374ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(506ns)} GenerateSessionVirtualId:988(1423ns) PrepareNewSessionHandshake:477(1039ns) {SwitchToState:179(44ns)}}}}}
2025-11-27T18:18:03.396426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471136706394245:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.396510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:03.396576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471136706394245:2048] [node 1] ICP22 created new session: [3:7577471141001361634:2048]
2025-11-27T18:18:03.396623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471141001361634:2048] [node 1] ICS09 handshake done sender: [3:7577471141001361606:2110] self: [3:7577471141001361619:0] peer: [1:7577471140011090981:0] socket: 355 qp: -1
2025-11-27T18:18:03.396658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471141001361634:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:03.396712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:03.460528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471141001361637:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59199
2025-11-27T18:18:03.478644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471141001361627:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.478691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471141001361627:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.499981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471141001361618:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:03.500292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471136706394246:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2460) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471141001361618:2115] held: no
2025-11-27T18:18:03.500598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471136706394246:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.500682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:03.500732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471136706394246:2049] [node 2] ICP22 created new session: [3:7577471141001361638:2049]
2025-11-27T18:18:03.500772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471141001361638:2049] [node 2] ICS09 handshake done sender: [3:7577471141001361618:2115] self: [3:7577471141001361624:0] peer: [2:7577471141752134927:0] socket: 373 qp: -1
2025-11-27T18:18:03.500813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471141001361638:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:03.500854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:03.522185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471141001361627:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:03.523996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471141001361609:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.524070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:03.524108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471141001361609:2107] [node 4] ICP22 created new session: [3:7577471141001361641:2107]
2025-11-27T18:18:03.524153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471141001361641:2107] [node 4] ICS09 handshake done sender: [3:7577471141001361627:2118] self: [3:7577471141001361626:0] peer: [4:7577471141814916305:0] socket: 404 qp: -1
2025-11-27T18:18:03.524176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471141001361641:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:03.524205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:03.570272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471144730014715:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.413483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471141752134924:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.472245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471137519948921:2049] [node 2] ICP22 created new session: [4:7577471141814916307:2049]
2025-11-27T18:18:03.472291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471141814916307:2049] [node 2] ICS09 handshake done sender: [4:7577471141814916285:2112] self: [4:7577471141814916295:0] peer: [2:7577471141752134921:0] socket: 362 qp: -1
2025-11-27T18:18:03.414275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471141752134928:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.570358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471144730014708:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:03.593847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471140435047334:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.597843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471144730014683:2091] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.597917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471140435047382:2085] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.597977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:03.598033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471140435047382:2085] [node 5] ICP22 created new session: [6:7577471144730014721:2085]
2025-11-27T18:18:03.598080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471144730014721:2085] [node 5] ICS09 handshake done sender: [6:7577471144730014708:2116] self: [6:7577471144730014711:0] peer: [5:7577471141300049071:0] socket: 395 qp: -1
2025-11-27T18:18:03.598119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471144730014721:2085] [node 5] ICS10 traffic start
2025-11-27T18:18:03.598158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:03.612036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471144730014712:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1099
2025-11-27T18:18:03.619517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471144730014712:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.619670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471140435047333:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.622946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471144730014723:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1099
2025-11-27T18:18:03.441646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471137457167560:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.441696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:03.441750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471137457167560:2049] [node 3] ICP22 created new session: [2:7577471141752134940:2049]
2025-11-27T18:18:03.441797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471141752134940:2049] [node 3] ICS09 handshake done sender: [2:7577471141752134928:2112] self: [2:7577471141752134927:0] peer: [3:7577471141001361624:0] socket: 372 qp: -1
2025-11-27T18:18:03.441836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471141752134940:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:03.441869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:03.443157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471141752134943:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
2025-11-27T18:18:03.443211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471141752134944:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
2025-11-27T18:18:03.447838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471141752134935:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.449566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471141752134922:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:03.449806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471141752134924:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:03.461725Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471141752134918:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(295657854ns) {Configure:104(295653145ns) {ProcessConfigured:135(44935968ns) {StartInitialHandshake:157(44933578ns) {DropHandshakes:470(2089ns) {DropIncomingHandshake:442(584ns) DropOutgoingHandshake:457(508ns)} GenerateSessionVirtualId:988(697ns) PrepareNewSessionHandshake:477(3388ns) {SwitchToState:179(1130ns)}}}}}
2025-11-27T18:18:03.473639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471141752134943:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.481723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471141752134919:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.485921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471141752134920:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.485984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:03.486031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471141752134920:2107] [node 4] ICP22 created new session: [2:7577471141752134948:2107]
2025-11-27T18:18:03.486080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471141752134948:2107] [node 4] ICS09 handshake done sender: [2:7577471141752134922:2110] self: [2:7577471141752134921:0] peer: [4:7577471141814916295:0] socket: 359 qp: -1
2025-11-27T18:18:03.486115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471141752134948:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:03.486150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:03.487602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:03.491105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471137457167560:2049] [node 3] ICP18 handshake (actor [2:7577471141752134943:2117]) is too old
2025-11-27T18:18:03.497946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471141752134933:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:03.524823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471141752134919:2108] [node 5] ICP22 created new session: [2:7577471141752134950:2108]
2025-11-27T18:18:03.526579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471137457167559:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.526629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:03.526676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471137457167559:2048] [node 1] ICP22 created new session: [2:7577471141752134953:2048]
2025-11-27T18:18:03.526719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471141752134953:2048] [node 1] ICS09 handshake done sender: [2:7577471141752134933:2115] self: [2:7577471141752134934:0] peer: [1:7577471144306058295:0] socket: 392 qp: -1
2025-11-27T18:18:03.526749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471141752134953:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:03.526779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:03.529507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471141752134950:2108] [node 5] ICS09 handshake done sender: [2:7577471141752134924:2111] self: [2:7577471141752134923:0] peer: [5:7577471141300049079:0] socket: 360 qp: -1
2025-11-27T18:18:03.534235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471141752134950:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:03.547924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:03.566130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471141752134952:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
2025-11-27T18:18:03.602787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471141752134919:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(120833045ns) {HandleHandshakeStatus:329(120830988ns) {DropIncomingHandshake:442(125ns) DropOutgoingHandshake:457(2923ns) SwitchToState:179(667ns) ProcessPendingSessionEvents:495(7115887ns) {ForwardSessionEventToSession:612(7058618ns) ForwardSessionEventToSession:612(5482ns) ForwardSessionEventToSession:612(10175ns) ForwardSessionEventToSession:612(2666ns) ForwardSessionEventToSession:612(2142ns) ForwardSessionEventToSession:612(4970ns) ForwardSessionEventToSession:612(2484ns) ForwardSessionEventToSession:612(8694ns) ForwardSessionEventToSession:612(2466ns) ForwardSessionEventToSession:612(3233ns)}}}
2025-11-27T18:18:03.614843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471141752134945:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.614868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471141752134945:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.615756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471141752134945:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:03.620494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471141752134918:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.620561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:03.620611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471141752134918:2109] [node 6] ICP22 created new session: [2:7577471141752134957:2109]
2025-11-27T18:18:03.620666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471141752134957:2109] [node 6] ICS09 handshake done sender: [2:7577471141752134945:2119] self: [2:7577471141752134937:0] peer: [6:7577471144730014719:0] socket: 356 qp: -1
2025-11-27T18:18:03.620699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471141752134957:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:03.620738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:03.472322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471141814916307:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:03.472363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:03.493097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471141814916310:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14379
2025-11-27T18:18:03.525775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471141814916310:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.525914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471137519948920:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:03.557431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471137519948922:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16359) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471141814916302:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.625403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471141300049097:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43330
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.626536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471144730014725:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1099
2025-11-27T18:18:03.557781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471137519948922:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(22672000ns) {HandleHandshakeStatus:395(22668448ns) {LogHandshakeFail:472(15690ns) {UpdateErrorStateLog:545(6292ns)} DropOutgoingHandshake:457(3428ns) ProcessPendingSessionEvents:495(79754ns) {DropSessionEvent:512(15246ns) DropSessionEvent:512(5997ns) DropSessionEvent:512(6312ns) DropSessionEvent:512(5808ns) DropSessionEvent:512(6683ns) DropSessionEvent:512(10090ns) DropSessionEvent:512(6545ns) DropSessionEvent:512(6511ns) DropSessionEvent:512(5294ns) DropSessionEvent:512(5559ns)}}}
2025-11-27T18:18:03.574989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471137519948920:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471141814916310:2120] held: no
2025-11-27T18:18:03.575303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471141814916312:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14379
2025-11-27T18:18:03.575937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471141814916313:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14379
2025-11-27T18:18:03.628292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471144730014715:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:03.628504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471144730014717:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:03.628779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471140435047334:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.628847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:03.628897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471140435047334:2049] [node 2] ICP22 created new session: [6:7577471144730014726:2049]
2025-11-27T18:18:03.628945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471144730014726:2049] [node 2] ICS09 handshake done sender: [6:7577471144730014717:2121] self: [6:7577471144730014719:0] peer: [2:7577471141752134937:0] socket: 384 qp: -1
2025-11-27T18:18:03.628980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471144730014726:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:03.629035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:03.635937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471144730014683:2091] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.635996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:03.636053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471144730014683:2091] [node 4] ICP22 created new session: [6:7577471144730014728:2091]
2025-11-27T18:18:03.636107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471144730014728:2091] [node 4] ICS09 handshake done sender: [6:7577471144730014715:2120] self: [6:7577471144730014720:0] peer: [4:7577471141814916301:0] socket: 423 qp: -1
2025-11-27T18:18:03.636145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471144730014728:2091] [node 4] ICS10 traffic start
2025-11-27T18:18:03.636187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:03.636396Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471144730014683:2091] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:03.625930Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471140011090921:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(145201224ns) {HandleHandshakeStatus:329(145198873ns) {DropIncomingHandshake:442(393ns) DropOutgoingHandshake:457(767ns) SwitchToState:179(754ns) ProcessPendingSessionEvents:495(25975218ns) {ForwardSessionEventToSession:612(14852335ns) ForwardSessionEventToSession:612(6261ns) ForwardSessionEventToSession:612(626228ns) ForwardSessionEventToSession:612(811121ns) ForwardSessionEventToSession:612(3378ns) ForwardSessionEventToSession:612(1781420ns) ForwardSessionEventToSession:612(4212ns) ForwardSessionEventToSession:612(3002ns) ForwardSessionEventToSession:612(3632735ns) ForwardSessionEventToSession:612(4568ns)}}}
2025-11-27T18:18:03.576406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471141814916302:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:03.626136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471144306058298:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.626162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471144306058298:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.626741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471140011090921:2048] [node 2] ICP18 handshake (actor [1:7577471144306058309:2120]) is too old
2025-11-27T18:18:03.576606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471137519948922:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.576668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:03.576723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471137519948922:2050] [node 3] ICP22 created new session: [4:7577471141814916314:2050]
2025-11-27T18:18:03.576782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471141814916314:2050] [node 3] ICS09 handshake done sender: [4:7577471141814916302:2117] self: [4:7577471141814916305:0] peer: [3:7577471141001361626:0] socket: 407 qp: -1
2025-11-27T18:18:03.576826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471141814916314:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:03.576874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:03.577149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471141814916310:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:03.594248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471137519948922:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17535667ns) {HandleHandshakeStatus:329(17532997ns) {DropIncomingHandshake:442(972ns) DropOutgoingHandshake:457(641ns) SwitchToState:179(686ns) ProcessPendingSessionEvents:495(1074ns)}}
2025-11-27T18:18:03.594628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471137519948920:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.594709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:03.594779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471137519948920:2048] [node 1] ICP22 created new session: [4:7577471141814916316:2048]
2025-11-27T18:18:03.601657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471141814916316:2048] [node 1] ICS09 handshake done sender: [4:7577471141814916310:2120] self: [4:7577471141814916311:0] peer: [1:7577471144306058303:0] socket: 422 qp: -1
2025-11-27T18:18:03.601708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471141814916316:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:03.601757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:03.614691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471141814916303:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:03.614729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471141814916303:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:03.618446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471141814916303:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:03.621949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471141814916280:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.636837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:03.638803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471141814916280:2108] [node 6] ICP22 created new session: [4:7577471141814916319:2108]
2025-11-27T18:18:03.642208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471144306058298:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:03.638875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471141814916319:2108] [node 6] ICS09 handshake done sender: [4:7577471141814916303:2118] self: [4:7577471141814916301:0] peer: [6:7577471144730014720:0] socket: 421 qp: -1
2025-11-27T18:18:03.638917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471141814916319:2108] [node 6] ICS10 traffic start
2025-11-27T18:18:03.638971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:03.639405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471141814916280:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(17249729ns) {HandleHandshakeStatus:329(17247753ns) {DropIncomingHandshake:442(10042ns) DropOutgoingHandshake:457(441ns) SwitchToState:179(160ns) ProcessPendingSessionEvents:495(91857ns) {ForwardSessionEventToSession:612(43240ns) ForwardSessionEventToSession:612(6442ns) ForwardSessionEventToSession:612(2994ns) ForwardSessionEventToSession:612(5538ns) ForwardSessionEventToSession:612(2772ns) ForwardSessionEventToSession:612(5048ns) ForwardSessionEventToSession:612(6187ns) ForwardSessionEventToSession:612(2916ns) ForwardSessionEventToSession:612(3065ns) ForwardSessionEventToSession:612(5496ns)}}}
2025-11-27T18:18:03.643515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471141300049058:2108] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:03.648880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471144730014730:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1099
2025-11-27T18:18:03.649365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471144306058286:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:03.649491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:03.650422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471144730014712:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:03.651126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471140435047333:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471144730014712:2118] held: no
2025-11-27T18:18:03.651236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471140435047333:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:03.651298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:03.651345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471140435047333:2048] [node 1] ICP22 created new session: [6:7577471144730014731:2048]
2025-11-27T18:18:03.649547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471144306058286:2110] [node 6] ICP22 created new session: [1:7577471144306058321:2110]
2025-11-27T18:18:03.649605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471144306058321:2110] [node 6] ICS09 handshake done sender: [1:7577471144306058298:2118] self: [1:7577471144306058294:0] peer: [6:7577471144730014724:0] socket: 410 qp: -1
2025-11-27T18:18:03.649644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471144306058321:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:03.649683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:03.652554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471144730014731:2048] [node 1] ICS09 handshake done sender: [6:7577471144730014712:2118] self: [6:7577471144730014724:0] peer: [1:7577471144306058294:0] socket: 417 qp: -1
2025-11-27T18:18:03.652604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471144730014731:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:03.652641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:03.708581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:03.708837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471141752134953:2048] [node 1] ICS01 socket: 392 reason# EndOfStream
2025-11-27T18:18:03.708877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471137457167559:2048] [node 1] ICP30 unregister session Session# [2:7577471141752134953:2048] VirtualId# [2:7577471141752134934:0]
2025-11-27T18:18:03.708915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471141752134953:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.708989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:03.709053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:03.718081Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:03.718807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471141001361634:2048] [node 1] ICS01 socket: 355 reason# EndOfStream
2025-11-27T18:18:03.718858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471136706394245:2048] [node 1] ICP30 unregister session Session# [3:7577471141001361634:2048] VirtualId# [3:7577471141001361619:0]
2025-11-27T18:18:03.718896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471141001361634:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.718979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:03.719025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:03.728266Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:03.728531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471141814916316:2048] [node 1] ICS01 socket: 422 reason# EndOfStream
2025-11-27T18:18:03.728588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471137519948920:2048] [node 1] ICP30 unregister session Session# [4:7577471141814916316:2048] VirtualId# [4:7577471141814916311:0]
2025-11-27T18:18:03.728625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471141814916316:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.728693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:03.728734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:03.731345Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:03.731591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471144730014731:2048] [node 1] ICS01 socket: 417 reason# EndOfStream
2025-11-27T18:18:03.731629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471140435047333:2048] [node 1] ICP30 unregister session Session# [6:7577471144730014731:2048] VirtualId# [6:7577471144730014724:0]
2025-11-27T18:18:03.731661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471144730014731:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.731722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:03.731757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:03.733619Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:03.733813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471141300049096:2048] [node 1] ICS01 socket: 393 reason# EndOfStream
2025-11-27T18:18:03.733846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471137005081698:2048] [node 1] ICP30 unregister session Session# [5:7577471141300049096:2048] VirtualId# [5:7577471141300049082:0]
2025-11-27T18:18:03.733878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471141300049096:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.733946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:03.733985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:03.837697Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:03.837908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471141001361638:2049] [node 2] ICS01 socket: 373 reason# EndOfStream
2025-11-27T18:18:03.837956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471136706394246:2049] [node 2] ICP30 unregister session Session# [3:7577471141001361638:2049] VirtualId# [3:7577471141001361624:0]
2025-11-27T18:18:03.837986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471141001361638:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.838077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:03.838134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:03.845974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:03.846213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471141814916307:2049] [node 2] ICS01 socket: 362 reason# EndOfStream
2025-11-27T18:18:03.846256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471137519948921:2049] [node 2] ICP30 unregister session Session# [4:7577471141814916307:2049] VirtualId# [4:7577471141814916295:0]
2025-11-27T18:18:03.846302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471141814916307:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.846379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:03.846423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:03.846621Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:03.846826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471141300049094:2049] [node 2] ICS01 socket: 375 reason# EndOfStream
2025-11-27T18:18:03.846903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471137005081699:2049] [node 2] ICP30 unregister session Session# [5:7577471141300049094:2049] VirtualId# [5:7577471141300049079:0]
2025-11-27T18:18:03.846951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471141300049094:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.847042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:03.847105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:03.851530Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:03.851786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471144730014726:2049] [node 2] ICS01 socket: 384 reason# EndOfStream
2025-11-27T18:18:03.863881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471140435047334:2049] [node 2] ICP30 unregister session Session# [6:7577471144730014726:2049] VirtualId# [6:7577471144730014719:0]
2025-11-27T18:18:03.865626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471144730014726:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.865740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:03.865798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:03.924786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:03.924469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:03.924706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471141300049087:2050] [node 3] ICS01 socket: 374 reason# EndOfStream
2025-11-27T18:18:03.924754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471137005081700:2050] [node 3] ICP30 unregister session Session# [5:7577471141300049087:2050] VirtualId# [5:7577471141300049075:0]
2025-11-27T18:18:03.924795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471141300049087:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.924892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:03.924939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:03.925789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:03.939328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471141814916314:2050] [node 3] ICS01 socket: 407 reason# EndOfStream
2025-11-27T18:18:03.939399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471137519948922:2050] [node 3] ICP30 unregister session Session# [4:7577471141814916314:2050] VirtualId# [4:7577471141814916305:0]
2025-11-27T18:18:03.939440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471141814916314:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.939533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:03.939588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:03.925148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471144730014714:2050] [node 3] ICS01 socket: 367 reason# EndOfStream
2025-11-27T18:18:03.925189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471140435047335:2050] [node 3] ICP30 unregister session Session# [6:7577471144730014714:2050] VirtualId# [6:7577471144730014709:0]
2025-11-27T18:18:03.925217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471144730014714:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:03.925315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:03.925369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:04.005501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:04.005769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471144730014728:2091] [node 4] ICS01 socket: 423 reason# EndOfStream
2025-11-27T18:18:04.005810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471144730014683:2091] [node 4] ICP30 unregister session Session# [6:7577471144730014728:2091] VirtualId# [6:7577471144730014720:0]
2025-11-27T18:18:04.005851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471144730014728:2091] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:04.005941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:04.005988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:04.006092Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:04.006280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471141300049085:2109] [node 4] ICS01 socket: 376 reason# EndOfStream
2025-11-27T18:18:04.006316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471141300049059:2109] [node 4] ICP30 unregister session Session# [5:7577471141300049085:2109] VirtualId# [5:7577471141300049076:0]
2025-11-27T18:18:04.006341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471141300049085:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:04.006398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:04.006433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:04.069083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:04.069351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471144730014721:2085] [node 5] ICS07 socket disconnect 395 reason# EndOfStream
2025-11-27T18:18:04.069380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471144730014721:2085] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:04.069457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:04.069497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471144730014721:2085] [node 5] ICS15 start handshake
2025-11-27T18:18:04.075861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471140435047382:2085] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17065) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:04.075937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471144730014721:2085] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:04.075977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471140435047382:2085] [node 5] ICP30 unregister session Session# [6:7577471144730014721:2085] VirtualId# [6:7577471144730014711:0]
2025-11-27T18:18:04.076035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:18:04.076070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471140435047382:2085] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:04.076084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:04.082697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471140435047382:2085] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:04.279293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471147260589025:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:04.279335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471147260589026:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:04.317848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471148549463044:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:04.317891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471148549463045:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:04.371358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471148084101782:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:04.371407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471148084101783:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:04.411290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471149415548657:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:04.411353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471149415548658:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:04.411383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471149415548659:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:04.431040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471147386892176:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:04.431123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471147386892177:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:04.431155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471147386892178:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:04.464649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471147094778139:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:04.464690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471147094778140:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:04.464713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471147094778141:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:05.042854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471153710516016:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:05.014529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471151389745453:2064] [node 4] ICP01 ready to work
2025-11-27T18:18:05.145717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471153710516017:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:05.133638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471151555556384:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:05.165294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471151389745464:2066] [node 5] ICP01 ready to work
2025-11-27T18:18:05.131876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471152379069143:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:05.152174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471152844430405:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:05.141999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471151555556385:2107] [node 6] ICP01 ready to work
2025-11-27T18:18:05.160023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471152379069144:2108] [node 6] ICP01 ready to work
2025-11-27T18:18:05.170654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471152379069145:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:05.231559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471147094778140:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(89394515ns) {Configure:104(89389104ns) {ProcessConfigured:135(72353ns) {StartInitialHandshake:157(69406ns) {DropHandshakes:470(2690ns) {DropIncomingHandshake:442(447ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(593ns) PrepareNewSessionHandshake:477(3047ns) {SwitchToState:179(935ns)}}}}}
2025-11-27T18:18:05.160376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471151555556386:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:05.160581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471152844430403:2108] [node 6] ICP01 ready to work
2025-11-27T18:18:05.243454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471149415548658:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(94223664ns) {Configure:104(94217772ns) {ProcessConfigured:135(67015ns) {StartInitialHandshake:157(64712ns) {DropHandshakes:470(2360ns) {DropIncomingHandshake:442(612ns) DropOutgoingHandshake:457(316ns)} GenerateSessionVirtualId:988(518ns) PrepareNewSessionHandshake:477(1616ns) {SwitchToState:179(400ns)}}}}}
2025-11-27T18:18:05.199216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471151681859530:2101] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.252529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471151389745501:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15412
2025-11-27T18:18:05.252667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471151389745502:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15412
2025-11-27T18:18:05.262958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471147260589026:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(120211659ns) {Configure:104(120204089ns) {ProcessConfigured:135(73598ns) {StartInitialHandshake:157(71129ns) {DropHandshakes:470(2923ns) {DropIncomingHandshake:442(493ns) DropOutgoingHandshake:457(791ns)} GenerateSessionVirtualId:988(783ns) PrepareNewSessionHandshake:477(1818ns) {SwitchToState:179(679ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.160600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471152844430404:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.209433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471151681859531:2100] [node 4] ICP01 ready to work
2025-11-27T18:18:05.183647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471148549463044:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34610393ns) {Configure:104(25674370ns) {ProcessConfigured:135(62682ns) {StartInitialHandshake:157(60638ns) {DropHandshakes:470(2232ns) {DropIncomingHandshake:442(604ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(1053ns) PrepareNewSessionHandshake:477(1620ns) {SwitchToState:179(545ns)}}}}}
2025-11-27T18:18:05.259360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471151389745501:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.255228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471153710516016:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(212113094ns) {Configure:104(212107969ns) {ProcessConfigured:135(13619435ns) {StartInitialHandshake:157(13615786ns) {DropHandshakes:470(2790ns) {DropIncomingHandshake:442(474ns) DropOutgoingHandshake:457(373ns)} GenerateSessionVirtualId:988(734ns) PrepareNewSessionHandshake:477(3007ns) {SwitchToState:179(1666ns)}}}}}
2025-11-27T18:18:05.250985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471147386892176:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(163951265ns) {Configure:104(135089752ns) {ProcessConfigured:135(68377ns) {StartInitialHandshake:157(66299ns) {DropHandshakes:470(2124ns) {DropIncomingHandshake:442(521ns) DropOutgoingHandshake:457(314ns)} GenerateSessionVirtualId:988(1273ns) PrepareNewSessionHandshake:477(1471ns) {SwitchToState:179(584ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.314048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471153710516027:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6928
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.322475Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471148084101782:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(173939929ns) {Configure:104(173933580ns) {ProcessConfigured:135(64294ns) {StartInitialHandshake:157(61118ns) {DropHandshakes:470(3123ns) {DropIncomingHandshake:442(1247ns) DropOutgoingHandshake:457(413ns)} GenerateSessionVirtualId:988(1286ns) PrepareNewSessionHandshake:477(2191ns) {SwitchToState:179(799ns)}}}}}
2025-11-27T18:18:05.322977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471152379069148:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48137
2025-11-27T18:18:05.278063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471151555556394:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51787
2025-11-27T18:18:05.301793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471147094778139:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(140364727ns) {Configure:104(140348133ns) {ProcessConfigured:135(9401402ns) {StartInitialHandshake:157(9396432ns) {DropHandshakes:470(2315ns) {DropIncomingHandshake:442(478ns) DropOutgoingHandshake:457(310ns)} GenerateSessionVirtualId:988(2242ns) PrepareNewSessionHandshake:477(4464ns) {SwitchToState:179(2240ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.251466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471151681859540:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62277
2025-11-27T18:18:05.323754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471147094778139:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.330472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471151555556388:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.337543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471153710516027:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.266160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471152844430414:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48250
2025-11-27T18:18:05.334312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471152379069148:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.323859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471151389745507:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15412
2025-11-27T18:18:05.339530Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471149415548657:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(191151294ns) {Configure:104(191144003ns) {ProcessConfigured:135(73065ns) {StartInitialHandshake:157(70481ns) {DropHandshakes:470(1607ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(263ns)} GenerateSessionVirtualId:988(675ns) PrepareNewSessionHandshake:477(1371ns) {SwitchToState:179(537ns)}}}}}
2025-11-27T18:18:05.369448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471149415548658:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.378096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471149415548659:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(228230236ns) {Configure:104(228223146ns) {ProcessConfigured:135(83756ns) {StartInitialHandshake:157(80469ns) {DropHandshakes:470(1309ns) {DropIncomingHandshake:442(429ns) DropOutgoingHandshake:457(186ns)} GenerateSessionVirtualId:988(928ns) PrepareNewSessionHandshake:477(2373ns) {SwitchToState:179(1371ns)}}}}}
2025-11-27T18:18:05.381695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471153710516025:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.381732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471153710516025:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.330512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471151555556388:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.337526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471151681859540:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.266222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471152844430415:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48250
2025-11-27T18:18:05.383120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471153710516031:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6928
2025-11-27T18:18:05.335677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471148084101782:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.331501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471151555556395:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51787
2025-11-27T18:18:05.289560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471152844430405:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(94017381ns) {Configure:104(94013505ns) {ProcessConfigured:135(32844228ns) {StartInitialHandshake:157(32841392ns) {DropHandshakes:470(2194ns) {DropIncomingHandshake:442(544ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(870ns) PrepareNewSessionHandshake:477(2734ns) {SwitchToState:179(951ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.425729Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471151681859530:2101] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(200271667ns) {Configure:104(200262402ns) {ProcessConfigured:135(12182697ns) {StartInitialHandshake:157(12174376ns) {DropHandshakes:470(2272ns) {DropIncomingHandshake:442(384ns) DropOutgoingHandshake:457(411ns)} GenerateSessionVirtualId:988(339ns) PrepareNewSessionHandshake:477(3519ns) {SwitchToState:179(2148ns)}}}}}
2025-11-27T18:18:05.388205Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471152379069144:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(181891821ns) {Configure:104(181886440ns) {ProcessConfigured:135(67310ns) {StartInitialHandshake:157(65584ns) {DropHandshakes:470(852ns) {DropIncomingHandshake:442(173ns) DropOutgoingHandshake:457(121ns)} GenerateSessionVirtualId:988(380ns) PrepareNewSessionHandshake:477(1815ns) {SwitchToState:179(592ns)}}}}}
2025-11-27T18:18:05.350874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471151389745453:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(336011908ns) {Configure:104(336006127ns) {ProcessConfigured:135(13558731ns) {StartInitialHandshake:157(13555246ns) {DropHandshakes:470(1816ns) {DropIncomingHandshake:442(284ns) DropOutgoingHandshake:457(368ns)} GenerateSessionVirtualId:988(1023ns) PrepareNewSessionHandshake:477(1671ns) {SwitchToState:179(221ns)}}}}}
2025-11-27T18:18:05.331568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471151555556396:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51787
2025-11-27T18:18:05.341521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471152844430415:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.401657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471153710516031:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.437111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471147386892178:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(305063521ns) {Configure:104(305052260ns) {ProcessConfigured:135(39369548ns) {StartInitialHandshake:157(39366828ns) {DropHandshakes:470(2416ns) {DropIncomingHandshake:442(856ns) DropOutgoingHandshake:457(395ns)} GenerateSessionVirtualId:988(382ns) PrepareNewSessionHandshake:477(2994ns) {SwitchToState:179(1692ns)}}}}}
2025-11-27T18:18:05.351030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471151389745507:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.402887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471153710516035:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6928
2025-11-27T18:18:05.403556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471153710516036:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6928
2025-11-27T18:18:05.417600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471153710516025:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:05.420001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471153710516035:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.420104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471153710516027:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:05.420734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471153710516016:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.420804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:05.420852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471153710516016:2109] [node 6] ICP22 created new session: [4:7577471153710516039:2109]
2025-11-27T18:18:05.420896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471153710516039:2109] [node 6] ICS09 handshake done sender: [4:7577471153710516025:2112] self: [4:7577471153710516021:0] peer: [6:7577471151389745512:0] socket: 375 qp: -1
2025-11-27T18:18:05.420931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471153710516039:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:05.420964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:05.438121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471149415548657:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.450200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471149415548658:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.450260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:05.455328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471153710516043:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6928
2025-11-27T18:18:05.457944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471149415548658:2049] [node 2] ICP22 created new session: [4:7577471153710516041:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.394934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471148084101783:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(240332920ns) {Configure:104(240326359ns) {ProcessConfigured:135(7440721ns) {StartInitialHandshake:157(7438530ns) {DropHandshakes:470(780ns) {DropIncomingHandshake:442(205ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:477(1961ns) {SwitchToState:179(184ns)}}}}}
2025-11-27T18:18:05.461559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471149415548657:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17943115ns) {IncomingHandshake:261(17941130ns) {DropIncomingHandshake:442(543ns) PrepareNewSessionHandshake:477(1351ns) {SwitchToState:179(504ns)} GenerateSessionVirtualId:988(1591ns) IssueIncomingHandshakeReply:193(17920479ns) {SwitchToState:179(1864ns)}}}
2025-11-27T18:18:05.403161Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471152379069143:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(243010577ns) {Configure:104(243002920ns) {ProcessConfigured:135(15947848ns) {StartInitialHandshake:157(15945656ns) {DropHandshakes:470(2332ns) {DropIncomingHandshake:442(381ns) DropOutgoingHandshake:457(404ns)} GenerateSessionVirtualId:988(817ns) PrepareNewSessionHandshake:477(2635ns) {SwitchToState:179(1103ns)}}}}}
2025-11-27T18:18:05.446223Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471152379069145:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(232423916ns) {Configure:104(232416080ns) {ProcessConfigured:135(96312ns) {StartInitialHandshake:157(92699ns) {DropHandshakes:470(5563ns) {DropIncomingHandshake:442(573ns) DropOutgoingHandshake:457(623ns)} GenerateSessionVirtualId:988(850ns) PrepareNewSessionHandshake:477(1667ns) {SwitchToState:179(448ns)}}}}}
2025-11-27T18:18:05.469613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471152379069162:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48137
2025-11-27T18:18:05.392376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471152844430413:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.392413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471152844430413:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.484053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471152379069163:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48137
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.401554Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471148549463045:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(257204776ns) {Configure:104(257198159ns) {ProcessConfigured:135(9019996ns) {StartInitialHandshake:157(9017827ns) {DropHandshakes:470(1212ns) {DropIncomingHandshake:442(280ns) DropOutgoingHandshake:457(277ns)} GenerateSessionVirtualId:988(403ns) PrepareNewSessionHandshake:477(3399ns) {SwitchToState:179(1446ns)}}}}}
2025-11-27T18:18:05.453299Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471147386892177:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(341584056ns) {Configure:104(341579198ns) {ProcessConfigured:135(14190758ns) {StartInitialHandshake:157(14188790ns) {DropHandshakes:470(2406ns) {DropIncomingHandshake:442(544ns) DropOutgoingHandshake:457(338ns)} GenerateSessionVirtualId:988(17036ns) PrepareNewSessionHandshake:477(1832ns) {SwitchToState:179(130ns)}}}}}
2025-11-27T18:18:05.352486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471151389745453:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.331840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471151555556388:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:05.406243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471152844430421:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48250
2025-11-27T18:18:05.480934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471153710516041:2049] [node 2] ICS09 handshake done sender: [4:7577471153710516027:2113] self: [4:7577471153710516030:0] peer: [2:7577471152844430410:0] socket: 380 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.486015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471151681859550:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62277
2025-11-27T18:18:05.406300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471152844430422:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48250
2025-11-27T18:18:05.406351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471152844430423:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48250
2025-11-27T18:18:05.410089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471152844430413:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:05.413300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471152844430409:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.413346Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471152844430409:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.413615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471152844430414:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.470224Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471152844430404:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(216281742ns) {Configure:104(216273283ns) {ProcessConfigured:135(59835ns) {StartInitialHandshake:157(57778ns) {DropHandshakes:470(1877ns) {DropIncomingHandshake:442(388ns) DropOutgoingHandshake:457(687ns)} GenerateSessionVirtualId:988(544ns) PrepareNewSessionHandshake:477(1454ns) {SwitchToState:179(424ns)}}}}}
2025-11-27T18:18:05.513730Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471152844430409:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:05.516137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471152844430421:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.516196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471152844430405:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.516251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:05.537832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471152844430422:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.332149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471151555556385:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.354410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471151389745514:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15412
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.332192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:05.493492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471153710516041:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:05.529327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471152379069154:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.366251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471151389745464:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(184685900ns) {Configure:104(184676389ns) {ProcessConfigured:135(87251ns) {StartInitialHandshake:157(84634ns) {DropHandshakes:470(9190ns) {DropIncomingHandshake:442(665ns) DropOutgoingHandshake:457(855ns)} GenerateSessionVirtualId:988(1018ns) PrepareNewSessionHandshake:477(1952ns) {SwitchToState:179(533ns)}}}}}
2025-11-27T18:18:05.501938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471151681859531:2100] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(281453140ns) {Configure:104(281446607ns) {ProcessConfigured:135(12315036ns) {StartInitialHandshake:157(12312187ns) {DropHandshakes:470(3132ns) {DropIncomingHandshake:442(962ns) DropOutgoingHandshake:457(329ns)} GenerateSessionVirtualId:988(933ns) PrepareNewSessionHandshake:477(2236ns) {SwitchToState:179(147ns)}}}}}
2025-11-27T18:18:05.540908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471152844430403:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.529386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471152379069154:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.493549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:05.332238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471151555556385:2107] [node 6] ICP22 created new session: [1:7577471151555556398:2107]
2025-11-27T18:18:05.502107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471151681859531:2100] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.373341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471151389745502:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.332278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471151555556398:2107] [node 6] ICS09 handshake done sender: [1:7577471151555556388:2110] self: [1:7577471151555556387:0] peer: [6:7577471151389745508:0] socket: 361 qp: -1
2025-11-27T18:18:05.545623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471152379069154:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:05.541010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:05.382334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471147094778141:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(239204210ns) {Configure:104(239197643ns) {ProcessConfigured:135(4766370ns) {StartInitialHandshake:157(4762875ns) {DropHandshakes:470(2349ns) {DropIncomingHandshake:442(554ns) DropOutgoingHandshake:457(382ns)} GenerateSessionVirtualId:988(1093ns) PrepareNewSessionHandshake:477(2974ns) {SwitchToState:179(1053ns)}}}}}
2025-11-27T18:18:05.493969Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471149415548658:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(55607992ns) {HandleHandshakeStatus:329(43604146ns) {DropIncomingHandshake:442(661ns) DropOutgoingHandshake:457(6784ns) SwitchToState:179(508ns) ProcessPendingSessionEvents:495(114256ns) {ForwardSessionEventToSession:612(57311ns) ForwardSessionEventToSession:612(3250ns) ForwardSessionEventToSession:612(4899ns) ForwardSessionEventToSession:612(2555ns) ForwardSessionEventToSession:612(5572ns) ForwardSessionEventToSession:612(3155ns) ForwardSessionEventToSession:612(2351ns) ForwardSessionEventToSession:612(7669ns) ForwardSessionEventToSession:612(10446ns) ForwardSessionEventToSession:612(2473ns)}}}
2025-11-27T18:18:05.534221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471151681859554:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62277
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.587689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471148084101782:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19402) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471152379069148:2113] held: no
2025-11-27T18:18:05.501676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471153710516035:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:05.541057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471152844430403:2108] [node 6] ICP22 created new session: [2:7577471152844430429:2108]
2025-11-27T18:18:05.332311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471151555556398:2107] [node 6] ICS10 traffic start
2025-11-27T18:18:05.382421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471147094778140:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.534283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471151681859555:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62277
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.541124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471152844430429:2108] [node 6] ICS09 handshake done sender: [2:7577471152844430409:2111] self: [2:7577471152844430408:0] peer: [6:7577471151389745519:0] socket: 368 qp: -1
2025-11-27T18:18:05.587745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471152379069159:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48137
2025-11-27T18:18:05.610327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471152379069163:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.626294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471152379069162:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.639740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471152379069144:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.639809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:05.649526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471148084101782:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(26719179ns) {HandleHandshakeStatus:395(26716467ns) {LogHandshakeFail:472(12593432ns) {UpdateErrorStateLog:545(5153ns)} DropOutgoingHandshake:457(1246ns) ProcessPendingSessionEvents:495(5759077ns) {DropSessionEvent:512(19210ns) DropSessionEvent:512(5747ns) DropSessionEvent:512(4393ns) DropSessionEvent:512(4631ns) DropSessionEvent:512(4520ns) DropSessionEvent:512(4602ns) DropSessionEvent:512(4961ns) DropSessionEvent:512(19517ns) DropSessionEvent:512(4344ns) DropSessionEvent:512(4759ns)}}}
2025-11-27T18:18:05.649634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471152379069144:2108] [node 6] ICP22 created new session: [3:7577471152379069167:2108]
2025-11-27T18:18:05.649683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471152379069167:2108] [node 6] ICS09 handshake done sender: [3:7577471152379069154:2114] self: [3:7577471152379069153:0] peer: [6:7577471151389745529:0] socket: 396 qp: -1
2025-11-27T18:18:05.649713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471152379069167:2108] [node 6] ICS10 traffic start
2025-11-27T18:18:05.649752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:05.650705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471152379069148:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:05.657558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471152379069168:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48137
2025-11-27T18:18:05.657909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471152379069171:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48137
2025-11-27T18:18:05.674704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471152379069168:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.675298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471152379069171:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.675920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471148084101782:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.332341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:05.502566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471153710516019:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.557835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471151681859540:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:05.393455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471151389745501:2110] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:05.354933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471151555556390:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51787
2025-11-27T18:18:05.358398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471151555556384:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(184730595ns) {Configure:104(184724355ns) {ProcessConfigured:135(79913ns) {StartInitialHandshake:157(77661ns) {DropHandshakes:470(3084ns) {DropIncomingHandshake:442(1050ns) DropOutgoingHandshake:457(1077ns)} GenerateSessionVirtualId:988(1033ns) PrepareNewSessionHandshake:477(2090ns) {SwitchToState:179(1057ns)}}}}}
2025-11-27T18:18:05.371936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471151555556392:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.371974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471151555556392:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.381527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471151555556390:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.383822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471151555556394:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.385733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471151555556395:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.390834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471147260589025:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(275176568ns) {Configure:104(275162953ns) {ProcessConfigured:135(12176593ns) {StartInitialHandshake:157(12174129ns) {DropHandshakes:470(1064ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(166ns)} GenerateSessionVirtualId:988(347ns) PrepareNewSessionHandshake:477(3605ns) {SwitchToState:179(1980ns)}}}}}
2025-11-27T18:18:05.397476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471151555556396:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.421671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471151555556401:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.421691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471151555556401:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.433522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471151555556401:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:05.434430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471151555556384:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.434474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:05.434521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471151555556384:2108] [node 4] ICP22 created new session: [1:7577471151555556410:2108]
2025-11-27T18:18:05.434565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471151555556410:2108] [node 4] ICS09 handshake done sender: [1:7577471151555556401:2116] self: [1:7577471151555556400:0] peer: [4:7577471153710516038:0] socket: 394 qp: -1
2025-11-27T18:18:05.434604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471151555556410:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:05.434639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:05.445637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471151555556385:2107] [node 6] ICP18 handshake (actor [1:7577471151555556396:2115]) is too old
2025-11-27T18:18:05.450415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [1:7577471151555556385:2107] [node 6] ICP35 event processing took too much time StateWork:294(31921396ns) {IncomingHandshake:261(31918305ns)}
2025-11-27T18:18:05.473901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471151555556412:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51787
2025-11-27T18:18:05.477358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471151555556392:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:05.497661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471151555556386:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(292082330ns) {Configure:104(292073672ns) {ProcessConfigured:135(22905534ns) {StartInitialHandshake:157(22902537ns) {DropHandshakes:470(3303967ns) {DropIncomingHandshake:442(1113ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(1372ns) PrepareNewSessionHandshake:477(3690ns) {SwitchToState:179(1609ns)}}}}}
2025-11-27T18:18:05.520893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471151555556412:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.521085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471151555556384:2108] [node 4] ICP18 handshake (actor [1:7577471151555556412:2118]) is too old
2025-11-27T18:18:05.525627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471147260589026:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.537675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:05.554897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471151555556404:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.554919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471151555556404:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.564003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471147260589026:2049] [node 3] ICP22 created new session: [1:7577471151555556417:2049]
2025-11-27T18:18:05.567130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471151555556404:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:05.578855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471151555556417:2049] [node 3] ICS09 handshake done sender: [1:7577471151555556392:2112] self: [1:7577471151555556391:0] peer: [3:7577471152379069150:0] socket: 376 qp: -1
2025-11-27T18:18:05.583381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471151555556417:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:05.587443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:05.604248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471147260589025:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.604305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:05.613594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471147260589025:2048] [node 2] ICP22 created new session: [1:7577471151555556420:2048]
2025-11-27T18:18:05.636465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471151555556414:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.636501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471151555556414:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.636650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471151555556420:2048] [node 2] ICS09 handshake done sender: [1:7577471151555556404:2117] self: [1:7577471151555556403:0] peer: [2:7577471152844430432:0] socket: 398 qp: -1
2025-11-27T18:18:05.647776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471151555556420:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:05.665033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:05.665400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471151555556414:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:05.673743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471147260589026:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(147898546ns) {HandleHandshakeStatus:329(147893644ns) {DropIncomingHandshake:442(15933ns) DropOutgoingHandshake:457(432ns) SwitchToState:179(499ns) ProcessPendingSessionEvents:495(11723211ns) {ForwardSessionEventToSession:612(11661434ns) ForwardSessionEventToSession:612(8847ns) ForwardSessionEventToSession:612(9987ns) ForwardSessionEventToSession:612(2927ns) ForwardSessionEventToSession:612(7448ns) ForwardSessionEventToSession:612(4014ns) ForwardSessionEventToSession:612(5837ns) ForwardSessionEventToSession:612(7717ns) ForwardSessionEventToSession:612(1604ns) ForwardSessionEventToSession:612(2646ns)}}}
2025-11-27T18:18:05.689563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:05.689642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471148084101782:2048] [node 1] ICP22 created new session: [3:7577471152379069173:2048]
2025-11-27T18:18:05.689688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471152379069173:2048] [node 1] ICS09 handshake done sender: [3:7577471152379069148:2113] self: [3:7577471152379069150:0] peer: [1:7577471151555556391:0] socket: 384 qp: -1
2025-11-27T18:18:05.689730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471152379069173:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:05.689785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:05.690024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471148084101783:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.692849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471152379069176:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48137
2025-11-27T18:18:05.701336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471152379069158:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.701373Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471152379069158:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.502588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471153710516019:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.541168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471152844430429:2108] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.709547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471152379069161:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.709598Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471152379069161:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.417457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471147094778139:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.719901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471152379069161:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:05.720816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471152379069145:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.720883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:05.720932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471152379069145:2107] [node 4] ICP22 created new session: [3:7577471152379069179:2107]
2025-11-27T18:18:05.720979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471152379069179:2107] [node 4] ICS09 handshake done sender: [3:7577471152379069161:2118] self: [3:7577471152379069160:0] peer: [4:7577471153710516052:0] socket: 416 qp: -1
2025-11-27T18:18:05.721027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471152379069179:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:05.721059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:05.721550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471152379069171:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:05.697401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471151555556386:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.558063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471151681859547:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.541209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:05.700477Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471147260589025:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(106966417ns) {HandleHandshakeStatus:329(106963754ns) {DropIncomingHandshake:442(13768ns) DropOutgoingHandshake:457(418ns) SwitchToState:179(442ns) ProcessPendingSessionEvents:495(6479166ns) {ForwardSessionEventToSession:612(6425668ns) ForwardSessionEventToSession:612(8480ns) ForwardSessionEventToSession:612(3091ns) ForwardSessionEventToSession:612(2554ns) ForwardSessionEventToSession:612(10613ns) ForwardSessionEventToSession:612(2570ns) ForwardSessionEventToSession:612(6312ns) ForwardSessionEventToSession:612(2673ns) ForwardSessionEventToSession:612(2332ns) ForwardSessionEventToSession:612(5562ns)}}}
2025-11-27T18:18:05.521428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471149415548657:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.418069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471151389745522:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15412
2025-11-27T18:18:05.723387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471148084101783:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.558094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471151681859547:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.525533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:05.542590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471152844430405:2107] [node 4] ICP22 created new session: [2:7577471152844430428:2107]
2025-11-27T18:18:05.701125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:05.420367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471151389745507:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:05.558938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471151681859547:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:05.729417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:05.553094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471148549463044:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.559199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471151681859530:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.709493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471151555556386:2109] [node 5] ICP22 created new session: [1:7577471151555556423:2109]
2025-11-27T18:18:05.559248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:05.559304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471151681859530:2101] [node 6] ICP22 created new session: [5:7577471151681859557:2101]
2025-11-27T18:18:05.723010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471151555556423:2109] [node 5] ICS09 handshake done sender: [1:7577471151555556414:2119] self: [1:7577471151555556413:0] peer: [5:7577471151681859562:0] socket: 420 qp: -1
2025-11-27T18:18:05.559352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471151681859557:2101] [node 6] ICS09 handshake done sender: [5:7577471151681859547:2114] self: [5:7577471151681859546:0] peer: [6:7577471151389745530:0] socket: 409 qp: -1
2025-11-27T18:18:05.734788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471151555556423:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:05.559383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471151681859557:2101] [node 6] ICS10 traffic start
2025-11-27T18:18:05.734845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:05.559418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:05.781723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471151555556386:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(84124976ns) {HandleHandshakeStatus:329(84122168ns) {DropIncomingHandshake:442(16032ns) DropOutgoingHandshake:457(958ns) SwitchToState:179(363ns) ProcessPendingSessionEvents:495(12360187ns) {ForwardSessionEventToSession:612(12293061ns) ForwardSessionEventToSession:612(11206ns) ForwardSessionEventToSession:612(5852ns) ForwardSessionEventToSession:612(2906ns) ForwardSessionEventToSession:612(4558ns) ForwardSessionEventToSession:612(5258ns) ForwardSessionEventToSession:612(3549ns) ForwardSessionEventToSession:612(3021ns) ForwardSessionEventToSession:612(8595ns) ForwardSessionEventToSession:612(10347ns)}}}
2025-11-27T18:18:05.566970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471151681859550:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.579438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471151681859531:2100] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.420556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471151389745453:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.579499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:05.420601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:05.579770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577471151681859530:2101] [node 6] ICP18 handshake (actor [5:7577471151681859550:2116]) is too old
2025-11-27T18:18:05.441345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:05.587820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471151681859531:2100] [node 4] ICP22 created new session: [5:7577471151681859561:2100]
2025-11-27T18:18:05.589432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471151681859554:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.445592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471151389745453:2064] [node 4] ICP22 created new session: [6:7577471151389745523:2064]
2025-11-27T18:18:05.589565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471147386892176:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.450343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471151389745523:2064] [node 4] ICS09 handshake done sender: [6:7577471151389745507:2114] self: [6:7577471151389745512:0] peer: [4:7577471153710516021:0] socket: 383 qp: -1
2025-11-27T18:18:05.597725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471151681859561:2100] [node 4] ICS09 handshake done sender: [5:7577471151681859540:2112] self: [5:7577471151681859553:0] peer: [4:7577471153710516018:0] socket: 369 qp: -1
2025-11-27T18:18:05.613847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471151681859561:2100] [node 4] ICS10 traffic start
2025-11-27T18:18:05.458025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471147094778139:2048] [node 1] ICP22 created new session: [6:7577471151389745521:2048]
2025-11-27T18:18:05.458079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471151389745523:2064] [node 4] ICS10 traffic start
2025-11-27T18:18:05.613897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:05.480801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:05.633993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471151681859531:2100] [node 4] ICP35 event processing took too much time PendingConnection:279(54336586ns) {HandleHandshakeStatus:329(54333951ns) {DropIncomingHandshake:442(581ns) DropOutgoingHandshake:457(9599ns) SwitchToState:179(430ns) ProcessPendingSessionEvents:495(127024ns) {ForwardSessionEventToSession:612(57508ns) ForwardSessionEventToSession:612(11035ns) ForwardSessionEventToSession:612(3475ns) ForwardSessionEventToSession:612(2645ns) ForwardSessionEventToSession:612(6193ns) ForwardSessionEventToSession:612(3167ns) ForwardSessionEventToSession:612(5340ns) ForwardSessionEventToSession:612(2742ns) ForwardSessionEventToSession:612(3002ns) ForwardSessionEventToSession:612(24027ns)}}}
2025-11-27T18:18:05.480910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471151389745521:2048] [node 1] ICS09 handshake done sender: [6:7577471151389745501:2110] self: [6:7577471151389745508:0] peer: [1:7577471151555556387:0] socket: 364 qp: -1
2025-11-27T18:18:05.482066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471151389745520:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15412
2025-11-27T18:18:05.634574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471151681859564:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62277
2025-11-27T18:18:05.482293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471151389745525:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15412
2025-11-27T18:18:05.666303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471151681859564:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.482402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471151389745526:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15412
2025-11-27T18:18:05.675993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471147386892178:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.488864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471151389745520:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.706271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471151681859560:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62277
2025-11-27T18:18:05.488974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471151389745525:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.710974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471151681859566:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62277
2025-11-27T18:18:05.493421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471151389745521:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:05.733673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471147386892176:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19402) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471151681859554:2118] held: no
2025-11-27T18:18:05.499555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471147094778141:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.745733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471151681859554:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:05.746589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471151681859560:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.499619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471151389745464:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.758430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471147386892176:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.553350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471152844430423:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.768974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:05.770260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471147386892177:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.563868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471152844430428:2107] [node 4] ICS09 handshake done sender: [2:7577471152844430413:2112] self: [2:7577471152844430410:0] peer: [4:7577471153710516030:0] socket: 379 qp: -1
2025-11-27T18:18:05.785943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471147386892176:2048] [node 1] ICP22 created new session: [5:7577471151681859567:2048]
2025-11-27T18:18:05.566224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471152844430428:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:05.791977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471151681859567:2048] [node 1] ICS09 handshake done sender: [5:7577471151681859554:2118] self: [5:7577471151681859562:0] peer: [1:7577471151555556413:0] socket: 421 qp: -1
2025-11-27T18:18:05.792442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471151681859569:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62277
2025-11-27T18:18:05.525589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471149415548657:2048] [node 1] ICP22 created new session: [4:7577471153710516047:2048]
2025-11-27T18:18:05.580616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:05.521459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:05.543453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471151389745502:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:05.554357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471151389745532:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15412
2025-11-27T18:18:05.571194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471147094778140:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.571252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:05.571302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471147094778140:2049] [node 2] ICP22 created new session: [6:7577471151389745534:2049]
2025-11-27T18:18:05.571344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471151389745534:2049] [node 2] ICS09 handshake done sender: [6:7577471151389745502:2111] self: [6:7577471151389745519:0] peer: [2:7577471152844430408:0] socket: 371 qp: -1
2025-11-27T18:18:05.571377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471151389745534:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:05.571405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:05.572784Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471147094778140:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:05.572895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471151389745520:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:05.573163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471151389745533:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15412
2025-11-27T18:18:05.578120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471147094778139:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(160454403ns) {HandleHandshakeStatus:329(160450915ns) {DropIncomingHandshake:442(1140ns) DropOutgoingHandshake:457(13509ns) SwitchToState:179(520ns) ProcessPendingSessionEvents:495(13667334ns) {ForwardSessionEventToSession:612(13601596ns) ForwardSessionEventToSession:612(5521ns) ForwardSessionEventToSession:612(3284ns) ForwardSessionEventToSession:612(21182ns) ForwardSessionEventToSession:612(2268ns) ForwardSessionEventToSession:612(5750ns) ForwardSessionEventToSession:612(5986ns) ForwardSessionEventToSession:612(2996ns) ForwardSessionEventToSession:612(5153ns) ForwardSessionEventToSession:612(2505ns)}}}
2025-11-27T18:18:05.585588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471151389745453:2064] [node 4] ICP35 event processing took too much time PendingConnection:279(134386826ns) {HandleHandshakeStatus:329(134384806ns) {DropIncomingHandshake:442(521ns) DropOutgoingHandshake:457(4465ns) SwitchToState:179(349ns) ProcessPendingSessionEvents:495(15632772ns) {ForwardSessionEventToSession:612(15576893ns) ForwardSessionEventToSession:612(6150ns) ForwardSessionEventToSession:612(7320ns) ForwardSessionEventToSession:612(2579ns) ForwardSessionEventToSession:612(5756ns) ForwardSessionEventToSession:612(2205ns) ForwardSessionEventToSession:612(2796ns) ForwardSessionEventToSession:612(11791ns) ForwardSessionEventToSession:612(2988ns) ForwardSessionEventToSession:612(2143ns)}}}
2025-11-27T18:18:05.585659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471147094778141:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.585704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:05.585747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471147094778141:2050] [node 3] ICP22 created new session: [6:7577471151389745536:2050]
2025-11-27T18:18:05.585789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471151389745536:2050] [node 3] ICS09 handshake done sender: [6:7577471151389745520:2119] self: [6:7577471151389745529:0] peer: [3:7577471152379069153:0] socket: 400 qp: -1
2025-11-27T18:18:05.585823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471151389745536:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:05.585855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:05.589102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471151389745525:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:05.613524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471151389745464:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.613588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:05.633480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471151389745464:2066] [node 5] ICP22 created new session: [6:7577471151389745538:2066]
2025-11-27T18:18:05.649456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471151389745538:2066] [node 5] ICS09 handshake done sender: [6:7577471151389745525:2121] self: [6:7577471151389745530:0] peer: [5:7577471151681859546:0] socket: 386 qp: -1
2025-11-27T18:18:05.675437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471151389745538:2066] [node 5] ICS10 traffic start
2025-11-27T18:18:05.685534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:05.739710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471151389745464:2066] [node 5] ICP35 event processing took too much time PendingConnection:279(130630279ns) {HandleHandshakeStatus:329(126022220ns) {DropIncomingHandshake:442(572ns) DropOutgoingHandshake:457(5944ns) SwitchToState:179(942ns) ProcessPendingSessionEvents:495(16544582ns) {ForwardSessionEventToSession:612(16497499ns) ForwardSessionEventToSession:612(9205ns) ForwardSessionEventToSession:612(2592ns) ForwardSessionEventToSession:612(2289ns) ForwardSessionEventToSession:612(5148ns) ForwardSessionEventToSession:612(2309ns) ForwardSessionEventToSession:612(7533ns) ForwardSessionEventToSession:612(2085ns) ForwardSessionEventToSession:612(3011ns) ForwardSessionEventToSession:612(4387ns)}}}
2025-11-27T18:18:05.739774Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471151389745464:2066] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:05.794002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471151681859567:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:05.794058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:05.525857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471153710516019:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:05.737373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471148084101783:2049] [node 2] ICP22 created new session: [3:7577471152379069181:2049]
2025-11-27T18:18:05.564144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471153710516047:2048] [node 1] ICS09 handshake done sender: [4:7577471153710516035:2117] self: [4:7577471153710516038:0] peer: [1:7577471151555556400:0] socket: 395 qp: -1
2025-11-27T18:18:05.564453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471153710516017:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.566347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471153710516047:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:05.566925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:05.573385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:05.579652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471153710516017:2108] [node 5] ICP22 created new session: [4:7577471153710516048:2108]
2025-11-27T18:18:05.596706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471153710516048:2108] [node 5] ICS09 handshake done sender: [4:7577471153710516019:2110] self: [4:7577471153710516018:0] peer: [5:7577471151681859553:0] socket: 360 qp: -1
2025-11-27T18:18:05.602789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471149415548657:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(99649882ns) {HandleHandshakeStatus:329(99646994ns) {DropIncomingHandshake:442(3011ns) DropOutgoingHandshake:457(8440ns) SwitchToState:179(510ns) ProcessPendingSessionEvents:495(14862316ns) {ForwardSessionEventToSession:612(14813767ns) ForwardSessionEventToSession:612(5676ns) ForwardSessionEventToSession:612(2779ns) ForwardSessionEventToSession:612(8717ns) ForwardSessionEventToSession:612(2378ns) ForwardSessionEventToSession:612(5342ns) ForwardSessionEventToSession:612(3151ns) ForwardSessionEventToSession:612(2690ns) ForwardSessionEventToSession:612(6222ns) ForwardSessionEventToSession:612(2483ns)}}}
2025-11-27T18:18:05.603988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471153710516048:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:05.632770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:05.665521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471153710516051:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6928
2025-11-27T18:18:05.693022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471153710516051:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.693186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471149415548659:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:05.699641Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471153710516017:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(134939767ns) {HandleHandshakeStatus:329(134937068ns) {DropIncomingHandshake:442(463ns) DropOutgoingHandshake:457(453ns) SwitchToState:179(328ns) ProcessPendingSessionEvents:495(13526770ns) {ForwardSessionEventToSession:612(13468541ns) ForwardSessionEventToSession:612(6401ns) ForwardSessionEventToSession:612(3324ns) ForwardSessionEventToSession:612(6106ns) ForwardSessionEventToSession:612(2626ns) ForwardSessionEventToSession:612(11779ns) ForwardSessionEventToSession:612(2898ns) ForwardSessionEventToSession:612(2540ns) ForwardSessionEventToSession:612(5632ns) ForwardSessionEventToSession:612(3008ns)}}}
2025-11-27T18:18:05.711576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471153710516053:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6928
2025-11-27T18:18:05.720518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471153710516051:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:05.731425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471149415548659:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2492) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471153710516051:2120] held: no
2025-11-27T18:18:05.738151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471149415548659:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.738200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:05.738240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471149415548659:2050] [node 3] ICP22 created new session: [4:7577471153710516054:2050]
2025-11-27T18:18:05.738280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471153710516054:2050] [node 3] ICS09 handshake done sender: [4:7577471153710516051:2120] self: [4:7577471153710516052:0] peer: [3:7577471152379069160:0] socket: 429 qp: -1
2025-11-27T18:18:05.738312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471153710516054:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:05.738342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:05.602126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471152844430437:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48250
2025-11-27T18:18:05.603910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471152844430422:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:05.609476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471152844430405:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(85524814ns) {HandleHandshakeStatus:329(85521202ns) {DropIncomingHandshake:442(13925ns) DropOutgoingHandshake:457(353ns) SwitchToState:179(639ns) ProcessPendingSessionEvents:495(144494ns) {ForwardSessionEventToSession:612(82088ns) ForwardSessionEventToSession:612(10483ns) ForwardSessionEventToSession:612(3661ns) ForwardSessionEventToSession:612(8936ns) ForwardSessionEventToSession:612(3310ns) ForwardSessionEventToSession:612(6025ns) ForwardSessionEventToSession:612(9644ns) ForwardSessionEventToSession:612(2754ns) ForwardSessionEventToSession:612(2883ns) ForwardSessionEventToSession:612(5135ns)}}}
2025-11-27T18:18:05.637380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471148549463044:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19402) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471152844430422:2117] held: no
2025-11-27T18:18:05.669693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471148549463044:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16160394ns) {HandleHandshakeStatus:395(16156909ns) {LogHandshakeFail:472(15958ns) {UpdateErrorStateLog:545(4917ns)} DropOutgoingHandshake:457(1626ns) ProcessPendingSessionEvents:495(11958622ns) {DropSessionEvent:512(10592ns) DropSessionEvent:512(5729ns) DropSessionEvent:512(5278ns) DropSessionEvent:512(5543ns) DropSessionEvent:512(5767ns) DropSessionEvent:512(20603ns) DropSessionEvent:512(5813ns) DropSessionEvent:512(4876ns) DropSessionEvent:512(4151ns) DropSessionEvent:512(6419530ns)}}}
2025-11-27T18:18:05.669745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471148549463044:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.669794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:05.669845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471148549463044:2048] [node 1] ICP22 created new session: [2:7577471152844430438:2048]
2025-11-27T18:18:05.669886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471152844430438:2048] [node 1] ICS09 handshake done sender: [2:7577471152844430422:2117] self: [2:7577471152844430432:0] peer: [1:7577471151555556403:0] socket: 406 qp: -1
2025-11-27T18:18:05.669927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471152844430438:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:05.669971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:05.691526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471152844430419:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.691550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471152844430419:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.693281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471152844430419:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:05.701619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471148549463045:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.706370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:05.725355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471148549463045:2049] [node 3] ICP22 created new session: [2:7577471152844430441:2049]
2025-11-27T18:18:05.725411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471152844430441:2049] [node 3] ICS09 handshake done sender: [2:7577471152844430419:2115] self: [2:7577471152844430418:0] peer: [3:7577471152379069175:0] socket: 427 qp: -1
2025-11-27T18:18:05.725451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471152844430441:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:05.725496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:05.725894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471148549463045:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(24100530ns) {HandleHandshakeStatus:329(24099023ns) {DropIncomingHandshake:442(12590ns) DropOutgoingHandshake:457(413ns) SwitchToState:179(766ns) ProcessPendingSessionEvents:495(100055ns) {ForwardSessionEventToSession:612(58653ns) ForwardSessionEventToSession:612(3259ns) ForwardSessionEventToSession:612(5039ns) ForwardSessionEventToSession:612(2403ns) ForwardSessionEventToSession:612(2774ns) ForwardSessionEventToSession:612(4586ns) ForwardSessionEventToSession:612(3086ns) ForwardSessionEventToSession:612(2623ns) ForwardSessionEventToSession:612(8039ns) ForwardSessionEventToSession:612(2083ns)}}}
2025-11-27T18:18:05.793819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471152844430425:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:05.793854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471152844430425:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:05.813302Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471152844430425:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:05.819062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471151681859564:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:05.821779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471147386892176:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(55474546ns) {HandleHandshakeStatus:329(55471795ns) {DropIncomingHandshake:442(652ns) DropOutgoingHandshake:457(309ns) SwitchToState:179(681ns) ProcessPendingSessionEvents:495(1466ns)}}
2025-11-27T18:18:05.821897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471151681859571:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62277
2025-11-27T18:18:05.822168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471147386892178:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.822246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:05.822293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471147386892178:2050] [node 3] ICP22 created new session: [5:7577471151681859572:2050]
2025-11-27T18:18:05.822365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471151681859572:2050] [node 3] ICS09 handshake done sender: [5:7577471151681859564:2121] self: [5:7577471151681859565:0] peer: [3:7577471152379069151:0] socket: 381 qp: -1
2025-11-27T18:18:05.822395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471151681859572:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:05.822454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:05.822754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471147386892178:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:05.824912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471151681859560:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:05.742235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471152379069181:2049] [node 2] ICS09 handshake done sender: [3:7577471152379069171:2122] self: [3:7577471152379069175:0] peer: [2:7577471152844430418:0] socket: 417 qp: -1
2025-11-27T18:18:05.742278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471152379069181:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:05.742332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:05.742797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471148084101783:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(19194074ns) {HandleHandshakeStatus:329(19192382ns) {DropIncomingHandshake:442(614ns) DropOutgoingHandshake:457(5955101ns) SwitchToState:179(1559ns) ProcessPendingSessionEvents:495(105116ns) {ForwardSessionEventToSession:612(51091ns) ForwardSessionEventToSession:612(13594ns) ForwardSessionEventToSession:612(2967ns) ForwardSessionEventToSession:612(2701ns) ForwardSessionEventToSession:612(5092ns) ForwardSessionEventToSession:612(2136ns) ForwardSessionEventToSession:612(5502ns) ForwardSessionEventToSession:612(2102ns) ForwardSessionEventToSession:612(2036ns) ForwardSessionEventToSession:612(10655ns)}}}
2025-11-27T18:18:05.793516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471152379069158:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:05.817435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471152379069143:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.817516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:05.820906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471152379069143:2109] [node 5] ICP22 created new session: [3:7577471152379069184:2109]
2025-11-27T18:18:05.829500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471152379069184:2109] [node 5] ICS09 handshake done sender: [3:7577471152379069158:2116] self: [3:7577471152379069151:0] peer: [5:7577471151681859565:0] socket: 363 qp: -1
2025-11-27T18:18:05.830444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471152379069184:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:05.830533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:05.830975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471152379069143:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(21172823ns) {HandleHandshakeStatus:329(21167821ns) {DropIncomingHandshake:442(25977ns) DropOutgoingHandshake:457(817ns) SwitchToState:179(928ns) ProcessPendingSessionEvents:495(107233ns) {ForwardSessionEventToSession:612(55895ns) ForwardSessionEventToSession:612(3272ns) ForwardSessionEventToSession:612(6338ns) ForwardSessionEventToSession:612(2613ns) ForwardSessionEventToSession:612(5505ns) ForwardSessionEventToSession:612(2512ns) ForwardSessionEventToSession:612(5861ns) ForwardSessionEventToSession:612(5398ns) ForwardSessionEventToSession:612(2491ns) ForwardSessionEventToSession:612(2646ns)}}}
2025-11-27T18:18:05.833079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471152844430404:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:05.833167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:05.833208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471152844430404:2109] [node 5] ICP22 created new session: [2:7577471152844430444:2109]
2025-11-27T18:18:05.833249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471152844430444:2109] [node 5] ICS09 handshake done sender: [2:7577471152844430425:2119] self: [2:7577471152844430424:0] peer: [5:7577471151681859568:0] socket: 387 qp: -1
2025-11-27T18:18:05.833477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471147386892177:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:05.833530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:05.833575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471147386892177:2049] [node 2] ICP22 created new session: [5:7577471151681859574:2049]
2025-11-27T18:18:05.833620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471151681859574:2049] [node 2] ICS09 handshake done sender: [5:7577471151681859560:2120] self: [5:7577471151681859568:0] peer: [2:7577471152844430424:0] socket: 423 qp: -1
2025-11-27T18:18:05.833649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471151681859574:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:05.833709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:05.839862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471152844430444:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:05.839963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:05.920448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:05.920668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471152379069173:2048] [node 1] ICS01 socket: 384 reason# EndOfStream
2025-11-27T18:18:05.920708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471148084101782:2048] [node 1] ICP30 unregister session Session# [3:7577471152379069173:2048] VirtualId# [3:7577471152379069150:0]
2025-11-27T18:18:05.920743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471152379069173:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:05.920814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:05.920850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:05.921385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:05.921624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471152844430438:2048] [node 1] ICS01 socket: 406 reason# EndOfStream
2025-11-27T18:18:05.921688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471148549463044:2048] [node 1] ICP30 unregister session Session# [2:7577471152844430438:2048] VirtualId# [2:7577471152844430432:0]
2025-11-27T18:18:05.921721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471152844430438:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:05.921806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:05.921847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:05.925085Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:05.925321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471151389745521:2048] [node 1] ICS01 socket: 364 reason# EndOfStream
2025-11-27T18:18:05.925365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471147094778139:2048] [node 1] ICP30 unregister session Session# [6:7577471151389745521:2048] VirtualId# [6:7577471151389745508:0]
2025-11-27T18:18:05.925414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471151389745521:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:05.925486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:05.925550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:05.933398Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:05.933660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471153710516047:2048] [node 1] ICS01 socket: 395 reason# EndOfStream
2025-11-27T18:18:05.933707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471149415548657:2048] [node 1] ICP30 unregister session Session# [4:7577471153710516047:2048] VirtualId# [4:7577471153710516038:0]
2025-11-27T18:18:05.933742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471153710516047:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:05.933822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:05.933880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:05.935866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:05.936071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471151681859567:2048] [node 1] ICS01 socket: 421 reason# EndOfStream
2025-11-27T18:18:05.936117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471147386892176:2048] [node 1] ICP30 unregister session Session# [5:7577471151681859567:2048] VirtualId# [5:7577471151681859562:0]
2025-11-27T18:18:05.936145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471151681859567:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:05.936203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:05.936240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:06.033445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:06.033668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471152379069181:2049] [node 2] ICS01 socket: 417 reason# EndOfStream
2025-11-27T18:18:06.033710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471148084101783:2049] [node 2] ICP30 unregister session Session# [3:7577471152379069181:2049] VirtualId# [3:7577471152379069175:0]
2025-11-27T18:18:06.033740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471152379069181:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:06.033811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:06.033858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:06.040915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:06.041140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471153710516041:2049] [node 2] ICS01 socket: 380 reason# EndOfStream
2025-11-27T18:18:06.041174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471149415548658:2049] [node 2] ICP30 unregister session Session# [4:7577471153710516041:2049] VirtualId# [4:7577471153710516030:0]
2025-11-27T18:18:06.041205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471153710516041:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:06.041295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:06.041340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:06.041426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:06.041586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471151389745534:2049] [node 2] ICS01 socket: 371 reason# EndOfStream
2025-11-27T18:18:06.041613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471147094778140:2049] [node 2] ICP30 unregister session Session# [6:7577471151389745534:2049] VirtualId# [6:7577471151389745519:0]
2025-11-27T18:18:06.041634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471151389745534:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:06.041679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:06.041713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:06.041791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:06.041945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471151681859574:2049] [node 2] ICS01 socket: 423 reason# EndOfStream
2025-11-27T18:18:06.041972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471147386892177:2049] [node 2] ICP30 unregister session Session# [5:7577471151681859574:2049] VirtualId# [5:7577471151681859568:0]
2025-11-27T18:18:06.041996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471151681859574:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:06.042038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:06.042070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:06.088949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:06.089180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471151681859572:2050] [node 3] ICS01 socket: 381 reason# EndOfStream
2025-11-27T18:18:06.089220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471147386892178:2050] [node 3] ICP30 unregister session Session# [5:7577471151681859572:2050] VirtualId# [5:7577471151681859565:0]
2025-11-27T18:18:06.089269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471151681859572:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:06.089334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:06.089369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:06.089454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:06.089638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471153710516054:2050] [node 3] ICS01 socket: 429 reason# EndOfStream
2025-11-27T18:18:06.089704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471149415548659:2050] [node 3] ICP30 unregister session Session# [4:7577471153710516054:2050] VirtualId# [4:7577471153710516052:0]
2025-11-27T18:18:06.089728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471153710516054:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:06.089794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:06.089829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:06.089907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:06.090054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471151389745536:2050] [node 3] ICS01 socket: 400 reason# EndOfStream
2025-11-27T18:18:06.090085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471147094778141:2050] [node 3] ICP30 unregister session Session# [6:7577471151389745536:2050] VirtualId# [6:7577471151389745529:0]
2025-11-27T18:18:06.090105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471151389745536:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:06.090167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:06.090201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:06.165596Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:06.165793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471151681859561:2100] [node 4] ICS01 socket: 369 reason# EndOfStream
2025-11-27T18:18:06.165839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471151681859531:2100] [node 4] ICP30 unregister session Session# [5:7577471151681859561:2100] VirtualId# [5:7577471151681859553:0]
2025-11-27T18:18:06.165873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471151681859561:2100] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:06.165957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:06.166004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:06.166106Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:06.166264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471151389745523:2064] [node 4] ICS07 socket disconnect 383 reason# EndOfStream
2025-11-27T18:18:06.166287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471151389745523:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:06.166345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:06.166396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471151389745523:2064] [node 4] ICS15 start handshake
2025-11-27T18:18:06.170778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471151389745453:2064] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4123) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:06.170842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471151389745523:2064] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:06.170885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471151389745453:2064] [node 4] ICP30 unregister session Session# [6:7577471151389745523:2064] VirtualId# [6:7577471151389745512:0]
2025-11-27T18:18:06.170931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:06.170965Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471151389745453:2064] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4123) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:06.170991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4123) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:06.181843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471151389745453:2064] [node 4] ICP33 wake up from error state
2025-11-27T18:18:06.218537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:06.218744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471151389745538:2066] [node 5] ICS01 socket: 386 reason# EndOfStream
2025-11-27T18:18:06.218785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471151389745464:2066] [node 5] ICP30 unregister session Session# [6:7577471151389745538:2066] VirtualId# [6:7577471151389745530:0]
2025-11-27T18:18:06.218817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471151389745538:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:06.218897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:06.218941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:06.393592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471155592558762:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:06.393699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471155592558763:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:06.425783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471156720090262:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:06.425831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471156720090263:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:06.445970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471157205625048:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:06.446011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471157205625049:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:06.477769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471156431161746:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:06.477831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471156431161747:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:06.478116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471156431161748:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:06.506802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471157973628786:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:06.506865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471157973628787:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:06.506893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471157973628788:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:06.629333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471154151704693:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:06.629465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471154151704694:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:06.641390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471154151704695:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:07.159881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471158446672040:2093] [node 5] ICP01 ready to work
2025-11-27T18:18:07.273462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471159887526121:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:07.306841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471157973628786:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(62173643ns) {Configure:104(62168620ns) {ProcessConfigured:135(83408ns) {StartInitialHandshake:157(80776ns) {DropHandshakes:470(2355ns) {DropIncomingHandshake:442(493ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(1188ns) PrepareNewSessionHandshake:477(1719ns) {SwitchToState:179(718ns)}}}}}
2025-11-27T18:18:07.341529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471161015057622:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:07.373368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471158446672046:2095] [node 4] ICP01 ready to work
2025-11-27T18:18:07.361505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471160726129106:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:07.307366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471162268596147:2108] [node 6] ICP01 ready to work
2025-11-27T18:18:07.307632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471162268596148:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:07.381594Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471162268596148:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(49397961ns) {Configure:104(49393100ns) {ProcessConfigured:135(3978669ns) {StartInitialHandshake:157(3976323ns) {DropHandshakes:470(2211ns) {DropIncomingHandshake:442(559ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(762ns) PrepareNewSessionHandshake:477(2502ns) {SwitchToState:179(1191ns)}}}}}
2025-11-27T18:18:07.361600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471159887526122:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:07.341974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471161015057623:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:07.415475Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471158446672040:2093] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25841819ns) {Configure:104(25836061ns) {ProcessConfigured:135(59639ns) {StartInitialHandshake:157(57611ns) {DropHandshakes:470(1838ns) {DropIncomingHandshake:442(479ns) DropOutgoingHandshake:457(451ns)} GenerateSessionVirtualId:988(503ns) PrepareNewSessionHandshake:477(875ns) {SwitchToState:179(322ns)}}}}}
2025-11-27T18:18:07.389758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471159887526123:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.409392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471160726129107:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:07.344047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471161015057626:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:07.430524Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471155592558762:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(37498648ns) {Configure:104(37496761ns) {ProcessConfigured:135(65761ns) {StartInitialHandshake:157(63195ns) {DropHandshakes:470(2350ns) {DropIncomingHandshake:442(561ns) DropOutgoingHandshake:457(362ns)} GenerateSessionVirtualId:988(1129ns) PrepareNewSessionHandshake:477(1945ns) {SwitchToState:179(994ns)}}}}}
2025-11-27T18:18:07.432827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471159887526132:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46902
2025-11-27T18:18:07.429790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471161015057626:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48490436ns) {Configure:104(48484224ns) {ProcessConfigured:135(36358685ns) {StartInitialHandshake:157(36355903ns) {DropHandshakes:470(5133ns) {DropIncomingHandshake:442(731ns) DropOutgoingHandshake:457(374ns)} GenerateSessionVirtualId:988(852ns) PrepareNewSessionHandshake:477(3625ns) {SwitchToState:179(1718ns)}}}}}
2025-11-27T18:18:07.449829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471159887526132:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.454602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471160726129118:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-27T18:18:07.459846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471157205625048:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(278691033ns) {Configure:104(278682034ns) {ProcessConfigured:135(10097171ns) {StartInitialHandshake:157(6055571ns) {DropHandshakes:470(3783ns) {DropIncomingHandshake:442(709ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(1868ns) PrepareNewSessionHandshake:477(2454ns) {SwitchToState:179(808ns)}}}}}
2025-11-27T18:18:07.460330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471161500592409:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:07.460604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471161500592410:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:07.460857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471161500592411:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:07.465433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471154151704695:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42479707ns) {Configure:104(42475725ns) {ProcessConfigured:135(56804ns) {StartInitialHandshake:157(54226ns) {DropHandshakes:470(2456ns) {DropIncomingHandshake:442(521ns) DropOutgoingHandshake:457(515ns)} GenerateSessionVirtualId:988(578ns) PrepareNewSessionHandshake:477(1347ns) {SwitchToState:179(415ns)}}}}}
2025-11-27T18:18:07.484483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471161500592410:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19547050ns) {Configure:104(19543528ns) {ProcessConfigured:135(79734ns) {StartInitialHandshake:157(76749ns) {DropHandshakes:470(6114ns) {DropIncomingHandshake:442(755ns) DropOutgoingHandshake:457(359ns)} GenerateSessionVirtualId:988(1054ns) PrepareNewSessionHandshake:477(1755ns) {SwitchToState:179(694ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.490409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471160726129122:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.496526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471161015057635:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39998
2025-11-27T18:18:07.496657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471161015057636:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39998
2025-11-27T18:18:07.496764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471161015057637:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39998
2025-11-27T18:18:07.494297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471161500592412:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31758
2025-11-27T18:18:07.494465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471161500592413:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31758
2025-11-27T18:18:07.494574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471161500592414:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31758
2025-11-27T18:18:07.495127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471161500592423:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31758
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.498059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471162268596160:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45370
2025-11-27T18:18:07.498206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471162268596161:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45370
2025-11-27T18:18:07.498323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471162268596162:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45370
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.500063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471161015057635:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.500198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471161015057636:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.517671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471162268596161:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.499649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471160726129123:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-27T18:18:07.517927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471160726129122:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.521595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471160726129123:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.521687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471160726129118:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.521805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471156431161747:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.521860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471156431161746:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.522296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471162268596160:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.522394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471157973628786:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.513784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471161500592412:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.513896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471161500592414:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.515775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471161500592427:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31758
2025-11-27T18:18:07.519782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471161500592423:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.522690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471161015057630:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.523982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471159887526129:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.524012Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471159887526129:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.524610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471159887526125:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.524626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471159887526125:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.525187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471159887526139:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46902
2025-11-27T18:18:07.522721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471161015057630:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.525348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471161015057630:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:07.525672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471160726129127:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-27T18:18:07.530167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471161500592427:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.530422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471161500592413:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.530559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471157205625049:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.530619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471157205625048:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.529923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471162268596162:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.530999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471157973628788:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.530907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471159887526140:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46902
2025-11-27T18:18:07.531701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471159887526141:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46902
2025-11-27T18:18:07.532074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471159887526139:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.532185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471159887526127:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.532203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471159887526127:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.533454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471160726129123:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:07.535444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471161015057637:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.535555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471161015057628:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.535574Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471161015057628:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.536215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471161015057622:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.536279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:07.536326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471161015057622:2107] [node 4] ICP22 created new session: [2:7577471161015057644:2107]
2025-11-27T18:18:07.536368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471161015057644:2107] [node 4] ICS09 handshake done sender: [2:7577471161015057630:2113] self: [2:7577471161015057629:0] peer: [4:7577471160726129125:0] socket: 398 qp: -1
2025-11-27T18:18:07.536396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471161015057644:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:07.536424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:07.533815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471156431161747:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.533878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:07.533925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471156431161747:2049] [node 2] ICP22 created new session: [4:7577471160726129129:2049]
2025-11-27T18:18:07.533966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471160726129129:2049] [node 2] ICS09 handshake done sender: [4:7577471160726129123:2118] self: [4:7577471160726129125:0] peer: [2:7577471161015057629:0] socket: 401 qp: -1
2025-11-27T18:18:07.534030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471160726129129:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:07.534059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.538200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471162268596166:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45370
2025-11-27T18:18:07.533876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471161500592418:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.533919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471161500592418:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.539649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471161500592433:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31758
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.540079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471162268596167:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45370
2025-11-27T18:18:07.540333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471159887526129:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:07.540817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471159887526125:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:07.541455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471162268596160:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:07.541734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471157973628786:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.541783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:07.541836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471157973628786:2048] [node 1] ICP22 created new session: [5:7577471162268596168:2048]
2025-11-27T18:18:07.541877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471162268596168:2048] [node 1] ICS09 handshake done sender: [5:7577471162268596160:2115] self: [5:7577471162268596164:0] peer: [1:7577471159887526124:0] socket: 374 qp: -1
2025-11-27T18:18:07.541905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471162268596168:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:07.541935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:07.544372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471159887526127:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.544714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471160726129128:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-27T18:18:07.545165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471159887526141:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.545275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471159887526140:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.545311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471159887526122:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.545378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:07.545424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471159887526122:2108] [node 4] ICP22 created new session: [1:7577471159887526146:2108]
2025-11-27T18:18:07.545463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471159887526146:2108] [node 4] ICS09 handshake done sender: [1:7577471159887526129:2112] self: [1:7577471159887526128:0] peer: [4:7577471160726129126:0] socket: 373 qp: -1
2025-11-27T18:18:07.545497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471159887526146:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:07.550527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471161015057628:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:07.546101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471161500592418:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:07.545553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:07.545957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471159887526121:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.545994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:07.546028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471159887526121:2107] [node 5] ICP22 created new session: [1:7577471159887526148:2107]
2025-11-27T18:18:07.546061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471159887526148:2107] [node 5] ICS09 handshake done sender: [1:7577471159887526125:2110] self: [1:7577471159887526124:0] peer: [5:7577471162268596164:0] socket: 371 qp: -1
2025-11-27T18:18:07.546093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471159887526148:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:07.546115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:07.550999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471155592558763:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.551040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:07.551084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471155592558763:2049] [node 3] ICP22 created new session: [1:7577471159887526150:2049]
2025-11-27T18:18:07.551144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471159887526150:2049] [node 3] ICS09 handshake done sender: [1:7577471159887526127:2111] self: [1:7577471159887526126:0] peer: [3:7577471161500592432:0] socket: 372 qp: -1
2025-11-27T18:18:07.551174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471159887526150:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:07.551203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:07.553472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471160726129118:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:07.554344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471161500592409:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.554406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:07.554448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471161500592409:2107] [node 5] ICP22 created new session: [3:7577471161500592436:2107]
2025-11-27T18:18:07.554494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471161500592436:2107] [node 5] ICS09 handshake done sender: [3:7577471161500592418:2115] self: [3:7577471161500592417:0] peer: [5:7577471162268596165:0] socket: 392 qp: -1
2025-11-27T18:18:07.554528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471161500592436:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:07.554557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:07.556081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471162268596162:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:07.556423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471156720090263:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.556494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:07.556535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471156720090263:2049] [node 3] ICP22 created new session: [2:7577471161015057649:2049]
2025-11-27T18:18:07.556572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471161015057649:2049] [node 3] ICS09 handshake done sender: [2:7577471161015057628:2112] self: [2:7577471161015057627:0] peer: [3:7577471161500592431:0] socket: 397 qp: -1
2025-11-27T18:18:07.556596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471161015057649:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:07.556618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:07.557998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471156431161746:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.558052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:07.558093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471156431161746:2048] [node 1] ICP22 created new session: [4:7577471160726129132:2048]
2025-11-27T18:18:07.558131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471160726129132:2048] [node 1] ICS09 handshake done sender: [4:7577471160726129118:2116] self: [4:7577471160726129126:0] peer: [1:7577471159887526128:0] socket: 376 qp: -1
2025-11-27T18:18:07.558164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471160726129132:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:07.558192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.560165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471162268596170:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45370
2025-11-27T18:18:07.560293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471157973628788:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.560330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:07.560365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471157973628788:2050] [node 3] ICP22 created new session: [5:7577471162268596173:2050]
2025-11-27T18:18:07.560399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471162268596173:2050] [node 3] ICS09 handshake done sender: [5:7577471162268596162:2117] self: [5:7577471162268596165:0] peer: [3:7577471161500592417:0] socket: 400 qp: -1
2025-11-27T18:18:07.560429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471162268596173:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:07.560454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:07.562202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471161500592413:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.562832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471158446672067:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40776
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.564492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471161500592435:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31758
2025-11-27T18:18:07.566602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471155592558763:2049] [node 3] ICP18 handshake (actor [1:7577471159887526140:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.568163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471158446672068:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40776
2025-11-27T18:18:07.568227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471158446672069:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40776
2025-11-27T18:18:07.568212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471162268596172:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45370
2025-11-27T18:18:07.569873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471157205625048:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.569927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:07.569977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471157205625048:2048] [node 1] ICP22 created new session: [3:7577471161500592439:2048]
2025-11-27T18:18:07.570020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471161500592439:2048] [node 1] ICS09 handshake done sender: [3:7577471161500592413:2112] self: [3:7577471161500592432:0] peer: [1:7577471159887526126:0] socket: 382 qp: -1
2025-11-27T18:18:07.570051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471161500592439:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:07.570082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:07.570618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471157205625049:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16028) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471161500592427:2119] held: no
2025-11-27T18:18:07.574661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471161500592427:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:07.574897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471162268596170:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.576725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471157973628787:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.577102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471161015057632:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.580777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471160726129135:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
2025-11-27T18:18:07.580887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471160726129136:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.582284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471157205625049:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.582328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:07.582369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471157205625049:2049] [node 2] ICP22 created new session: [3:7577471161500592441:2049]
2025-11-27T18:18:07.582423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471161500592441:2049] [node 2] ICS09 handshake done sender: [3:7577471161500592427:2119] self: [3:7577471161500592431:0] peer: [2:7577471161015057627:0] socket: 403 qp: -1
2025-11-27T18:18:07.582453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471161500592441:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:07.582482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:07.581239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471158446672068:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.577130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471161015057632:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.585016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471160726129135:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.581637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471162268596177:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45370
2025-11-27T18:18:07.581891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471161015057632:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.587203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471162268596172:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.587296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471162268596154:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.587313Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471162268596154:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.587738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471162268596170:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:07.585455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471158446672067:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.585844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471154151704694:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.585905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471158446672040:2093] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.586113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471158446672070:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40776
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.591195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471157973628787:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.591240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:07.591285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471157973628787:2049] [node 2] ICP22 created new session: [5:7577471162268596179:2049]
2025-11-27T18:18:07.591324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471162268596179:2049] [node 2] ICS09 handshake done sender: [5:7577471162268596170:2120] self: [5:7577471162268596176:0] peer: [2:7577471161015057631:0] socket: 366 qp: -1
2025-11-27T18:18:07.591356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471162268596179:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:07.591385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:07.591774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471162268596148:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.590727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471158446672073:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40776
2025-11-27T18:18:07.592854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471158446672074:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40776
2025-11-27T18:18:07.593645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471161015057634:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.593684Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471161015057634:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.594235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471161015057623:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.594274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:07.594320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471161015057623:2108] [node 5] ICP22 created new session: [2:7577471161015057654:2108]
2025-11-27T18:18:07.594357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471161015057654:2108] [node 5] ICS09 handshake done sender: [2:7577471161015057632:2114] self: [2:7577471161015057631:0] peer: [5:7577471162268596176:0] socket: 418 qp: -1
2025-11-27T18:18:07.594384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471161015057654:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:07.594417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:07.595958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471160726129117:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.595982Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471160726129117:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.595623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471158446672069:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.595737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471158446672046:2095] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.597065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471158446672076:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40776
2025-11-27T18:18:07.598795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471160726129117:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:07.599292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471158446672069:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:07.599475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471158446672046:2095] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.599546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:07.599592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471158446672046:2095] [node 4] ICP22 created new session: [6:7577471158446672077:2095]
2025-11-27T18:18:07.599631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471158446672077:2095] [node 4] ICS09 handshake done sender: [6:7577471158446672069:2117] self: [6:7577471158446672075:0] peer: [4:7577471160726129116:0] socket: 377 qp: -1
2025-11-27T18:18:07.599658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471158446672077:2095] [node 4] ICS10 traffic start
2025-11-27T18:18:07.599707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:07.600686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471160726129136:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.600731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471160726129107:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.600772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:07.600809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471160726129107:2110] [node 6] ICP22 created new session: [4:7577471160726129139:2110]
2025-11-27T18:18:07.600856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471160726129139:2110] [node 6] ICS09 handshake done sender: [4:7577471160726129117:2115] self: [4:7577471160726129116:0] peer: [6:7577471158446672075:0] socket: 370 qp: -1
2025-11-27T18:18:07.600886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471160726129139:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:07.600926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:07.602899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471156431161748:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.604146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471161015057653:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39998
2025-11-27T18:18:07.604406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471161015057657:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39998
2025-11-27T18:18:07.604696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471160726129115:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.604719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471160726129115:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.604743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471161015057634:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:07.605580Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471160726129115:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:07.605787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471160726129106:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.605834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:07.605868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471160726129106:2108] [node 5] ICP22 created new session: [4:7577471160726129143:2108]
2025-11-27T18:18:07.605899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471160726129143:2108] [node 5] ICS09 handshake done sender: [4:7577471160726129115:2114] self: [4:7577471160726129114:0] peer: [5:7577471162268596181:0] socket: 419 qp: -1
2025-11-27T18:18:07.605927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471160726129143:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:07.605968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:07.606820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471161500592420:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.606845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471161500592420:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.603524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471162268596154:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:07.609210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471159887526131:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.609236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471159887526131:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.608036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471161015057653:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.608286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471161015057626:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.608315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:07.608355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471161015057626:2109] [node 6] ICP22 created new session: [2:7577471161015057658:2109]
2025-11-27T18:18:07.608390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471161015057658:2109] [node 6] ICS09 handshake done sender: [2:7577471161015057634:2115] self: [2:7577471161015057633:0] peer: [6:7577471158446672071:0] socket: 395 qp: -1
2025-11-27T18:18:07.608429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471161015057658:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:07.608454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:07.608922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471156720090262:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.610534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471158446672073:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.610614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471158446672070:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.610878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471158446672067:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:07.611454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471162268596147:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.611503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:07.611538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471162268596147:2108] [node 6] ICP22 created new session: [5:7577471162268596183:2108]
2025-11-27T18:18:07.611576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471162268596183:2108] [node 6] ICS09 handshake done sender: [5:7577471162268596154:2113] self: [5:7577471162268596153:0] peer: [6:7577471158446672072:0] socket: 420 qp: -1
2025-11-27T18:18:07.611602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471162268596183:2108] [node 6] ICS10 traffic start
2025-11-27T18:18:07.611628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.612795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471160726129145:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8262
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.613070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471161015057657:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.613347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471161015057661:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39998
2025-11-27T18:18:07.613395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471161015057626:2109] [node 6] ICP18 handshake (actor [2:7577471161015057657:2120]) is too old
2025-11-27T18:18:07.613889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471161500592420:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:07.614193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471161500592410:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.614241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:07.614274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471161500592410:2108] [node 4] ICP22 created new session: [3:7577471161500592445:2108]
2025-11-27T18:18:07.614312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471161500592445:2108] [node 4] ICS09 handshake done sender: [3:7577471161500592420:2116] self: [3:7577471161500592419:0] peer: [4:7577471160726129141:0] socket: 391 qp: -1
2025-11-27T18:18:07.614333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471161500592445:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:07.614359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:07.615151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471159887526131:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:07.615513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471160726129136:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:07.616610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471155592558762:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.616667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:07.616709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471155592558762:2048] [node 2] ICP22 created new session: [1:7577471159887526156:2048]
2025-11-27T18:18:07.616753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471159887526156:2048] [node 2] ICS09 handshake done sender: [1:7577471159887526131:2113] self: [1:7577471159887526130:0] peer: [2:7577471161015057660:0] socket: 426 qp: -1
2025-11-27T18:18:07.616778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471159887526156:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:07.616803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:07.617793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471161015057653:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:07.617997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471156431161748:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.618037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:07.618072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471156431161748:2050] [node 3] ICP22 created new session: [4:7577471160726129146:2050]
2025-11-27T18:18:07.618107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471160726129146:2050] [node 3] ICS09 handshake done sender: [4:7577471160726129136:2122] self: [4:7577471160726129141:0] peer: [3:7577471161500592419:0] socket: 423 qp: -1
2025-11-27T18:18:07.618133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471160726129146:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:07.618158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:07.620322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471156720090262:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.620369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:07.620403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471156720090262:2048] [node 1] ICP22 created new session: [2:7577471161015057662:2048]
2025-11-27T18:18:07.620441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471161015057662:2048] [node 1] ICS09 handshake done sender: [2:7577471161015057653:2119] self: [2:7577471161015057660:0] peer: [1:7577471159887526130:0] socket: 431 qp: -1
2025-11-27T18:18:07.620463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471161015057662:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:07.620489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:07.622715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471158446672040:2093] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.622784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:07.622825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471158446672040:2093] [node 5] ICP22 created new session: [6:7577471158446672081:2093]
2025-11-27T18:18:07.622872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471158446672081:2093] [node 5] ICS09 handshake done sender: [6:7577471158446672067:2115] self: [6:7577471158446672072:0] peer: [5:7577471162268596153:0] socket: 381 qp: -1
2025-11-27T18:18:07.622899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471158446672081:2093] [node 5] ICS10 traffic start
2025-11-27T18:18:07.622931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.623419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471158446672080:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40776
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.624176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471162268596182:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45370
2025-11-27T18:18:07.624572Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471162268596148:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471162268596172:2121] held: no
2025-11-27T18:18:07.624924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471162268596172:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:07.625966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471154151704693:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.626029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471154151704695:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:07.626086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471154151704694:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16028) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471158446672068:2116] held: no
2025-11-27T18:18:07.626980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471162268596148:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.627019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:07.627052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471162268596148:2109] [node 4] ICP22 created new session: [5:7577471162268596185:2109]
2025-11-27T18:18:07.627094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471162268596185:2109] [node 4] ICS09 handshake done sender: [5:7577471162268596172:2121] self: [5:7577471162268596181:0] peer: [4:7577471160726129114:0] socket: 421 qp: -1
2025-11-27T18:18:07.627126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471162268596185:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:07.627158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:07.628390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471158446672068:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:07.628582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471159887526134:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.628608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471159887526134:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.629410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471159887526134:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:07.629586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471159887526123:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.629613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:07.629651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471159887526123:2109] [node 6] ICP22 created new session: [1:7577471159887526159:2109]
2025-11-27T18:18:07.629684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471159887526159:2109] [node 6] ICS09 handshake done sender: [1:7577471159887526134:2115] self: [1:7577471159887526133:0] peer: [6:7577471158446672083:0] socket: 427 qp: -1
2025-11-27T18:18:07.629711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471159887526159:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:07.629738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:07.630541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471161500592422:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:07.630566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471161500592422:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:07.631412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471161500592422:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:07.631635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471161500592411:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:07.631680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:07.631718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471161500592411:2109] [node 6] ICP22 created new session: [3:7577471161500592448:2109]
2025-11-27T18:18:07.631749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471161500592448:2109] [node 6] ICS09 handshake done sender: [3:7577471161500592422:2117] self: [3:7577471161500592421:0] peer: [6:7577471158446672084:0] socket: 424 qp: -1
2025-11-27T18:18:07.631772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471161500592448:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:07.631799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:07.632627Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471158446672040:2093] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:07.632879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471154151704694:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.632908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:07.632946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471154151704694:2049] [node 2] ICP22 created new session: [6:7577471158446672087:2049]
2025-11-27T18:18:07.632993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471158446672087:2049] [node 2] ICS09 handshake done sender: [6:7577471158446672068:2116] self: [6:7577471158446672071:0] peer: [2:7577471161015057633:0] socket: 368 qp: -1
2025-11-27T18:18:07.633019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471158446672087:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:07.633050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.635570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471158446672085:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40776
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.635959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471159887526161:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46902
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:07.636322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471158446672086:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40776
2025-11-27T18:18:07.636668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471154151704695:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3708) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471158446672070:2118] held: no
2025-11-27T18:18:07.636847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471158446672070:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:07.637111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471158446672073:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:07.637381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471154151704695:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.637421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:07.637462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471154151704695:2050] [node 3] ICP22 created new session: [6:7577471158446672090:2050]
2025-11-27T18:18:07.637502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471158446672090:2050] [node 3] ICS09 handshake done sender: [6:7577471158446672070:2118] self: [6:7577471158446672084:0] peer: [3:7577471161500592421:0] socket: 425 qp: -1
2025-11-27T18:18:07.637529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471158446672090:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:07.637558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:07.637877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471154151704693:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:07.637908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:07.637936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471154151704693:2048] [node 1] ICP22 created new session: [6:7577471158446672092:2048]
2025-11-27T18:18:07.637970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471158446672092:2048] [node 1] ICS09 handshake done sender: [6:7577471158446672073:2119] self: [6:7577471158446672083:0] peer: [1:7577471159887526133:0] socket: 428 qp: -1
2025-11-27T18:18:07.637991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471158446672092:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:07.638013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:07.664825Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:07.665073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471161015057662:2048] [node 1] ICS01 socket: 431 reason# EndOfStream
2025-11-27T18:18:07.665119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471156720090262:2048] [node 1] ICP30 unregister session Session# [2:7577471161015057662:2048] VirtualId# [2:7577471161015057660:0]
2025-11-27T18:18:07.665156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471161015057662:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.665230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:07.665282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:07.667648Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:07.667850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471161500592439:2048] [node 1] ICS01 socket: 382 reason# EndOfStream
2025-11-27T18:18:07.667882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471157205625048:2048] [node 1] ICP30 unregister session Session# [3:7577471161500592439:2048] VirtualId# [3:7577471161500592432:0]
2025-11-27T18:18:07.667910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471161500592439:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.667971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:07.668005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:07.672728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:07.672963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471162268596168:2048] [node 1] ICS01 socket: 374 reason# EndOfStream
2025-11-27T18:18:07.673008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471157973628786:2048] [node 1] ICP30 unregister session Session# [5:7577471162268596168:2048] VirtualId# [5:7577471162268596164:0]
2025-11-27T18:18:07.673039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471162268596168:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.673106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:07.673147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:07.673223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:07.673401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471160726129132:2048] [node 1] ICS01 socket: 376 reason# EndOfStream
2025-11-27T18:18:07.673427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471156431161746:2048] [node 1] ICP30 unregister session Session# [4:7577471160726129132:2048] VirtualId# [4:7577471160726129126:0]
2025-11-27T18:18:07.673453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471160726129132:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.673505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:07.673540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:07.674866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:07.675070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471158446672092:2048] [node 1] ICS01 socket: 428 reason# EndOfStream
2025-11-27T18:18:07.675103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471154151704693:2048] [node 1] ICP30 unregister session Session# [6:7577471158446672092:2048] VirtualId# [6:7577471158446672083:0]
2025-11-27T18:18:07.675145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471158446672092:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.675203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:07.675236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:07.731206Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:07.731432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471161500592441:2049] [node 2] ICS01 socket: 403 reason# EndOfStream
2025-11-27T18:18:07.731474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471157205625049:2049] [node 2] ICP30 unregister session Session# [3:7577471161500592441:2049] VirtualId# [3:7577471161500592431:0]
2025-11-27T18:18:07.731510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471161500592441:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.731574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:07.731615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:07.737622Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:07.737844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471160726129129:2049] [node 2] ICS01 socket: 401 reason# EndOfStream
2025-11-27T18:18:07.737886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471156431161747:2049] [node 2] ICP30 unregister session Session# [4:7577471160726129129:2049] VirtualId# [4:7577471160726129125:0]
2025-11-27T18:18:07.737918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471160726129129:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.737998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:07.738041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:07.738128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:07.738300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471162268596179:2049] [node 2] ICS01 socket: 366 reason# EndOfStream
2025-11-27T18:18:07.738333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471157973628787:2049] [node 2] ICP30 unregister session Session# [5:7577471162268596179:2049] VirtualId# [5:7577471162268596176:0]
2025-11-27T18:18:07.738356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471162268596179:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.738406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:07.738440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:07.738562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:07.738737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471158446672087:2049] [node 2] ICS01 socket: 368 reason# EndOfStream
2025-11-27T18:18:07.738766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471154151704694:2049] [node 2] ICP30 unregister session Session# [6:7577471158446672087:2049] VirtualId# [6:7577471158446672071:0]
2025-11-27T18:18:07.738786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471158446672087:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.738829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:07.738860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:07.793633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:07.793858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471162268596173:2050] [node 3] ICS01 socket: 400 reason# EndOfStream
2025-11-27T18:18:07.793899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471157973628788:2050] [node 3] ICP30 unregister session Session# [5:7577471162268596173:2050] VirtualId# [5:7577471162268596165:0]
2025-11-27T18:18:07.793931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471162268596173:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.794023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:07.794071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:07.794165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:07.794351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471160726129146:2050] [node 3] ICS01 socket: 423 reason# EndOfStream
2025-11-27T18:18:07.794391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471156431161748:2050] [node 3] ICP30 unregister session Session# [4:7577471160726129146:2050] VirtualId# [4:7577471160726129141:0]
2025-11-27T18:18:07.794418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471160726129146:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.794460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:07.794497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:07.794655Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:07.794834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471158446672090:2050] [node 3] ICS01 socket: 425 reason# EndOfStream
2025-11-27T18:18:07.794866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471154151704695:2050] [node 3] ICP30 unregister session Session# [6:7577471158446672090:2050] VirtualId# [6:7577471158446672084:0]
2025-11-27T18:18:07.794886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471158446672090:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.794934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:07.794966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:07.821451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:07.821667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471162268596185:2109] [node 4] ICS01 socket: 421 reason# EndOfStream
2025-11-27T18:18:07.821713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471162268596148:2109] [node 4] ICP30 unregister session Session# [5:7577471162268596185:2109] VirtualId# [5:7577471162268596181:0]
2025-11-27T18:18:07.821744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471162268596185:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.821832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:07.821877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:07.821979Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:07.822146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471158446672077:2095] [node 4] ICS01 socket: 377 reason# EndOfStream
2025-11-27T18:18:07.822177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471158446672046:2095] [node 4] ICP30 unregister session Session# [6:7577471158446672077:2095] VirtualId# [6:7577471158446672075:0]
2025-11-27T18:18:07.822206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471158446672077:2095] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.822266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:07.822301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:07.874122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:07.874472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471158446672081:2093] [node 5] ICS01 socket: 381 reason# EndOfStream
2025-11-27T18:18:07.874559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471158446672040:2093] [node 5] ICP30 unregister session Session# [6:7577471158446672081:2093] VirtualId# [6:7577471158446672072:0]
2025-11-27T18:18:07.874642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471158446672081:2093] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:07.874749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:07.874836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:07.963774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471160800618834:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:07.963814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471160800618835:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:07.986590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471159105844286:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:07.986634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471159105844287:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:08.017926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471163198453093:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:08.017978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471163198453094:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:08.058173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471162417458496:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:08.058225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471162417458497:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:08.058251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471162417458498:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:08.093270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471163574589162:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:08.093323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471163574589163:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:08.093602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471163574589164:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:08.124676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471165310324906:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:08.124769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471165310324907:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:08.124796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471165310324908:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:08.280909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471165095586193:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:08.287739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471165310324967:2105] [node 5] ICP01 ready to work
2025-11-27T18:18:08.393562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471163198453159:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:08.327395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471163400811645:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:08.393332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471165095586194:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:08.420613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471163198453160:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:08.429822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471163400811647:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:08.437820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471165095586195:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:08.417793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471165310324969:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:08.449533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471162417458559:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:08.433512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471163198453161:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:08.449371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471163574589226:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:08.441382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471163400811646:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:08.457538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471163574589225:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:08.449796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471162417458560:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:08.474006Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471165095586193:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(26579452ns) {Configure:104(26570773ns) {ProcessConfigured:135(61549ns) {StartInitialHandshake:157(59752ns) {DropHandshakes:470(1495ns) {DropIncomingHandshake:442(309ns) DropOutgoingHandshake:457(513ns)} GenerateSessionVirtualId:988(784ns) PrepareNewSessionHandshake:477(4641ns) {SwitchToState:179(562ns)}}}}}
2025-11-27T18:18:08.487029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471162417458497:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28341078ns) {Configure:104(28338559ns) {ProcessConfigured:135(35201ns) {StartInitialHandshake:157(33848ns) {DropHandshakes:470(979ns) {DropIncomingHandshake:442(325ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(390ns) PrepareNewSessionHandshake:477(611ns) {SwitchToState:179(127ns)}}}}}
2025-11-27T18:18:08.489364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471163400811647:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19069416ns) {Configure:104(19065756ns) {ProcessConfigured:135(423527ns) {StartInitialHandshake:157(421123ns) {DropHandshakes:470(3157ns) {DropIncomingHandshake:442(707ns) DropOutgoingHandshake:457(632ns)} GenerateSessionVirtualId:988(632ns) PrepareNewSessionHandshake:477(2346ns) {SwitchToState:179(880ns)}}}}}
2025-11-27T18:18:08.493938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471163198453094:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51789295ns) {Configure:104(51785204ns) {ProcessConfigured:135(53292ns) {StartInitialHandshake:157(50487ns) {DropHandshakes:470(1336ns) {DropIncomingHandshake:442(375ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(924ns) PrepareNewSessionHandshake:477(1218ns) {SwitchToState:179(382ns)}}}}}
2025-11-27T18:18:08.505382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471160800618834:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(180940418ns) {Configure:104(180931481ns) {ProcessConfigured:135(4792655ns) {StartInitialHandshake:157(4790338ns) {DropHandshakes:470(1038ns) {DropIncomingHandshake:442(275ns) DropOutgoingHandshake:457(218ns)} GenerateSessionVirtualId:988(392ns) PrepareNewSessionHandshake:477(2407ns) {SwitchToState:179(897ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.491364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471165310324967:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(154002036ns) {Configure:104(153995877ns) {ProcessConfigured:135(3583558ns) {StartInitialHandshake:157(3581760ns) {DropHandshakes:470(1301ns) {DropIncomingHandshake:442(381ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(537ns) PrepareNewSessionHandshake:477(2265ns) {SwitchToState:179(1009ns)}}}}}
2025-11-27T18:18:08.510405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471165310324969:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(44385692ns) {Configure:104(44381085ns) {ProcessConfigured:135(54066ns) {StartInitialHandshake:157(48864ns) {DropHandshakes:470(3167ns) {DropIncomingHandshake:442(751ns) DropOutgoingHandshake:457(621ns)} GenerateSessionVirtualId:988(593ns) PrepareNewSessionHandshake:477(1893ns) {SwitchToState:179(913ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.506041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471165095586202:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8285
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.526225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471165310324977:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-27T18:18:08.506550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471165095586207:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8285
2025-11-27T18:18:08.541645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471163198453172:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16893
2025-11-27T18:18:08.549550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471163198453173:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16893
2025-11-27T18:18:08.542294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471162417458574:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13354
2025-11-27T18:18:08.526329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471165310324978:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-27T18:18:08.558727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471163574589164:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(77259736ns) {Configure:104(77251053ns) {ProcessConfigured:135(19816972ns) {StartInitialHandshake:157(19812096ns) {DropHandshakes:470(2568ns) {DropIncomingHandshake:442(613ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(737ns) PrepareNewSessionHandshake:477(3184ns) {SwitchToState:179(1607ns)}}}}}
2025-11-27T18:18:08.557610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471163198453172:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.542737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471159105844286:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(63996060ns) {Configure:104(63987230ns) {ProcessConfigured:135(30179019ns) {StartInitialHandshake:157(30176642ns) {DropHandshakes:470(6091944ns) {DropIncomingHandshake:442(276ns) DropOutgoingHandshake:457(1700ns)} GenerateSessionVirtualId:988(1213ns) PrepareNewSessionHandshake:477(4125ns) {SwitchToState:179(1894ns)}}}}}
2025-11-27T18:18:08.542955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471165095586202:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.564182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471163198453173:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.562382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471165310324907:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55270692ns) {Configure:104(55263673ns) {ProcessConfigured:135(58834ns) {StartInitialHandshake:157(56432ns) {DropHandshakes:470(2223ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(399ns) PrepareNewSessionHandshake:477(1478ns) {SwitchToState:179(268ns)}}}}}
2025-11-27T18:18:08.563762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471165310324978:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.564159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471165310324908:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.542399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471162417458575:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13354
2025-11-27T18:18:08.564643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471163198453093:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.559018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471163574589163:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(101508132ns) {Configure:104(101489592ns) {ProcessConfigured:135(1253816ns) {StartInitialHandshake:157(1250185ns) {DropHandshakes:470(2676ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(352ns)} GenerateSessionVirtualId:988(990ns) PrepareNewSessionHandshake:477(3146ns) {SwitchToState:179(1397ns)}}}}}
2025-11-27T18:18:08.542473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471162417458576:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13354
2025-11-27T18:18:08.566204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471165095586197:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.568053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471163198453163:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.557841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471162417458574:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.566245Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471165095586197:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.558593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471162417458575:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.585230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471165310324977:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.585422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471165310324969:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.573351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471165095586201:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.573380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471165095586201:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.573995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471165095586207:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.574449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471165095586201:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:08.575455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471165095586197:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:08.568087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471163198453163:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.583874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471165095586194:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.568616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471162417458498:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.583947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:08.580778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471163198453165:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.594954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471163574589226:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29252356ns) {Configure:104(29244811ns) {ProcessConfigured:135(41173ns) {StartInitialHandshake:157(39232ns) {DropHandshakes:470(648ns) {DropIncomingHandshake:442(155ns) DropOutgoingHandshake:457(161ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:477(990ns) {SwitchToState:179(298ns)}}}}}
2025-11-27T18:18:08.583997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471165095586194:2107] [node 4] ICP22 created new session: [1:7577471165095586215:2107]
2025-11-27T18:18:08.584045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471165095586215:2107] [node 4] ICS09 handshake done sender: [1:7577471165095586201:2112] self: [1:7577471165095586200:0] peer: [4:7577471162417458578:0] socket: 383 qp: -1
2025-11-27T18:18:08.584083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471165095586215:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:08.584122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:08.580798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471163198453165:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.568697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471162417458496:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.568800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471162417458576:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.598522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471163400811646:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31739867ns) {Configure:104(31735303ns) {ProcessConfigured:135(1564219ns) {StartInitialHandshake:157(1562536ns) {DropHandshakes:470(3080ns) {DropIncomingHandshake:442(522ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(652ns) PrepareNewSessionHandshake:477(2216ns) {SwitchToState:179(866ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.600633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471163574589235:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63506
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.600784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471163574589236:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63506
2025-11-27T18:18:08.585963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471160800618835:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.600914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471163574589237:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63506
2025-11-27T18:18:08.609552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471163574589236:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.609690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471163574589235:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.586011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:08.580504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471162417458580:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13354
2025-11-27T18:18:08.598782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471163198453093:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471163198453172:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.614786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471165310324989:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.616310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471165310324990:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-27T18:18:08.610297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471163198453163:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:08.586053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471160800618835:2049] [node 3] ICP22 created new session: [1:7577471165095586217:2049]
2025-11-27T18:18:08.586096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471165095586217:2049] [node 3] ICS09 handshake done sender: [1:7577471165095586197:2110] self: [1:7577471165095586196:0] peer: [3:7577471163198453174:0] socket: 381 qp: -1
2025-11-27T18:18:08.586131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471165095586217:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:08.586158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:08.620502Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471163574589225:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(44518443ns) {Configure:104(44514623ns) {ProcessConfigured:135(12460070ns) {StartInitialHandshake:157(12457197ns) {DropHandshakes:470(2104ns) {DropIncomingHandshake:442(528ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(595ns) PrepareNewSessionHandshake:477(2553ns) {SwitchToState:179(850ns)}}}}}
2025-11-27T18:18:08.620690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471163574589237:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.596894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471162417458564:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.617655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471165310324991:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-27T18:18:08.613823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471165095586221:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8285
2025-11-27T18:18:08.596923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471162417458564:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.622568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471163574589162:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.625733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471163574589226:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.629235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471163198453165:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:08.617762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471165310324978:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.632329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471165095586221:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.630249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471163198453161:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.599632Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471162417458496:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471162417458575:2116] held: no
2025-11-27T18:18:08.621693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471165310324908:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.642234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471163400811661:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65114
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.642608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471163400811663:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65114
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.634422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471165095586199:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.630307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:08.599727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471162417458575:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:08.650641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471163400811661:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.650801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471163400811663:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.650898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471163400811651:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.650934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471163400811651:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.651696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471159105844286:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.621729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.646180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471163574589247:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63506
2025-11-27T18:18:08.606751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471162417458496:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.634469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471165095586199:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.654252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471163400811651:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.632749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471163198453161:2110] [node 6] ICP22 created new session: [3:7577471163198453182:2110]
2025-11-27T18:18:08.648268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471163574589248:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63506
2025-11-27T18:18:08.649849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471165095586199:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:08.621787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471165310324908:2050] [node 3] ICP22 created new session: [6:7577471165310324992:2050]
2025-11-27T18:18:08.634129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471163198453182:2110] [node 6] ICS09 handshake done sender: [3:7577471163198453163:2111] self: [3:7577471163198453162:0] peer: [6:7577471165310324986:0] socket: 378 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.654615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471163400811645:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.606796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:08.674703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471165095586193:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.674764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:08.674851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471165095586193:2108] [node 5] ICP22 created new session: [1:7577471165095586225:2108]
2025-11-27T18:18:08.674896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471165095586225:2108] [node 5] ICS09 handshake done sender: [1:7577471165095586199:2111] self: [1:7577471165095586198:0] peer: [5:7577471163574589243:0] socket: 382 qp: -1
2025-11-27T18:18:08.674928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471165095586225:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:08.674966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:08.687312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471165095586206:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.687348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471165095586206:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.649005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471163574589235:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:08.606832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471162417458496:2048] [node 1] ICP22 created new session: [4:7577471162417458582:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.635893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471163198453182:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:08.621829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471165310324992:2050] [node 3] ICS09 handshake done sender: [6:7577471165310324978:2114] self: [6:7577471165310324986:0] peer: [3:7577471163198453162:0] socket: 386 qp: -1
2025-11-27T18:18:08.654664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:08.649143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471163574589226:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.654710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471163400811645:2107] [node 4] ICP22 created new session: [2:7577471163400811667:2107]
2025-11-27T18:18:08.606871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471162417458582:2048] [node 1] ICS09 handshake done sender: [4:7577471162417458575:2116] self: [4:7577471162417458578:0] peer: [1:7577471165095586200:0] socket: 395 qp: -1
2025-11-27T18:18:08.644484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:08.621865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471165310324992:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:08.649174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:08.649225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471163574589226:2108] [node 4] ICP22 created new session: [5:7577471163574589249:2108]
2025-11-27T18:18:08.649297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471163574589249:2108] [node 4] ICS09 handshake done sender: [5:7577471163574589235:2114] self: [5:7577471163574589244:0] peer: [4:7577471162417458565:0] socket: 402 qp: -1
2025-11-27T18:18:08.649339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471163574589249:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:08.649371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:08.666999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471163574589245:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63506
2025-11-27T18:18:08.667115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471163574589236:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:08.667355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471163574589162:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.667420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:08.667479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471163574589162:2048] [node 1] ICP22 created new session: [5:7577471163574589252:2048]
2025-11-27T18:18:08.667530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471163574589252:2048] [node 1] ICS09 handshake done sender: [5:7577471163574589236:2115] self: [5:7577471163574589243:0] peer: [1:7577471165095586198:0] socket: 403 qp: -1
2025-11-27T18:18:08.667564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471163574589252:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:08.667608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:08.703495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471163574589226:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(26579576ns) {HandleHandshakeStatus:329(26577786ns) {DropIncomingHandshake:442(268ns) DropOutgoingHandshake:457(2517ns) SwitchToState:179(659ns) ProcessPendingSessionEvents:495(25781818ns) {ForwardSessionEventToSession:612(9846641ns) ForwardSessionEventToSession:612(4867ns) ForwardSessionEventToSession:612(6401648ns) ForwardSessionEventToSession:612(7319ns) ForwardSessionEventToSession:612(2931ns) ForwardSessionEventToSession:612(5905018ns) ForwardSessionEventToSession:612(4195ns) ForwardSessionEventToSession:612(3585138ns) ForwardSessionEventToSession:612(5057ns) ForwardSessionEventToSession:612(3069ns)}}}
2025-11-27T18:18:08.719305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471165095586206:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.719619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471165095586224:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8285
2025-11-27T18:18:08.719707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471165095586228:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8285
2025-11-27T18:18:08.654766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471163400811667:2107] [node 4] ICS09 handshake done sender: [2:7577471163400811651:2111] self: [2:7577471163400811650:0] peer: [4:7577471162417458592:0] socket: 410 qp: -1
2025-11-27T18:18:08.606899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471162417458582:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:08.622210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:08.659545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471163198453176:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16893
2025-11-27T18:18:08.719166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471163574589245:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.721553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471163574589164:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.606921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:08.654807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471163400811667:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:08.632329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471165310324988:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-27T18:18:08.685476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471163198453160:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.726270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471160800618834:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.654839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:08.685503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:08.632516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471165310324977:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:08.607775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471162417458584:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13354
2025-11-27T18:18:08.726311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:08.682992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471163400811662:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65114
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.619227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471162417458564:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.737292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471163574589255:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63506
2025-11-27T18:18:08.685537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471163198453160:2108] [node 4] ICP22 created new session: [3:7577471163198453184:2108]
2025-11-27T18:18:08.633188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471165310324969:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.740014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471163574589164:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64833) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471163574589245:2118] held: no
2025-11-27T18:18:08.712050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471163400811649:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.624325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471162417458586:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13354
2025-11-27T18:18:08.728175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471160800618834:2048] [node 2] ICP22 created new session: [1:7577471165095586229:2048]
2025-11-27T18:18:08.633248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:08.755441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471163574589245:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:08.712075Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471163400811649:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.685575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471163198453184:2108] [node 4] ICS09 handshake done sender: [3:7577471163198453165:2112] self: [3:7577471163198453164:0] peer: [4:7577471162417458577:0] socket: 379 qp: -1
2025-11-27T18:18:08.729131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471165095586229:2048] [node 2] ICS09 handshake done sender: [1:7577471165095586206:2115] self: [1:7577471165095586205:0] peer: [2:7577471163400811665:0] socket: 400 qp: -1
2025-11-27T18:18:08.729800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471165095586229:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:08.729846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:08.730421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471165095586204:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.730446Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471165095586204:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.731082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471165095586224:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.732663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471165095586204:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:08.741224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471165095586195:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.741315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:08.741370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471165095586195:2109] [node 6] ICP22 created new session: [1:7577471165095586233:2109]
2025-11-27T18:18:08.742524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471165095586233:2109] [node 6] ICS09 handshake done sender: [1:7577471165095586204:2114] self: [1:7577471165095586203:0] peer: [6:7577471165310324999:0] socket: 373 qp: -1
2025-11-27T18:18:08.755096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471165095586233:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:08.755153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:08.760017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471160800618834:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:08.633314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471165310324969:2107] [node 4] ICP22 created new session: [6:7577471165310324994:2107]
2025-11-27T18:18:08.685605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471163198453184:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.758346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471163574589242:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.724693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471163400811662:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.633798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471162417458559:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.758382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471163574589242:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.758542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471163574589164:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.633368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471165310324994:2107] [node 4] ICS09 handshake done sender: [6:7577471165310324977:2113] self: [6:7577471165310324987:0] peer: [4:7577471162417458563:0] socket: 385 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.633860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:08.765647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:08.685636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:08.765711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471163574589164:2050] [node 3] ICP22 created new session: [5:7577471163574589256:2050]
2025-11-27T18:18:08.765756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471163574589256:2050] [node 3] ICS09 handshake done sender: [5:7577471163574589245:2118] self: [5:7577471163574589254:0] peer: [3:7577471163198453166:0] socket: 389 qp: -1
2025-11-27T18:18:08.765791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471163574589256:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:08.765826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:08.775956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471163574589258:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63506
2025-11-27T18:18:08.633904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471162417458559:2109] [node 6] ICP22 created new session: [4:7577471162417458587:2109]
2025-11-27T18:18:08.645076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471162417458566:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.645096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471162417458566:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.646513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471162417458584:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.646768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471162417458574:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:08.647208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471162417458498:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.647239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:08.647269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471162417458498:2050] [node 3] ICP22 created new session: [4:7577471162417458590:2050]
2025-11-27T18:18:08.647301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471162417458590:2050] [node 3] ICS09 handshake done sender: [4:7577471162417458574:2115] self: [4:7577471162417458577:0] peer: [3:7577471163198453164:0] socket: 392 qp: -1
2025-11-27T18:18:08.647324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471162417458590:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:08.647351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:08.647653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471162417458497:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.648528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471162417458566:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:08.649626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471162417458560:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.649649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:08.649813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471162417458587:2109] [node 6] ICS09 handshake done sender: [4:7577471162417458564:2111] self: [4:7577471162417458563:0] peer: [6:7577471165310324987:0] socket: 375 qp: -1
2025-11-27T18:18:08.649843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471162417458560:2108] [node 5] ICP22 created new session: [4:7577471162417458593:2108]
2025-11-27T18:18:08.649851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471162417458587:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:08.655447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471162417458593:2108] [node 5] ICS09 handshake done sender: [4:7577471162417458566:2112] self: [4:7577471162417458565:0] peer: [5:7577471163574589244:0] socket: 376 qp: -1
2025-11-27T18:18:08.655487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471162417458593:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:08.653804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:08.672284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:08.672763Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471162417458560:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(22902646ns) {HandleHandshakeStatus:329(22900781ns) {DropIncomingHandshake:442(149ns) DropOutgoingHandshake:457(181ns) SwitchToState:179(95ns) ProcessPendingSessionEvents:495(97284ns) {ForwardSessionEventToSession:612(50356ns) ForwardSessionEventToSession:612(6573ns) ForwardSessionEventToSession:612(2040ns) ForwardSessionEventToSession:612(2739ns) ForwardSessionEventToSession:612(5432ns) ForwardSessionEventToSession:612(4814ns) ForwardSessionEventToSession:612(2325ns) ForwardSessionEventToSession:612(4306ns) ForwardSessionEventToSession:612(2428ns) ForwardSessionEventToSession:612(4444ns)}}}
2025-11-27T18:18:08.681618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471162417458559:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(47619275ns) {HandleHandshakeStatus:329(47615934ns) {DropIncomingHandshake:442(12438ns) DropOutgoingHandshake:457(320ns) SwitchToState:179(380ns) ProcessPendingSessionEvents:495(6948155ns) {ForwardSessionEventToSession:612(6881372ns) ForwardSessionEventToSession:612(5808ns) ForwardSessionEventToSession:612(7337ns) ForwardSessionEventToSession:612(5463ns) ForwardSessionEventToSession:612(2787ns) ForwardSessionEventToSession:612(5291ns) ForwardSessionEventToSession:612(3283ns) ForwardSessionEventToSession:612(2338ns) ForwardSessionEventToSession:612(5168ns) ForwardSessionEventToSession:612(2703ns)}}}
2025-11-27T18:18:08.681868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471162417458497:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23294) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471162417458584:2119] held: no
2025-11-27T18:18:08.682526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471162417458596:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13354
2025-11-27T18:18:08.684050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471162417458584:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:08.684468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471162417458497:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.684508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:08.684553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471162417458497:2049] [node 2] ICP22 created new session: [4:7577471162417458597:2049]
2025-11-27T18:18:08.684595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471162417458597:2049] [node 2] ICS09 handshake done sender: [4:7577471162417458584:2119] self: [4:7577471162417458592:0] peer: [2:7577471163400811650:0] socket: 414 qp: -1
2025-11-27T18:18:08.684626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471162417458597:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:08.684654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:08.724870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471163400811670:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65114
2025-11-27T18:18:08.725274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471163400811672:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65114
2025-11-27T18:18:08.725846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471163400811649:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:08.731669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471159105844287:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.731704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:08.732919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471163400811661:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:08.742049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471159105844287:2049] [node 3] ICP22 created new session: [2:7577471163400811675:2049]
2025-11-27T18:18:08.742705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471163400811675:2049] [node 3] ICS09 handshake done sender: [2:7577471163400811649:2110] self: [2:7577471163400811648:0] peer: [3:7577471163198453190:0] socket: 411 qp: -1
2025-11-27T18:18:08.742731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471163400811675:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:08.742764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:08.743015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471163400811669:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65114
2025-11-27T18:18:08.745402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471159105844286:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.745457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:08.745500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471159105844286:2048] [node 1] ICP22 created new session: [2:7577471163400811677:2048]
2025-11-27T18:18:08.745545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471163400811677:2048] [node 1] ICS09 handshake done sender: [2:7577471163400811661:2115] self: [2:7577471163400811665:0] peer: [1:7577471165095586205:0] socket: 372 qp: -1
2025-11-27T18:18:08.745582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471163400811677:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:08.745648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:08.746144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471163400811653:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.746166Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471163400811653:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.747440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471159105844287:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:08.769072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471163400811670:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.770338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471163400811653:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:08.772144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471163400811647:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.772206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:08.772258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471163400811647:2109] [node 6] ICP22 created new session: [2:7577471163400811682:2109]
2025-11-27T18:18:08.772313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471163400811682:2109] [node 6] ICS09 handshake done sender: [2:7577471163400811653:2112] self: [2:7577471163400811652:0] peer: [6:7577471165310325000:0] socket: 412 qp: -1
2025-11-27T18:18:08.772342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471163400811682:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:08.772397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:08.633400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471165310324994:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.686351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471163198453179:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16893
2025-11-27T18:18:08.686473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471163198453180:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16893
2025-11-27T18:18:08.793483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471163574589258:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.686756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471163198453172:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:08.633439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:08.698289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471163198453093:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.698335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:08.698386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471163198453093:2048] [node 1] ICP22 created new session: [3:7577471163198453188:2048]
2025-11-27T18:18:08.698442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471163198453188:2048] [node 1] ICS09 handshake done sender: [3:7577471163198453172:2115] self: [3:7577471163198453174:0] peer: [1:7577471165095586196:0] socket: 384 qp: -1
2025-11-27T18:18:08.698477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471163198453188:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:08.698551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:08.654389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471165310324988:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.798084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471163400811657:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.798120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471163400811657:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.797337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471163574589163:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.700153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471163198453186:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16893
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.713674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471165310324990:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.720662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471165310324906:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.733097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471165310324907:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.740383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471165310324998:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-27T18:18:08.742781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471165310324998:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.753536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471165310324967:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.759667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471165310324906:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23840) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471165310324988:2117] held: no
2025-11-27T18:18:08.805125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471163574589242:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:08.770660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471165310325003:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-27T18:18:08.807071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471163400811657:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:08.707733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471163198453180:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:08.772012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471165310324990:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:08.821924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471163400811646:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.822005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:08.822061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471163400811646:2108] [node 5] ICP22 created new session: [2:7577471163400811685:2108]
2025-11-27T18:18:08.822122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471163400811685:2108] [node 5] ICS09 handshake done sender: [2:7577471163400811657:2114] self: [2:7577471163400811656:0] peer: [5:7577471163574589260:0] socket: 377 qp: -1
2025-11-27T18:18:08.822157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471163400811685:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:08.822185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:08.813539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471163574589261:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63506
2025-11-27T18:18:08.825610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471163574589225:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.825677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:08.825738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471163574589225:2109] [node 6] ICP22 created new session: [5:7577471163574589262:2109]
2025-11-27T18:18:08.825783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471163574589262:2109] [node 6] ICS09 handshake done sender: [5:7577471163574589242:2117] self: [5:7577471163574589241:0] peer: [6:7577471165310325002:0] socket: 421 qp: -1
2025-11-27T18:18:08.825813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471163574589262:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:08.825868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:08.827377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471163574589258:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:08.709196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471163198453094:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:08.828138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471163574589163:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.793979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471165310325001:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-27T18:18:08.715843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471163198453179:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.718494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471163198453161:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(76231999ns) {HandleHandshakeStatus:329(76229935ns) {DropIncomingHandshake:442(12010ns) DropOutgoingHandshake:457(455ns) SwitchToState:179(696ns) ProcessPendingSessionEvents:495(29176522ns) {ForwardSessionEventToSession:612(13373204ns) ForwardSessionEventToSession:612(8863ns) ForwardSessionEventToSession:612(1181801ns) ForwardSessionEventToSession:612(3193ns) ForwardSessionEventToSession:612(5816697ns) ForwardSessionEventToSession:612(18423ns) ForwardSessionEventToSession:612(2877ns) ForwardSessionEventToSession:612(4164682ns) ForwardSessionEventToSession:612(1608209ns) ForwardSessionEventToSession:612(3444ns)}}}
2025-11-27T18:18:08.722303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471163198453167:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:08.722322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471163198453167:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:08.723057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471163198453167:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:08.724296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471163198453159:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:08.726179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:08.727938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471163198453159:2109] [node 5] ICP22 created new session: [3:7577471163198453194:2109]
2025-11-27T18:18:08.728592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471163198453193:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16893
2025-11-27T18:18:08.728842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471163198453194:2109] [node 5] ICS09 handshake done sender: [3:7577471163198453167:2113] self: [3:7577471163198453166:0] peer: [5:7577471163574589254:0] socket: 426 qp: -1
2025-11-27T18:18:08.729051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471163198453194:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:08.729089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:08.735485Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471163198453180:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:08.735719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471163198453094:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.735788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:08.735839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471163198453094:2049] [node 2] ICP22 created new session: [3:7577471163198453196:2049]
2025-11-27T18:18:08.735881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471163198453196:2049] [node 2] ICS09 handshake done sender: [3:7577471163198453180:2119] self: [3:7577471163198453190:0] peer: [2:7577471163400811648:0] socket: 425 qp: -1
2025-11-27T18:18:08.735917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471163198453196:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:08.735946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:08.828190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:08.828239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471163574589163:2049] [node 2] ICP22 created new session: [5:7577471163574589264:2049]
2025-11-27T18:18:08.828283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471163574589264:2049] [node 2] ICS09 handshake done sender: [5:7577471163574589258:2122] self: [5:7577471163574589260:0] peer: [2:7577471163400811656:0] socket: 431 qp: -1
2025-11-27T18:18:08.828332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471163574589264:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:08.828360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:08.806369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471165310324907:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.806422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:08.806476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471165310324907:2049] [node 2] ICP22 created new session: [6:7577471165310325004:2049]
2025-11-27T18:18:08.806526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471165310325004:2049] [node 2] ICS09 handshake done sender: [6:7577471165310324990:2119] self: [6:7577471165310325000:0] peer: [2:7577471163400811652:0] socket: 419 qp: -1
2025-11-27T18:18:08.806562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471165310325004:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:08.806598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:08.814023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471165310324907:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:08.814783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471165310324988:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:08.815205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471165310324906:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.815249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:08.815289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471165310324906:2048] [node 1] ICP22 created new session: [6:7577471165310325007:2048]
2025-11-27T18:18:08.815338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471165310325007:2048] [node 1] ICS09 handshake done sender: [6:7577471165310324988:2117] self: [6:7577471165310324999:0] peer: [1:7577471165095586203:0] socket: 401 qp: -1
2025-11-27T18:18:08.815370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471165310325007:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:08.815398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:08.815746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471165310325006:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-27T18:18:08.816374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471165310324998:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:08.816600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471165310324967:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:08.816634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:08.816673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471165310324967:2105] [node 5] ICP22 created new session: [6:7577471165310325009:2105]
2025-11-27T18:18:08.816716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471165310325009:2105] [node 5] ICS09 handshake done sender: [6:7577471165310324998:2121] self: [6:7577471165310325002:0] peer: [5:7577471163574589241:0] socket: 415 qp: -1
2025-11-27T18:18:08.816743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471165310325009:2105] [node 5] ICS10 traffic start
2025-11-27T18:18:08.816773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:08.869446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:08.869693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471163400811677:2048] [node 1] ICS01 socket: 372 reason# EndOfStream
2025-11-27T18:18:08.869740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471159105844286:2048] [node 1] ICP30 unregister session Session# [2:7577471163400811677:2048] VirtualId# [2:7577471163400811665:0]
2025-11-27T18:18:08.869794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471163400811677:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:08.869871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:08.869924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:08.877516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:08.877737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471163198453188:2048] [node 1] ICS01 socket: 384 reason# EndOfStream
2025-11-27T18:18:08.877854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471163198453093:2048] [node 1] ICP30 unregister session Session# [3:7577471163198453188:2048] VirtualId# [3:7577471163198453174:0]
2025-11-27T18:18:08.877888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471163198453188:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:08.877958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:08.877997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:08.885426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:08.885661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471162417458582:2048] [node 1] ICS01 socket: 395 reason# EndOfStream
2025-11-27T18:18:08.885702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471162417458496:2048] [node 1] ICP30 unregister session Session# [4:7577471162417458582:2048] VirtualId# [4:7577471162417458578:0]
2025-11-27T18:18:08.885735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471162417458582:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:08.885811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:08.885860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:08.889552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:08.889902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471165310325007:2048] [node 1] ICS01 socket: 401 reason# EndOfStream
2025-11-27T18:18:08.887292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:08.903100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471165310324906:2048] [node 1] ICP30 unregister session Session# [6:7577471165310325007:2048] VirtualId# [6:7577471165310324999:0]
2025-11-27T18:18:08.903175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471165310325007:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:08.903271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:08.903318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:08.887522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471163574589252:2048] [node 1] ICS01 socket: 403 reason# EndOfStream
2025-11-27T18:18:08.887556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471163574589162:2048] [node 1] ICP30 unregister session Session# [5:7577471163574589252:2048] VirtualId# [5:7577471163574589243:0]
2025-11-27T18:18:08.887587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471163574589252:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:08.887661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:08.887701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:08.936395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:08.936645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471163198453196:2049] [node 2] ICS01 socket: 425 reason# EndOfStream
2025-11-27T18:18:08.936682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471163198453094:2049] [node 2] ICP30 unregister session Session# [3:7577471163198453196:2049] VirtualId# [3:7577471163198453190:0]
2025-11-27T18:18:08.936713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471163198453196:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:08.936792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:08.936837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:08.946303Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:08.946521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471165310325004:2049] [node 2] ICS07 socket disconnect 419 reason# EndOfStream
2025-11-27T18:18:08.946552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471165310325004:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:08.946613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:08.946671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471165310325004:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:08.945746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:08.945973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471163574589264:2049] [node 2] ICS01 socket: 431 reason# EndOfStream
2025-11-27T18:18:08.946009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471163574589163:2049] [node 2] ICP30 unregister session Session# [5:7577471163574589264:2049] VirtualId# [5:7577471163574589260:0]
2025-11-27T18:18:08.946047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471163574589264:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:08.946146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:08.946199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:08.948606Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471165310324907:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23294) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:08.948670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471165310325004:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:08.948702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471165310324907:2049] [node 2] ICP30 unregister session Session# [6:7577471165310325004:2049] VirtualId# [6:7577471165310325000:0]
2025-11-27T18:18:08.948742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:08.948780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471165310324907:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23294) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:08.948793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23294) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:08.949475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:08.949658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471162417458597:2049] [node 2] ICS01 socket: 414 reason# EndOfStream
2025-11-27T18:18:08.949695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471162417458497:2049] [node 2] ICP30 unregister session Session# [4:7577471162417458597:2049] VirtualId# [4:7577471162417458592:0]
2025-11-27T18:18:08.949722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471162417458597:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:08.949799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:08.949838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:08.961587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471165310324907:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:09.036462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:09.036728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471163574589256:2050] [node 3] ICS01 socket: 389 reason# EndOfStream
2025-11-27T18:18:09.036767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471163574589164:2050] [node 3] ICP30 unregister session Session# [5:7577471163574589256:2050] VirtualId# [5:7577471163574589254:0]
2025-11-27T18:18:09.036798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471163574589256:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:09.036878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:09.036939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:09.038582Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:09.038788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471162417458590:2050] [node 3] ICS01 socket: 392 reason# EndOfStream
2025-11-27T18:18:09.038829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471162417458498:2050] [node 3] ICP30 unregister session Session# [4:7577471162417458590:2050] VirtualId# [4:7577471162417458577:0]
2025-11-27T18:18:09.038856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471162417458590:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:09.038929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:09.038978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:09.039072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:09.039251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471165310324992:2050] [node 3] ICS01 socket: 386 reason# EndOfStream
2025-11-27T18:18:09.039284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471165310324908:2050] [node 3] ICP30 unregister session Session# [6:7577471165310324992:2050] VirtualId# [6:7577471165310324986:0]
2025-11-27T18:18:09.039306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471165310324992:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:09.039355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:09.039403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:09.094320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:09.094573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471163574589249:2108] [node 4] ICS07 socket disconnect 402 reason# EndOfStream
2025-11-27T18:18:09.094633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471163574589249:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:09.094746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:09.094808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471163574589249:2108] [node 4] ICS15 start handshake
2025-11-27T18:18:09.096030Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471163574589226:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10804) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:09.096096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471163574589249:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:09.096136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471163574589226:2108] [node 4] ICP30 unregister session Session# [5:7577471163574589249:2108] VirtualId# [5:7577471163574589244:0]
2025-11-27T18:18:09.096191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:09.096234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471163574589226:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10804) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:09.096248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10804) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:09.097612Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:09.097835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471165310324994:2107] [node 4] ICS01 socket: 385 reason# EndOfStream
2025-11-27T18:18:09.097876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471165310324969:2107] [node 4] ICP30 unregister session Session# [6:7577471165310324994:2107] VirtualId# [6:7577471165310324987:0]
2025-11-27T18:18:09.097912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471165310324994:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:09.097991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:09.098032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:09.106241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471163574589226:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:18:09.165518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:09.165762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471165310325009:2105] [node 5] ICS01 socket: 415 reason# EndOfStream
2025-11-27T18:18:09.165800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471165310324967:2105] [node 5] ICP30 unregister session Session# [6:7577471165310325009:2105] VirtualId# [6:7577471165310325002:0]
2025-11-27T18:18:09.165831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471165310325009:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:09.165940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:09.165985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:09.291575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471169857377994:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:09.291620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471169857377995:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:09.317064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471168782098226:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:09.317534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471168782098225:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:09.342469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471170234102145:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:09.342510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471170234102146:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:09.369374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471168617984437:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:09.369450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471168617984439:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:09.369493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471168617984438:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:09.394276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471167688175639:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:09.394327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471167688175640:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:09.394361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471167688175641:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:09.436689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471169158766496:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:09.436730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471169158766497:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:09.436757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471169158766498:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:09.697788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471169158766558:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:09.716772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471169857378060:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:09.717070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471169857378061:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:09.727319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471168782098287:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:09.749105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471170234102209:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:09.722528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471169857378062:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:09.786365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471168782098291:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:09.876797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471168782098292:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:09.887456Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471169857378062:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(65760156ns) {Configure:104(65754166ns) {ProcessConfigured:135(64669ns) {StartInitialHandshake:157(61458ns) {DropHandshakes:470(2474ns) {DropIncomingHandshake:442(850ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(1190ns) PrepareNewSessionHandshake:477(1795ns) {SwitchToState:179(438ns)}}}}}
2025-11-27T18:18:09.842293Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471169158766496:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20764785ns) {Configure:104(20760004ns) {ProcessConfigured:135(19441086ns) {StartInitialHandshake:157(19438071ns) {DropHandshakes:470(2243ns) {DropIncomingHandshake:442(498ns) DropOutgoingHandshake:457(318ns)} GenerateSessionVirtualId:988(1153ns) PrepareNewSessionHandshake:477(1821ns) {SwitchToState:179(306ns)}}}}}
2025-11-27T18:18:09.878853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471168782098285:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5445
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:09.885672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471169158766569:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:09.883680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471170234102212:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:09.898114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471168617984501:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:09.891897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471167688175703:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:09.944155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471168617984502:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:09.946739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471168617984439:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(160333186ns) {Configure:104(160328538ns) {ProcessConfigured:135(64959ns) {StartInitialHandshake:157(62816ns) {DropHandshakes:470(1924ns) {DropIncomingHandshake:442(458ns) DropOutgoingHandshake:457(288ns)} GenerateSessionVirtualId:988(730ns) PrepareNewSessionHandshake:477(1102ns) {SwitchToState:179(306ns)}}}}}
2025-11-27T18:18:09.910218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471170234102213:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:09.914551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471169857378067:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15615
2025-11-27T18:18:09.901398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471167688175704:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:09.970897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471168617984438:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(197244679ns) {Configure:104(197233923ns) {ProcessConfigured:135(82117ns) {StartInitialHandshake:157(79180ns) {DropHandshakes:470(2976ns) {DropIncomingHandshake:442(590ns) DropOutgoingHandshake:457(740ns)} GenerateSessionVirtualId:988(797ns) PrepareNewSessionHandshake:477(1521ns) {SwitchToState:179(400ns)}}}}}
2025-11-27T18:18:09.975762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471170234102145:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(65269241ns) {Configure:104(65264421ns) {ProcessConfigured:135(48108ns) {StartInitialHandshake:157(45728ns) {DropHandshakes:470(1595ns) {DropIncomingHandshake:442(325ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(629ns) PrepareNewSessionHandshake:477(953ns) {SwitchToState:179(280ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:09.976021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471170234102214:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29301
2025-11-27T18:18:09.938456Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471168782098225:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(220197636ns) {Configure:104(220190710ns) {ProcessConfigured:135(67796ns) {StartInitialHandshake:157(65932ns) {DropHandshakes:470(2321ns) {DropIncomingHandshake:442(748ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(1091ns) PrepareNewSessionHandshake:477(1734ns) {SwitchToState:179(541ns)}}}}}
2025-11-27T18:18:09.977327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471169857378067:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:09.978413Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471169857378061:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(202605739ns) {Configure:104(202598647ns) {ProcessConfigured:135(53358ns) {StartInitialHandshake:157(51285ns) {DropHandshakes:470(1722ns) {DropIncomingHandshake:442(329ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(698ns) PrepareNewSessionHandshake:477(964ns) {SwitchToState:179(265ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:09.981440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471170234102214:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.010981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471170234102209:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(261579356ns) {Configure:104(261571894ns) {ProcessConfigured:135(16214126ns) {StartInitialHandshake:157(16208694ns) {DropHandshakes:470(3892ns) {DropIncomingHandshake:442(1294ns) DropOutgoingHandshake:457(353ns)} GenerateSessionVirtualId:988(1487ns) PrepareNewSessionHandshake:477(3511ns) {SwitchToState:179(1655ns)}}}}}
2025-11-27T18:18:10.013436Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471169857377995:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(312413184ns) {Configure:104(312409987ns) {ProcessConfigured:135(31631189ns) {StartInitialHandshake:157(15943704ns) {DropHandshakes:470(3803ns) {DropIncomingHandshake:442(931ns) DropOutgoingHandshake:457(343ns)} GenerateSessionVirtualId:988(3275ns) PrepareNewSessionHandshake:477(2596ns) {SwitchToState:179(1197ns)}}}}}
2025-11-27T18:18:09.986821Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471167688175639:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(76686225ns) {Configure:104(76677239ns) {ProcessConfigured:135(74264ns) {StartInitialHandshake:157(72035ns) {DropHandshakes:470(2412ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(1139ns) PrepareNewSessionHandshake:477(1389ns) {SwitchToState:179(319ns)}}}}}
2025-11-27T18:18:10.025523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471169857378064:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.025572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471169857378064:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.027340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471173453733870:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32100
2025-11-27T18:18:10.027481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471173453733871:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32100
2025-11-27T18:18:10.028724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471169857378064:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:10.029219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471169857378066:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.029247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471169857378066:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.029780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471169857378062:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.029822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:10.029864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471169857378062:2108] [node 4] ICP22 created new session: [1:7577471174152345372:2108]
2025-11-27T18:18:10.029908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471174152345372:2108] [node 4] ICS09 handshake done sender: [1:7577471169857378064:2111] self: [1:7577471169857378063:0] peer: [4:7577471168617984515:0] socket: 389 qp: -1
2025-11-27T18:18:10.029936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471174152345372:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:10.029974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.006957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471168617984514:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48403
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.028066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471174529069522:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29301
2025-11-27T18:18:10.032839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471173453733870:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.044012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471169857378066:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:10.051867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471168782098226:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(99663695ns) {Configure:104(99660140ns) {ProcessConfigured:135(57500057ns) {StartInitialHandshake:157(57498545ns) {DropHandshakes:470(2040ns) {DropIncomingHandshake:442(517ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(615ns) PrepareNewSessionHandshake:477(2705ns) {SwitchToState:179(1033ns)}}}}}
2025-11-27T18:18:10.017839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471168617984514:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:09.987170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471167688175700:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46191
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.032193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471174529069522:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.033035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471173453733871:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:09.988241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471167688175711:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46191
2025-11-27T18:18:09.988372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471167688175712:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46191
2025-11-27T18:18:09.990074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471167688175700:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:09.990151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471167688175711:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:09.997899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471167688175639:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.031982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471171983143014:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46191
2025-11-27T18:18:10.041439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471170234102216:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.041476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471170234102216:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.044590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471174529069525:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29301
2025-11-27T18:18:10.017972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471168617984437:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.046445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471174152345377:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15615
2025-11-27T18:18:10.061585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471168782098285:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.046844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471170234102216:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:10.062237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471168782098293:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5445
2025-11-27T18:18:10.033313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471169158766498:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.048465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471174152345376:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15615
2025-11-27T18:18:10.058411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471169857378062:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:10.064854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471169857378055:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.064882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471169857378055:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.068135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471169857378060:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.068198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:10.068257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471169857378060:2109] [node 5] ICP22 created new session: [1:7577471174152345379:2109]
2025-11-27T18:18:10.068300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471174152345379:2109] [node 5] ICS09 handshake done sender: [1:7577471169857378066:2112] self: [1:7577471169857378065:0] peer: [5:7577471167688175717:0] socket: 390 qp: -1
2025-11-27T18:18:10.068341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471174152345379:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:10.068391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:10.069153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471174152345378:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15615
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.101641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471167688175712:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.027806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471172912951813:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48403
2025-11-27T18:18:10.036861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471173453733872:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32100
2025-11-27T18:18:10.062509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471168782098225:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.097523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471174529069525:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.099640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471169857378071:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.117116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471167688175704:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(176791967ns) {Configure:104(176786738ns) {ProcessConfigured:135(17367964ns) {StartInitialHandshake:157(17364583ns) {DropHandshakes:470(2817ns) {DropIncomingHandshake:442(460ns) DropOutgoingHandshake:457(637ns)} GenerateSessionVirtualId:988(609ns) PrepareNewSessionHandshake:477(3882ns) {SwitchToState:179(782ns)}}}}}
2025-11-27T18:18:10.121395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471167688175708:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.032668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471168617984514:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:10.062645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471173077065596:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5445
2025-11-27T18:18:10.037151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471173453733868:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32100
2025-11-27T18:18:10.099684Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471169857378071:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.121432Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471167688175708:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.037504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471168617984501:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(89122315ns) {Configure:104(89115458ns) {ProcessConfigured:135(18809754ns) {StartInitialHandshake:157(18805648ns) {DropHandshakes:470(3181641ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(2372ns)} GenerateSessionVirtualId:988(2536ns) PrepareNewSessionHandshake:477(3256ns) {SwitchToState:179(1604ns)}}}}}
2025-11-27T18:18:10.130141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471170234102212:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.134700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471167688175641:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.114086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471173077065596:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.037548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471168617984437:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.039345Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471169158766497:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(237519888ns) {Configure:104(237508161ns) {ProcessConfigured:135(16020926ns) {StartInitialHandshake:157(16017683ns) {DropHandshakes:470(1975ns) {DropIncomingHandshake:442(401ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(533ns) PrepareNewSessionHandshake:477(2936ns) {SwitchToState:179(1591ns)}}}}}
2025-11-27T18:18:10.130211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:10.117737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471168782098287:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(219375977ns) {Configure:104(219369371ns) {ProcessConfigured:135(43137ns) {StartInitialHandshake:157(40825ns) {DropHandshakes:470(1090ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(377ns) PrepareNewSessionHandshake:477(1450ns) {SwitchToState:179(418ns)}}}}}
2025-11-27T18:18:10.043515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471173453733875:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32100
2025-11-27T18:18:10.043998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471169158766558:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.047893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471173453733868:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.052148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471169158766569:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.098875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471173453733872:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.099068Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471173453733870:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:10.099247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471173453733878:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32100
2025-11-27T18:18:10.099326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471169158766496:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.099784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471169158766498:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.110859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:10.113064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471173453733882:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32100
2025-11-27T18:18:10.120536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471169158766498:2050] [node 3] ICP22 created new session: [6:7577471173453733880:2050]
2025-11-27T18:18:10.126488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471173453733868:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:10.126606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471173453733880:2050] [node 3] ICS09 handshake done sender: [6:7577471173453733870:2115] self: [6:7577471173453733873:0] peer: [3:7577471170234102215:0] socket: 408 qp: -1
2025-11-27T18:18:10.130045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471173453733880:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:10.149435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:10.149936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471169158766498:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(49952889ns) {HandleHandshakeStatus:329(49950910ns) {DropIncomingHandshake:442(486ns) DropOutgoingHandshake:457(5161ns) SwitchToState:179(224ns) ProcessPendingSessionEvents:495(112766ns) {ForwardSessionEventToSession:612(61580ns) ForwardSessionEventToSession:612(7179ns) ForwardSessionEventToSession:612(2920ns) ForwardSessionEventToSession:612(6030ns) ForwardSessionEventToSession:612(5358ns) ForwardSessionEventToSession:612(5126ns) ForwardSessionEventToSession:612(5296ns) ForwardSessionEventToSession:612(2745ns) ForwardSessionEventToSession:612(2686ns) ForwardSessionEventToSession:612(4993ns)}}}
2025-11-27T18:18:10.150085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471169158766569:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.150125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:10.150182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471169158766569:2110] [node 4] ICP22 created new session: [6:7577471173453733886:2110]
2025-11-27T18:18:10.150227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471173453733886:2110] [node 4] ICS09 handshake done sender: [6:7577471173453733868:2114] self: [6:7577471173453733877:0] peer: [4:7577471168617984509:0] socket: 396 qp: -1
2025-11-27T18:18:10.150264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471173453733886:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:10.150293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:10.150856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471173453733878:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.151267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471173453733883:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32100
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.109484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471169857378055:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:10.134770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471167688175639:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65340) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471167688175711:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.117296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471174152345377:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.037596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:10.137866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471170234102212:2110] [node 6] ICP22 created new session: [3:7577471174529069528:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.117948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471168782098293:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.165837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471173453733885:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32100
2025-11-27T18:18:10.137708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471167688175711:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:10.037644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471168617984437:2048] [node 1] ICP22 created new session: [4:7577471172912951815:2048]
2025-11-27T18:18:10.165903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471173453733872:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:10.118861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471169857378060:2109] [node 5] ICP18 handshake (actor [1:7577471174152345377:2117]) is too old
2025-11-27T18:18:10.156081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471174529069528:2110] [node 6] ICS09 handshake done sender: [3:7577471170234102216:2112] self: [3:7577471170234102215:0] peer: [6:7577471173453733873:0] socket: 397 qp: -1
2025-11-27T18:18:10.126874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471174152345378:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.137845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471169857378071:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:10.141560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471169857377994:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.141630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:10.149329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471169857377994:2048] [node 2] ICP22 created new session: [1:7577471174152345384:2048]
2025-11-27T18:18:10.157382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471174152345384:2048] [node 2] ICS09 handshake done sender: [1:7577471169857378055:2107] self: [1:7577471169857378054:0] peer: [2:7577471173077065597:0] socket: 378 qp: -1
2025-11-27T18:18:10.157782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471169857378061:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.161333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471174152345384:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:10.161390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.118323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471173077065601:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5445
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.162432Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471167688175640:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(220861167ns) {Configure:104(216493180ns) {ProcessConfigured:135(9731344ns) {StartInitialHandshake:157(9728371ns) {DropHandshakes:470(2740ns) {DropIncomingHandshake:442(1000ns) DropOutgoingHandshake:457(383ns)} GenerateSessionVirtualId:988(1075ns) PrepareNewSessionHandshake:477(1955ns) {SwitchToState:179(109ns)}}}}}
2025-11-27T18:18:10.161857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471169857377994:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(20096494ns) {HandleHandshakeStatus:329(20093402ns) {DropIncomingHandshake:442(280ns) DropOutgoingHandshake:457(326ns) SwitchToState:179(571ns) ProcessPendingSessionEvents:495(108275ns) {ForwardSessionEventToSession:612(59439ns) ForwardSessionEventToSession:612(5876ns) ForwardSessionEventToSession:612(6012ns) ForwardSessionEventToSession:612(1949ns) ForwardSessionEventToSession:612(5041ns) ForwardSessionEventToSession:612(3482ns) ForwardSessionEventToSession:612(2195ns) ForwardSessionEventToSession:612(5911ns) ForwardSessionEventToSession:612(2240ns) ForwardSessionEventToSession:612(5266ns)}}}
2025-11-27T18:18:10.037697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471172912951815:2048] [node 1] ICS09 handshake done sender: [4:7577471168617984514:2115] self: [4:7577471168617984515:0] peer: [1:7577471169857378063:0] socket: 400 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.171142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471169158766497:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.164129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:10.171884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471169857378061:2110] [node 6] ICP22 created new session: [1:7577471174152345385:2110]
2025-11-27T18:18:10.178332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471169857377994:2048] [node 2] ICP18 handshake (actor [1:7577471174152345378:2118]) is too old
2025-11-27T18:18:10.183038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471174152345385:2110] [node 6] ICS09 handshake done sender: [1:7577471169857378071:2114] self: [1:7577471169857378070:0] peer: [6:7577471173453733879:0] socket: 414 qp: -1
2025-11-27T18:18:10.191810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471174152345385:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:10.193940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:10.214155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471174152345388:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15615
2025-11-27T18:18:10.156130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471174529069528:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:10.126051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471168782098291:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(210756090ns) {Configure:104(210747525ns) {ProcessConfigured:135(7119631ns) {StartInitialHandshake:157(7117086ns) {DropHandshakes:470(984ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(129ns)} GenerateSessionVirtualId:988(256ns) PrepareNewSessionHandshake:477(1291ns) {SwitchToState:179(236ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.169566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471167688175708:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:10.037731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471172912951815:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:10.162625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:10.251541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471174152345388:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.197705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471169158766496:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.137429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471168782098285:2107] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.169737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471167688175639:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.037766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.169781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:10.169825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471167688175639:2048] [node 1] ICP22 created new session: [5:7577471171983143021:2048]
2025-11-27T18:18:10.169867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471171983143021:2048] [node 1] ICS09 handshake done sender: [5:7577471167688175711:2114] self: [5:7577471167688175717:0] peer: [1:7577471169857378065:0] socket: 401 qp: -1
2025-11-27T18:18:10.169899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471171983143021:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:10.169937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:10.178002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471171983143024:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46191
2025-11-27T18:18:10.189183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471167688175703:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.189246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:10.189328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471167688175703:2110] [node 6] ICP22 created new session: [5:7577471171983143025:2110]
2025-11-27T18:18:10.189376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471171983143025:2110] [node 6] ICS09 handshake done sender: [5:7577471167688175708:2112] self: [5:7577471167688175707:0] peer: [6:7577471173453733876:0] socket: 404 qp: -1
2025-11-27T18:18:10.189409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471171983143025:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:10.189440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:10.249588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471171983143024:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.259800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471171983143028:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46191
2025-11-27T18:18:10.265745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471171983143029:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46191
2025-11-27T18:18:10.266071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471167688175704:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.198397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471173453733871:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:10.073524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471168617984510:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.151548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471168782098225:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.193848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471170234102212:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(63512430ns) {HandleHandshakeStatus:329(63509407ns) {DropIncomingHandshake:442(15612ns) DropOutgoingHandshake:457(446ns) SwitchToState:179(591ns) ProcessPendingSessionEvents:495(132741ns) {ForwardSessionEventToSession:612(82637ns) ForwardSessionEventToSession:612(7133ns) ForwardSessionEventToSession:612(3469ns) ForwardSessionEventToSession:612(3017ns) ForwardSessionEventToSession:612(7682ns) ForwardSessionEventToSession:612(3148ns) ForwardSessionEventToSession:612(2794ns) ForwardSessionEventToSession:612(7160ns) ForwardSessionEventToSession:612(2522ns) ForwardSessionEventToSession:612(4623ns)}}}
2025-11-27T18:18:10.201475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:10.157519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:10.073570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471168617984510:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.281377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471174152345370:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.196646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471174529069527:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29301
2025-11-27T18:18:10.270811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471167688175712:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.285097Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471167688175641:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30066) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471167688175712:2115] held: no
2025-11-27T18:18:10.285210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471167688175641:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.285305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:10.285360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471167688175641:2050] [node 3] ICP22 created new session: [5:7577471171983143031:2050]
2025-11-27T18:18:10.101227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471172912951818:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48403
2025-11-27T18:18:10.206070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471173453733889:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32100
2025-11-27T18:18:10.217335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471170234102218:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.162274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471168782098225:2048] [node 1] ICP22 created new session: [2:7577471173077065608:2048]
2025-11-27T18:18:10.281439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471174152345370:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.184333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471173077065608:2048] [node 1] ICS09 handshake done sender: [2:7577471168782098285:2107] self: [2:7577471173077065597:0] peer: [1:7577471169857378054:0] socket: 382 qp: -1
2025-11-27T18:18:10.207937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471169158766496:2048] [node 1] ICP22 created new session: [6:7577471173453733890:2048]
2025-11-27T18:18:10.109694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471168617984510:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:10.286643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471171983143031:2050] [node 3] ICS09 handshake done sender: [5:7577471167688175712:2115] self: [5:7577471171983143020:0] peer: [3:7577471170234102217:0] socket: 402 qp: -1
2025-11-27T18:18:10.217386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471170234102218:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.283485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471169857378061:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(103734648ns) {HandleHandshakeStatus:329(103732277ns) {DropIncomingHandshake:442(16242ns) DropOutgoingHandshake:457(379ns) SwitchToState:179(564ns) ProcessPendingSessionEvents:495(36568671ns) {ForwardSessionEventToSession:612(21219875ns) ForwardSessionEventToSession:612(4818040ns) ForwardSessionEventToSession:612(5949ns) ForwardSessionEventToSession:612(3216ns) ForwardSessionEventToSession:612(505933ns) ForwardSessionEventToSession:612(3569ns) ForwardSessionEventToSession:612(8627753ns) ForwardSessionEventToSession:612(6646ns) ForwardSessionEventToSession:612(2257ns) ForwardSessionEventToSession:612(1353157ns)}}}
2025-11-27T18:18:10.286691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471171983143031:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:10.232881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471170234102218:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:10.222882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471173453733890:2048] [node 1] ICS09 handshake done sender: [6:7577471173453733872:2117] self: [6:7577471173453733879:0] peer: [1:7577471169857378070:0] socket: 416 qp: -1
2025-11-27T18:18:10.186046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471173077065608:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.133799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471168617984502:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.286740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:10.222932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471173453733890:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:10.286140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471174152345370:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:10.251153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471174529069529:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29301
2025-11-27T18:18:10.189978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471168782098297:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.287202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471171983143029:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.139328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:10.307118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471169857377995:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.251680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471174529069527:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.190004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471168782098297:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.226093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471169158766558:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.287494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471167688175640:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.303687Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471167688175704:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5053) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471171983143024:2119] held: no
2025-11-27T18:18:10.152925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471168617984502:2109] [node 6] ICP22 created new session: [4:7577471172912951820:2109]
2025-11-27T18:18:10.254785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471170234102213:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.316181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:10.226160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:10.186095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:10.316224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471169857377995:2049] [node 3] ICP22 created new session: [1:7577471174152345391:2049]
2025-11-27T18:18:10.317659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471167688175640:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17685) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471171983143029:2121] held: no
2025-11-27T18:18:10.164078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471172912951820:2109] [node 6] ICS09 handshake done sender: [4:7577471168617984510:2114] self: [4:7577471168617984509:0] peer: [6:7577471173453733877:0] socket: 395 qp: -1
2025-11-27T18:18:10.164141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471172912951820:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:10.166412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:10.179289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471172912951818:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.185329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471172912951823:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48403
2025-11-27T18:18:10.185386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471172912951824:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48403
2025-11-27T18:18:10.206468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471168617984502:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(72475959ns) {HandleHandshakeStatus:329(72473275ns) {DropIncomingHandshake:442(254ns) DropOutgoingHandshake:457(446ns) SwitchToState:179(459ns) ProcessPendingSessionEvents:495(14472213ns) {ForwardSessionEventToSession:612(14421659ns) ForwardSessionEventToSession:612(4953ns) ForwardSessionEventToSession:612(3256ns) ForwardSessionEventToSession:612(7216ns) ForwardSessionEventToSession:612(2787ns) ForwardSessionEventToSession:612(4493ns) ForwardSessionEventToSession:612(2189ns) ForwardSessionEventToSession:612(7171ns) ForwardSessionEventToSession:612(4119ns) ForwardSessionEventToSession:612(2923ns)}}}
2025-11-27T18:18:10.206776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471168617984502:2109] [node 6] ICP18 handshake (actor [4:7577471172912951818:2118]) is too old
2025-11-27T18:18:10.227817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471172912951824:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.245538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471172912951823:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.245688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471168617984438:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.273591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471172912951828:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48403
2025-11-27T18:18:10.277814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471172912951828:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.278969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471172912951814:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.279004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471172912951814:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.284012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471172912951814:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:10.288216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471172912951827:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48403
2025-11-27T18:18:10.289057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471168617984439:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.290411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471168617984501:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.290459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:10.254862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:10.290503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471168617984501:2108] [node 5] ICP22 created new session: [4:7577471172912951831:2108]
2025-11-27T18:18:10.254907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471170234102213:2109] [node 5] ICP22 created new session: [3:7577471174529069533:2109]
2025-11-27T18:18:10.290552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471172912951831:2108] [node 5] ICS09 handshake done sender: [4:7577471172912951814:2117] self: [4:7577471172912951812:0] peer: [5:7577471171983143030:0] socket: 417 qp: -1
2025-11-27T18:18:10.254949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471174529069533:2109] [node 5] ICS09 handshake done sender: [3:7577471170234102218:2113] self: [3:7577471170234102217:0] peer: [5:7577471171983143020:0] socket: 398 qp: -1
2025-11-27T18:18:10.290591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471172912951831:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:10.254979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471174529069533:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:10.290620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:10.255008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:10.294898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471172912951823:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:10.260190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471170234102146:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.307975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471168617984438:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17685) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471172912951823:2119] held: no
2025-11-27T18:18:10.308066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471168617984438:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.269834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471174529069529:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.308101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:10.269975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471170234102145:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:10.308144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471168617984438:2049] [node 2] ICP22 created new session: [4:7577471172912951833:2049]
2025-11-27T18:18:10.282884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471174529069538:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29301
2025-11-27T18:18:10.308179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471172912951833:2049] [node 2] ICS09 handshake done sender: [4:7577471172912951823:2119] self: [4:7577471172912951825:0] peer: [2:7577471173077065602:0] socket: 421 qp: -1
2025-11-27T18:18:10.292613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471174529069520:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.308210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471172912951833:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:10.292635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471174529069520:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.308234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:10.308752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471172912951835:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48403
2025-11-27T18:18:10.294441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471174529069529:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:10.311339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471172912951828:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:10.301777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471170234102145:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.312645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471168617984439:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.301867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:10.327043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:10.301923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471170234102145:2048] [node 1] ICP22 created new session: [3:7577471174529069541:2048]
2025-11-27T18:18:10.327737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471168617984439:2050] [node 3] ICP22 created new session: [4:7577471172912951836:2050]
2025-11-27T18:18:10.301966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471174529069541:2048] [node 1] ICS09 handshake done sender: [3:7577471174529069529:2119] self: [3:7577471174529069537:0] peer: [1:7577471169857378073:0] socket: 436 qp: -1
2025-11-27T18:18:10.327780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471172912951836:2050] [node 3] ICS09 handshake done sender: [4:7577471172912951828:2122] self: [4:7577471172912951830:0] peer: [3:7577471170234102223:0] socket: 449 qp: -1
2025-11-27T18:18:10.301997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471174529069541:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:10.327807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471172912951836:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:10.302051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:10.327873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:10.307742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471174529069539:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29301
2025-11-27T18:18:10.309483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471174529069520:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:10.326270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471174152345391:2049] [node 3] ICS09 handshake done sender: [1:7577471174152345370:2115] self: [1:7577471169857378073:0] peer: [3:7577471174529069537:0] socket: 425 qp: -1
2025-11-27T18:18:10.309741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471170234102209:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.327511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471174152345391:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:10.309823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:10.309869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471170234102209:2108] [node 4] ICP22 created new session: [3:7577471174529069543:2108]
2025-11-27T18:18:10.327562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:10.309917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471174529069543:2108] [node 4] ICS09 handshake done sender: [3:7577471174529069520:2115] self: [3:7577471170234102223:0] peer: [4:7577471172912951830:0] socket: 447 qp: -1
2025-11-27T18:18:10.309949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471174529069543:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:10.330769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471169857377995:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(26800715ns) {HandleHandshakeStatus:329(23453824ns) {DropIncomingHandshake:442(18591ns) DropOutgoingHandshake:457(308ns) SwitchToState:179(374ns) ProcessPendingSessionEvents:495(439553ns) {ForwardSessionEventToSession:612(391853ns) ForwardSessionEventToSession:612(4292ns) ForwardSessionEventToSession:612(5402ns) ForwardSessionEventToSession:612(4951ns) ForwardSessionEventToSession:612(2500ns) ForwardSessionEventToSession:612(5242ns) ForwardSessionEventToSession:612(3017ns) ForwardSessionEventToSession:612(7995ns) ForwardSessionEventToSession:612(3991ns) ForwardSessionEventToSession:612(2768ns)}}}
2025-11-27T18:18:10.226195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471169158766558:2107] [node 5] ICP22 created new session: [6:7577471173453733892:2107]
2025-11-27T18:18:10.226235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471173453733892:2107] [node 5] ICS09 handshake done sender: [6:7577471173453733871:2116] self: [6:7577471173453733876:0] peer: [5:7577471167688175707:0] socket: 411 qp: -1
2025-11-27T18:18:10.309981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:10.226264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471173453733892:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:10.310366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471174529069527:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:10.226298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:10.314086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471170234102146:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.226513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471169158766558:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:10.314139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:10.227007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471173453733878:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:10.314179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471170234102146:2049] [node 2] ICP22 created new session: [3:7577471174529069545:2049]
2025-11-27T18:18:10.227753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:10.314217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471174529069545:2049] [node 2] ICS09 handshake done sender: [3:7577471174529069527:2118] self: [3:7577471174529069536:0] peer: [2:7577471168782098298:0] socket: 431 qp: -1
2025-11-27T18:18:10.255789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471169158766497:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.314248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471174529069545:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:10.255863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:10.314274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:10.255909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471169158766497:2049] [node 2] ICP22 created new session: [6:7577471173453733895:2049]
2025-11-27T18:18:10.255947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471173453733895:2049] [node 2] ICS09 handshake done sender: [6:7577471173453733878:2120] self: [6:7577471173453733888:0] peer: [2:7577471168782098296:0] socket: 428 qp: -1
2025-11-27T18:18:10.255971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471173453733895:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:10.256015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:10.289602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471169158766496:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(78529165ns) {HandleHandshakeStatus:329(78525417ns) {DropIncomingHandshake:442(713ns) DropOutgoingHandshake:457(4905ns) SwitchToState:179(485ns) ProcessPendingSessionEvents:495(25144516ns) {ForwardSessionEventToSession:612(10809023ns) ForwardSessionEventToSession:612(5989ns) ForwardSessionEventToSession:612(6525005ns) ForwardSessionEventToSession:612(7814ns) ForwardSessionEventToSession:612(2811ns) ForwardSessionEventToSession:612(1189399ns) ForwardSessionEventToSession:612(3462ns) ForwardSessionEventToSession:612(972600ns) ForwardSessionEventToSession:612(5605379ns) ForwardSessionEventToSession:612(2822ns)}}}
2025-11-27T18:18:10.289665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471169158766496:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:10.193879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471168782098297:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:10.230159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471168782098225:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(78408312ns) {HandleHandshakeStatus:329(78401490ns) {DropIncomingHandshake:442(454ns) DropOutgoingHandshake:457(1757553ns) SwitchToState:179(1221ns) ProcessPendingSessionEvents:495(122109ns) {ForwardSessionEventToSession:612(64234ns) ForwardSessionEventToSession:612(12161ns) ForwardSessionEventToSession:612(3104ns) ForwardSessionEventToSession:612(2970ns) ForwardSessionEventToSession:612(9093ns) ForwardSessionEventToSession:612(2839ns) ForwardSessionEventToSession:612(8705ns) ForwardSessionEventToSession:612(2626ns) ForwardSessionEventToSession:612(2160ns) ForwardSessionEventToSession:612(5579ns)}}}
2025-11-27T18:18:10.230492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471168782098292:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.230530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:10.230577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471168782098292:2110] [node 6] ICP22 created new session: [2:7577471173077065614:2110]
2025-11-27T18:18:10.230622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471173077065614:2110] [node 6] ICS09 handshake done sender: [2:7577471168782098297:2114] self: [2:7577471168782098296:0] peer: [6:7577471173453733888:0] socket: 422 qp: -1
2025-11-27T18:18:10.230652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471173077065614:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:10.230707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:10.231811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471173077065612:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5445
2025-11-27T18:18:10.231869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471173077065613:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5445
2025-11-27T18:18:10.253469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471173077065613:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.253564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471173077065603:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.253581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471173077065603:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.260480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471173077065603:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:10.277879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471168782098292:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:10.284093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471173077065595:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.284131Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471173077065595:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.285989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471168782098287:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.288480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:10.289463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471168782098287:2108] [node 4] ICP22 created new session: [2:7577471173077065619:2108]
2025-11-27T18:18:10.292538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471173077065607:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:10.292565Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471173077065607:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:10.294971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471173077065619:2108] [node 4] ICS09 handshake done sender: [2:7577471173077065603:2118] self: [2:7577471173077065602:0] peer: [4:7577471172912951825:0] socket: 388 qp: -1
2025-11-27T18:18:10.295339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471173077065595:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:10.295559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471173077065607:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:10.295870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471168782098291:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.295951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:10.296338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471168782098291:2109] [node 5] ICP22 created new session: [2:7577471173077065622:2109]
2025-11-27T18:18:10.303966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471173077065619:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:10.305678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471168782098226:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:10.305762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:10.305810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471168782098226:2049] [node 3] ICP22 created new session: [2:7577471173077065623:2049]
2025-11-27T18:18:10.305855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471173077065623:2049] [node 3] ICS09 handshake done sender: [2:7577471173077065595:2115] self: [2:7577471168782098298:0] peer: [3:7577471174529069536:0] socket: 423 qp: -1
2025-11-27T18:18:10.305889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471173077065623:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:10.305924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:10.307198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:10.312473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471173077065622:2109] [node 5] ICS09 handshake done sender: [2:7577471173077065607:2119] self: [2:7577471173077065604:0] peer: [5:7577471171983143033:0] socket: 444 qp: -1
2025-11-27T18:18:10.312520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471173077065622:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:10.318253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:10.320311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471168782098287:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(35548222ns) {HandleHandshakeStatus:329(35545291ns) {DropIncomingHandshake:442(18909ns) DropOutgoingHandshake:457(253ns) SwitchToState:179(285ns) ProcessPendingSessionEvents:495(1816992ns) {ForwardSessionEventToSession:612(1766017ns) ForwardSessionEventToSession:612(7688ns) ForwardSessionEventToSession:612(3473ns) ForwardSessionEventToSession:612(8276ns) ForwardSessionEventToSession:612(5960ns) ForwardSessionEventToSession:612(2286ns) ForwardSessionEventToSession:612(5552ns) ForwardSessionEventToSession:612(1850ns) ForwardSessionEventToSession:612(1914ns) ForwardSessionEventToSession:612(5344ns)}}}
2025-11-27T18:18:10.332687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471168782098291:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(32744500ns) {HandleHandshakeStatus:329(32742936ns) {DropIncomingHandshake:442(15004ns) DropOutgoingHandshake:457(358ns) SwitchToState:179(568ns) ProcessPendingSessionEvents:495(1005395ns) {ForwardSessionEventToSession:612(954852ns) ForwardSessionEventToSession:612(6704ns) ForwardSessionEventToSession:612(2541ns) ForwardSessionEventToSession:612(5450ns) ForwardSessionEventToSession:612(4444ns) ForwardSessionEventToSession:612(2667ns) ForwardSessionEventToSession:612(9264ns) ForwardSessionEventToSession:612(2426ns) ForwardSessionEventToSession:612(2751ns) ForwardSessionEventToSession:612(5677ns)}}}
2025-11-27T18:18:10.317817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471171983143034:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46191
2025-11-27T18:18:10.318072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471171983143035:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46191
2025-11-27T18:18:10.322072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471171983143029:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:10.322429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471171983143024:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:10.322850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471167688175640:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.322921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:10.322971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471167688175640:2049] [node 2] ICP22 created new session: [5:7577471171983143036:2049]
2025-11-27T18:18:10.323013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471171983143036:2049] [node 2] ICS09 handshake done sender: [5:7577471171983143029:2121] self: [5:7577471171983143033:0] peer: [2:7577471173077065604:0] socket: 445 qp: -1
2025-11-27T18:18:10.323049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471171983143036:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:10.323089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:10.323568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471167688175704:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:10.323620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:10.323666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471167688175704:2109] [node 4] ICP22 created new session: [5:7577471171983143038:2109]
2025-11-27T18:18:10.323711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471171983143038:2109] [node 4] ICS09 handshake done sender: [5:7577471171983143024:2119] self: [5:7577471171983143030:0] peer: [4:7577471172912951812:0] socket: 437 qp: -1
2025-11-27T18:18:10.323748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471171983143038:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:10.323774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:10.334789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471168617984439:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16479557ns) {HandleHandshakeStatus:329(16478010ns) {DropIncomingHandshake:442(566ns) DropOutgoingHandshake:457(4164ns) SwitchToState:179(442ns) ProcessPendingSessionEvents:495(1084527ns) {ForwardSessionEventToSession:612(1024412ns) ForwardSessionEventToSession:612(6063ns) ForwardSessionEventToSession:612(2630ns) ForwardSessionEventToSession:612(7490ns) ForwardSessionEventToSession:612(2932ns) ForwardSessionEventToSession:612(1880ns) ForwardSessionEventToSession:612(8584ns) ForwardSessionEventToSession:612(2523ns) ForwardSessionEventToSession:612(12718ns) ForwardSessionEventToSession:612(2073ns)}}}
2025-11-27T18:18:10.381438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:10.381688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471173077065608:2048] [node 1] ICS07 socket disconnect 382 reason# EndOfStream
2025-11-27T18:18:10.381721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471173077065608:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.381798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:10.381840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471173077065608:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:10.385472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471168782098225:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65340) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:10.385519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471173077065608:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:10.389704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:10.389887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471174529069541:2048] [node 1] ICS01 socket: 436 reason# EndOfStream
2025-11-27T18:18:10.389914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471170234102145:2048] [node 1] ICP30 unregister session Session# [3:7577471174529069541:2048] VirtualId# [3:7577471174529069537:0]
2025-11-27T18:18:10.389934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471174529069541:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.389985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:10.390014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:10.389587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:10.389816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471171983143021:2048] [node 1] ICS01 socket: 401 reason# EndOfStream
2025-11-27T18:18:10.389857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471167688175639:2048] [node 1] ICP30 unregister session Session# [5:7577471171983143021:2048] VirtualId# [5:7577471167688175717:0]
2025-11-27T18:18:10.389888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471171983143021:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.389961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:10.390001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:10.390143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:10.390116Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:10.390383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471173453733890:2048] [node 1] ICS01 socket: 416 reason# EndOfStream
2025-11-27T18:18:10.390421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471169158766496:2048] [node 1] ICP30 unregister session Session# [6:7577471173453733890:2048] VirtualId# [6:7577471173453733879:0]
2025-11-27T18:18:10.390281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471172912951815:2048] [node 1] ICS01 socket: 400 reason# EndOfStream
2025-11-27T18:18:10.390310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471168617984437:2048] [node 1] ICP30 unregister session Session# [4:7577471172912951815:2048] VirtualId# [4:7577471168617984515:0]
2025-11-27T18:18:10.390454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471173453733890:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.391213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471168782098225:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65340) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:10.390333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471172912951815:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.390531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:10.395133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471173077065608:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:10.390384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:10.390453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:10.390582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:10.450933Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:10.451210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471174529069545:2049] [node 2] ICS01 socket: 431 reason# EndOfStream
2025-11-27T18:18:10.451247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471170234102146:2049] [node 2] ICP30 unregister session Session# [3:7577471174529069545:2049] VirtualId# [3:7577471174529069536:0]
2025-11-27T18:18:10.451276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471174529069545:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.451354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:10.451398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:10.456320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:10.456494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471173453733895:2049] [node 2] ICS01 socket: 428 reason# EndOfStream
2025-11-27T18:18:10.456526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471169158766497:2049] [node 2] ICP30 unregister session Session# [6:7577471173453733895:2049] VirtualId# [6:7577471173453733888:0]
2025-11-27T18:18:10.456550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471173453733895:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.456606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:10.456639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:10.456143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:10.456347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471172912951833:2049] [node 2] ICS01 socket: 421 reason# EndOfStream
2025-11-27T18:18:10.456389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471168617984438:2049] [node 2] ICP30 unregister session Session# [4:7577471172912951833:2049] VirtualId# [4:7577471172912951825:0]
2025-11-27T18:18:10.456417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471172912951833:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.456494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:10.456537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:10.456619Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:10.456784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471171983143036:2049] [node 2] ICS01 socket: 445 reason# EndOfStream
2025-11-27T18:18:10.456814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471167688175640:2049] [node 2] ICP30 unregister session Session# [5:7577471171983143036:2049] VirtualId# [5:7577471171983143033:0]
2025-11-27T18:18:10.456835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471171983143036:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.456884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:10.456922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:10.550947Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:10.551181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471173453733880:2050] [node 3] ICS01 socket: 408 reason# EndOfStream
2025-11-27T18:18:10.551230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471169158766498:2050] [node 3] ICP30 unregister session Session# [6:7577471173453733880:2050] VirtualId# [6:7577471173453733873:0]
2025-11-27T18:18:10.551277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471173453733880:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.551377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:10.551434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:10.553632Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:10.553828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471172912951836:2050] [node 3] ICS07 socket disconnect 449 reason# EndOfStream
2025-11-27T18:18:10.553860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471172912951836:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.553962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:10.554019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471172912951836:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:10.553710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:10.553896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471171983143031:2050] [node 3] ICS01 socket: 402 reason# EndOfStream
2025-11-27T18:18:10.553939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471167688175641:2050] [node 3] ICP30 unregister session Session# [5:7577471171983143031:2050] VirtualId# [5:7577471171983143020:0]
2025-11-27T18:18:10.553972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471171983143031:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.554041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:10.554079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:10.554939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471168617984439:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30066) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:10.554994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471172912951836:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:10.555034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471168617984439:2050] [node 3] ICP30 unregister session Session# [4:7577471172912951836:2050] VirtualId# [4:7577471172912951830:0]
2025-11-27T18:18:10.555072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:10.555121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471168617984439:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30066) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:10.555144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30066) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:10.611458Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:10.611460Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:10.611757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471171983143038:2109] [node 4] ICS01 socket: 437 reason# EndOfStream
2025-11-27T18:18:10.611796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471167688175704:2109] [node 4] ICP30 unregister session Session# [5:7577471171983143038:2109] VirtualId# [5:7577471171983143030:0]
2025-11-27T18:18:10.611821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471171983143038:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.611875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:10.611911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:10.611676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471173453733886:2110] [node 4] ICS01 socket: 396 reason# EndOfStream
2025-11-27T18:18:10.611721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471169158766569:2110] [node 4] ICP30 unregister session Session# [6:7577471173453733886:2110] VirtualId# [6:7577471173453733877:0]
2025-11-27T18:18:10.611759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471173453733886:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.611843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:10.611906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:10.689945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:10.690205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471173453733892:2107] [node 5] ICS01 socket: 411 reason# EndOfStream
2025-11-27T18:18:10.690261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471169158766558:2107] [node 5] ICP30 unregister session Session# [6:7577471173453733892:2107] VirtualId# [6:7577471173453733876:0]
2025-11-27T18:18:10.690292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471173453733892:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:10.690375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:10.690415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:10.809718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471173677591545:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:10.809771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471173677591546:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:10.857834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471173236949489:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:10.857892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471173236949490:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:10.881619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471172992811627:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:10.881671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471172992811628:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:10.928851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471174108410486:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:10.928895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471174108410487:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:10.928923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471174108410488:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:10.955460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471173541263829:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:10.955500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471173541263830:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:10.955525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471173541263831:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:11.013241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471176809673888:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:11.013304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471176809673889:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:11.013346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471176809673890:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:11.653370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471176809673948:2106] [node 5] ICP01 ready to work
2025-11-27T18:18:11.674000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471178403377846:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:11.593189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471177287778986:2107] [node 6] ICP01 ready to work
2025-11-27T18:18:11.685854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471177836231190:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:11.666481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471177531916848:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:11.701615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471177972558906:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:11.699107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471177972558905:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:11.701481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471177972558904:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:11.669946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471176809673951:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.675073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471177287778987:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:11.701619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471177531916849:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.735568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471177972558907:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19539
2025-11-27T18:18:11.693687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471177836231189:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:11.702269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471177287778988:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.753935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471176809673959:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45156
2025-11-27T18:18:11.742896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471173541263829:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(215711857ns) {Configure:104(215706922ns) {ProcessConfigured:135(68354ns) {StartInitialHandshake:157(65291ns) {DropHandshakes:470(2065ns) {DropIncomingHandshake:442(452ns) DropOutgoingHandshake:457(316ns)} GenerateSessionVirtualId:988(1352ns) PrepareNewSessionHandshake:477(1725ns) {SwitchToState:179(682ns)}}}}}
2025-11-27T18:18:11.748676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471177531916856:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:11.778470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471173236949490:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(312329880ns) {Configure:104(312326061ns) {ProcessConfigured:135(59776ns) {StartInitialHandshake:157(56033ns) {DropHandshakes:470(2845ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(873ns) PrepareNewSessionHandshake:477(1722ns) {SwitchToState:179(497ns)}}}}}
2025-11-27T18:18:11.735640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471177972558908:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19539
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.799029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471173541263830:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49409632ns) {Configure:104(49404583ns) {ProcessConfigured:135(63059ns) {StartInitialHandshake:157(60270ns) {DropHandshakes:470(4735ns) {DropIncomingHandshake:442(634ns) DropOutgoingHandshake:457(289ns)} GenerateSessionVirtualId:988(1025ns) PrepareNewSessionHandshake:477(1802ns) {SwitchToState:179(500ns)}}}}}
2025-11-27T18:18:11.767415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471173677591546:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(37948621ns) {Configure:104(37936459ns) {ProcessConfigured:135(63813ns) {StartInitialHandshake:157(60592ns) {DropHandshakes:470(2677ns) {DropIncomingHandshake:442(564ns) DropOutgoingHandshake:457(372ns)} GenerateSessionVirtualId:988(675ns) PrepareNewSessionHandshake:477(1374ns) {SwitchToState:179(316ns)}}}}}
2025-11-27T18:18:11.771259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471173677591545:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(101649629ns) {Configure:104(101644091ns) {ProcessConfigured:135(59295ns) {StartInitialHandshake:157(56108ns) {DropHandshakes:470(2294ns) {DropIncomingHandshake:442(515ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(746ns) PrepareNewSessionHandshake:477(2334ns) {SwitchToState:179(682ns)}}}}}
2025-11-27T18:18:11.771755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471177972558913:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19539
2025-11-27T18:18:11.777743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471177972558907:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.754012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471176809673960:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45156
2025-11-27T18:18:11.769533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471178403377850:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.793716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471177531916860:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65073
2025-11-27T18:18:11.775106Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471176809673948:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27249772ns) {Configure:104(27243644ns) {ProcessConfigured:135(54296ns) {StartInitialHandshake:157(51766ns) {DropHandshakes:470(5241ns) {DropIncomingHandshake:442(480ns) DropOutgoingHandshake:457(310ns)} GenerateSessionVirtualId:988(688ns) PrepareNewSessionHandshake:477(1621ns) {SwitchToState:179(496ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.831322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471172992811628:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(164896328ns) {Configure:104(164882267ns) {ProcessConfigured:135(65140ns) {StartInitialHandshake:157(62422ns) {DropHandshakes:470(2733ns) {DropIncomingHandshake:442(567ns) DropOutgoingHandshake:457(377ns)} GenerateSessionVirtualId:988(1119ns) PrepareNewSessionHandshake:477(2285ns) {SwitchToState:179(628ns)}}}}}
2025-11-27T18:18:11.833480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471177287778993:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50732
2025-11-27T18:18:11.814427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471174108410488:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(144751497ns) {Configure:104(144747778ns) {ProcessConfigured:135(62889ns) {StartInitialHandshake:157(59928ns) {DropHandshakes:470(2256ns) {DropIncomingHandshake:442(528ns) DropOutgoingHandshake:457(325ns)} GenerateSessionVirtualId:988(1075ns) PrepareNewSessionHandshake:477(1527ns) {SwitchToState:179(504ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.820038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471173541263831:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(133869961ns) {Configure:104(133864552ns) {ProcessConfigured:135(9895723ns) {StartInitialHandshake:157(9891472ns) {DropHandshakes:470(2181ns) {DropIncomingHandshake:442(511ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(645ns) PrepareNewSessionHandshake:477(4014ns) {SwitchToState:179(2548ns)}}}}}
2025-11-27T18:18:11.805776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471176809673963:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45156
2025-11-27T18:18:11.845843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471176809673959:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.853762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471176809673963:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.850647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471177972558908:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.864606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471176809673890:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(162427226ns) {Configure:104(162420328ns) {ProcessConfigured:135(35538ns) {StartInitialHandshake:157(34396ns) {DropHandshakes:470(2035ns) {DropIncomingHandshake:442(396ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(502ns) PrepareNewSessionHandshake:477(661ns) {SwitchToState:179(147ns)}}}}}
2025-11-27T18:18:11.842534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471177836231202:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49228
2025-11-27T18:18:11.814600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471178403377849:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20997
2025-11-27T18:18:11.850746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471177531916860:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.865494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471176809673960:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.865722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471176809673948:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.867268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471176809673969:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45156
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.863135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471172992811627:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(187398753ns) {Configure:104(187394005ns) {ProcessConfigured:135(71750ns) {StartInitialHandshake:157(69285ns) {DropHandshakes:470(2745ns) {DropIncomingHandshake:442(551ns) DropOutgoingHandshake:457(792ns)} GenerateSessionVirtualId:988(962ns) PrepareNewSessionHandshake:477(1972ns) {SwitchToState:179(564ns)}}}}}
2025-11-27T18:18:11.837808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471178403377849:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.868198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471176809673888:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:11.862264Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471177836231190:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(104694329ns) {Configure:104(104688765ns) {ProcessConfigured:135(69114ns) {StartInitialHandshake:157(66206ns) {DropHandshakes:470(3138ns) {DropIncomingHandshake:442(639ns) DropOutgoingHandshake:457(675ns)} GenerateSessionVirtualId:988(971ns) PrepareNewSessionHandshake:477(1477ns) {SwitchToState:179(465ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.884665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471178403377855:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20997
2025-11-27T18:18:11.873990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471177972558910:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.864587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471177287778993:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.897491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471177531916848:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(115540425ns) {Configure:104(115518882ns) {ProcessConfigured:135(42210479ns) {StartInitialHandshake:157(42207898ns) {DropHandshakes:470(2213ns) {DropIncomingHandshake:442(507ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(788ns) PrepareNewSessionHandshake:477(5835785ns) {SwitchToState:179(2806ns)}}}}}
2025-11-27T18:18:11.903723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471177531916858:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.903766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471177531916858:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:11.905991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471177531916856:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(136980829ns) {Configure:104(136975421ns) {ProcessConfigured:135(66357ns) {StartInitialHandshake:157(64338ns) {DropHandshakes:470(2498ns) {DropIncomingHandshake:442(582ns) DropOutgoingHandshake:457(327ns)} GenerateSessionVirtualId:988(924ns) PrepareNewSessionHandshake:477(2002ns) {SwitchToState:179(684ns)}}}}}
2025-11-27T18:18:11.906174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471177531916853:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.906194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471177531916853:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:11.906831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471177531916861:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65073
2025-11-27T18:18:11.907099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471173236949489:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:11.885968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471176809673960:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.865962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471177836231194:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.874027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471177972558910:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:11.864679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471172992811628:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:11.875015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471177287779000:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50732
2025-11-27T18:18:11.895788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471177287779001:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50732
2025-11-27T18:18:11.895856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471177287779002:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50732
2025-11-27T18:18:11.898203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471177287778986:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(167926993ns) {Configure:104(167918432ns) {ProcessConfigured:135(66739ns) {StartInitialHandshake:157(64363ns) {DropHandshakes:470(2316ns) {DropIncomingHandshake:442(498ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(798ns) PrepareNewSessionHandshake:477(3786ns) {SwitchToState:179(347ns)}}}}}
2025-11-27T18:18:11.902276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471177287779002:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.930773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471177287779005:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50732
2025-11-27T18:18:11.942234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471177287778993:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:11.865981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471177836231194:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:11.887650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471176809673889:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(185094809ns) {Configure:104(185088931ns) {ProcessConfigured:135(71436ns) {StartInitialHandshake:157(68271ns) {DropHandshakes:470(2422ns) {DropIncomingHandshake:442(714ns) DropOutgoingHandshake:457(265ns)} GenerateSessionVirtualId:988(934ns) PrepareNewSessionHandshake:477(1715ns) {SwitchToState:179(569ns)}}}}}
2025-11-27T18:18:11.920589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471177531916853:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:11.933690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471177531916858:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:11.918364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471174108410486:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(163975138ns) {Configure:104(163953428ns) {ProcessConfigured:135(10495289ns) {StartInitialHandshake:157(10491747ns) {DropHandshakes:470(1681ns) {DropIncomingHandshake:442(235ns) DropOutgoingHandshake:457(554ns)} GenerateSessionVirtualId:988(1510ns) PrepareNewSessionHandshake:477(2569ns) {SwitchToState:179(1017ns)}}}}}
2025-11-27T18:18:11.883392Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471177972558904:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(113572471ns) {Configure:104(113567302ns) {ProcessConfigured:135(60607ns) {StartInitialHandshake:157(58249ns) {DropHandshakes:470(2807ns) {DropIncomingHandshake:442(969ns) DropOutgoingHandshake:457(477ns)} GenerateSessionVirtualId:988(601ns) PrepareNewSessionHandshake:477(2037ns) {SwitchToState:179(953ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.887832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471176809673889:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:11.866906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471177836231194:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.931076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471178403377846:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(158499512ns) {Configure:104(158494381ns) {ProcessConfigured:135(13420804ns) {StartInitialHandshake:157(13418733ns) {DropHandshakes:470(1972ns) {DropIncomingHandshake:442(419ns) DropOutgoingHandshake:457(325ns)} GenerateSessionVirtualId:988(1276ns) PrepareNewSessionHandshake:477(3356ns) {SwitchToState:179(1906ns)}}}}}
2025-11-27T18:18:11.960869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471177531916861:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.883938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471177972558921:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19539
2025-11-27T18:18:11.954247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471172992811628:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:11.885803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471177972558913:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.889969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471177972558910:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:11.937801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471177972558905:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(131085913ns) {Configure:104(131080507ns) {ProcessConfigured:135(8085229ns) {StartInitialHandshake:157(8083209ns) {DropHandshakes:470(2130ns) {DropIncomingHandshake:442(330ns) DropOutgoingHandshake:457(271ns)} GenerateSessionVirtualId:988(506ns) PrepareNewSessionHandshake:477(2808ns) {SwitchToState:179(962ns)}}}}}
2025-11-27T18:18:11.961684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471177972558916:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.961712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471177972558916:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:11.963490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471177972558916:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:11.973300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471177972558906:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:11.982015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471177972558921:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.982153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.910920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471176809673948:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:11.958776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471178403377855:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.964161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471177531916870:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65073
2025-11-27T18:18:11.867930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471177836231189:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:11.960815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471177287779001:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.986446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471177972558906:2109] [node 6] ICP22 created new session: [1:7577471177972558931:2109]
2025-11-27T18:18:11.918034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471176809673973:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45156
2025-11-27T18:18:11.964548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471177287778992:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.958943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471174108410488:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:11.994729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471177531916849:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:11.867971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:11.918525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471176809673974:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45156
2025-11-27T18:18:11.996350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471177972558912:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.873320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471177836231202:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.981931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471178403377850:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(155873112ns) {Configure:104(155866105ns) {ProcessConfigured:135(8414687ns) {StartInitialHandshake:157(8411306ns) {DropHandshakes:470(1517ns) {DropIncomingHandshake:442(606ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(742ns) PrepareNewSessionHandshake:477(3131ns) {SwitchToState:179(1208ns)}}}}}
2025-11-27T18:18:11.994979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471177531916870:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.964570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471177287778992:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.965241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471177287779000:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.965585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471172992811627:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:11.973511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471177287778992:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:11.982211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:11.986286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471172992811628:2049] [node 2] ICP22 created new session: [3:7577471177287779011:2049]
2025-11-27T18:18:11.995007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471177287779011:2049] [node 2] ICS09 handshake done sender: [3:7577471177287778993:2112] self: [3:7577471177287778999:0] peer: [2:7577471177531916857:0] socket: 409 qp: -1
2025-11-27T18:18:11.996558Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471177287778987:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(201339384ns) {Configure:104(201334445ns) {ProcessConfigured:135(11099587ns) {StartInitialHandshake:157(11095973ns) {DropHandshakes:470(2200ns) {DropIncomingHandshake:442(478ns) DropOutgoingHandshake:457(341ns)} GenerateSessionVirtualId:988(1873ns) PrepareNewSessionHandshake:477(3292ns) {SwitchToState:179(1900ns)}}}}}
2025-11-27T18:18:12.003882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471177287779011:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:12.003951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:12.013461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471177287778988:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:12.013588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:12.013638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471177287778988:2109] [node 4] ICP22 created new session: [3:7577471181582746310:2109]
2025-11-27T18:18:12.013686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471181582746310:2109] [node 4] ICS09 handshake done sender: [3:7577471177287778992:2111] self: [3:7577471177287778991:0] peer: [4:7577471178403377862:0] socket: 411 qp: -1
2025-11-27T18:18:12.013723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471181582746310:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:12.013768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:12.025748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471181582746314:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50732
2025-11-27T18:18:11.996402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471177972558912:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.933983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:11.995019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:11.983859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471178403377863:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20997
2025-11-27T18:18:11.881823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471177836231189:2109] [node 6] ICP22 created new session: [5:7577471177836231206:2109]
2025-11-27T18:18:12.035355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471181582746309:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50732
2025-11-27T18:18:11.942704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471176809673948:2106] [node 5] ICP22 created new session: [6:7577471176809673975:2106]
2025-11-27T18:18:12.038597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471177287779000:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:11.998473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471173236949490:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:12.018096Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471174108410488:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471178403377855:2114] held: no
2025-11-27T18:18:11.905384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471177836231206:2109] [node 6] ICS09 handshake done sender: [5:7577471177836231194:2112] self: [5:7577471177836231193:0] peer: [6:7577471176809673968:0] socket: 393 qp: -1
2025-11-27T18:18:12.002689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471173677591545:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:11.946080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471176809673963:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:12.039581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471181582746309:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:12.018277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471178403377855:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:11.908301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471177836231206:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:11.999174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471177531916849:2109] [node 6] ICP22 created new session: [2:7577471177531916873:2109]
2025-11-27T18:18:12.002745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:12.041280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471172992811627:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:12.033489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471174108410488:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:11.953415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471176809673975:2106] [node 5] ICS09 handshake done sender: [6:7577471176809673960:2113] self: [6:7577471176809673968:0] peer: [5:7577471177836231193:0] socket: 396 qp: -1
2025-11-27T18:18:12.009252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:11.910834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:12.002796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471173677591545:2048] [node 2] ICP22 created new session: [1:7577471182267526229:2048]
2025-11-27T18:18:12.042397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:12.033559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:11.959526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471176809673959:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:11.962965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471176809673975:2106] [node 5] ICS10 traffic start
2025-11-27T18:18:11.963021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:11.965970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471176809673889:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:12.002838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471182267526229:2048] [node 2] ICS09 handshake done sender: [1:7577471177972558916:2115] self: [1:7577471177972558915:0] peer: [2:7577471177531916867:0] socket: 399 qp: -1
2025-11-27T18:18:12.009338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471173236949490:2049] [node 3] ICP22 created new session: [2:7577471181826884170:2049]
2025-11-27T18:18:11.995138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471177836231209:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49228
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:12.033619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471174108410488:2050] [node 3] ICP22 created new session: [4:7577471182698345162:2050]
2025-11-27T18:18:12.044387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471172992811628:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(81229093ns) {HandleHandshakeStatus:329(81225369ns) {DropIncomingHandshake:442(512ns) DropOutgoingHandshake:457(6082ns) SwitchToState:179(1099ns) ProcessPendingSessionEvents:495(2738621ns) {ForwardSessionEventToSession:612(2684949ns) ForwardSessionEventToSession:612(8538ns) ForwardSessionEventToSession:612(2060ns) ForwardSessionEventToSession:612(2919ns) ForwardSessionEventToSession:612(5477ns) ForwardSessionEventToSession:612(5642ns) ForwardSessionEventToSession:612(11515ns) ForwardSessionEventToSession:612(2630ns) ForwardSessionEventToSession:612(1827ns) ForwardSessionEventToSession:612(4623ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:12.013294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471177531916873:2109] [node 6] ICS09 handshake done sender: [2:7577471177531916853:2111] self: [2:7577471177531916852:0] peer: [6:7577471176809673972:0] socket: 389 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:11.966036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:12.002868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471182267526229:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:12.049599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471177287779004:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.018567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471177836231189:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(119659385ns) {HandleHandshakeStatus:329(119656827ns) {DropIncomingHandshake:442(260ns) DropOutgoingHandshake:457(460ns) SwitchToState:179(136ns) ProcessPendingSessionEvents:495(18043968ns) {ForwardSessionEventToSession:612(12457487ns) ForwardSessionEventToSession:612(15516ns) ForwardSessionEventToSession:612(7326ns) ForwardSessionEventToSession:612(2645ns) ForwardSessionEventToSession:612(1802448ns) ForwardSessionEventToSession:612(3698ns) ForwardSessionEventToSession:612(3294ns) ForwardSessionEventToSession:612(2328654ns) ForwardSessionEventToSession:612(4035ns) ForwardSessionEventToSession:612(1396227ns)}}}
2025-11-27T18:18:12.002909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:12.017388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471177531916873:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:11.966084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471176809673889:2049] [node 2] ICP22 created new session: [6:7577471176809673977:2049]
2025-11-27T18:18:12.033663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471182698345162:2050] [node 3] ICS09 handshake done sender: [4:7577471178403377855:2114] self: [4:7577471178403377862:0] peer: [3:7577471177287778991:0] socket: 412 qp: -1
2025-11-27T18:18:12.027421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577471177836231189:2109] [node 6] ICP18 handshake (actor [5:7577471177836231202:2115]) is too old
2025-11-27T18:18:12.049634Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471177287779004:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:12.033693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471182698345162:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:12.004165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471177972558931:2109] [node 6] ICS09 handshake done sender: [1:7577471177972558910:2112] self: [1:7577471177972558909:0] peer: [6:7577471176809673967:0] socket: 398 qp: -1
2025-11-27T18:18:12.052058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471177836231209:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.009493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471177972558931:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:12.017435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:11.966136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471176809673977:2049] [node 2] ICS09 handshake done sender: [6:7577471176809673959:2112] self: [6:7577471176809673972:0] peer: [2:7577471177531916852:0] socket: 395 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:12.044217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471172992811627:2048] [node 1] ICP22 created new session: [3:7577471181582746316:2048]
2025-11-27T18:18:12.054074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471173541263829:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:12.033722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:11.966185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471176809673977:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:12.056584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471182131198508:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49228
2025-11-27T18:18:12.010132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:12.032307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471181826884170:2049] [node 3] ICS09 handshake done sender: [2:7577471177531916858:2112] self: [2:7577471177531916857:0] peer: [3:7577471177287778999:0] socket: 404 qp: -1
2025-11-27T18:18:12.046546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471182698345166:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20997
2025-11-27T18:18:11.973023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471176809673888:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:12.035668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471181826884170:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:12.054163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471177287779004:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:12.057180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471182698345166:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.056717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471182131198509:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49228
2025-11-27T18:18:11.975075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:12.022691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471177972558929:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19539
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:12.060567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471182131198508:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:12.054414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471177287778986:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:12.060650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471177836231209:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:12.037431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:12.059285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471174108410487:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:12.054481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:12.035248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471177972558912:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:12.067214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471173541263829:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:11.986129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:12.038094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471177531916864:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65073
2025-11-27T18:18:12.066294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471178403377860:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:11.997675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471176809673948:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(88668945ns) {HandleHandshakeStatus:329(86544061ns) {DropIncomingHandshake:442(495ns) DropOutgoingHandshake:457(9365101ns) SwitchToState:179(1322ns) ProcessPendingSessionEvents:495(9558451ns) {ForwardSessionEventToSession:612(9505282ns) ForwardSessionEventToSession:612(5140ns) ForwardSessionEventToSession:612(3049ns) ForwardSessionEventToSession:612(8863ns) ForwardSessionEventToSession:612(5581ns) ForwardSessionEventToSession:612(2455ns) ForwardSessionEventToSession:612(4859ns) ForwardSessionEventToSession:612(2734ns) ForwardSessionEventToSession:612(5656ns) ForwardSessionEventToSession:612(2827ns)}}}
2025-11-27T18:18:12.067288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:12.054530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471177287778986:2107] [node 6] ICP22 created new session: [3:7577471181582746319:2107]
2025-11-27T18:18:12.037890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471177972558906:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(74102822ns) {HandleHandshakeStatus:329(74100583ns) {DropIncomingHandshake:442(15375ns) DropOutgoingHandshake:457(198ns) SwitchToState:179(258ns) ProcessPendingSessionEvents:495(949495ns) {ForwardSessionEventToSession:612(918100ns) ForwardSessionEventToSession:612(2557ns) ForwardSessionEventToSession:612(6529ns) ForwardSessionEventToSession:612(2196ns) ForwardSessionEventToSession:612(3306ns) ForwardSessionEventToSession:612(1555ns) ForwardSessionEventToSession:612(1537ns) ForwardSessionEventToSession:612(4909ns) ForwardSessionEventToSession:612(1409ns) ForwardSessionEventToSession:612(2091ns)}}}
2025-11-27T18:18:12.066313Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471178403377860:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:12.042330Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471177531916849:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(26696606ns) {HandleHandshakeStatus:329(26692825ns) {DropIncomingHandshake:442(786ns) DropOutgoingHandshake:457(1281ns) SwitchToState:179(632ns) ProcessPendingSessionEvents:495(3791942ns) {ForwardSessionEventToSession:612(50370ns) ForwardSessionEventToSession:612(3689627ns) ForwardSessionEventToSession:612(4597ns) ForwardSessionEventToSession:612(7257ns) ForwardSessionEventToSession:612(5512ns) ForwardSessionEventToSession:612(3028ns) ForwardSessionEventToSession:612(8650ns) ForwardSessionEventToSession:612(2375ns) ForwardSessionEventToSession:612(2601ns) ForwardSessionEventToSession:612(5028ns)}}}
2025-11-27T18:18:12.054574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471181582746319:2107] [node 6] ICS09 handshake done sender: [3:7577471177287779004:2117] self: [3:7577471177287779003:0] peer: [6:7577471181104641279:0] socket: 425 qp: -1
2025-11-27T18:18:12.038940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471173677591546:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:12.069178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471178403377860:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:11.998514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471176809673888:2048] [node 1] ICP22 created new session: [6:7577471176809673980:2048]
2025-11-27T18:18:12.067334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471173541263829:2048] [node 1] ICP22 created new session: [5:7577471182131198510:2048]
2025-11-27T18:18:12.045497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471173236949489:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21324) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471177531916860:2113] held: no
2025-11-27T18:18:12.054606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471181582746319:2107] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:12.038983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:12.001739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471176809673889:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(35558977ns) {HandleHandshakeStatus:329(35556491ns) {DropIncomingHandshake:442(488ns) DropOutgoingHandshake:457(5583ns) SwitchToState:179(367ns) ProcessPendingSessionEvents:495(6440431ns) {ForwardSessionEventToSession:612(6387074ns) ForwardSessionEventToSession:612(8170ns) ForwardSessionEventToSession:612(3131ns) ForwardSessionEventToSession:612(3500ns) ForwardSessionEventToSession:612(5711ns) ForwardSessionEventToSession:612(2483ns) ForwardSessionEventToSession:612(5390ns) ForwardSessionEventToSession:612(2188ns) ForwardSessionEventToSession:612(4929ns) ForwardSessionEventToSession:612(5499ns)}}}
2025-11-27T18:18:12.054658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:12.059197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471181582746316:2048] [node 1] ICS09 handshake done sender: [3:7577471177287779000:2114] self: [3:7577471177287779009:0] peer: [1:7577471177972558911:0] socket: 420 qp: -1
2025-11-27T18:18:12.060334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471181582746316:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:12.061941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:12.095376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471177287779010:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.095416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471177287779010:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:12.100719Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471177287779010:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:12.102195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471172992811627:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(59128679ns) {HandleHandshakeStatus:329(57494709ns) {DropIncomingHandshake:442(453ns) DropOutgoingHandshake:457(4770ns) SwitchToState:179(627ns) ProcessPendingSessionEvents:495(7897395ns) {ForwardSessionEventToSession:612(3342566ns) ForwardSessionEventToSession:612(11397ns) ForwardSessionEventToSession:612(3470ns) ForwardSessionEventToSession:612(2552ns) ForwardSessionEventToSession:612(637507ns) ForwardSessionEventToSession:612(2575ns) ForwardSessionEventToSession:612(2211466ns) ForwardSessionEventToSession:612(21164ns) ForwardSessionEventToSession:612(2545ns) ForwardSessionEventToSession:612(1637772ns)}}}
2025-11-27T18:18:12.104565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471177287778987:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:12.104922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:12.104980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471177287778987:2108] [node 5] ICP22 created new session: [3:7577471181582746323:2108]
2025-11-27T18:18:12.109794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471181582746323:2108] [node 5] ICS09 handshake done sender: [3:7577471177287779010:2119] self: [3:7577471177287779006:0] peer: [5:7577471182131198515:0] socket: 442 qp: -1
2025-11-27T18:18:12.110104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471181582746323:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:12.111888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:12.069463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471178403377846:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:12.045651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471177531916872:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65073
2025-11-27T18:18:12.046233Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471177531916860:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:12.046779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471173236949489:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:12.046819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:12.046861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471173236949489:2048] [node 1] ICP22 created new session: [2:7577471181826884176:2048]
2025-11-27T18:18:12.046901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471181826884176:2048] [node 1] ICS09 handshake done sender: [2:7577471177531916860:2113] self: [2:7577471177531916867:0] peer: [1:7577471177972558915:0] socket: 403 qp: -1
2025-11-27T18:18:12.046928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471181826884176:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:12.046958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:12.058574Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471173236949490:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(58313130ns) {HandleHandshakeStatus:329(58310715ns) {DropIncomingHandshake:442(1461ns) DropOutgoingHandshake:457(465ns) SwitchToState:179(490ns) ProcessPendingSessionEvents:495(9085902ns) {ForwardSessionEventToSession:612(8117698ns) ForwardSessionEventToSession:612(10163ns) ForwardSessionEventToSession:612(3145ns) ForwardSessionEventToSession:612(5874ns) ForwardSessionEventToSession:612(5923ns) ForwardSessionEventToSession:612(2602ns) ForwardSessionEventToSession:612(9313ns) ForwardSessionEventToSession:612(2231ns) ForwardSessionEventToSession:612(2359ns) ForwardSessionEventToSession:612(910071ns)}}}
2025-11-27T18:18:12.058802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471173236949490:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:12.061832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471177531916863:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.061858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471177531916863:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:12.070409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471177531916866:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.070433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471177531916866:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:12.091828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471177531916863:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:12.091980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471177531916866:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:12.099558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471177531916848:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:12.099610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:12.099649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471177531916848:2108] [node 4] ICP22 created new session: [2:7577471181826884180:2108]
2025-11-27T18:18:12.100728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471181826884180:2108] [node 4] ICS09 handshake done sender: [2:7577471177531916863:2115] self: [2:7577471177531916862:0] peer: [4:7577471182698345167:0] socket: 388 qp: -1
2025-11-27T18:18:12.100923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471177531916856:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:12.101566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471181826884180:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:12.101615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:12.102716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:12.104962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471177531916856:2107] [node 5] ICP22 created new session: [2:7577471181826884182:2107]
2025-11-27T18:18:12.106166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471181826884182:2107] [node 5] ICS09 handshake done sender: [2:7577471177531916866:2117] self: [2:7577471177531916865:0] peer: [5:7577471182131198512:0] socket: 445 qp: -1
2025-11-27T18:18:12.110637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471181826884182:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:12.110681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:12.015365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471176809673980:2048] [node 1] ICS09 handshake done sender: [6:7577471176809673963:2115] self: [6:7577471176809673967:0] peer: [1:7577471177972558909:0] socket: 400 qp: -1
2025-11-27T18:18:12.067375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471182131198510:2048] [node 1] ICS09 handshake done sender: [5:7577471177836231209:2117] self: [5:7577471182131198507:0] peer: [1:7577471177972558922:0] socket: 439 qp: -1
2025-11-27T18:18:12.067405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471182131198510:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:12.069516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:12.039016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471173677591546:2049] [node 3] ICP22 created new session: [1:7577471182267526234:2049]
2025-11-27T18:18:12.069559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471178403377846:2110] [node 6] ICP22 created new session: [4:7577471182698345170:2110]
2025-11-27T18:18:12.015401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471176809673980:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:12.141388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471177287778987:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(26505760ns) {HandleHandshakeStatus:329(24849605ns) {DropIncomingHandshake:442(15062ns) DropOutgoingHandshake:457(262ns) SwitchToState:179(402ns) ProcessPendingSessionEvents:495(15946908ns) {ForwardSessionEventToSession:612(11980123ns) ForwardSessionEventToSession:612(3796ns) ForwardSessionEventToSession:612(1873ns) ForwardSessionEventToSession:612(3921874ns) ForwardSessionEventToSession:612(5679ns) ForwardSessionEventToSession:612(5354ns) ForwardSessionEventToSession:612(3414ns) ForwardSessionEventToSession:612(5673ns) ForwardSessionEventToSession:612(5144ns) ForwardSessionEventToSession:612(2583ns)}}}
2025-11-27T18:18:12.067453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:12.039075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471182267526234:2049] [node 3] ICS09 handshake done sender: [1:7577471177972558912:2113] self: [1:7577471177972558911:0] peer: [3:7577471177287779009:0] socket: 419 qp: -1
2025-11-27T18:18:12.069598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471182698345170:2110] [node 6] ICS09 handshake done sender: [4:7577471178403377860:2116] self: [4:7577471178403377857:0] peer: [6:7577471181104641282:0] socket: 415 qp: -1
2025-11-27T18:18:12.069624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471182698345170:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:12.069657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:12.072581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471182698345169:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20997
2025-11-27T18:18:12.089420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471182698345164:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20997
2025-11-27T18:18:12.092765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471182698345169:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.092883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471182698345164:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.093576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471174108410486:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:12.104865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471182698345175:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20997
2025-11-27T18:18:12.112664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471174108410487:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471182698345166:2120] held: no
2025-11-27T18:18:12.125331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471174108410486:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21324) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471182698345164:2119] held: no
2025-11-27T18:18:12.134541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471182698345166:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:12.141599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471174108410487:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:12.145570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471182698345176:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20997
2025-11-27T18:18:12.157352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:12.039105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471182267526234:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:12.039136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:12.041736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471177972558929:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.055407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471177972558923:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.055428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471177972558923:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:12.057651Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471177972558923:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:12.073676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471177972558904:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:12.091001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:12.091055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471177972558904:2107] [node 5] ICP22 created new session: [1:7577471182267526238:2107]
2025-11-27T18:18:12.092113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471182267526238:2107] [node 5] ICS09 handshake done sender: [1:7577471177972558923:2117] self: [1:7577471177972558922:0] peer: [5:7577471182131198507:0] socket: 438 qp: -1
2025-11-27T18:18:12.094537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471182267526238:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:12.095031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:12.098248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471177972558926:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.098275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471177972558926:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:12.101931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471177972558926:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:12.106008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471177972558905:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:12.110241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471177972558904:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(40177574ns) {HandleHandshakeStatus:329(37285782ns) {DropIncomingHandshake:442(23742ns) DropOutgoingHandshake:457(291ns) SwitchToState:179(597ns) ProcessPendingSessionEvents:495(3660853ns) {ForwardSessionEventToSession:612(1762777ns) ForwardSessionEventToSession:612(9235ns) ForwardSessionEventToSession:612(2980ns) ForwardSessionEventToSession:612(213880ns) ForwardSessionEventToSession:612(304161ns) ForwardSessionEventToSession:612(5259ns) ForwardSessionEventToSession:612(585077ns) ForwardSessionEventToSession:612(3061ns) ForwardSessionEventToSession:612(756361ns) ForwardSessionEventToSession:612(4071ns)}}}
2025-11-27T18:18:12.027223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:12.111243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:12.111303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471177972558905:2108] [node 4] ICP22 created new session: [1:7577471182267526241:2108]
2025-11-27T18:18:12.035394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471181104641278:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45156
2025-11-27T18:18:12.112947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471182267526241:2108] [node 4] ICS09 handshake done sender: [1:7577471177972558926:2118] self: [1:7577471177972558925:0] peer: [4:7577471182698345174:0] socket: 408 qp: -1
2025-11-27T18:18:12.038621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471181104641278:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.113698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471182267526241:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:12.038773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471176809673890:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:12.121450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:12.058417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471181104641280:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45156
2025-11-27T18:18:12.153280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471177972558905:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(32721306ns) {HandleHandshakeStatus:329(31846646ns) {DropIncomingHandshake:442(14341ns) DropOutgoingHandshake:457(300ns) SwitchToState:179(395ns) ProcessPendingSessionEvents:495(3340687ns) {ForwardSessionEventToSession:612(3290236ns) ForwardSessionEventToSession:612(11457ns) ForwardSessionEventToSession:612(2909ns) ForwardSessionEventToSession:612(5282ns) ForwardSessionEventToSession:612(4404ns) ForwardSessionEventToSession:612(2277ns) ForwardSessionEventToSession:612(2703ns) ForwardSessionEventToSession:612(5200ns) ForwardSessionEventToSession:612(2380ns) ForwardSessionEventToSession:612(4287ns)}}}
2025-11-27T18:18:12.060472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471176809673888:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(69388580ns) {HandleHandshakeStatus:329(69384937ns) {DropIncomingHandshake:442(675ns) DropOutgoingHandshake:457(5693ns) SwitchToState:179(463ns) ProcessPendingSessionEvents:495(4960055ns) {ForwardSessionEventToSession:612(1368208ns) ForwardSessionEventToSession:612(11136ns) ForwardSessionEventToSession:612(2946ns) ForwardSessionEventToSession:612(2029442ns) ForwardSessionEventToSession:612(2970ns) ForwardSessionEventToSession:612(3347ns) ForwardSessionEventToSession:612(743444ns) ForwardSessionEventToSession:612(7862ns) ForwardSessionEventToSession:612(2734ns) ForwardSessionEventToSession:612(774165ns)}}}
2025-11-27T18:18:12.065492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471176809673890:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471181104641278:2121] held: no
2025-11-27T18:18:12.065698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471181104641280:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.065896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471176809673951:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:12.078598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471176809673951:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1362) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471181104641280:2122] held: no
2025-11-27T18:18:12.094673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471181104641283:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45156
2025-11-27T18:18:12.095485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471181104641281:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45156
2025-11-27T18:18:12.098188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471181104641280:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:12.100352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471181104641278:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:12.103394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471176809673951:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:12.103432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:12.103571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471176809673890:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:12.103608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:12.103667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471176809673890:2050] [node 3] ICP22 created new session: [6:7577471181104641285:2050]
2025-11-27T18:18:12.103717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471181104641285:2050] [node 3] ICS09 handshake done sender: [6:7577471181104641278:2121] self: [6:7577471181104641279:0] peer: [3:7577471177287779003:0] socket: 441 qp: -1
2025-11-27T18:18:12.103746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471181104641285:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:12.103776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:12.104661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471176809673951:2108] [node 4] ICP22 created new session: [6:7577471181104641284:2108]
2025-11-27T18:18:12.109795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471181104641284:2108] [node 4] ICS09 handshake done sender: [6:7577471181104641280:2122] self: [6:7577471181104641282:0] peer: [4:7577471178403377857:0] socket: 444 qp: -1
2025-11-27T18:18:12.111938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471181104641284:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:12.111986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:12.141683Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471177531916848:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(21811338ns) {HandleHandshakeStatus:329(21805910ns) {DropIncomingHandshake:442(11862ns) DropOutgoingHandshake:457(304ns) SwitchToState:179(597ns) ProcessPendingSessionEvents:495(15238562ns) {ForwardSessionEventToSession:612(4576505ns) ForwardSessionEventToSession:612(598205ns) ForwardSessionEventToSession:612(540084ns) ForwardSessionEventToSession:612(829710ns) ForwardSessionEventToSession:612(5305ns) ForwardSessionEventToSession:612(11656ns) ForwardSessionEventToSession:612(682809ns) ForwardSessionEventToSession:612(5493ns) ForwardSessionEventToSession:612(3760ns) ForwardSessionEventToSession:612(7966302ns)}}}
2025-11-27T18:18:12.145697Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471177531916856:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(34007910ns) {HandleHandshakeStatus:329(34005871ns) {DropIncomingHandshake:442(13485ns) DropOutgoingHandshake:457(357ns) SwitchToState:179(450ns) ProcessPendingSessionEvents:495(21058969ns) {ForwardSessionEventToSession:612(8773297ns) ForwardSessionEventToSession:612(4238ns) ForwardSessionEventToSession:612(3331ns) ForwardSessionEventToSession:612(3292697ns) ForwardSessionEventToSession:612(4360840ns) ForwardSessionEventToSession:612(4385587ns) ForwardSessionEventToSession:612(4814ns) ForwardSessionEventToSession:612(3134ns) ForwardSessionEventToSession:612(205038ns) ForwardSessionEventToSession:612(3221ns)}}}
2025-11-27T18:18:12.067822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471173541263830:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:12.073356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471182131198506:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49228
2025-11-27T18:18:12.091059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471182131198506:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.092301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471173541263831:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:12.102812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471182131198514:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49228
2025-11-27T18:18:12.105244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471182131198516:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49228
2025-11-27T18:18:12.112710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471182131198514:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.113372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471173541263830:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471182131198508:2119] held: no
2025-11-27T18:18:12.113420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471182131198508:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:12.129659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471182131198517:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49228
2025-11-27T18:18:12.129887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471173541263831:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471182131198506:2118] held: no
2025-11-27T18:18:12.134571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471173541263830:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:12.138297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:12.149645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471173541263830:2049] [node 2] ICP22 created new session: [5:7577471182131198518:2049]
2025-11-27T18:18:12.158369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471182131198518:2049] [node 2] ICS09 handshake done sender: [5:7577471182131198508:2119] self: [5:7577471182131198512:0] peer: [2:7577471177531916865:0] socket: 414 qp: -1
2025-11-27T18:18:12.158414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471182131198506:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:12.158425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471182131198518:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:12.158471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:12.158521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471177836231190:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:12.158700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471173541263830:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(24027640ns) {HandleHandshakeStatus:329(24025588ns) {DropIncomingHandshake:442(831ns) DropOutgoingHandshake:457(314ns) SwitchToState:179(814ns) ProcessPendingSessionEvents:495(795ns)}}
2025-11-27T18:18:12.161050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471174108410487:2049] [node 2] ICP22 created new session: [4:7577471182698345177:2049]
2025-11-27T18:18:12.165536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471173541263831:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:12.166370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:12.166423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471173541263831:2050] [node 3] ICP22 created new session: [5:7577471182131198521:2050]
2025-11-27T18:18:12.166486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471182131198521:2050] [node 3] ICS09 handshake done sender: [5:7577471182131198506:2118] self: [5:7577471182131198515:0] peer: [3:7577471177287779006:0] socket: 443 qp: -1
2025-11-27T18:18:12.166527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471182131198521:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:12.166572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:12.167384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471178403377865:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:12.167428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471178403377865:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:12.168138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471182698345177:2049] [node 2] ICS09 handshake done sender: [4:7577471182698345166:2120] self: [4:7577471182698345167:0] peer: [2:7577471177531916862:0] socket: 416 qp: -1
2025-11-27T18:18:12.171709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471182698345164:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:12.171958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471182698345177:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:12.172038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:12.172346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471174108410486:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:12.172391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:12.172420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471174108410486:2048] [node 1] ICP22 created new session: [4:7577471182698345179:2048]
2025-11-27T18:18:12.172450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471182698345179:2048] [node 1] ICS09 handshake done sender: [4:7577471182698345164:2119] self: [4:7577471182698345174:0] peer: [1:7577471177972558925:0] socket: 423 qp: -1
2025-11-27T18:18:12.172468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471182698345179:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:12.172493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:12.173463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471174108410487:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(30552187ns) {HandleHandshakeStatus:329(30550411ns) {DropIncomingHandshake:442(1373ns) DropOutgoingHandshake:457(252ns) SwitchToState:179(438ns) ProcessPendingSessionEvents:495(1169ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:12.175740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471178403377865:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:12.175471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471182131198523:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49228
2025-11-27T18:18:12.176234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471182131198514:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:12.176466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471177836231190:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:12.176531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:12.176576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471177836231190:2108] [node 4] ICP22 created new session: [5:7577471182131198524:2108]
2025-11-27T18:18:12.176617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471182131198524:2108] [node 4] ICS09 handshake done sender: [5:7577471182131198514:2121] self: [5:7577471182131198520:0] peer: [4:7577471178403377864:0] socket: 384 qp: -1
2025-11-27T18:18:12.176646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471182131198524:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:12.176680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:12.178305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471178403377850:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:12.178432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:12.178487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471178403377850:2108] [node 5] ICP22 created new session: [4:7577471182698345182:2108]
2025-11-27T18:18:12.178545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471182698345182:2108] [node 5] ICS09 handshake done sender: [4:7577471178403377865:2118] self: [4:7577471178403377864:0] peer: [5:7577471182131198520:0] socket: 453 qp: -1
2025-11-27T18:18:12.178572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471182698345182:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:12.178620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:12.216945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:12.217181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471181826884176:2048] [node 1] ICS01 socket: 403 reason# EndOfStream
2025-11-27T18:18:12.217230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471173236949489:2048] [node 1] ICP30 unregister session Session# [2:7577471181826884176:2048] VirtualId# [2:7577471177531916867:0]
2025-11-27T18:18:12.217284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471181826884176:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.217369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:12.217411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:12.218549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:12.218748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471181582746316:2048] [node 1] ICS07 socket disconnect 420 reason# EndOfStream
2025-11-27T18:18:12.218780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471181582746316:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.218851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:12.218897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471181582746316:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:12.222148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471172992811627:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21324) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:12.222216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471181582746316:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:12.226832Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:12.225327Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:12.225596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471176809673980:2048] [node 1] ICS07 socket disconnect 400 reason# EndOfStream
2025-11-27T18:18:12.225632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471176809673980:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.225715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:12.225756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471176809673980:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:12.228400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:12.228838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471182131198510:2048] [node 1] ICS01 socket: 439 reason# EndOfStream
2025-11-27T18:18:12.228871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471173541263829:2048] [node 1] ICP30 unregister session Session# [5:7577471182131198510:2048] VirtualId# [5:7577471182131198507:0]
2025-11-27T18:18:12.228903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471182131198510:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.229006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:12.229055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:12.230298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471182698345179:2048] [node 1] ICS01 socket: 423 reason# EndOfStream
2025-11-27T18:18:12.237087Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471172992811627:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21324) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:12.236432Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471176809673888:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21324) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:12.230352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471174108410486:2048] [node 1] ICP30 unregister session Session# [4:7577471182698345179:2048] VirtualId# [4:7577471182698345174:0]
2025-11-27T18:18:12.230378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471182698345179:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.230448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:12.230491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:12.238446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471181582746316:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:12.236507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471176809673980:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:12.238503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471172992811627:2048] [node 1] ICP30 unregister session Session# [3:7577471181582746316:2048] VirtualId# [3:7577471177287779009:0]
2025-11-27T18:18:12.238556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:12.238593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471172992811627:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:12.238607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:12.236537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471176809673888:2048] [node 1] ICP30 unregister session Session# [6:7577471176809673980:2048] VirtualId# [6:7577471176809673967:0]
2025-11-27T18:18:12.236600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:12.236638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471176809673888:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:12.236654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21324) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:12.249465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471176809673888:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:12.249716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471172992811627:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:12.309601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:12.309825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471177287779011:2049] [node 2] ICS01 socket: 409 reason# EndOfStream
2025-11-27T18:18:12.309865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471172992811628:2049] [node 2] ICP30 unregister session Session# [3:7577471177287779011:2049] VirtualId# [3:7577471177287778999:0]
2025-11-27T18:18:12.309900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471177287779011:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.309981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:12.310027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:12.311736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:12.311946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471182131198518:2049] [node 2] ICS01 socket: 414 reason# EndOfStream
2025-11-27T18:18:12.311980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471173541263830:2049] [node 2] ICP30 unregister session Session# [5:7577471182131198518:2049] VirtualId# [5:7577471182131198512:0]
2025-11-27T18:18:12.312008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471182131198518:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.312077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:12.312114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:12.312922Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:12.313120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471182698345177:2049] [node 2] ICS01 socket: 416 reason# EndOfStream
2025-11-27T18:18:12.313150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471174108410487:2049] [node 2] ICP30 unregister session Session# [4:7577471182698345177:2049] VirtualId# [4:7577471182698345167:0]
2025-11-27T18:18:12.313181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471182698345177:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.313246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:12.313550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:12.313919Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:12.314094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471176809673977:2049] [node 2] ICS01 socket: 395 reason# EndOfStream
2025-11-27T18:18:12.314125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471176809673889:2049] [node 2] ICP30 unregister session Session# [6:7577471176809673977:2049] VirtualId# [6:7577471176809673972:0]
2025-11-27T18:18:12.314154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471176809673977:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.314212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:12.314248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:12.371287Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:12.371522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471181104641285:2050] [node 3] ICS01 socket: 441 reason# EndOfStream
2025-11-27T18:18:12.371566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471176809673890:2050] [node 3] ICP30 unregister session Session# [6:7577471181104641285:2050] VirtualId# [6:7577471181104641279:0]
2025-11-27T18:18:12.371605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471181104641285:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.371686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:12.371725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:12.371833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:12.371999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471182131198521:2050] [node 3] ICS01 socket: 443 reason# EndOfStream
2025-11-27T18:18:12.372031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471173541263831:2050] [node 3] ICP30 unregister session Session# [5:7577471182131198521:2050] VirtualId# [5:7577471182131198515:0]
2025-11-27T18:18:12.372056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471182131198521:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.372109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:12.372141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:12.372260Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:12.372448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471182698345162:2050] [node 3] ICS01 socket: 412 reason# EndOfStream
2025-11-27T18:18:12.372475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471174108410488:2050] [node 3] ICP30 unregister session Session# [4:7577471182698345162:2050] VirtualId# [4:7577471178403377862:0]
2025-11-27T18:18:12.372495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471182698345162:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.372536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:12.372570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:12.478538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:12.478774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471182131198524:2108] [node 4] ICS01 socket: 384 reason# EndOfStream
2025-11-27T18:18:12.478808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471177836231190:2108] [node 4] ICP30 unregister session Session# [5:7577471182131198524:2108] VirtualId# [5:7577471182131198520:0]
2025-11-27T18:18:12.478840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471182131198524:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.479162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:12.479223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:12.484006Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:12.484286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471181104641284:2108] [node 4] ICS01 socket: 444 reason# EndOfStream
2025-11-27T18:18:12.484341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471176809673951:2108] [node 4] ICP30 unregister session Session# [6:7577471181104641284:2108] VirtualId# [6:7577471181104641282:0]
2025-11-27T18:18:12.484404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471181104641284:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.484536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:12.484626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:12.565432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:12.565669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471176809673975:2106] [node 5] ICS01 socket: 396 reason# EndOfStream
2025-11-27T18:18:12.565709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471176809673948:2106] [node 5] ICP30 unregister session Session# [6:7577471176809673975:2106] VirtualId# [6:7577471176809673968:0]
2025-11-27T18:18:12.565742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471176809673975:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:12.565845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:12.565887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:12.717641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471180796326934:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:12.717686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471180796326935:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:12.746693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471180991254467:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:12.746733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471180991254468:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:12.780934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471179855778682:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:12.781000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471179855778683:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:12.810332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471181541269395:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:12.810390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471181541269396:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:12.811289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471181541269397:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:12.849559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471183362706014:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:12.849607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471183362706015:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:12.849635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471183362706016:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:12.887567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471183196626798:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:12.887611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471183196626799:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:12.887641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471183196626800:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:13.022111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471187491594156:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:13.082365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471184150746041:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:13.031315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471185091294292:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:13.215692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471187657673374:2108] [node 6] ICP01 ready to work
2025-11-27T18:18:13.215211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471185091294294:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:13.215831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471184150746043:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:13.224338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471185836236757:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:13.233418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471187491594161:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:13.223419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471185286221826:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:13.245288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471187657673375:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:13.223982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471185286221827:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:13.223528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471185091294295:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:13.264706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471183196626800:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48570838ns) {Configure:104(48563638ns) {ProcessConfigured:135(1854384ns) {StartInitialHandshake:157(1850748ns) {DropHandshakes:470(2648ns) {DropIncomingHandshake:442(430ns) DropOutgoingHandshake:457(439ns)} GenerateSessionVirtualId:988(1007ns) PrepareNewSessionHandshake:477(3120ns) {SwitchToState:179(1309ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.236957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471184150746044:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.250716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471185091294292:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(219104779ns) {Configure:104(219100136ns) {ProcessConfigured:135(8528104ns) {StartInitialHandshake:157(8526523ns) {DropHandshakes:470(2071ns) {DropIncomingHandshake:442(514ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(1192ns) PrepareNewSessionHandshake:477(2023ns) {SwitchToState:179(136ns)}}}}}
2025-11-27T18:18:13.253370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471185836236758:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.229794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471185286221828:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:13.290697Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471183362706016:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(77158389ns) {Configure:104(77149358ns) {ProcessConfigured:135(52430ns) {StartInitialHandshake:157(50667ns) {DropHandshakes:470(2609ns) {DropIncomingHandshake:442(564ns) DropOutgoingHandshake:457(305ns)} GenerateSessionVirtualId:988(1012ns) PrepareNewSessionHandshake:477(1412ns) {SwitchToState:179(637ns)}}}}}
2025-11-27T18:18:13.281227Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471179855778682:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(63408048ns) {Configure:104(63401972ns) {ProcessConfigured:135(59519ns) {StartInitialHandshake:157(57102ns) {DropHandshakes:470(2067ns) {DropIncomingHandshake:442(581ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(1027ns) PrepareNewSessionHandshake:477(1478ns) {SwitchToState:179(549ns)}}}}}
2025-11-27T18:18:13.291212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471187657673376:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10551
2025-11-27T18:18:13.279926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471185836236762:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39248
2025-11-27T18:18:13.292778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471185286221826:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(36126449ns) {Configure:104(36121638ns) {ProcessConfigured:135(37559ns) {StartInitialHandshake:157(35298ns) {DropHandshakes:470(934ns) {DropIncomingHandshake:442(277ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(368ns) PrepareNewSessionHandshake:477(556ns) {SwitchToState:179(279ns)}}}}}
2025-11-27T18:18:13.293737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471180991254467:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(251471205ns) {Configure:104(251462912ns) {ProcessConfigured:135(18765869ns) {StartInitialHandshake:157(11847168ns) {DropHandshakes:470(2544ns) {DropIncomingHandshake:442(97ns) DropOutgoingHandshake:457(169ns)} GenerateSessionVirtualId:988(1127ns) PrepareNewSessionHandshake:477(2703ns) {SwitchToState:179(1238ns)}}}}}
2025-11-27T18:18:13.273744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471185091294298:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60252
2025-11-27T18:18:13.285577Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471183196626798:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(120434435ns) {Configure:104(120429190ns) {ProcessConfigured:135(10754065ns) {StartInitialHandshake:157(10749887ns) {DropHandshakes:470(2644ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(367ns)} GenerateSessionVirtualId:988(944ns) PrepareNewSessionHandshake:477(4106ns) {SwitchToState:179(2076ns)}}}}}
2025-11-27T18:18:13.302109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471183196626799:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(135272114ns) {Configure:104(135266823ns) {ProcessConfigured:135(75484ns) {StartInitialHandshake:157(73199ns) {DropHandshakes:470(2317ns) {DropIncomingHandshake:442(816ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(1093ns) PrepareNewSessionHandshake:477(1508ns) {SwitchToState:179(496ns)}}}}}
2025-11-27T18:18:13.277025Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471180796326934:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(228256746ns) {Configure:104(228059517ns) {ProcessConfigured:135(25901221ns) {StartInitialHandshake:157(25898464ns) {DropHandshakes:470(2547ns) {DropIncomingHandshake:442(798ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(694ns) PrepareNewSessionHandshake:477(2255ns) {SwitchToState:179(1147ns)}}}}}
2025-11-27T18:18:13.291305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471187657673379:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10551
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.331977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471187657673379:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.294232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471179855778683:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(98510652ns) {Configure:104(98504656ns) {ProcessConfigured:135(8113305ns) {StartInitialHandshake:157(8110901ns) {DropHandshakes:470(1811ns) {DropIncomingHandshake:442(440ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(263ns) PrepareNewSessionHandshake:477(1943ns) {SwitchToState:179(1249ns)}}}}}
2025-11-27T18:18:13.301997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471185091294298:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.325742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471180991254468:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(114986866ns) {Configure:104(114980007ns) {ProcessConfigured:135(62810ns) {StartInitialHandshake:157(60096ns) {DropHandshakes:470(2244ns) {DropIncomingHandshake:442(562ns) DropOutgoingHandshake:457(343ns)} GenerateSessionVirtualId:988(1635ns) PrepareNewSessionHandshake:477(2032ns) {SwitchToState:179(593ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.333695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471184150746041:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(203346523ns) {Configure:104(203341174ns) {ProcessConfigured:135(12700947ns) {StartInitialHandshake:157(12698943ns) {DropHandshakes:470(1686ns) {DropIncomingHandshake:442(403ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(525ns) PrepareNewSessionHandshake:477(3137ns) {SwitchToState:179(1522ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.332044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471185091294304:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60252
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.353101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471185286221838:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35171
2025-11-27T18:18:13.353248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471185286221839:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35171
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.364464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471187491594174:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18978
2025-11-27T18:18:13.340014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471184150746054:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7180
2025-11-27T18:18:13.346609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471180796326935:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(260995244ns) {Configure:104(260988505ns) {ProcessConfigured:135(8439317ns) {StartInitialHandshake:157(8436794ns) {DropHandshakes:470(2072ns) {DropIncomingHandshake:442(517ns) DropOutgoingHandshake:457(292ns)} GenerateSessionVirtualId:988(915ns) PrepareNewSessionHandshake:477(3470ns) {SwitchToState:179(1457ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.348365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471187657673375:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(72038320ns) {Configure:104(72032591ns) {ProcessConfigured:135(11999464ns) {StartInitialHandshake:157(11996528ns) {DropHandshakes:470(2248ns) {DropIncomingHandshake:442(566ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(1210ns) PrepareNewSessionHandshake:477(2418ns) {SwitchToState:179(1130ns)}}}}}
2025-11-27T18:18:13.362431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471187657673374:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(108825951ns) {Configure:104(108820711ns) {ProcessConfigured:135(102004ns) {StartInitialHandshake:157(99335ns) {DropHandshakes:470(1669ns) {DropIncomingHandshake:442(158ns) DropOutgoingHandshake:457(464ns)} GenerateSessionVirtualId:988(416ns) PrepareNewSessionHandshake:477(1030ns) {SwitchToState:179(321ns)}}}}}
2025-11-27T18:18:13.363169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471187657673387:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10551
2025-11-27T18:18:13.367810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471187657673376:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.348252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471185836236762:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.355081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471185091294295:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(113183750ns) {Configure:104(113176195ns) {ProcessConfigured:135(11536648ns) {StartInitialHandshake:157(11534432ns) {DropHandshakes:470(4920987ns) {DropIncomingHandshake:442(2941ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(2199ns) PrepareNewSessionHandshake:477(3202ns) {SwitchToState:179(1802ns)}}}}}
2025-11-27T18:18:13.387278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471187657673387:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.355563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471184150746056:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7180
2025-11-27T18:18:13.374248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471181541269396:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(153987086ns) {Configure:104(153980496ns) {ProcessConfigured:135(8531225ns) {StartInitialHandshake:157(8526769ns) {DropHandshakes:470(1847ns) {DropIncomingHandshake:442(368ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(1634ns) PrepareNewSessionHandshake:477(2390ns) {SwitchToState:179(1302ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.374715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471185286221839:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.386649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471187491594169:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18978
No RDMA context found for GID: 2025-11-27T18:18:13.365415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471184150746056:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:13.383507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471181541269397:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(162780175ns) {Configure:104(162774289ns) {ProcessConfigured:135(10649278ns) {StartInitialHandshake:157(10645097ns) {DropHandshakes:470(2226ns) {DropIncomingHandshake:442(597ns) DropOutgoingHandshake:457(353ns)} GenerateSessionVirtualId:988(1999ns) PrepareNewSessionHandshake:477(3199ns) {SwitchToState:179(1954ns)}}}}}
2025-11-27T18:18:13.391238Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471183362706014:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(127676464ns) {Configure:104(127672576ns) {ProcessConfigured:135(8461560ns) {StartInitialHandshake:157(8458982ns) {DropHandshakes:470(2548ns) {DropIncomingHandshake:442(835ns) DropOutgoingHandshake:457(600ns)} GenerateSessionVirtualId:988(1570ns) PrepareNewSessionHandshake:477(3572ns) {SwitchToState:179(2050ns)}}}}}
2025-11-27T18:18:13.373897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471185091294311:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60252
2025-11-27T18:18:13.405895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471185286221838:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.449394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471185286221835:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.449439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471185286221835:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.449899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471185286221830:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.449917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471185286221830:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.450363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471185286221837:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35171
2025-11-27T18:18:13.450681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471185286221835:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.378890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471184150746054:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.409910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471183362706014:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.405907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471187491594169:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.379725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471185091294306:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.379750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471185091294306:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.380988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471185091294306:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:13.381219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471180796326935:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.381819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471185091294304:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.386360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:13.391393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471185091294303:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60252
2025-11-27T18:18:13.392520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471180796326935:2049] [node 3] ICP22 created new session: [1:7577471185091294314:2049]
2025-11-27T18:18:13.405461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471185091294311:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.411190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471185091294300:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.411221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471185091294300:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.412611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471185091294300:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:13.412639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471185091294303:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.412741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471185091294314:2049] [node 3] ICS09 handshake done sender: [1:7577471185091294306:2115] self: [1:7577471185091294305:0] peer: [3:7577471184150746057:0] socket: 422 qp: -1
2025-11-27T18:18:13.421576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471185091294294:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(171452675ns) {Configure:104(171447040ns) {ProcessConfigured:135(15054250ns) {StartInitialHandshake:157(15051630ns) {DropHandshakes:470(1399ns) {DropIncomingHandshake:442(218ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:477(6934ns) {SwitchToState:179(2276ns)}}}}}
2025-11-27T18:18:13.424204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471185091294314:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:13.433557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:13.441870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471185091294292:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.441935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:13.441996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471185091294292:2108] [node 5] ICP22 created new session: [1:7577471185091294321:2108]
2025-11-27T18:18:13.442039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471185091294321:2108] [node 5] ICS09 handshake done sender: [1:7577471185091294300:2112] self: [1:7577471185091294297:0] peer: [5:7577471187657673391:0] socket: 399 qp: -1
2025-11-27T18:18:13.449394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471185091294321:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:13.449452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.430312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471185836236760:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.406035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471187491594174:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.406173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471187491594161:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.424457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471187491594156:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.426370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471187491594172:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18978
2025-11-27T18:18:13.441350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471187491594172:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.449391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471183196626800:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.457825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471187491594179:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18978
2025-11-27T18:18:13.457881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471187491594180:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18978
2025-11-27T18:18:13.458136Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471187491594156:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(18280170ns) {IncomingHandshake:261(18278998ns) {DropIncomingHandshake:442(778ns) PrepareNewSessionHandshake:477(969ns) {SwitchToState:179(318ns)} GenerateSessionVirtualId:988(436ns) IssueIncomingHandshakeReply:193(60150ns) {SwitchToState:179(499ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.430347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471185836236760:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.378990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471179855778682:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.489635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471180796326935:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(108223068ns) {HandleHandshakeStatus:329(108220189ns) {DropIncomingHandshake:442(759ns) DropOutgoingHandshake:457(568ns) SwitchToState:179(3336ns) ProcessPendingSessionEvents:495(12464424ns) {ForwardSessionEventToSession:612(12402629ns) ForwardSessionEventToSession:612(6873ns) ForwardSessionEventToSession:612(9871ns) ForwardSessionEventToSession:612(3292ns) ForwardSessionEventToSession:612(2711ns) ForwardSessionEventToSession:612(5319ns) ForwardSessionEventToSession:612(2665ns) ForwardSessionEventToSession:612(2693ns) ForwardSessionEventToSession:612(11137ns) ForwardSessionEventToSession:612(2767ns)}}}
2025-11-27T18:18:13.489726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471180796326935:2049] [node 3] ICP18 handshake (actor [1:7577471185091294304:2114]) is too old
2025-11-27T18:18:13.412301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471187657673393:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10551
2025-11-27T18:18:13.384344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471184150746055:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7180
2025-11-27T18:18:13.413706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471184150746055:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.425477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471179855778683:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.451074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471184150746048:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.451110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471184150746048:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.461719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471184150746043:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(224346749ns) {Configure:104(224340509ns) {ProcessConfigured:135(23747084ns) {StartInitialHandshake:157(23744196ns) {DropHandshakes:470(1836ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(773ns) PrepareNewSessionHandshake:477(3364ns) {SwitchToState:179(1991ns)}}}}}
2025-11-27T18:18:13.462081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471184150746048:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:13.467238Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471184150746044:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(201281391ns) {Configure:104(201275415ns) {ProcessConfigured:135(21402567ns) {StartInitialHandshake:157(21400035ns) {DropHandshakes:470(2341ns) {DropIncomingHandshake:442(344ns) DropOutgoingHandshake:457(365ns)} GenerateSessionVirtualId:988(717ns) PrepareNewSessionHandshake:477(2295ns) {SwitchToState:179(1230ns)}}}}}
2025-11-27T18:18:13.477848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471185286221828:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(232480502ns) {Configure:104(232474474ns) {ProcessConfigured:135(19541541ns) {StartInitialHandshake:157(19539103ns) {DropHandshakes:470(2781ns) {DropIncomingHandshake:442(561ns) DropOutgoingHandshake:457(670ns)} GenerateSessionVirtualId:988(593ns) PrepareNewSessionHandshake:477(3161ns) {SwitchToState:179(763ns)}}}}}
2025-11-27T18:18:13.493814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471185836236760:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:13.505305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471187491594177:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18978
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.421978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471183362706015:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(198141009ns) {Configure:104(198135626ns) {ProcessConfigured:135(15324687ns) {StartInitialHandshake:157(15322713ns) {DropHandshakes:470(1472ns) {DropIncomingHandshake:442(630ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(1760ns) PrepareNewSessionHandshake:477(2696ns) {SwitchToState:179(1495ns)}}}}}
2025-11-27T18:18:13.518121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471179855778682:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23787) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471184150746056:2116] held: no
2025-11-27T18:18:13.478040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471185286221830:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.521766Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471185091294292:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(79703956ns) {HandleHandshakeStatus:329(79700441ns) {DropIncomingHandshake:442(12513ns) DropOutgoingHandshake:457(372ns) SwitchToState:179(524ns) ProcessPendingSessionEvents:495(44132060ns) {ForwardSessionEventToSession:612(35934820ns) ForwardSessionEventToSession:612(10878ns) ForwardSessionEventToSession:612(502551ns) ForwardSessionEventToSession:612(4350ns) ForwardSessionEventToSession:612(2702ns) ForwardSessionEventToSession:612(4218452ns) ForwardSessionEventToSession:612(4425ns) ForwardSessionEventToSession:612(3052ns) ForwardSessionEventToSession:612(12004ns) ForwardSessionEventToSession:612(3105ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.526954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471187491594172:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:13.527128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471187491594174:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.422463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471183362706015:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.485372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471185286221827:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(187297721ns) {Configure:104(187292298ns) {ProcessConfigured:135(19944515ns) {StartInitialHandshake:157(19941835ns) {DropHandshakes:470(2557ns) {DropIncomingHandshake:442(471ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(1358ns) PrepareNewSessionHandshake:477(2980ns) {SwitchToState:179(1753ns)}}}}}
2025-11-27T18:18:13.522830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471185286221837:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.522986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471180991254468:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.525726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471185091294299:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.501601Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471185836236758:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(190365709ns) {Configure:104(190357758ns) {ProcessConfigured:135(1473896ns) {StartInitialHandshake:157(1471557ns) {DropHandshakes:470(2469ns) {DropIncomingHandshake:442(607ns) DropOutgoingHandshake:457(391ns)} GenerateSessionVirtualId:988(1558ns) PrepareNewSessionHandshake:477(2719ns) {SwitchToState:179(1537ns)}}}}}
2025-11-27T18:18:13.527662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471187491594181:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18978
2025-11-27T18:18:13.523043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:13.523102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471180991254468:2049] [node 3] ICP22 created new session: [2:7577471185286221851:2049]
2025-11-27T18:18:13.523146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471185286221851:2049] [node 3] ICS09 handshake done sender: [2:7577471185286221835:2112] self: [2:7577471185286221834:0] peer: [3:7577471184150746059:0] socket: 416 qp: -1
2025-11-27T18:18:13.523174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471185286221851:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:13.523205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:13.523553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471185286221826:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.523612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:13.523654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471185286221826:2108] [node 5] ICP22 created new session: [2:7577471185286221853:2108]
2025-11-27T18:18:13.523695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471185286221853:2108] [node 5] ICS09 handshake done sender: [2:7577471185286221830:2110] self: [2:7577471185286221829:0] peer: [5:7577471187657673395:0] socket: 421 qp: -1
2025-11-27T18:18:13.523727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471185286221853:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:13.523752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:13.524247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471185286221841:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35171
2025-11-27T18:18:13.525118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471185286221850:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35171
2025-11-27T18:18:13.525189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471180991254467:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.509438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471185836236768:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39248
2025-11-27T18:18:13.527161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471184150746041:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.429517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471187657673386:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.528692Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471187491594156:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14121) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471187491594174:2117] held: no
2025-11-27T18:18:13.525758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471185091294299:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.527209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.564686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471185286221850:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.429545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471187657673386:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.538735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471185836236768:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.528448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471184150746058:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7180
2025-11-27T18:18:13.541384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471185091294299:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:13.534489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471183196626800:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.434178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471187657673379:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:13.547612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471185836236770:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39248
2025-11-27T18:18:13.566114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471185286221826:2108] [node 5] ICP18 handshake (actor [2:7577471185286221850:2119]) is too old
2025-11-27T18:18:13.538971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:13.564655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471185091294309:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.534248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471184150746041:2109] [node 6] ICP22 created new session: [3:7577471184150746067:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.434491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471183362706014:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.575707Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471185286221841:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.540890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471187491594169:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:13.564700Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471185091294309:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.552973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471185836236775:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39248
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.434544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:13.547504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471184150746067:2109] [node 6] ICS09 handshake done sender: [3:7577471184150746048:2112] self: [3:7577471184150746045:0] peer: [6:7577471187491594178:0] socket: 406 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.599983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471180991254467:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23787) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471185286221837:2113] held: no
2025-11-27T18:18:13.541296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471187491594156:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.553620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471185836236757:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.583305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471180796326934:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.553808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471184150746067:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:13.434584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471183362706014:2048] [node 1] ICP22 created new session: [5:7577471187657673397:2048]
2025-11-27T18:18:13.541369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471183196626800:2050] [node 3] ICP22 created new session: [6:7577471187491594182:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.553920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471181541269395:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.606231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471185286221844:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.583419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471185091294309:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:13.434632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471187657673397:2048] [node 1] ICS09 handshake done sender: [5:7577471187657673379:2113] self: [5:7577471187657673391:0] peer: [1:7577471185091294297:0] socket: 403 qp: -1
2025-11-27T18:18:13.555693Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471179855778683:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25481) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471184150746055:2115] held: no
2025-11-27T18:18:13.556236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:13.571745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471185836236770:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.606264Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471185286221844:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.545926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471187491594161:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.434666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471187657673397:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:13.584124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471185091294318:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.613238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471185286221856:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35171
2025-11-27T18:18:13.582310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471184150746056:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:13.583485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471185836236777:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39248
2025-11-27T18:18:13.545997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:13.434700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:13.584162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471185091294318:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.594096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471184150746063:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7180
2025-11-27T18:18:13.596356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:13.616704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471185286221848:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.546045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471187491594161:2110] [node 4] ICP22 created new session: [6:7577471187491594183:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:13.473447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471187657673386:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:13.597921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471185091294295:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.616743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471185286221848:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.485567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471187657673374:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.603872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471179855778683:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(19955043ns) {HandleHandshakeStatus:395(19952455ns) {LogHandshakeFail:472(12943ns) {UpdateErrorStateLog:545(4599ns)} DropOutgoingHandshake:457(2113ns) ProcessPendingSessionEvents:495(11778803ns) {DropSessionEvent:512(14325ns) DropSessionEvent:512(5265579ns) DropSessionEvent:512(5069ns) DropSessionEvent:512(4977ns) DropSessionEvent:512(5452ns) DropSessionEvent:512(8952ns) DropSessionEvent:512(6187ns) DropSessionEvent:512(4412ns) DropSessionEvent:512(4400ns) DropSessionEvent:512(6450742ns)}}}
2025-11-27T18:18:13.546089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471187491594183:2110] [node 4] ICS09 handshake done sender: [6:7577471187491594169:2114] self: [6:7577471187491594175:0] peer: [4:7577471185836236759:0] socket: 400 qp: -1
2025-11-27T18:18:13.597982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:13.596434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471185836236757:2109] [node 6] ICP22 created new session: [4:7577471185836236779:2109]
2025-11-27T18:18:13.603916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471179855778682:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.605671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:13.612692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471179855778682:2048] [node 1] ICP22 created new session: [3:7577471184150746071:2048]
2025-11-27T18:18:13.614957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471184150746064:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.614992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471184150746064:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.617224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471184150746055:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:13.617828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471184150746071:2048] [node 1] ICS09 handshake done sender: [3:7577471184150746056:2116] self: [3:7577471184150746057:0] peer: [1:7577471185091294305:0] socket: 423 qp: -1
2025-11-27T18:18:13.620095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471179855778683:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.620156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:13.620198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471179855778683:2049] [node 2] ICP22 created new session: [3:7577471184150746072:2049]
2025-11-27T18:18:13.620250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471184150746072:2049] [node 2] ICS09 handshake done sender: [3:7577471184150746055:2115] self: [3:7577471184150746059:0] peer: [2:7577471185286221834:0] socket: 418 qp: -1
2025-11-27T18:18:13.620281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471184150746072:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:13.620326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:13.624449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471184150746071:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:13.624498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:13.633415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471184150746064:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:13.646347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471184150746041:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(105955820ns) {HandleHandshakeStatus:329(105953607ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(394ns) SwitchToState:179(180ns) ProcessPendingSessionEvents:495(30857276ns) {ForwardSessionEventToSession:612(3109787ns) ForwardSessionEventToSession:612(1254565ns) ForwardSessionEventToSession:612(3692ns) ForwardSessionEventToSession:612(2650ns) ForwardSessionEventToSession:612(10410677ns) ForwardSessionEventToSession:612(2522ns) ForwardSessionEventToSession:612(724612ns) ForwardSessionEventToSession:612(4387147ns) ForwardSessionEventToSession:612(5470ns) ForwardSessionEventToSession:612(10935058ns)}}}
2025-11-27T18:18:13.647309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471184150746043:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.647897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:13.649341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471184150746043:2107] [node 4] ICP22 created new session: [3:7577471184150746077:2107]
2025-11-27T18:18:13.652603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471179855778682:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(44155960ns) {HandleHandshakeStatus:329(44154268ns) {DropIncomingHandshake:442(473ns) DropOutgoingHandshake:457(311ns) SwitchToState:179(280ns) ProcessPendingSessionEvents:495(832ns)}}
2025-11-27T18:18:13.654702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471184150746077:2107] [node 4] ICS09 handshake done sender: [3:7577471184150746064:2119] self: [3:7577471184150746060:0] peer: [4:7577471185836236782:0] socket: 410 qp: -1
2025-11-27T18:18:13.659637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471184150746077:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:13.659694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:13.666628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471184150746065:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.666656Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471184150746065:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.685610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471184150746043:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(38974162ns) {HandleHandshakeStatus:329(38971453ns) {DropIncomingHandshake:442(863ns) DropOutgoingHandshake:457(472ns) SwitchToState:179(467ns) ProcessPendingSessionEvents:495(19377123ns) {ForwardSessionEventToSession:612(10143058ns) ForwardSessionEventToSession:612(4477496ns) ForwardSessionEventToSession:612(3131ns) ForwardSessionEventToSession:612(4707811ns) ForwardSessionEventToSession:612(4438ns) ForwardSessionEventToSession:612(5984ns) ForwardSessionEventToSession:612(5537ns) ForwardSessionEventToSession:612(3161ns) ForwardSessionEventToSession:612(2183ns) ForwardSessionEventToSession:612(6121ns)}}}
2025-11-27T18:18:13.686874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471184150746075:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7180
2025-11-27T18:18:13.691308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471184150746043:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:13.617180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471185286221844:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:13.596483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471185836236779:2109] [node 6] ICS09 handshake done sender: [4:7577471185836236760:2111] self: [4:7577471185836236759:0] peer: [6:7577471187491594175:0] socket: 397 qp: -1
2025-11-27T18:18:13.546119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471187491594183:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:13.626541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471185286221848:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:13.546172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:13.546632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471187491594179:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.632933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471185286221837:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:13.547738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:13.638262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471185286221827:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.550259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471183196626798:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.640496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:13.555533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471187491594182:2050] [node 3] ICS09 handshake done sender: [6:7577471187491594172:2116] self: [6:7577471187491594178:0] peer: [3:7577471184150746045:0] socket: 411 qp: -1
2025-11-27T18:18:13.646330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471185286221827:2109] [node 6] ICP22 created new session: [2:7577471185286221862:2109]
2025-11-27T18:18:13.561313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471187491594156:2107] [node 5] ICP22 created new session: [6:7577471187491594186:2107]
2025-11-27T18:18:13.649390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471185286221862:2109] [node 6] ICS09 handshake done sender: [2:7577471185286221844:2117] self: [2:7577471185286221842:0] peer: [6:7577471187491594191:0] socket: 454 qp: -1
2025-11-27T18:18:13.561373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471187491594182:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:13.649430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471185286221862:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:13.567590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:13.577468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471187491594186:2107] [node 5] ICS09 handshake done sender: [6:7577471187491594174:2117] self: [6:7577471187491594176:0] peer: [5:7577471187657673385:0] socket: 428 qp: -1
2025-11-27T18:18:13.647826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471180991254467:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.649634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:13.579681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471187491594189:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18978
2025-11-27T18:18:13.650094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471180991254467:2048] [node 1] ICP22 created new session: [2:7577471185286221863:2048]
2025-11-27T18:18:13.583838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471187491594186:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:13.650889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:13.589969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:13.655838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471185286221828:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.600467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471187491594189:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.655897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:13.602673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471183196626799:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.655959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471185286221828:2107] [node 4] ICP22 created new session: [2:7577471185286221865:2107]
2025-11-27T18:18:13.608345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471187491594192:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18978
2025-11-27T18:18:13.656014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471185286221865:2107] [node 4] ICS09 handshake done sender: [2:7577471185286221848:2118] self: [2:7577471185286221847:0] peer: [4:7577471185836236783:0] socket: 409 qp: -1
2025-11-27T18:18:13.609230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471187491594179:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:13.656054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471185286221865:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:13.609607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471183196626798:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.656086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:13.609682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:13.609724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471183196626798:2048] [node 1] ICP22 created new session: [6:7577471187491594193:2048]
2025-11-27T18:18:13.659771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471185286221863:2048] [node 1] ICS09 handshake done sender: [2:7577471185286221837:2113] self: [2:7577471185286221855:0] peer: [1:7577471185091294296:0] socket: 415 qp: -1
2025-11-27T18:18:13.609767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471187491594193:2048] [node 1] ICS09 handshake done sender: [6:7577471187491594179:2119] self: [6:7577471187491594187:0] peer: [1:7577471185091294316:0] socket: 441 qp: -1
2025-11-27T18:18:13.661048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471185286221863:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:13.609800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471187491594193:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:13.666069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:13.609853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:13.697602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471180991254467:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(44221809ns) {HandleHandshakeStatus:329(44219097ns) {DropIncomingHandshake:442(1955ns) DropOutgoingHandshake:457(260ns) SwitchToState:179(9203ns) ProcessPendingSessionEvents:495(843ns)}}
2025-11-27T18:18:13.622249Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471187491594156:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(76401739ns) {HandleHandshakeStatus:329(76399833ns) {DropIncomingHandshake:442(3027ns) DropOutgoingHandshake:457(191ns) SwitchToState:179(365ns) ProcessPendingSessionEvents:495(883ns)}}
2025-11-27T18:18:13.698198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471185286221827:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(42847774ns) {HandleHandshakeStatus:329(42844740ns) {DropIncomingHandshake:442(14070ns) DropOutgoingHandshake:457(287ns) SwitchToState:179(457ns) ProcessPendingSessionEvents:495(20925386ns) {ForwardSessionEventToSession:612(450163ns) ForwardSessionEventToSession:612(1146102ns) ForwardSessionEventToSession:612(3622976ns) ForwardSessionEventToSession:612(817530ns) ForwardSessionEventToSession:612(3899ns) ForwardSessionEventToSession:612(2888ns) ForwardSessionEventToSession:612(5356924ns) ForwardSessionEventToSession:612(2530ns) ForwardSessionEventToSession:612(6935838ns) ForwardSessionEventToSession:612(2570768ns)}}}
2025-11-27T18:18:13.641493Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471183196626800:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(93033550ns) {HandleHandshakeStatus:329(87689498ns) {DropIncomingHandshake:442(2959ns) DropOutgoingHandshake:457(6536ns) SwitchToState:179(734ns) ProcessPendingSessionEvents:495(27145417ns) {ForwardSessionEventToSession:612(3857190ns) ForwardSessionEventToSession:612(5121ns) ForwardSessionEventToSession:612(805583ns) ForwardSessionEventToSession:612(4386ns) ForwardSessionEventToSession:612(3307ns) ForwardSessionEventToSession:612(5764946ns) ForwardSessionEventToSession:612(2114063ns) ForwardSessionEventToSession:612(7228437ns) ForwardSessionEventToSession:612(4195ns) ForwardSessionEventToSession:612(3033ns)}}}
2025-11-27T18:18:13.598025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471185091294295:2107] [node 4] ICP22 created new session: [1:7577471185091294328:2107]
2025-11-27T18:18:13.649510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471187491594195:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18978
2025-11-27T18:18:13.660305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471187491594189:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:13.598066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471185091294328:2107] [node 4] ICS09 handshake done sender: [1:7577471185091294309:2116] self: [1:7577471185091294308:0] peer: [4:7577471185836236774:0] socket: 433 qp: -1
2025-11-27T18:18:13.669612Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471183196626799:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25481) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471187491594189:2122] held: no
2025-11-27T18:18:13.702798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471184150746065:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:13.598105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471185091294328:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:13.669703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471183196626799:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.493913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471187657673400:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10551
2025-11-27T18:18:13.598129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:13.669763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:13.602197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:13.505386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:13.669807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471183196626799:2049] [node 2] ICP22 created new session: [6:7577471187491594196:2049]
2025-11-27T18:18:13.604122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471180796326934:2048] [node 2] ICP22 created new session: [1:7577471185091294327:2048]
2025-11-27T18:18:13.669847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471187491594196:2049] [node 2] ICS09 handshake done sender: [6:7577471187491594189:2122] self: [6:7577471187491594191:0] peer: [2:7577471185286221842:0] socket: 407 qp: -1
2025-11-27T18:18:13.508002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471187657673374:2108] [node 6] ICP22 created new session: [5:7577471187657673402:2108]
2025-11-27T18:18:13.605965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471185091294327:2048] [node 2] ICS09 handshake done sender: [1:7577471185091294299:2111] self: [1:7577471185091294296:0] peer: [2:7577471185286221855:0] socket: 398 qp: -1
2025-11-27T18:18:13.669889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471187491594196:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:13.509832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471187657673402:2108] [node 6] ICS09 handshake done sender: [5:7577471187657673386:2116] self: [5:7577471187657673385:0] peer: [6:7577471187491594176:0] socket: 427 qp: -1
2025-11-27T18:18:13.606005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471185091294327:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:13.669917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:13.517340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471187657673402:2108] [node 6] ICS10 traffic start
2025-11-27T18:18:13.606069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:13.534224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471187657673387:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:13.606390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471185091294318:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:13.534415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:13.569656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471183362706015:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.614817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471185091294326:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60252
2025-11-27T18:18:13.569753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:13.569800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471183362706015:2049] [node 2] ICP22 created new session: [5:7577471187657673405:2049]
2025-11-27T18:18:13.617649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471185091294294:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.569848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471187657673405:2049] [node 2] ICS09 handshake done sender: [5:7577471187657673387:2117] self: [5:7577471187657673395:0] peer: [2:7577471185286221829:0] socket: 425 qp: -1
2025-11-27T18:18:13.625361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:13.569886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471187657673405:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:13.637833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471185091294294:2109] [node 6] ICP22 created new session: [1:7577471185091294332:2109]
2025-11-27T18:18:13.569921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:13.646161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471185091294332:2109] [node 6] ICS09 handshake done sender: [1:7577471185091294318:2118] self: [1:7577471185091294316:0] peer: [6:7577471187491594187:0] socket: 429 qp: -1
2025-11-27T18:18:13.572289Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471187657673374:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(86469956ns) {HandleHandshakeStatus:329(86466402ns) {DropIncomingHandshake:442(14035ns) DropOutgoingHandshake:457(385ns) SwitchToState:179(498ns) ProcessPendingSessionEvents:495(10897531ns) {ForwardSessionEventToSession:612(10826169ns) ForwardSessionEventToSession:612(11946ns) ForwardSessionEventToSession:612(3629ns) ForwardSessionEventToSession:612(3779ns) ForwardSessionEventToSession:612(8668ns) ForwardSessionEventToSession:612(8632ns) ForwardSessionEventToSession:612(8272ns) ForwardSessionEventToSession:612(3720ns) ForwardSessionEventToSession:612(2160ns) ForwardSessionEventToSession:612(9758ns)}}}
2025-11-27T18:18:13.646221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471185091294332:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:13.629657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471187657673407:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10551
2025-11-27T18:18:13.647967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:13.630387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471187657673407:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.660533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471180796326934:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(72288238ns) {HandleHandshakeStatus:329(64700221ns) {DropIncomingHandshake:442(13958ns) DropOutgoingHandshake:457(624ns) SwitchToState:179(606ns) ProcessPendingSessionEvents:495(25185513ns) {ForwardSessionEventToSession:612(11316112ns) ForwardSessionEventToSession:612(4998961ns) ForwardSessionEventToSession:612(4203ns) ForwardSessionEventToSession:612(3049ns) ForwardSessionEventToSession:612(4112375ns) ForwardSessionEventToSession:612(3345528ns) ForwardSessionEventToSession:612(3375ns) ForwardSessionEventToSession:612(503742ns) ForwardSessionEventToSession:612(2796ns) ForwardSessionEventToSession:612(877695ns)}}}
2025-11-27T18:18:13.596524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471185836236779:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:13.596601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:13.597155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471185836236757:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(43312913ns) {HandleHandshakeStatus:329(43309918ns) {DropIncomingHandshake:442(13178ns) DropOutgoingHandshake:457(392ns) SwitchToState:179(238ns) ProcessPendingSessionEvents:495(111532ns) {ForwardSessionEventToSession:612(62906ns) ForwardSessionEventToSession:612(3669ns) ForwardSessionEventToSession:612(2820ns) ForwardSessionEventToSession:612(8602ns) ForwardSessionEventToSession:612(2785ns) ForwardSessionEventToSession:612(5957ns) ForwardSessionEventToSession:612(5269ns) ForwardSessionEventToSession:612(2719ns) ForwardSessionEventToSession:612(5497ns) ForwardSessionEventToSession:612(2427ns)}}}
2025-11-27T18:18:13.630552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471187657673375:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.601826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471185836236781:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39248
2025-11-27T18:18:13.602267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471185836236775:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.604251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471185836236768:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:13.604367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471181541269397:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-27T18:18:13.604610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471185836236781:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:13.604716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471181541269396:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.606284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471181541269395:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23787) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471185836236768:2114] held: no
2025-11-27T18:18:13.612683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471181541269395:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.617683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:13.622216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471181541269395:2048] [node 1] ICP22 created new session: [4:7577471185836236784:2048]
2025-11-27T18:18:13.706589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471184150746044:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.630954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471185836236772:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.706680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:13.630978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471185836236772:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:13.706722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471184150746044:2108] [node 5] ICP22 created new session: [3:7577471184150746080:2108]
2025-11-27T18:18:13.706767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471184150746080:2108] [node 5] ICS09 handshake done sender: [3:7577471184150746065:2120] self: [3:7577471184150746061:0] peer: [5:7577471187657673410:0] socket: 419 qp: -1
2025-11-27T18:18:13.631712Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471185836236772:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:13.706811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471184150746080:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:13.632002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471185836236758:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:13.706864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:13.632091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:13.634127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471187657673404:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10551
2025-11-27T18:18:13.633180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471185836236784:2048] [node 1] ICS09 handshake done sender: [4:7577471185836236768:2114] self: [4:7577471185836236774:0] peer: [1:7577471185091294308:0] socket: 434 qp: -1
2025-11-27T18:18:13.633278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471185836236785:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39248
2025-11-27T18:18:13.637750Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471187657673375:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20633) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471187657673407:2122] held: no
2025-11-27T18:18:13.633685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471185836236758:2108] [node 5] ICP22 created new session: [4:7577471185836236789:2108]
2025-11-27T18:18:13.649928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471187657673404:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:13.638736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471185836236784:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:13.657528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471183362706016:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:13.638780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:13.678157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471187657673409:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10551
2025-11-27T18:18:13.639270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471185836236789:2108] [node 5] ICS09 handshake done sender: [4:7577471185836236772:2117] self: [4:7577471185836236771:0] peer: [5:7577471187657673408:0] socket: 390 qp: -1
2025-11-27T18:18:13.697438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471187657673407:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:13.703318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471187657673375:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.704380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:13.707622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471187657673375:2109] [node 4] ICP22 created new session: [5:7577471187657673411:2109]
2025-11-27T18:18:13.708186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471187657673411:2109] [node 4] ICS09 handshake done sender: [5:7577471187657673407:2122] self: [5:7577471187657673408:0] peer: [4:7577471185836236771:0] socket: 462 qp: -1
2025-11-27T18:18:13.708234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471187657673411:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:13.708771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:13.708907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471187657673412:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10551
2025-11-27T18:18:13.698004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471185091294294:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(63281867ns) {HandleHandshakeStatus:329(63277654ns) {DropIncomingHandshake:442(419ns) DropOutgoingHandshake:457(650ns) SwitchToState:179(736ns) ProcessPendingSessionEvents:495(21192420ns) {ForwardSessionEventToSession:612(6628106ns) ForwardSessionEventToSession:612(4602ns) ForwardSessionEventToSession:612(2768ns) ForwardSessionEventToSession:612(714404ns) ForwardSessionEventToSession:612(2485ns) ForwardSessionEventToSession:612(11106495ns) ForwardSessionEventToSession:612(8274ns) ForwardSessionEventToSession:612(3096ns) ForwardSessionEventToSession:612(2698766ns) ForwardSessionEventToSession:612(4175ns)}}}
2025-11-27T18:18:13.698712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471185091294294:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:13.711009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471187657673404:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:13.646490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471185836236789:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:13.711409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471183362706016:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.711588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:13.711697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471183362706016:2050] [node 3] ICP22 created new session: [5:7577471187657673414:2050]
2025-11-27T18:18:13.647934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:13.711740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471187657673414:2050] [node 3] ICS09 handshake done sender: [5:7577471187657673404:2121] self: [5:7577471187657673410:0] peer: [3:7577471184150746061:0] socket: 450 qp: -1
2025-11-27T18:18:13.648376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471185836236786:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39248
2025-11-27T18:18:13.711771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471187657673414:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:13.649805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471185836236775:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:13.655028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471185836236781:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:13.711809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:13.658155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471181541269396:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.659955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471181541269395:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(35214132ns) {HandleHandshakeStatus:329(35212341ns) {DropIncomingHandshake:442(397ns) DropOutgoingHandshake:457(287ns) SwitchToState:179(434ns) ProcessPendingSessionEvents:495(844ns)}}
2025-11-27T18:18:13.662198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:13.662251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471181541269396:2049] [node 2] ICP22 created new session: [4:7577471185836236792:2049]
2025-11-27T18:18:13.662293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471185836236792:2049] [node 2] ICS09 handshake done sender: [4:7577471185836236781:2120] self: [4:7577471185836236783:0] peer: [2:7577471185286221847:0] socket: 405 qp: -1
2025-11-27T18:18:13.662324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471185836236792:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:13.662353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:13.662883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471181541269397:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:13.662957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:13.662996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471181541269397:2050] [node 3] ICP22 created new session: [4:7577471185836236794:2050]
2025-11-27T18:18:13.663041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471185836236794:2050] [node 3] ICS09 handshake done sender: [4:7577471185836236775:2118] self: [4:7577471185836236782:0] peer: [3:7577471184150746060:0] socket: 453 qp: -1
2025-11-27T18:18:13.663070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471185836236794:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:13.713215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471183362706016:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:13.663094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:13.703589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471185836236758:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(39433401ns) {HandleHandshakeStatus:329(39425482ns) {DropIncomingHandshake:442(13040ns) DropOutgoingHandshake:457(395ns) SwitchToState:179(380ns) ProcessPendingSessionEvents:495(15839595ns) {ForwardSessionEventToSession:612(5757447ns) ForwardSessionEventToSession:612(5368ns) ForwardSessionEventToSession:612(2538ns) ForwardSessionEventToSession:612(3550731ns) ForwardSessionEventToSession:612(4497ns) ForwardSessionEventToSession:612(1262427ns) ForwardSessionEventToSession:612(961802ns) ForwardSessionEventToSession:612(2718ns) ForwardSessionEventToSession:612(4246656ns) ForwardSessionEventToSession:612(4016ns)}}}
2025-11-27T18:18:13.749538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:13.749823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471185286221863:2048] [node 1] ICS01 socket: 415 reason# EndOfStream
2025-11-27T18:18:13.749868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471180991254467:2048] [node 1] ICP30 unregister session Session# [2:7577471185286221863:2048] VirtualId# [2:7577471185286221855:0]
2025-11-27T18:18:13.749918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471185286221863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.750000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:13.750333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:13.750043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:13.750538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471184150746071:2048] [node 1] ICS01 socket: 423 reason# EndOfStream
2025-11-27T18:18:13.750579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471179855778682:2048] [node 1] ICP30 unregister session Session# [3:7577471184150746071:2048] VirtualId# [3:7577471184150746057:0]
2025-11-27T18:18:13.750601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471184150746071:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.750671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:13.750704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:13.752212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:13.752410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471185836236784:2048] [node 1] ICS01 socket: 434 reason# EndOfStream
2025-11-27T18:18:13.752450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471181541269395:2048] [node 1] ICP30 unregister session Session# [4:7577471185836236784:2048] VirtualId# [4:7577471185836236774:0]
2025-11-27T18:18:13.752481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471185836236784:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.752553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:13.753238Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:13.753464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471187657673397:2048] [node 1] ICS01 socket: 403 reason# EndOfStream
2025-11-27T18:18:13.753504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471183362706014:2048] [node 1] ICP30 unregister session Session# [5:7577471187657673397:2048] VirtualId# [5:7577471187657673391:0]
2025-11-27T18:18:13.753533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471187657673397:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.753593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:13.753626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:13.752603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:13.754850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:13.755053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471187491594193:2048] [node 1] ICS01 socket: 441 reason# EndOfStream
2025-11-27T18:18:13.755089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471183196626798:2048] [node 1] ICP30 unregister session Session# [6:7577471187491594193:2048] VirtualId# [6:7577471187491594187:0]
2025-11-27T18:18:13.755116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471187491594193:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.755177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:13.755244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:13.814021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:13.814237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471184150746072:2049] [node 2] ICS01 socket: 418 reason# EndOfStream
2025-11-27T18:18:13.814272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471179855778683:2049] [node 2] ICP30 unregister session Session# [3:7577471184150746072:2049] VirtualId# [3:7577471184150746059:0]
2025-11-27T18:18:13.814295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471184150746072:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.814373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:13.814481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:13.817863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:13.819344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:13.820416Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:13.819572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471187657673405:2049] [node 2] ICS01 socket: 425 reason# EndOfStream
2025-11-27T18:18:13.819611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471183362706015:2049] [node 2] ICP30 unregister session Session# [5:7577471187657673405:2049] VirtualId# [5:7577471187657673395:0]
2025-11-27T18:18:13.819639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471187657673405:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.819731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:13.820614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471187491594196:2049] [node 2] ICS01 socket: 407 reason# EndOfStream
2025-11-27T18:18:13.818084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471185836236792:2049] [node 2] ICS01 socket: 405 reason# EndOfStream
2025-11-27T18:18:13.818143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471181541269396:2049] [node 2] ICP30 unregister session Session# [4:7577471185836236792:2049] VirtualId# [4:7577471185836236783:0]
2025-11-27T18:18:13.818177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471185836236792:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.818256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:13.818300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:13.819767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:13.820652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471183196626799:2049] [node 2] ICP30 unregister session Session# [6:7577471187491594196:2049] VirtualId# [6:7577471187491594191:0]
2025-11-27T18:18:13.820682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471187491594196:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.820744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:13.820777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:13.898256Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:13.898509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471185836236794:2050] [node 3] ICS01 socket: 453 reason# EndOfStream
2025-11-27T18:18:13.898541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471181541269397:2050] [node 3] ICP30 unregister session Session# [4:7577471185836236794:2050] VirtualId# [4:7577471185836236782:0]
2025-11-27T18:18:13.898563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471185836236794:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.898954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:13.899040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:13.905547Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:13.905732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471187657673414:2050] [node 3] ICS07 socket disconnect 450 reason# EndOfStream
2025-11-27T18:18:13.905871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471187657673414:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.905954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:13.905998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471187657673414:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:13.907133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471183362706016:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3100) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:13.907190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471187657673414:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:13.907224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471183362706016:2050] [node 3] ICP30 unregister session Session# [5:7577471187657673414:2050] VirtualId# [5:7577471187657673410:0]
2025-11-27T18:18:13.907271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:13.907311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471183362706016:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3100) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:13.907325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3100) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:13.905429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:13.905714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471187491594182:2050] [node 3] ICS01 socket: 411 reason# EndOfStream
2025-11-27T18:18:13.905756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471183196626800:2050] [node 3] ICP30 unregister session Session# [6:7577471187491594182:2050] VirtualId# [6:7577471187491594178:0]
2025-11-27T18:18:13.905795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471187491594182:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.905896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:13.905942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:13.923085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471183362706016:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:13.979334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:13.979570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471187657673411:2109] [node 4] ICS01 socket: 462 reason# EndOfStream
2025-11-27T18:18:13.979611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471187657673375:2109] [node 4] ICP30 unregister session Session# [5:7577471187657673411:2109] VirtualId# [5:7577471187657673408:0]
2025-11-27T18:18:13.979647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471187657673411:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.979713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:13.979752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:13.985858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:13.986110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471187491594183:2110] [node 4] ICS01 socket: 400 reason# EndOfStream
2025-11-27T18:18:13.986148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471187491594161:2110] [node 4] ICP30 unregister session Session# [6:7577471187491594183:2110] VirtualId# [6:7577471187491594175:0]
2025-11-27T18:18:13.986181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471187491594183:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:13.986260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:13.986320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:14.030631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:14.030903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471187491594186:2107] [node 5] ICS01 socket: 428 reason# EndOfStream
2025-11-27T18:18:14.030945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471187491594156:2107] [node 5] ICP30 unregister session Session# [6:7577471187491594186:2107] VirtualId# [6:7577471187491594176:0]
2025-11-27T18:18:14.030977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471187491594186:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:14.031041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:14.031079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:14.192884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471190620318098:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:14.192935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471190620318099:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:14.215528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471188962041498:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:14.215586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471188962041499:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:14.239906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471192165920247:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:14.239963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471192165920248:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:14.277000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471192045694688:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:14.277064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471192045694689:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:14.277095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471192045694690:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:14.292620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471191003974176:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:14.292670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471191003974177:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:14.292719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471191003974178:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:14.337378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471190786092990:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:14.337443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471190786092991:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:14.337489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471190786092992:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:14.777683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471190620318166:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:14.835089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471190620318167:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:14.819897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471192165920310:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:14.834874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471192045694760:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:14.837804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471192165920311:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:14.836114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471191003974239:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:14.839172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471191003974240:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:14.866542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471190786092991:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(231461126ns) {Configure:104(231442110ns) {ProcessConfigured:135(72431ns) {StartInitialHandshake:157(69787ns) {DropHandshakes:470(3616ns) {DropIncomingHandshake:442(1122ns) DropOutgoingHandshake:457(690ns)} GenerateSessionVirtualId:988(699ns) PrepareNewSessionHandshake:477(2068ns) {SwitchToState:179(885ns)}}}}}
2025-11-27T18:18:14.853553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471192165920312:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:14.877275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471190786093055:2094] [node 4] ICP01 ready to work
2025-11-27T18:18:14.876981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471192045694689:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(38063630ns) {Configure:104(38054403ns) {ProcessConfigured:135(94461ns) {StartInitialHandshake:157(91761ns) {DropHandshakes:470(3576ns) {DropIncomingHandshake:442(737ns) DropOutgoingHandshake:457(491ns)} GenerateSessionVirtualId:988(1226ns) PrepareNewSessionHandshake:477(2262ns) {SwitchToState:179(935ns)}}}}}
2025-11-27T18:18:14.901878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471188962041498:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(230932628ns) {Configure:104(218252515ns) {ProcessConfigured:135(84018ns) {StartInitialHandshake:157(81288ns) {DropHandshakes:470(3648ns) {DropIncomingHandshake:442(742ns) DropOutgoingHandshake:457(479ns)} GenerateSessionVirtualId:988(1216ns) PrepareNewSessionHandshake:477(2117ns) {SwitchToState:179(736ns)}}}}}
2025-11-27T18:18:14.902399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471188962041564:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:14.902747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471188962041565:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:14.903020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471188962041566:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:14.917500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471192045694763:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:14.932802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471190786093056:2097] [node 5] ICP01 ready to work
2025-11-27T18:18:14.926607Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471192165920247:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(67290294ns) {Configure:104(67283118ns) {ProcessConfigured:135(69663ns) {StartInitialHandshake:157(66635ns) {DropHandshakes:470(7480ns) {DropIncomingHandshake:442(814ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(1284ns) PrepareNewSessionHandshake:477(1590ns) {SwitchToState:179(413ns)}}}}}
2025-11-27T18:18:14.964569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471192045694760:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(43461672ns) {Configure:104(43457544ns) {ProcessConfigured:135(14167071ns) {StartInitialHandshake:157(14164774ns) {DropHandshakes:470(3282ns) {DropIncomingHandshake:442(719ns) DropOutgoingHandshake:457(501ns)} GenerateSessionVirtualId:988(868ns) PrepareNewSessionHandshake:477(4253ns) {SwitchToState:179(2083ns)}}}}}
2025-11-27T18:18:14.951379Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471188962041564:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(46166469ns) {Configure:104(46162735ns) {ProcessConfigured:135(94764ns) {StartInitialHandshake:157(89508ns) {DropHandshakes:470(1363ns) {DropIncomingHandshake:442(433ns) DropOutgoingHandshake:457(327ns)} GenerateSessionVirtualId:988(472ns) PrepareNewSessionHandshake:477(2016ns) {SwitchToState:179(791ns)}}}}}
2025-11-27T18:18:14.978282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471190786093055:2094] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(28721511ns) {Configure:104(28715395ns) {ProcessConfigured:135(51786ns) {StartInitialHandshake:157(50157ns) {DropHandshakes:470(5408ns) {DropIncomingHandshake:442(805ns) DropOutgoingHandshake:457(443ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(533ns) {SwitchToState:179(143ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:14.981950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471188962041569:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:14.961709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471190620318099:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(126061615ns) {Configure:104(126051554ns) {ProcessConfigured:135(15685318ns) {StartInitialHandshake:157(15682565ns) {DropHandshakes:470(3341ns) {DropIncomingHandshake:442(851ns) DropOutgoingHandshake:457(373ns)} GenerateSessionVirtualId:988(1040ns) PrepareNewSessionHandshake:477(4756ns) {SwitchToState:179(2044ns)}}}}}
2025-11-27T18:18:14.982693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471188962041569:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:14.982999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471188962041578:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:14.962186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471190620318170:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:14.990600Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471192165920310:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(53363481ns) {Configure:104(53360043ns) {ProcessConfigured:135(17750739ns) {StartInitialHandshake:157(17748657ns) {DropHandshakes:470(1293ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(595ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(2630ns) {SwitchToState:179(1304ns)}}}}}
2025-11-27T18:18:15.006176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471190786092992:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20737353ns) {Configure:104(20733646ns) {ProcessConfigured:135(11307136ns) {StartInitialHandshake:157(11304169ns) {DropHandshakes:470(2886ns) {DropIncomingHandshake:442(820ns) DropOutgoingHandshake:457(406ns)} GenerateSessionVirtualId:988(2022ns) PrepareNewSessionHandshake:477(3608ns) {SwitchToState:179(2163ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.009338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471195298941550:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54396
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.013177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471193257008877:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
2025-11-27T18:18:14.996636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471190620318177:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19527
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:14.996802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471190620318178:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19527
2025-11-27T18:18:14.996854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471190620318179:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19527
2025-11-27T18:18:14.997721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471190620318177:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.029440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471193257008878:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.033007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471192165920311:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(34665302ns) {Configure:104(34657909ns) {ProcessConfigured:135(62579ns) {StartInitialHandshake:157(59715ns) {DropHandshakes:470(2437ns) {DropIncomingHandshake:442(716ns) DropOutgoingHandshake:457(476ns)} GenerateSessionVirtualId:988(816ns) PrepareNewSessionHandshake:477(1377ns) {SwitchToState:179(364ns)}}}}}
2025-11-27T18:18:15.035916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471196460887619:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57930
2025-11-27T18:18:15.036409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471196460887619:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.030549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471193257008877:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.030693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471188962041498:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.036920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471190620318179:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.037064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471190620318172:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.037086Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471190620318172:2116] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.038364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471193257008880:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
2025-11-27T18:18:15.041824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471190620318172:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:15.042194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471190620318098:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.043866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:15.043943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471190620318098:2048] [node 2] ICP22 created new session: [1:7577471194915285484:2048]
2025-11-27T18:18:15.043997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471194915285484:2048] [node 2] ICS09 handshake done sender: [1:7577471190620318172:2116] self: [1:7577471190620318171:0] peer: [2:7577471193257008879:0] socket: 415 qp: -1
2025-11-27T18:18:15.044036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471194915285484:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:15.044077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.041960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471195298941550:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.045958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471195298941552:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54396
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.046827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471196460887620:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57930
2025-11-27T18:18:15.047005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471196460887621:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57930
2025-11-27T18:18:15.047117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471196460887622:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57930
2025-11-27T18:18:15.046960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471192045694775:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18258
2025-11-27T18:18:15.047104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471196340662072:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18258
2025-11-27T18:18:15.047187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471196340662073:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18258
2025-11-27T18:18:15.070366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471188962041578:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.065419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471196460887620:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.073421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471196460887622:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.070749Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471188962041498:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18252) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471193257008877:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.089631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471193257008878:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.089777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471193257008877:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.090206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471188962041571:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.079031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471196340662072:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.079164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471196340662073:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.079291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471192045694775:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.079619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471196340662074:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18258
2025-11-27T18:18:15.079724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471192045694688:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.079830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471192045694689:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.101337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471196460887621:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.090238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471188962041571:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.116778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471192165920247:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.116852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471192165920248:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.090838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471188962041498:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.090890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:15.090944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471188962041498:2048] [node 1] ICP22 created new session: [2:7577471193257008882:2048]
2025-11-27T18:18:15.090988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471193257008882:2048] [node 1] ICS09 handshake done sender: [2:7577471193257008877:2118] self: [2:7577471193257008879:0] peer: [1:7577471190620318171:0] socket: 418 qp: -1
2025-11-27T18:18:15.091024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471193257008882:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:15.091057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:15.097570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471193257008881:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57599
2025-11-27T18:18:15.101910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471188962041571:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:15.117439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471188962041564:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.117508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.117553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471188962041564:2107] [node 4] ICP22 created new session: [2:7577471193257008888:2107]
2025-11-27T18:18:15.107187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471195298941553:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54396
2025-11-27T18:18:15.125748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471196340662078:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18258
2025-11-27T18:18:15.126294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471190620318178:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.126442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471190620318174:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.126465Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471190620318174:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.137857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471196340662074:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.141838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471195081060364:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13418
2025-11-27T18:18:15.117606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471193257008888:2107] [node 4] ICS09 handshake done sender: [2:7577471188962041571:2114] self: [2:7577471188962041570:0] peer: [4:7577471196340662077:0] socket: 408 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.153207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471195298941553:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.162030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471191003974177:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.168994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471192045694775:2124] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:15.117645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471193257008888:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:15.169898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471192045694689:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471192045694775:2124] held: no
2025-11-27T18:18:15.170020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471192045694689:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.170071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:15.170122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471192045694689:2049] [node 2] ICP22 created new session: [4:7577471196340662079:2049]
2025-11-27T18:18:15.170166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471196340662079:2049] [node 2] ICS09 handshake done sender: [4:7577471192045694775:2124] self: [4:7577471196340662077:0] peer: [2:7577471188962041570:0] socket: 409 qp: -1
2025-11-27T18:18:15.117681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:15.118291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471188962041568:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.118323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471188962041568:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.131448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471193257008881:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.131910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471188962041568:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:15.145736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471188962041499:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.145825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:15.145885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471188962041499:2049] [node 3] ICP22 created new session: [2:7577471193257008893:2049]
2025-11-27T18:18:15.145934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471193257008893:2049] [node 3] ICS09 handshake done sender: [2:7577471188962041568:2112] self: [2:7577471188962041567:0] peer: [3:7577471196460887629:0] socket: 407 qp: -1
2025-11-27T18:18:15.145970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471193257008893:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:15.146006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:15.169535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471188962041573:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.169567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471188962041573:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.149848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471196460887630:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57930
2025-11-27T18:18:15.170199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471196340662079:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.170234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:15.170914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471192045694690:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.181070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471196340662083:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18258
2025-11-27T18:18:15.179136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471190620318169:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.179179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471190620318169:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.179730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471194915285487:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19527
2025-11-27T18:18:15.180523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471190620318174:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:15.180949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471190620318169:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.186724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471190620318166:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.182393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471195298941552:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.182703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471195298941555:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54396
2025-11-27T18:18:15.182345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471195081060364:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.183265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471191003974176:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.187355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471195298941556:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54396
2025-11-27T18:18:15.183127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471188962041573:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:15.183518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471188962041575:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.182493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471190786092991:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.183540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471188962041575:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.185471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471195081060366:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13418
2025-11-27T18:18:15.183918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471188962041565:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.187495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:15.183990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:15.201630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471195081060364:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:15.205628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471192165920318:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.205671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471192165920318:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.206518Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471192165920248:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471196460887621:2117] held: no
2025-11-27T18:18:15.206807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471196460887621:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:15.187559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471190620318166:2107] [node 4] ICP22 created new session: [1:7577471194915285491:2107]
2025-11-27T18:18:15.184046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471188962041565:2108] [node 5] ICP22 created new session: [2:7577471193257008896:2108]
2025-11-27T18:18:15.196119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471196340662072:2125] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:15.211056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471192045694688:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.211121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:15.211177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471192045694688:2048] [node 1] ICP22 created new session: [4:7577471196340662084:2048]
2025-11-27T18:18:15.211243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471196340662084:2048] [node 1] ICS09 handshake done sender: [4:7577471196340662072:2125] self: [4:7577471196340662075:0] peer: [1:7577471190620318173:0] socket: 432 qp: -1
2025-11-27T18:18:15.211281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471196340662084:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:15.211332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:15.211610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471192045694688:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:15.184101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471193257008896:2108] [node 5] ICS09 handshake done sender: [2:7577471188962041573:2115] self: [2:7577471188962041572:0] peer: [5:7577471195298941554:0] socket: 441 qp: -1
2025-11-27T18:18:15.184135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471193257008896:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:15.184169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:15.187976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471188962041575:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:15.207048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471188962041566:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.207132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:15.207190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471188962041566:2109] [node 6] ICP22 created new session: [2:7577471193257008899:2109]
2025-11-27T18:18:15.207241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471193257008899:2109] [node 6] ICS09 handshake done sender: [2:7577471188962041575:2116] self: [2:7577471188962041574:0] peer: [6:7577471195081060365:0] socket: 444 qp: -1
2025-11-27T18:18:15.207276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471193257008899:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:15.207320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.218867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471192165920248:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.218933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:15.218991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471192165920248:2049] [node 2] ICP22 created new session: [3:7577471196460887633:2049]
2025-11-27T18:18:15.219042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471196460887633:2049] [node 2] ICS09 handshake done sender: [3:7577471196460887621:2117] self: [3:7577471196460887629:0] peer: [2:7577471188962041567:0] socket: 427 qp: -1
2025-11-27T18:18:15.219076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471196460887633:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:15.219112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:15.219600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471196460887631:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57930
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.221971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471196340662087:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18258
2025-11-27T18:18:15.220018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471190786092991:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.220071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:15.220126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471190786092991:2049] [node 2] ICP22 created new session: [6:7577471195081060369:2049]
2025-11-27T18:18:15.220163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471195081060369:2049] [node 2] ICS09 handshake done sender: [6:7577471195081060364:2115] self: [6:7577471195081060365:0] peer: [2:7577471188962041574:0] socket: 446 qp: -1
2025-11-27T18:18:15.220188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471195081060369:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:15.220234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:15.187613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471194915285491:2107] [node 4] ICS09 handshake done sender: [1:7577471190620318174:2117] self: [1:7577471190620318173:0] peer: [4:7577471196340662075:0] socket: 416 qp: -1
2025-11-27T18:18:15.222493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471196340662074:2127] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:15.224811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471192045694690:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.224858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:15.224906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471192045694690:2050] [node 3] ICP22 created new session: [4:7577471196340662088:2050]
2025-11-27T18:18:15.224960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471196340662088:2050] [node 3] ICS09 handshake done sender: [4:7577471196340662074:2127] self: [4:7577471196340662081:0] peer: [3:7577471192165920317:0] socket: 437 qp: -1
2025-11-27T18:18:15.224990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471196340662088:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:15.225024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:15.221531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471192165920318:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:15.187644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471194915285491:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:15.190805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:15.194033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471190620318099:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.194103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:15.194170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471190620318099:2049] [node 3] ICP22 created new session: [1:7577471194915285493:2049]
2025-11-27T18:18:15.194214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471194915285493:2049] [node 3] ICS09 handshake done sender: [1:7577471190620318169:2115] self: [1:7577471190620318168:0] peer: [3:7577471196460887628:0] socket: 414 qp: -1
2025-11-27T18:18:15.194249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471194915285493:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:15.194287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:15.196847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471190620318176:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.196873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471190620318176:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.209714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471190620318176:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:15.224563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471194915285487:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.231525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471190620318167:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.231582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:15.231634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471190620318167:2108] [node 5] ICP22 created new session: [1:7577471194915285497:2108]
2025-11-27T18:18:15.231717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471194915285497:2108] [node 5] ICS09 handshake done sender: [1:7577471190620318176:2118] self: [1:7577471190620318175:0] peer: [5:7577471195298941557:0] socket: 413 qp: -1
2025-11-27T18:18:15.231751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471194915285497:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:15.231808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:15.235282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471192165920310:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.235352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:15.235397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471192165920310:2107] [node 4] ICP22 created new session: [3:7577471196460887636:2107]
2025-11-27T18:18:15.235444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471196460887636:2107] [node 4] ICS09 handshake done sender: [3:7577471192165920318:2112] self: [3:7577471192165920317:0] peer: [4:7577471196340662081:0] socket: 436 qp: -1
2025-11-27T18:18:15.235477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471196460887636:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:15.235514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:15.236243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471196460887622:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:15.234675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471190620318099:2049] [node 3] ICP18 handshake (actor [1:7577471194915285487:2123]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.236836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471191003974177:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471195298941553:2117] held: no
2025-11-27T18:18:15.237058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471195298941555:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.238471Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471190786092991:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:15.238832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471195081060367:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13418
2025-11-27T18:18:15.239083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471195081060368:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13418
2025-11-27T18:18:15.240696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471195298941553:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.242982Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471191003974176:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18252) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471195298941552:2116] held: no
2025-11-27T18:18:15.243135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471191003974177:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.243196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:15.243248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471191003974177:2049] [node 2] ICP22 created new session: [5:7577471195298941560:2049]
2025-11-27T18:18:15.243291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471195298941560:2049] [node 2] ICS09 handshake done sender: [5:7577471195298941553:2117] self: [5:7577471195298941554:0] peer: [2:7577471188962041572:0] socket: 443 qp: -1
2025-11-27T18:18:15.243330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471195298941560:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:15.243357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:15.243984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471191003974239:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.244076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471195298941558:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54396
2025-11-27T18:18:15.244972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471195081060367:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.246195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471190786092990:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.247500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471190620318181:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.247540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471190620318181:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.248355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471192165920247:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.248414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:15.248468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471192165920247:2048] [node 1] ICP22 created new session: [3:7577471196460887639:2048]
2025-11-27T18:18:15.248510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471196460887639:2048] [node 1] ICS09 handshake done sender: [3:7577471196460887622:2118] self: [3:7577471196460887628:0] peer: [1:7577471190620318168:0] socket: 428 qp: -1
2025-11-27T18:18:15.248544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471196460887639:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:15.248579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:15.248805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471192165920247:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:15.249540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471190620318181:2122] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.249955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471195298941559:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54396
2025-11-27T18:18:15.250458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471195298941552:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.254920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471196460887638:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57930
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.255584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471192045694769:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.255621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471192045694769:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.256263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471196340662090:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18258
2025-11-27T18:18:15.256662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471192045694769:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:15.256927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471192045694760:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.257004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:15.257058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471192045694760:2108] [node 5] ICP22 created new session: [4:7577471196340662092:2108]
2025-11-27T18:18:15.257098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471196340662092:2108] [node 5] ICS09 handshake done sender: [4:7577471192045694769:2122] self: [4:7577471192045694768:0] peer: [5:7577471195298941564:0] socket: 401 qp: -1
2025-11-27T18:18:15.257137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471196340662092:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:15.257169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.251699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471190620318170:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.251289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471195081060368:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.251760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:15.251821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471190620318170:2109] [node 6] ICP22 created new session: [1:7577471194915285500:2109]
2025-11-27T18:18:15.251474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471191003974176:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.251869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471194915285500:2109] [node 6] ICS09 handshake done sender: [1:7577471190620318181:2122] self: [1:7577471190620318180:0] peer: [6:7577471195081060372:0] socket: 412 qp: -1
2025-11-27T18:18:15.251937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471194915285500:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:15.251989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:15.263545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471196340662090:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.263650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471192045694760:2108] [node 5] ICP18 handshake (actor [4:7577471196340662090:2131]) is too old
2025-11-27T18:18:15.264329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471196460887638:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.251540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:15.251585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471191003974176:2048] [node 1] ICP22 created new session: [5:7577471195298941565:2048]
2025-11-27T18:18:15.251627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471195298941565:2048] [node 1] ICS09 handshake done sender: [5:7577471195298941552:2116] self: [5:7577471195298941557:0] peer: [1:7577471190620318175:0] socket: 431 qp: -1
2025-11-27T18:18:15.251659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471195298941565:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:15.251689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:15.260682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471195298941567:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54396
2025-11-27T18:18:15.263377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471195298941555:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.265107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471190786093055:2094] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.265195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471195081060373:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13418
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.265828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471195081060374:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13418
2025-11-27T18:18:15.265949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471195081060375:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13418
2025-11-27T18:18:15.266263Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471195081060367:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:15.266556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471192045694771:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.266587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471192045694771:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.267481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471192045694771:2123] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.267839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471195081060375:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.267985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471190786092990:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.268045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:15.268094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471190786092990:2048] [node 1] ICP22 created new session: [6:7577471195081060378:2048]
2025-11-27T18:18:15.268141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471195081060378:2048] [node 1] ICS09 handshake done sender: [6:7577471195081060367:2117] self: [6:7577471195081060372:0] peer: [1:7577471190620318180:0] socket: 435 qp: -1
2025-11-27T18:18:15.268173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471195081060378:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:15.268225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:15.269249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471195081060377:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13418
2025-11-27T18:18:15.270034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471192045694763:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.270114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:15.270161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471192045694763:2109] [node 6] ICP22 created new session: [4:7577471196340662095:2109]
2025-11-27T18:18:15.270199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471196340662095:2109] [node 6] ICS09 handshake done sender: [4:7577471192045694771:2123] self: [4:7577471192045694770:0] peer: [6:7577471195081060376:0] socket: 403 qp: -1
2025-11-27T18:18:15.270232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471196340662095:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:15.270264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:15.272356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471195298941559:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.272537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471191003974239:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.272580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:15.272624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471191003974239:2108] [node 4] ICP22 created new session: [5:7577471195298941568:2108]
2025-11-27T18:18:15.272690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471195298941568:2108] [node 4] ICS09 handshake done sender: [5:7577471195298941555:2118] self: [5:7577471195298941564:0] peer: [4:7577471192045694768:0] socket: 439 qp: -1
2025-11-27T18:18:15.272725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471195298941568:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:15.272754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:15.272984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471191003974239:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:15.273587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471191003974178:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:15.274325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471196460887616:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.274359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471196460887616:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.275375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471196460887616:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:15.275670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471192165920311:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.275728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:15.275779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471192165920311:2108] [node 5] ICP22 created new session: [3:7577471196460887644:2108]
2025-11-27T18:18:15.275822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471196460887644:2108] [node 5] ICS09 handshake done sender: [3:7577471196460887616:2113] self: [3:7577471196460887615:0] peer: [5:7577471195298941570:0] socket: 411 qp: -1
2025-11-27T18:18:15.275852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471196460887644:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:15.275879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:15.277709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471190786092992:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.277929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471195298941571:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54396
2025-11-27T18:18:15.278841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471195298941559:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:15.279232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471195081060374:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.279329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471190786093055:2094] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21063) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471195081060368:2118] held: no
2025-11-27T18:18:15.279589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471195081060368:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:15.281673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471196460887618:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.281709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471196460887618:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.283361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471196460887618:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:15.284467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471192165920312:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.284523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:15.284561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471192165920312:2109] [node 6] ICP22 created new session: [3:7577471196460887647:2109]
2025-11-27T18:18:15.284613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471196460887647:2109] [node 6] ICS09 handshake done sender: [3:7577471196460887618:2114] self: [3:7577471196460887617:0] peer: [6:7577471195081060380:0] socket: 459 qp: -1
2025-11-27T18:18:15.284011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471190786093055:2094] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.284647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471196460887647:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:15.284674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:15.284083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:15.284138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471190786093055:2094] [node 4] ICP22 created new session: [6:7577471195081060381:2094]
2025-11-27T18:18:15.285081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471195081060381:2094] [node 4] ICS09 handshake done sender: [6:7577471195081060368:2118] self: [6:7577471195081060376:0] peer: [4:7577471192045694770:0] socket: 455 qp: -1
2025-11-27T18:18:15.285127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471195081060381:2094] [node 4] ICS10 traffic start
2025-11-27T18:18:15.285171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:15.287423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471190786093056:2097] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.291158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471190786092992:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19170) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471195081060375:2121] held: no
2025-11-27T18:18:15.291419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471195081060383:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13418
2025-11-27T18:18:15.291741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471191003974178:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.291807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:15.291855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471191003974178:2050] [node 3] ICP22 created new session: [5:7577471195298941572:2050]
2025-11-27T18:18:15.291900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471195298941572:2050] [node 3] ICS09 handshake done sender: [5:7577471195298941559:2121] self: [5:7577471195298941570:0] peer: [3:7577471196460887615:0] socket: 457 qp: -1
2025-11-27T18:18:15.291948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471195298941572:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:15.291985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:15.292265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471191003974178:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:15.293108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471191003974250:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:15.293140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471191003974250:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:15.293750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471195081060375:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:15.295136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471191003974250:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:15.295183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471190786092992:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.295232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:15.295287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471190786092992:2050] [node 3] ICP22 created new session: [6:7577471195081060386:2050]
2025-11-27T18:18:15.295330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471195081060386:2050] [node 3] ICS09 handshake done sender: [6:7577471195081060375:2121] self: [6:7577471195081060380:0] peer: [3:7577471196460887617:0] socket: 463 qp: -1
2025-11-27T18:18:15.295364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471195081060386:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:15.295393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:15.296064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471195081060385:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13418
2025-11-27T18:18:15.296901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471191003974240:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:15.297159Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471195081060374:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:15.297041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:15.297113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471191003974240:2109] [node 6] ICP22 created new session: [5:7577471195298941575:2109]
2025-11-27T18:18:15.297161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471195298941575:2109] [node 6] ICS09 handshake done sender: [5:7577471191003974250:2114] self: [5:7577471191003974249:0] peer: [6:7577471195081060384:0] socket: 434 qp: -1
2025-11-27T18:18:15.297201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471195298941575:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:15.297236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:15.303776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471190786093056:2097] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:15.303878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:15.303951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471190786093056:2097] [node 5] ICP22 created new session: [6:7577471195081060388:2097]
2025-11-27T18:18:15.304010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471195081060388:2097] [node 5] ICS09 handshake done sender: [6:7577471195081060374:2120] self: [6:7577471195081060384:0] peer: [5:7577471191003974249:0] socket: 462 qp: -1
2025-11-27T18:18:15.304057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471195081060388:2097] [node 5] ICS10 traffic start
2025-11-27T18:18:15.304102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:15.304422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471190786093056:2097] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:15.329144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:15.329447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471193257008882:2048] [node 1] ICS01 socket: 418 reason# EndOfStream
2025-11-27T18:18:15.329504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471188962041498:2048] [node 1] ICP30 unregister session Session# [2:7577471193257008882:2048] VirtualId# [2:7577471193257008879:0]
2025-11-27T18:18:15.329549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471193257008882:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.329632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:15.329673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:15.329436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:15.329657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471196460887639:2048] [node 1] ICS01 socket: 428 reason# EndOfStream
2025-11-27T18:18:15.329713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471192165920247:2048] [node 1] ICP30 unregister session Session# [3:7577471196460887639:2048] VirtualId# [3:7577471196460887628:0]
2025-11-27T18:18:15.329743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471196460887639:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.329812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:15.329869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:15.339854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:15.340100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471195298941565:2048] [node 1] ICS01 socket: 431 reason# EndOfStream
2025-11-27T18:18:15.340147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471191003974176:2048] [node 1] ICP30 unregister session Session# [5:7577471195298941565:2048] VirtualId# [5:7577471195298941557:0]
2025-11-27T18:18:15.340192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471195298941565:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.340276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:15.340346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:15.341647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:15.341878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471196340662084:2048] [node 1] ICS01 socket: 432 reason# EndOfStream
2025-11-27T18:18:15.341931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471192045694688:2048] [node 1] ICP30 unregister session Session# [4:7577471196340662084:2048] VirtualId# [4:7577471196340662075:0]
2025-11-27T18:18:15.341963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471196340662084:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.342034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:15.342077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:15.340782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:15.341013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471195081060378:2048] [node 1] ICS01 socket: 435 reason# EndOfStream
2025-11-27T18:18:15.341048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471190786092990:2048] [node 1] ICP30 unregister session Session# [6:7577471195081060378:2048] VirtualId# [6:7577471195081060372:0]
2025-11-27T18:18:15.341075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471195081060378:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.341137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:15.341171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:15.391198Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:15.391445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471196460887633:2049] [node 2] ICS01 socket: 427 reason# EndOfStream
2025-11-27T18:18:15.391499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471192165920248:2049] [node 2] ICP30 unregister session Session# [3:7577471196460887633:2049] VirtualId# [3:7577471196460887629:0]
2025-11-27T18:18:15.391546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471196460887633:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.391640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:15.391690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:15.394515Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:15.394751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471196340662079:2049] [node 2] ICS01 socket: 409 reason# EndOfStream
2025-11-27T18:18:15.394812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471192045694689:2049] [node 2] ICP30 unregister session Session# [4:7577471196340662079:2049] VirtualId# [4:7577471196340662077:0]
2025-11-27T18:18:15.394846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471196340662079:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.394914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:15.394962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:15.395752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:15.395946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471195298941560:2049] [node 2] ICS01 socket: 443 reason# EndOfStream
2025-11-27T18:18:15.395988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471191003974177:2049] [node 2] ICP30 unregister session Session# [5:7577471195298941560:2049] VirtualId# [5:7577471195298941554:0]
2025-11-27T18:18:15.396024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471195298941560:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.396104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:15.396163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:15.405684Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:15.405940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471195081060369:2049] [node 2] ICS07 socket disconnect 446 reason# EndOfStream
2025-11-27T18:18:15.405973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471195081060369:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.406056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:15.406098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471195081060369:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:15.410081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471190786092991:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65504) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:15.410154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471195081060369:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:15.410189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471190786092991:2049] [node 2] ICP30 unregister session Session# [6:7577471195081060369:2049] VirtualId# [6:7577471195081060365:0]
2025-11-27T18:18:15.410240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:15.410280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471190786092991:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:15.410295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:15.422244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471190786092991:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:15.484035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:15.484275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471196340662088:2050] [node 3] ICS01 socket: 437 reason# EndOfStream
2025-11-27T18:18:15.484328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471192045694690:2050] [node 3] ICP30 unregister session Session# [4:7577471196340662088:2050] VirtualId# [4:7577471196340662081:0]
2025-11-27T18:18:15.484370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471196340662088:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.484448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:15.484497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:15.485210Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:15.485435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471195298941572:2050] [node 3] ICS01 socket: 457 reason# EndOfStream
2025-11-27T18:18:15.485480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471191003974178:2050] [node 3] ICP30 unregister session Session# [5:7577471195298941572:2050] VirtualId# [5:7577471195298941570:0]
2025-11-27T18:18:15.485507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471195298941572:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.485565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:15.485599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:15.489678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:15.489888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471195081060386:2050] [node 3] ICS01 socket: 463 reason# EndOfStream
2025-11-27T18:18:15.489926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471190786092992:2050] [node 3] ICP30 unregister session Session# [6:7577471195081060386:2050] VirtualId# [6:7577471195081060380:0]
2025-11-27T18:18:15.489966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471195081060386:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.490041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:15.490083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:15.521351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:15.521585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471195298941568:2108] [node 4] ICS01 socket: 439 reason# EndOfStream
2025-11-27T18:18:15.521630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471191003974239:2108] [node 4] ICP30 unregister session Session# [5:7577471195298941568:2108] VirtualId# [5:7577471195298941564:0]
2025-11-27T18:18:15.521670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471195298941568:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.521769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:15.521828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:15.529451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:15.529714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471195081060381:2094] [node 4] ICS01 socket: 455 reason# EndOfStream
2025-11-27T18:18:15.529769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471190786093055:2094] [node 4] ICP30 unregister session Session# [6:7577471195081060381:2094] VirtualId# [6:7577471195081060376:0]
2025-11-27T18:18:15.529804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471195081060381:2094] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.529895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:15.529949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:15.585694Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:15.585938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471195081060388:2097] [node 5] ICS07 socket disconnect 462 reason# EndOfStream
2025-11-27T18:18:15.585970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471195081060388:2097] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:15.586069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:15.586120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471195081060388:2097] [node 5] ICS15 start handshake
2025-11-27T18:18:15.587308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471190786093056:2097] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31956) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:15.587368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471195081060388:2097] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:15.587411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471190786093056:2097] [node 5] ICP30 unregister session Session# [6:7577471195081060388:2097] VirtualId# [6:7577471195081060384:0]
2025-11-27T18:18:15.587460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:18:15.587494Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471190786093056:2097] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31956) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:15.587509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31956) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:15.714304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471195774067687:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:15.714364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471195774067688:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:15.785601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471192851148593:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:15.785661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471192851148594:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:15.817941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471193988338491:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:15.818007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471193988338492:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:15.855404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471194179858712:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:15.855451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471194179858713:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:15.855481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471194179858714:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:15.897157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471195923159755:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:15.897203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471195923159756:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:15.897233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471195923159757:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:15.914116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471193260817470:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:15.914162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471193260817471:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:15.914197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471193260817472:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:16.409753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471200069035044:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:16.453706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471200069035052:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:16.454151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471200069035053:2112] [node 6] ICP01 ready to work
2025-11-27T18:18:16.432332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471197555784820:2098] [node 5] ICP01 ready to work
2025-11-27T18:18:16.466785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471197555784824:2100] [node 4] ICP01 ready to work
2025-11-27T18:18:16.584302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471197555784820:2098] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(132881664ns) {Configure:104(132872469ns) {ProcessConfigured:135(67900ns) {StartInitialHandshake:157(65373ns) {DropHandshakes:470(6529ns) {DropIncomingHandshake:442(816ns) DropOutgoingHandshake:457(740ns)} GenerateSessionVirtualId:988(979ns) PrepareNewSessionHandshake:477(1802ns) {SwitchToState:179(299ns)}}}}}
2025-11-27T18:18:16.567579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471197146115954:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:16.593276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471198283305868:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:16.598991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471198474826071:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:16.582161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471197146115953:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:16.613874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471198474826072:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:16.628238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471200218127132:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:16.626903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471193988338491:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(26850250ns) {Configure:104(26840246ns) {ProcessConfigured:135(13753523ns) {StartInitialHandshake:157(13750860ns) {DropHandshakes:470(2106ns) {DropIncomingHandshake:442(431ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(1133ns) PrepareNewSessionHandshake:477(5421ns) {SwitchToState:179(2700ns)}}}}}
2025-11-27T18:18:16.630490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471200218127135:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:16.600054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471197146115955:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.654076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471195923159756:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21553806ns) {Configure:104(21549778ns) {ProcessConfigured:135(63620ns) {StartInitialHandshake:157(60928ns) {DropHandshakes:470(3267ns) {DropIncomingHandshake:442(750ns) DropOutgoingHandshake:457(464ns)} GenerateSessionVirtualId:988(1162ns) PrepareNewSessionHandshake:477(4427ns) {SwitchToState:179(651ns)}}}}}
2025-11-27T18:18:16.656645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471200218127136:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13065
2025-11-27T18:18:16.656953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471200218127141:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13065
2025-11-27T18:18:16.645855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471198283305872:2108] [node 6] ICP01 ready to work
2025-11-27T18:18:16.646280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471198283305873:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.678329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471195923159755:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(188009860ns) {Configure:104(188003760ns) {ProcessConfigured:135(75600ns) {StartInitialHandshake:157(72997ns) {DropHandshakes:470(3325ns) {DropIncomingHandshake:442(971ns) DropOutgoingHandshake:457(567ns)} GenerateSessionVirtualId:988(886ns) PrepareNewSessionHandshake:477(1879ns) {SwitchToState:179(613ns)}}}}}
2025-11-27T18:18:16.686944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471198474826071:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(69728630ns) {Configure:104(69724807ns) {ProcessConfigured:135(19362860ns) {StartInitialHandshake:157(19360991ns) {DropHandshakes:470(2477ns) {DropIncomingHandshake:442(727ns) DropOutgoingHandshake:457(363ns)} GenerateSessionVirtualId:988(789ns) PrepareNewSessionHandshake:477(3866ns) {SwitchToState:179(2415ns)}}}}}
2025-11-27T18:18:16.702021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471198474826077:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3114
2025-11-27T18:18:16.702246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471198474826078:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3114
2025-11-27T18:18:16.702315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471198474826079:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3114
2025-11-27T18:18:16.702446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471198474826080:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3114
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.706222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471193260817472:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(209335658ns) {Configure:104(209330465ns) {ProcessConfigured:135(63807ns) {StartInitialHandshake:157(61752ns) {DropHandshakes:470(2137ns) {DropIncomingHandshake:442(417ns) DropOutgoingHandshake:457(456ns)} GenerateSessionVirtualId:988(603ns) PrepareNewSessionHandshake:477(1417ns) {SwitchToState:179(452ns)}}}}}
2025-11-27T18:18:16.709777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471197555784844:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36164
2025-11-27T18:18:16.711232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471200069035053:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(243638581ns) {Configure:104(243632289ns) {ProcessConfigured:135(5648417ns) {StartInitialHandshake:157(5645940ns) {DropHandshakes:470(3072ns) {DropIncomingHandshake:442(707ns) DropOutgoingHandshake:457(564ns)} GenerateSessionVirtualId:988(743ns) PrepareNewSessionHandshake:477(3954ns) {SwitchToState:179(1461ns)}}}}}
2025-11-27T18:18:16.712531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471200069035062:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38642
2025-11-27T18:18:16.682768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471193988338492:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(201743428ns) {Configure:104(201733735ns) {ProcessConfigured:135(78036ns) {StartInitialHandshake:157(75933ns) {DropHandshakes:470(3088ns) {DropIncomingHandshake:442(566ns) DropOutgoingHandshake:457(524ns)} GenerateSessionVirtualId:988(793ns) PrepareNewSessionHandshake:477(1695ns) {SwitchToState:179(522ns)}}}}}
2025-11-27T18:18:16.713854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471198474826077:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.712660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471200069035063:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38642
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.722455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471198474826079:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.691130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471200218127136:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.712719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471200069035064:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38642
2025-11-27T18:18:16.684998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471198283305876:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.747160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471192851148593:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(144735749ns) {Configure:104(144729798ns) {ProcessConfigured:135(13482289ns) {StartInitialHandshake:157(13479346ns) {DropHandshakes:470(3353ns) {DropIncomingHandshake:442(783ns) DropOutgoingHandshake:457(461ns)} GenerateSessionVirtualId:988(918ns) PrepareNewSessionHandshake:477(5948ns) {SwitchToState:179(3459ns)}}}}}
2025-11-27T18:18:16.751170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471197146115970:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32123
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.734472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471197555784844:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.734682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471197555784824:2100] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.721758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471195774067687:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(280478813ns) {Configure:104(280468451ns) {ProcessConfigured:135(25182680ns) {StartInitialHandshake:157(25179854ns) {DropHandshakes:470(3007ns) {DropIncomingHandshake:442(678ns) DropOutgoingHandshake:457(523ns)} GenerateSessionVirtualId:988(875ns) PrepareNewSessionHandshake:477(3066ns) {SwitchToState:179(1790ns)}}}}}
2025-11-27T18:18:16.729804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471200069035062:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.753075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471200069035064:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.722670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471198474826088:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3114
2025-11-27T18:18:16.691078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471198283305881:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.783444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471200218127150:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13065
2025-11-27T18:18:16.713147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471198283305872:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(64701455ns) {Configure:104(64698228ns) {ProcessConfigured:135(21921301ns) {StartInitialHandshake:157(21918562ns) {DropHandshakes:470(3576832ns) {DropIncomingHandshake:442(1345ns) DropOutgoingHandshake:457(118ns)} GenerateSessionVirtualId:988(1169ns) PrepareNewSessionHandshake:477(3931ns) {SwitchToState:179(1667ns)}}}}}
2025-11-27T18:18:16.725860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471198283305881:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.752647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471198283305887:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47686
2025-11-27T18:18:16.766963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471198283305876:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.767372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471193988338492:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.773999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471198283305887:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.774096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471193988338491:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.771626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471200069035063:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.774424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471197146115970:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.791368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471200218127150:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.773425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471198474826082:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.801404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471198283305891:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47686
2025-11-27T18:18:16.792997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471197555784846:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36164
2025-11-27T18:18:16.793373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471200218127132:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.801946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471200218127141:2132] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.802072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471195923159756:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.785915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471200069035050:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.773475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471198474826082:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:16.798442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471197146115957:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.783890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471194179858713:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(302297072ns) {Configure:104(302292679ns) {ProcessConfigured:135(17878397ns) {StartInitialHandshake:157(17874377ns) {DropHandshakes:470(2847ns) {DropIncomingHandshake:442(813ns) DropOutgoingHandshake:457(482ns)} GenerateSessionVirtualId:988(1433ns) PrepareNewSessionHandshake:477(4119ns) {SwitchToState:179(1900ns)}}}}}
2025-11-27T18:18:16.784156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471198474826088:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.784429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471194179858712:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.798056Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471198474826082:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:16.785944Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471200069035050:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:16.791329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471200069035050:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:16.791674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471200069035044:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.791739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:16.791800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471200069035044:2107] [node 4] ICP22 created new session: [1:7577471200069035070:2107]
2025-11-27T18:18:16.793091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471200069035046:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.793124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471200069035046:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:16.797511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471200069035070:2107] [node 4] ICS09 handshake done sender: [1:7577471200069035050:2111] self: [1:7577471200069035047:0] peer: [4:7577471198474826093:0] socket: 435 qp: -1
2025-11-27T18:18:16.797561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471200069035070:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:16.797611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:16.808507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471197555784844:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.798485Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471197146115957:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:16.800789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471197146115967:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32123
2025-11-27T18:18:16.801940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471197146115967:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.802070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471197146115957:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:16.804882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471197146115961:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.804905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471197146115961:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:16.805555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471192851148594:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.805609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:16.805648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471192851148594:2049] [node 3] ICP22 created new session: [2:7577471197146115974:2049]
2025-11-27T18:18:16.805694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471197146115974:2049] [node 3] ICS09 handshake done sender: [2:7577471197146115957:2111] self: [2:7577471197146115956:0] peer: [3:7577471198283305889:0] socket: 407 qp: -1
2025-11-27T18:18:16.805723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471197146115974:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:16.805752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:16.803598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471198283305876:2131] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.809986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471198474826076:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.810018Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471198474826076:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.813249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471200069035046:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.809722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471200218127154:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13065
2025-11-27T18:18:16.815016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471195774067688:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.815083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:16.815127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471195774067688:2049] [node 3] ICP22 created new session: [1:7577471200069035074:2049]
2025-11-27T18:18:16.815172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471200069035074:2049] [node 3] ICS09 handshake done sender: [1:7577471200069035046:2108] self: [1:7577471200069035045:0] peer: [3:7577471198283305890:0] socket: 434 qp: -1
2025-11-27T18:18:16.815201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471200069035074:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:16.815233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.824179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471198474826080:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.824285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471198474826078:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.825534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471198474826072:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.825610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:16.825663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471198474826072:2109] [node 6] ICP22 created new session: [4:7577471198474826096:2109]
2025-11-27T18:18:16.825709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471198474826096:2109] [node 6] ICS09 handshake done sender: [4:7577471198474826082:2116] self: [4:7577471198474826081:0] peer: [6:7577471197555784845:0] socket: 427 qp: -1
2025-11-27T18:18:16.825751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471198474826096:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:16.825809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:16.826416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471198474826095:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3114
2025-11-27T18:18:16.803836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471193988338492:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:16.814374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471197555784824:2100] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:16.813470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471197146115973:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32123
2025-11-27T18:18:16.812560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471200218127155:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13065
2025-11-27T18:18:16.803902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:16.803953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471193988338492:2049] [node 2] ICP22 created new session: [3:7577471198283305892:2049]
2025-11-27T18:18:16.804011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471198283305892:2049] [node 2] ICS09 handshake done sender: [3:7577471198283305876:2131] self: [3:7577471198283305889:0] peer: [2:7577471197146115956:0] socket: 410 qp: -1
2025-11-27T18:18:16.804051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471198283305892:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:16.804090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:16.816318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471198283305895:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47686
2025-11-27T18:18:16.827402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471200218127151:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13065
2025-11-27T18:18:16.814438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:16.816106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471197146115961:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:16.843219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471193988338491:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62102) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471198283305887:2136] held: no
2025-11-27T18:18:16.843425Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471198283305887:2136] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:16.850677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471193988338491:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:16.850738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:16.850789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471193988338491:2048] [node 1] ICP22 created new session: [3:7577471198283305897:2048]
2025-11-27T18:18:16.850832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471198283305897:2048] [node 1] ICS09 handshake done sender: [3:7577471198283305887:2136] self: [3:7577471198283305890:0] peer: [1:7577471200069035045:0] socket: 441 qp: -1
2025-11-27T18:18:16.850866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471198283305897:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:16.850898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:16.855812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471200218127154:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.855954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471200218127151:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.856049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471200218127141:2132] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:16.856317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471195923159757:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.856386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471195923159755:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.839515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471194179858712:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62102) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471198474826088:2118] held: no
2025-11-27T18:18:16.840011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471197146115953:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.857618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471195923159756:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.849868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471198474826088:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:16.814484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471197555784824:2100] [node 4] ICP22 created new session: [6:7577471197555784848:2100]
2025-11-27T18:18:16.860886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471200069035055:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.874050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471194179858714:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.874131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471194179858713:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.874169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471194179858712:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:16.874211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:16.874255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471194179858712:2048] [node 1] ICP22 created new session: [4:7577471198474826103:2048]
2025-11-27T18:18:16.874297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471198474826103:2048] [node 1] ICS09 handshake done sender: [4:7577471198474826088:2118] self: [4:7577471198474826093:0] peer: [1:7577471200069035047:0] socket: 438 qp: -1
2025-11-27T18:18:16.874331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471198474826103:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:16.874378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.866869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.814529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471197555784848:2100] [node 4] ICS09 handshake done sender: [6:7577471197555784844:2115] self: [6:7577471197555784845:0] peer: [4:7577471198474826081:0] socket: 429 qp: -1
2025-11-27T18:18:16.860053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471198283305880:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.881532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471198474826076:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.840294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471197146115973:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.860921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471200069035055:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:16.890850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471198474826071:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.890927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:16.890977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471198474826071:2108] [node 5] ICP22 created new session: [4:7577471198474826106:2108]
2025-11-27T18:18:16.891034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471198474826106:2108] [node 5] ICS09 handshake done sender: [4:7577471198474826076:2111] self: [4:7577471198474826075:0] peer: [5:7577471200218127152:0] socket: 426 qp: -1
2025-11-27T18:18:16.891064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471198474826106:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:16.891102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:16.873740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471195923159756:2049] [node 2] ICP22 created new session: [5:7577471200218127158:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.814573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471197555784848:2100] [node 4] ICS10 traffic start
2025-11-27T18:18:16.841050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:16.860089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471198283305880:2133] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.866953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471200069035055:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:16.876029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471200218127158:2049] [node 2] ICS09 handshake done sender: [5:7577471200218127141:2132] self: [5:7577471200218127153:0] peer: [2:7577471197146115960:0] socket: 411 qp: -1
2025-11-27T18:18:16.893850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471198474826105:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3114
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.814608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.850086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471197146115953:2108] [node 5] ICP22 created new session: [2:7577471197146115980:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.876068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471200218127158:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:16.873547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471198283305880:2133] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:16.907985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471198474826078:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:16.853178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471197146115980:2108] [node 5] ICS09 handshake done sender: [2:7577471197146115961:2113] self: [2:7577471197146115960:0] peer: [5:7577471200218127153:0] socket: 409 qp: -1
2025-11-27T18:18:16.895371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471200069035052:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.910228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471198474826108:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3114
2025-11-27T18:18:16.912744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471194179858713:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:16.912789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:16.912837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471194179858713:2049] [node 2] ICP22 created new session: [4:7577471198474826109:2049]
2025-11-27T18:18:16.912880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471198474826109:2049] [node 2] ICS09 handshake done sender: [4:7577471198474826078:2113] self: [4:7577471198474826102:0] peer: [2:7577471197146115958:0] socket: 421 qp: -1
2025-11-27T18:18:16.912911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471198474826109:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:16.912959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:16.943112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471198474826080:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:16.876106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:16.876462Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471195923159756:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(18676309ns) {HandleHandshakeStatus:329(18674208ns) {DropIncomingHandshake:442(560ns) DropOutgoingHandshake:457(5941ns) SwitchToState:179(421ns) ProcessPendingSessionEvents:495(96910ns) {ForwardSessionEventToSession:612(48856ns) ForwardSessionEventToSession:612(3877ns) ForwardSessionEventToSession:612(3020ns) ForwardSessionEventToSession:612(10293ns) ForwardSessionEventToSession:612(3395ns) ForwardSessionEventToSession:612(2962ns) ForwardSessionEventToSession:612(5458ns) ForwardSessionEventToSession:612(2775ns) ForwardSessionEventToSession:612(5648ns) ForwardSessionEventToSession:612(2991ns)}}}
2025-11-27T18:18:16.877410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471200218127160:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13065
2025-11-27T18:18:16.882461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471200218127162:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13065
2025-11-27T18:18:16.886892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471200218127161:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13065
2025-11-27T18:18:16.917391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471200218127154:2137] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:16.924179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471200218127150:2135] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:16.942595Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471200218127132:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471200218127150:2135] held: no
2025-11-27T18:18:16.942752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471200218127151:2136] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:16.857011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471197146115980:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:16.861773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:16.879023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471197146115959:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.822540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471197555784847:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36164
2025-11-27T18:18:16.840950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471197555784847:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.877780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471197555784851:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36164
2025-11-27T18:18:16.878158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471193260817471:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.878345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471197555784852:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36164
2025-11-27T18:18:16.882020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471197555784854:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36164
2025-11-27T18:18:16.930171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471197555784851:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.930347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471197555784847:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:16.941653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471197555784852:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.947461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471195923159757:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:16.879048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471197146115959:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:16.954382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471194179858714:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:16.954447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:16.954494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471194179858714:2050] [node 3] ICP22 created new session: [4:7577471198474826111:2050]
2025-11-27T18:18:16.954542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471198474826111:2050] [node 3] ICS09 handshake done sender: [4:7577471198474826080:2115] self: [4:7577471198474826101:0] peer: [3:7577471198283305874:0] socket: 423 qp: -1
2025-11-27T18:18:16.954572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471198474826111:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:16.954601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:16.901714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:16.894116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471198283305875:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.947542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.955228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471193260817471:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:16.947592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471195923159757:2050] [node 3] ICP22 created new session: [5:7577471200218127164:2050]
2025-11-27T18:18:16.894142Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471198283305875:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:16.947639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471200218127164:2050] [node 3] ICS09 handshake done sender: [5:7577471200218127154:2137] self: [5:7577471200218127156:0] peer: [3:7577471198283305879:0] socket: 452 qp: -1
2025-11-27T18:18:16.947841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471200218127132:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:16.947904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:16.947964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471200218127132:2108] [node 4] ICP22 created new session: [5:7577471200218127165:2108]
2025-11-27T18:18:16.948019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471200218127165:2108] [node 4] ICS09 handshake done sender: [5:7577471200218127150:2135] self: [5:7577471200218127152:0] peer: [4:7577471198474826075:0] socket: 437 qp: -1
2025-11-27T18:18:16.948054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471200218127165:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:16.948093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:16.948530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471195923159755:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:16.948601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:16.948637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471195923159755:2048] [node 1] ICP22 created new session: [5:7577471200218127167:2048]
2025-11-27T18:18:16.948679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471200218127167:2048] [node 1] ICS09 handshake done sender: [5:7577471200218127151:2136] self: [5:7577471200218127157:0] peer: [1:7577471200069035054:0] socket: 440 qp: -1
2025-11-27T18:18:16.948706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471200218127167:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:16.948732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:16.953350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471200218127164:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:16.953424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:16.953691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471195923159757:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:16.978927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471200218127143:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.978956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471200218127143:2133] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:16.979961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471200218127143:2133] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:16.980212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471200218127135:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.980281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:16.980342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471200218127135:2110] [node 6] ICP22 created new session: [5:7577471200218127172:2110]
2025-11-27T18:18:16.980388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471200218127172:2110] [node 6] ICS09 handshake done sender: [5:7577471200218127143:2133] self: [5:7577471200218127142:0] peer: [6:7577471197555784860:0] socket: 422 qp: -1
2025-11-27T18:18:16.980425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471200218127172:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:16.980475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:16.879589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471197146115963:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.905484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471198283305873:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.905559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:16.905617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471198283305873:2109] [node 5] ICP22 created new session: [3:7577471198283305902:2109]
2025-11-27T18:18:16.905666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471198283305902:2109] [node 5] ICS09 handshake done sender: [3:7577471198283305880:2133] self: [3:7577471198283305879:0] peer: [5:7577471200218127156:0] socket: 428 qp: -1
2025-11-27T18:18:16.905711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471198283305902:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:16.905746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:16.924388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471198283305875:2130] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:16.929526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471198283305900:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47686
2025-11-27T18:18:16.930969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471198283305868:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.931011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:16.931061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471198283305868:2107] [node 4] ICP22 created new session: [3:7577471198283305904:2107]
2025-11-27T18:18:16.931122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471198283305904:2107] [node 4] ICS09 handshake done sender: [3:7577471198283305875:2130] self: [3:7577471198283305874:0] peer: [4:7577471198474826101:0] socket: 419 qp: -1
2025-11-27T18:18:16.931161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471198283305904:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:16.931201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:16.943471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471198283305900:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.943602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471198283305868:2107] [node 4] ICP18 handshake (actor [3:7577471198283305900:2139]) is too old
2025-11-27T18:18:16.993132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471198283305886:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.993177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471198283305886:2135] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.909761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471200069035052:2109] [node 5] ICP22 created new session: [1:7577471200069035078:2109]
2025-11-27T18:18:16.910887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471200069035057:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.879606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471197146115963:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:16.955286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:16.955319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471193260817471:2049] [node 2] ICP22 created new session: [6:7577471197555784856:2049]
2025-11-27T18:18:16.955366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471197555784856:2049] [node 2] ICS09 handshake done sender: [6:7577471197555784847:2117] self: [6:7577471197555784853:0] peer: [2:7577471197146115962:0] socket: 456 qp: -1
2025-11-27T18:18:16.955394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471197555784856:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:16.955421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:16.955615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471193260817471:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:16.969659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471193260817470:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.969741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471193260817472:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.970183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471197555784855:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36164
2025-11-27T18:18:16.978192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471197555784855:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.978447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471197555784820:2098] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.994476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471197555784820:2098] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2355) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471197555784855:2121] held: no
2025-11-27T18:18:16.994733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471197555784861:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36164
2025-11-27T18:18:16.994838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471197555784862:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36164
2025-11-27T18:18:16.998328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471197555784851:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:16.910911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471200069035057:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:16.911773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471200069035079:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38642
2025-11-27T18:18:16.912142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471200069035078:2109] [node 5] ICS09 handshake done sender: [1:7577471200069035055:2113] self: [1:7577471200069035054:0] peer: [5:7577471200218127157:0] socket: 436 qp: -1
2025-11-27T18:18:16.912183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471200069035078:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:16.912230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:16.932171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471200069035057:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:16.956656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471200069035052:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(61076292ns) {HandleHandshakeStatus:329(61073541ns) {DropIncomingHandshake:442(527ns) DropOutgoingHandshake:457(3076ns) SwitchToState:179(736ns) ProcessPendingSessionEvents:495(15106678ns) {ForwardSessionEventToSession:612(15049306ns) ForwardSessionEventToSession:612(5730ns) ForwardSessionEventToSession:612(6858ns) ForwardSessionEventToSession:612(4754ns) ForwardSessionEventToSession:612(2352ns) ForwardSessionEventToSession:612(10974ns) ForwardSessionEventToSession:612(2924ns) ForwardSessionEventToSession:612(4953ns) ForwardSessionEventToSession:612(2655ns) ForwardSessionEventToSession:612(2024ns)}}}
2025-11-27T18:18:16.961983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471195774067687:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.973368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:16.976138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471200069035059:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.976165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471200069035059:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:16.984946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471195774067687:2048] [node 2] ICP22 created new session: [1:7577471200069035082:2048]
2025-11-27T18:18:16.993281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471200069035079:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.995543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471200069035082:2048] [node 2] ICS09 handshake done sender: [1:7577471200069035057:2114] self: [1:7577471200069035056:0] peer: [2:7577471197146115986:0] socket: 433 qp: -1
2025-11-27T18:18:16.995641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471200069035059:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:16.996908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471200069035082:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:16.996965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:17.000531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471200069035053:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:17.000584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:17.000628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471200069035053:2112] [node 6] ICP22 created new session: [1:7577471200069035085:2112]
2025-11-27T18:18:17.000672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471200069035085:2112] [node 6] ICS09 handshake done sender: [1:7577471200069035059:2115] self: [1:7577471200069035058:0] peer: [6:7577471197555784858:0] socket: 403 qp: -1
2025-11-27T18:18:17.000703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471200069035085:2112] [node 6] ICS10 traffic start
2025-11-27T18:18:17.000728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:16.880121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471197146115979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32123
2025-11-27T18:18:16.880331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471192851148593:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:16.880746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471197146115982:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32123
2025-11-27T18:18:16.889786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471197146115953:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(49513458ns) {HandleHandshakeStatus:329(49509923ns) {DropIncomingHandshake:442(15972ns) DropOutgoingHandshake:457(258ns) SwitchToState:179(664ns) ProcessPendingSessionEvents:495(13603241ns) {ForwardSessionEventToSession:612(13542788ns) ForwardSessionEventToSession:612(7447ns) ForwardSessionEventToSession:612(8848ns) ForwardSessionEventToSession:612(3302ns) ForwardSessionEventToSession:612(2701ns) ForwardSessionEventToSession:612(6136ns) ForwardSessionEventToSession:612(3990ns) ForwardSessionEventToSession:612(5839ns) ForwardSessionEventToSession:612(5805ns) ForwardSessionEventToSession:612(3453ns)}}}
2025-11-27T18:18:16.889972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471197146115963:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:16.890463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471197146115959:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:16.917636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471197146115979:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:16.920529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471197146115954:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.920574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:16.920626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471197146115954:2107] [node 4] ICP22 created new session: [2:7577471197146115987:2107]
2025-11-27T18:18:16.920669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471197146115987:2107] [node 4] ICS09 handshake done sender: [2:7577471197146115959:2112] self: [2:7577471197146115958:0] peer: [4:7577471198474826102:0] socket: 408 qp: -1
2025-11-27T18:18:16.920708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471197146115987:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:16.920741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:16.921007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471197146115954:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:16.921406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471197146115955:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.921445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:16.921484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471197146115955:2110] [node 6] ICP22 created new session: [2:7577471197146115989:2110]
2025-11-27T18:18:16.921520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471197146115989:2110] [node 6] ICS09 handshake done sender: [2:7577471197146115963:2114] self: [2:7577471197146115962:0] peer: [6:7577471197555784853:0] socket: 454 qp: -1
2025-11-27T18:18:16.921553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471197146115989:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:16.921583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:16.923577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471197146115955:2110] [node 6] ICP18 handshake (actor [2:7577471197146115979:2119]) is too old
2025-11-27T18:18:16.923933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471197146115991:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32123
2025-11-27T18:18:16.950779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471197146115973:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:16.974463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471192851148593:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:16.974512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:16.974573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471192851148593:2048] [node 1] ICP22 created new session: [2:7577471197146115992:2048]
2025-11-27T18:18:16.974618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471197146115992:2048] [node 1] ICS09 handshake done sender: [2:7577471197146115973:2118] self: [2:7577471197146115986:0] peer: [1:7577471200069035056:0] socket: 450 qp: -1
2025-11-27T18:18:16.974744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471197146115992:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:16.974785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:16.998843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471198283305886:2135] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:16.999348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471198283305872:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:16.999388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:16.999437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471198283305872:2108] [node 6] ICP22 created new session: [3:7577471198283305908:2108]
2025-11-27T18:18:16.999483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471198283305908:2108] [node 6] ICS09 handshake done sender: [3:7577471198283305886:2135] self: [3:7577471198283305884:0] peer: [6:7577471197555784859:0] socket: 420 qp: -1
2025-11-27T18:18:16.999521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471198283305908:2108] [node 6] ICS10 traffic start
2025-11-27T18:18:16.999571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:17.000218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471198283305907:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:17.008857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471197555784864:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36164
2025-11-27T18:18:17.009234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471197555784855:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:17.009415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471193260817470:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:17.009469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:17.009527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471193260817470:2048] [node 1] ICP22 created new session: [6:7577471201850752161:2048]
2025-11-27T18:18:17.009581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471201850752161:2048] [node 1] ICS09 handshake done sender: [6:7577471197555784851:2118] self: [6:7577471197555784858:0] peer: [1:7577471200069035058:0] socket: 425 qp: -1
2025-11-27T18:18:17.009615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471201850752161:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:17.009652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:17.006743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471195774067687:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(35240317ns) {HandleHandshakeStatus:329(35238090ns) {DropIncomingHandshake:442(707ns) DropOutgoingHandshake:457(562ns) SwitchToState:179(608ns) ProcessPendingSessionEvents:495(106767ns) {ForwardSessionEventToSession:612(54645ns) ForwardSessionEventToSession:612(2566ns) ForwardSessionEventToSession:612(3369ns) ForwardSessionEventToSession:612(5987ns) ForwardSessionEventToSession:612(3492ns) ForwardSessionEventToSession:612(6087ns) ForwardSessionEventToSession:612(3357ns) ForwardSessionEventToSession:612(5112ns) ForwardSessionEventToSession:612(8558ns) ForwardSessionEventToSession:612(2519ns)}}}
2025-11-27T18:18:17.007268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471195774067687:2048] [node 2] ICP18 handshake (actor [1:7577471200069035079:2119]) is too old
2025-11-27T18:18:17.011476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471197555784820:2098] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:17.011524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:17.011568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471197555784820:2098] [node 5] ICP22 created new session: [6:7577471201850752163:2098]
2025-11-27T18:18:17.011613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471201850752163:2098] [node 5] ICS09 handshake done sender: [6:7577471197555784855:2121] self: [6:7577471197555784860:0] peer: [5:7577471200218127142:0] socket: 417 qp: -1
2025-11-27T18:18:17.011642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471201850752163:2098] [node 5] ICS10 traffic start
2025-11-27T18:18:17.011672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:17.013871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471197555784852:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:17.014302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471193260817472:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:17.014346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:17.014393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471193260817472:2050] [node 3] ICP22 created new session: [6:7577471201850752165:2050]
2025-11-27T18:18:17.014433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471201850752165:2050] [node 3] ICS09 handshake done sender: [6:7577471197555784852:2119] self: [6:7577471197555784859:0] peer: [3:7577471198283305884:0] socket: 464 qp: -1
2025-11-27T18:18:17.014460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471201850752165:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:17.014500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:17.016720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471198283305872:2108] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:17.057481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:17.057794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471197146115992:2048] [node 1] ICS01 socket: 450 reason# EndOfStream
2025-11-27T18:18:17.057846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471192851148593:2048] [node 1] ICP30 unregister session Session# [2:7577471197146115992:2048] VirtualId# [2:7577471197146115986:0]
2025-11-27T18:18:17.057882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471197146115992:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.057958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:17.058000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:17.061877Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:17.062145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471198283305897:2048] [node 1] ICS01 socket: 441 reason# EndOfStream
2025-11-27T18:18:17.062195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471193988338491:2048] [node 1] ICP30 unregister session Session# [3:7577471198283305897:2048] VirtualId# [3:7577471198283305890:0]
2025-11-27T18:18:17.062233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471198283305897:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.062305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:17.062345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:17.069239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:17.069487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471201850752161:2048] [node 1] ICS01 socket: 425 reason# EndOfStream
2025-11-27T18:18:17.069536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471193260817470:2048] [node 1] ICP30 unregister session Session# [6:7577471201850752161:2048] VirtualId# [6:7577471197555784858:0]
2025-11-27T18:18:17.069572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471201850752161:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.069656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:17.069706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:17.069794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:17.069967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471200218127167:2048] [node 1] ICS01 socket: 440 reason# EndOfStream
2025-11-27T18:18:17.069995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471195923159755:2048] [node 1] ICP30 unregister session Session# [5:7577471200218127167:2048] VirtualId# [5:7577471200218127157:0]
2025-11-27T18:18:17.070021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471200218127167:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.069313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:17.070068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:17.069518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471198474826103:2048] [node 1] ICS01 socket: 438 reason# EndOfStream
2025-11-27T18:18:17.070101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:17.069555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471194179858712:2048] [node 1] ICP30 unregister session Session# [4:7577471198474826103:2048] VirtualId# [4:7577471198474826093:0]
2025-11-27T18:18:17.069586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471198474826103:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.069658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:17.069699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:17.148376Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:17.148605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471198283305892:2049] [node 2] ICS01 socket: 410 reason# EndOfStream
2025-11-27T18:18:17.148646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471193988338492:2049] [node 2] ICP30 unregister session Session# [3:7577471198283305892:2049] VirtualId# [3:7577471198283305889:0]
2025-11-27T18:18:17.148684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471198283305892:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.148754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:17.148794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:17.162212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:17.162386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471200218127158:2049] [node 2] ICS01 socket: 411 reason# EndOfStream
2025-11-27T18:18:17.162419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471195923159756:2049] [node 2] ICP30 unregister session Session# [5:7577471200218127158:2049] VirtualId# [5:7577471200218127153:0]
2025-11-27T18:18:17.162452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471200218127158:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.162504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:17.162537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:17.161651Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:17.161876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471198474826109:2049] [node 2] ICS01 socket: 421 reason# EndOfStream
2025-11-27T18:18:17.161942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471194179858713:2049] [node 2] ICP30 unregister session Session# [4:7577471198474826109:2049] VirtualId# [4:7577471198474826102:0]
2025-11-27T18:18:17.161979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471198474826109:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.162061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:17.162114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:17.162615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:17.162788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471197555784856:2049] [node 2] ICS01 socket: 456 reason# EndOfStream
2025-11-27T18:18:17.162813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471193260817471:2049] [node 2] ICP30 unregister session Session# [6:7577471197555784856:2049] VirtualId# [6:7577471197555784853:0]
2025-11-27T18:18:17.162839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471197555784856:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.162882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:17.162914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:17.245603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:17.245839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471198474826111:2050] [node 3] ICS01 socket: 423 reason# EndOfStream
2025-11-27T18:18:17.245884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471194179858714:2050] [node 3] ICP30 unregister session Session# [4:7577471198474826111:2050] VirtualId# [4:7577471198474826101:0]
2025-11-27T18:18:17.245922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471198474826111:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.246015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:17.246063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:17.246159Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:17.246349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471201850752165:2050] [node 3] ICS01 socket: 464 reason# EndOfStream
2025-11-27T18:18:17.246376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471193260817472:2050] [node 3] ICP30 unregister session Session# [6:7577471201850752165:2050] VirtualId# [6:7577471197555784859:0]
2025-11-27T18:18:17.246401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471201850752165:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.246460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:17.246508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:17.246686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:17.246870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471200218127164:2050] [node 3] ICS07 socket disconnect 452 reason# EndOfStream
2025-11-27T18:18:17.246896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471200218127164:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.246960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:17.246998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471200218127164:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:17.249195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471195923159757:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18106) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:17.249242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471200218127164:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:17.249308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471195923159757:2050] [node 3] ICP30 unregister session Session# [5:7577471200218127164:2050] VirtualId# [5:7577471200218127156:0]
2025-11-27T18:18:17.249353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:17.249402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471195923159757:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:17.249420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:17.263861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471195923159757:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:17.304593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:17.304759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471197555784848:2100] [node 4] ICS01 socket: 429 reason# EndOfStream
2025-11-27T18:18:17.304791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471197555784824:2100] [node 4] ICP30 unregister session Session# [6:7577471197555784848:2100] VirtualId# [6:7577471197555784845:0]
2025-11-27T18:18:17.304816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471197555784848:2100] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.304863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:17.304896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:17.309122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:17.309372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471200218127165:2108] [node 4] ICS01 socket: 437 reason# EndOfStream
2025-11-27T18:18:17.309426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471200218127132:2108] [node 4] ICP30 unregister session Session# [5:7577471200218127165:2108] VirtualId# [5:7577471200218127152:0]
2025-11-27T18:18:17.309463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471200218127165:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.309544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:17.309588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:17.429813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:17.430062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471201850752163:2098] [node 5] ICS01 socket: 417 reason# EndOfStream
2025-11-27T18:18:17.430107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471197555784820:2098] [node 5] ICP30 unregister session Session# [6:7577471201850752163:2098] VirtualId# [6:7577471197555784860:0]
2025-11-27T18:18:17.430144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471201850752163:2098] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:17.430230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:17.430275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:17.597603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471204327570795:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:17.597673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471204327570796:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:17.636961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471201581567887:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:17.637016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471201581567888:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:17.670637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471204781762391:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:17.670692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471204781762392:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:17.702903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471203259868037:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:17.702944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471203259868038:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:17.702972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471203259868039:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:17.731640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471202589687223:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:17.731718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471202589687224:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:17.731766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471202589687225:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:17.758152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471201800613428:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:17.758192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471201800613429:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:17.758225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471201800613430:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:18.090547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471207554835389:2101] [node 6] ICP01 ready to work
2025-11-27T18:18:18.195753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471208622538156:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:18.139752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471206884654544:2071] [node 6] ICP01 ready to work
2025-11-27T18:18:18.134243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471205876535243:2106] [node 4] ICP01 ready to work
2025-11-27T18:18:18.210137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471206095580742:2061] [node 5] ICP01 ready to work
2025-11-27T18:18:18.213391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471209076729761:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:18.221461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471205876535248:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:18.249362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471208622538157:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:18.298173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471209076729763:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.305471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471207554835389:2101] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(214612761ns) {Configure:104(214606246ns) {ProcessConfigured:135(18847171ns) {StartInitialHandshake:157(18843360ns) {DropHandshakes:470(3076ns) {DropIncomingHandshake:442(707ns) DropOutgoingHandshake:457(410ns)} GenerateSessionVirtualId:988(902ns) PrepareNewSessionHandshake:477(3544ns) {SwitchToState:179(1784ns)}}}}}
2025-11-27T18:18:18.248553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471206884654553:2074] [node 4] ICP01 ready to work
2025-11-27T18:18:18.304975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471209076729764:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31842
2025-11-27T18:18:18.321316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471205876535249:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:18.286344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471208622538158:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:18.329380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471209076729762:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:18.341537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471207554835402:2102] [node 5] ICP01 ready to work
2025-11-27T18:18:18.368826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471209076729762:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19964080ns) {Configure:104(19958163ns) {ProcessConfigured:135(76687ns) {StartInitialHandshake:157(71231ns) {DropHandshakes:470(3007ns) {DropIncomingHandshake:442(657ns) DropOutgoingHandshake:457(490ns)} GenerateSessionVirtualId:988(995ns) PrepareNewSessionHandshake:477(1982ns) {SwitchToState:179(744ns)}}}}}
2025-11-27T18:18:18.379566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471201800613428:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(145681458ns) {Configure:104(145671409ns) {ProcessConfigured:135(81414ns) {StartInitialHandshake:157(77989ns) {DropHandshakes:470(3362ns) {DropIncomingHandshake:442(752ns) DropOutgoingHandshake:457(536ns)} GenerateSessionVirtualId:988(1077ns) PrepareNewSessionHandshake:477(1947ns) {SwitchToState:179(627ns)}}}}}
2025-11-27T18:18:18.385621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471206884654544:2071] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(126554014ns) {Configure:104(126548904ns) {ProcessConfigured:135(21520716ns) {StartInitialHandshake:157(21518703ns) {DropHandshakes:470(3672ns) {DropIncomingHandshake:442(1217ns) DropOutgoingHandshake:457(419ns)} GenerateSessionVirtualId:988(596ns) PrepareNewSessionHandshake:477(2663ns) {SwitchToState:179(511ns)}}}}}
2025-11-27T18:18:18.387421Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471201800613429:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(229472669ns) {Configure:104(229467463ns) {ProcessConfigured:135(79401ns) {StartInitialHandshake:157(77233ns) {DropHandshakes:470(2807ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(484ns)} GenerateSessionVirtualId:988(791ns) PrepareNewSessionHandshake:477(1709ns) {SwitchToState:179(636ns)}}}}}
2025-11-27T18:18:18.347639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471203259868037:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(253039613ns) {Configure:104(253034175ns) {ProcessConfigured:135(49480ns) {StartInitialHandshake:157(46629ns) {DropHandshakes:470(3399ns) {DropIncomingHandshake:442(777ns) DropOutgoingHandshake:457(526ns)} GenerateSessionVirtualId:988(648ns) PrepareNewSessionHandshake:477(2147ns) {SwitchToState:179(756ns)}}}}}
2025-11-27T18:18:18.399361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471206095580796:2087] [node 4] ICP01 ready to work
2025-11-27T18:18:18.400612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471201581567888:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(303000932ns) {Configure:104(286965234ns) {ProcessConfigured:135(77230ns) {StartInitialHandshake:157(74763ns) {DropHandshakes:470(3175ns) {DropIncomingHandshake:442(809ns) DropOutgoingHandshake:457(417ns)} GenerateSessionVirtualId:988(882ns) PrepareNewSessionHandshake:477(3222ns) {SwitchToState:179(1780ns)}}}}}
2025-11-27T18:18:18.418326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471209076729764:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.452083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471206095580798:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-27T18:18:18.425354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471205876535252:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46633
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.458582Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471202589687223:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(200387264ns) {Configure:104(200377910ns) {ProcessConfigured:135(94760ns) {StartInitialHandshake:157(92386ns) {DropHandshakes:470(3149ns) {DropIncomingHandshake:442(765ns) DropOutgoingHandshake:457(346ns)} GenerateSessionVirtualId:988(1017ns) PrepareNewSessionHandshake:477(2153ns) {SwitchToState:179(677ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.478959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471202589687225:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(365209514ns) {Configure:104(365203602ns) {ProcessConfigured:135(13232591ns) {StartInitialHandshake:157(13227810ns) {DropHandshakes:470(1996ns) {DropIncomingHandshake:442(577ns) DropOutgoingHandshake:457(455ns)} GenerateSessionVirtualId:988(2900ns) PrepareNewSessionHandshake:477(3637ns) {SwitchToState:179(1282ns)}}}}}
2025-11-27T18:18:18.479098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471206884654591:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23068
2025-11-27T18:18:18.479258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471206884654592:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23068
2025-11-27T18:18:18.433802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471209076729776:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31842
2025-11-27T18:18:18.425543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471205876535253:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46633
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.494301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471207554835409:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4396
2025-11-27T18:18:18.473799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471206095580796:2087] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(42048636ns) {Configure:104(42041806ns) {ProcessConfigured:135(10711822ns) {StartInitialHandshake:157(10709670ns) {DropHandshakes:470(760ns) {DropIncomingHandshake:442(210ns) DropOutgoingHandshake:457(387ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(3805ns) {SwitchToState:179(1243ns)}}}}}
2025-11-27T18:18:18.480464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471209076729776:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.480691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471206884654590:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23068
2025-11-27T18:18:18.490341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471206884654592:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.499621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471208622538157:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(179941239ns) {Configure:104(179927040ns) {ProcessConfigured:135(62736ns) {StartInitialHandshake:157(60226ns) {DropHandshakes:470(2088ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(484ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:477(829ns) {SwitchToState:179(306ns)}}}}}
2025-11-27T18:18:18.425595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471205876535254:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46633
2025-11-27T18:18:18.475282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471206095580798:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.502477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471204781762392:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.451514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471205876535252:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.537293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471207554835401:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.537349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471207554835401:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.519042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471206884654590:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.485477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471206095580799:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.490929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471205876535253:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.566321Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471204327570795:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(289747184ns) {Configure:104(289739864ns) {ProcessConfigured:135(5974905ns) {StartInitialHandshake:157(5973168ns) {DropHandshakes:470(3276ns) {DropIncomingHandshake:442(887ns) DropOutgoingHandshake:457(536ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:477(5579ns) {SwitchToState:179(1408ns)}}}}}
2025-11-27T18:18:18.567217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471208622538163:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13610
2025-11-27T18:18:18.567381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471208622538164:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13610
2025-11-27T18:18:18.567464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471208622538165:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13610
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.556190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471207554835409:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.560210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471209076729770:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.510902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471206095580796:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.509646Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471205876535243:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(255885844ns) {Configure:104(255874217ns) {ProcessConfigured:135(27992438ns) {StartInitialHandshake:157(27989143ns) {DropHandshakes:470(3656ns) {DropIncomingHandshake:442(741ns) DropOutgoingHandshake:457(543ns)} GenerateSessionVirtualId:988(733ns) PrepareNewSessionHandshake:477(5957ns) {SwitchToState:179(3220ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.523379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471202589687225:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.559970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471207554835401:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:18.511132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471206095580804:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.529696Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471202589687224:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(136799422ns) {Configure:104(136791885ns) {ProcessConfigured:135(78433ns) {StartInitialHandshake:157(75951ns) {DropHandshakes:470(6736ns) {DropIncomingHandshake:442(946ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(911ns) PrepareNewSessionHandshake:477(1884ns) {SwitchToState:179(633ns)}}}}}
2025-11-27T18:18:18.560253Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471209076729770:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.542036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471201581567887:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(194894563ns) {Configure:104(194887290ns) {ProcessConfigured:135(7852233ns) {StartInitialHandshake:157(7849323ns) {DropHandshakes:470(3102ns) {DropIncomingHandshake:442(637ns) DropOutgoingHandshake:457(588ns)} GenerateSessionVirtualId:988(1051ns) PrepareNewSessionHandshake:477(3320ns) {SwitchToState:179(1906ns)}}}}}
2025-11-27T18:18:18.595669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471208622538163:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.565646Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471209076729761:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(236041136ns) {Configure:104(236033300ns) {ProcessConfigured:135(68232ns) {StartInitialHandshake:157(66003ns) {DropHandshakes:470(5022ns) {DropIncomingHandshake:442(537ns) DropOutgoingHandshake:457(362ns)} GenerateSessionVirtualId:988(710ns) PrepareNewSessionHandshake:477(2030ns) {SwitchToState:179(723ns)}}}}}
2025-11-27T18:18:18.571372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471209076729770:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:18.583371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471209076729782:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31842
2025-11-27T18:18:18.583465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471209076729783:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31842
2025-11-27T18:18:18.583512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471209076729784:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31842
2025-11-27T18:18:18.605251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471209076729763:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.607704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:18.618280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471209076729763:2108] [node 5] ICP22 created new session: [3:7577471209076729786:2108]
2025-11-27T18:18:18.529816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471202589687224:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.542171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471205876535256:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.514789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471206095580799:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.598952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471208622538156:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(97196733ns) {Configure:104(97192312ns) {ProcessConfigured:135(61751ns) {StartInitialHandshake:157(59281ns) {DropHandshakes:470(1944ns) {DropIncomingHandshake:442(389ns) DropOutgoingHandshake:457(312ns)} GenerateSessionVirtualId:988(750ns) PrepareNewSessionHandshake:477(1991ns) {SwitchToState:179(534ns)}}}}}
2025-11-27T18:18:18.605402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471207554835402:2102] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(229715279ns) {Configure:104(229708864ns) {ProcessConfigured:135(19203634ns) {StartInitialHandshake:157(19201375ns) {DropHandshakes:470(2532ns) {DropIncomingHandshake:442(522ns) DropOutgoingHandshake:457(526ns)} GenerateSessionVirtualId:988(448ns) PrepareNewSessionHandshake:477(6155ns) {SwitchToState:179(3624ns)}}}}}
2025-11-27T18:18:18.541585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471206884654591:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.542192Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471205876535256:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.514930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471201800613429:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.638621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471209076729776:2126] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:18.600727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471204327570796:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(311905550ns) {Configure:104(311900963ns) {ProcessConfigured:135(20047932ns) {StartInitialHandshake:157(20044137ns) {DropHandshakes:470(2900ns) {DropIncomingHandshake:442(670ns) DropOutgoingHandshake:457(616ns)} GenerateSessionVirtualId:988(864ns) PrepareNewSessionHandshake:477(8795790ns) {SwitchToState:179(5568ns)}}}}}
2025-11-27T18:18:18.606756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471207554835413:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4396
2025-11-27T18:18:18.544745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471205876535256:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.533521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471206095580804:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.638673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471209076729786:2108] [node 5] ICS09 handshake done sender: [3:7577471209076729770:2124] self: [3:7577471209076729769:0] peer: [5:7577471206884654600:0] socket: 412 qp: -1
2025-11-27T18:18:18.621523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471207554835389:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.601343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471208622538165:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.543407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471206884654603:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23068
2025-11-27T18:18:18.540362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471201800613430:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(415154132ns) {Configure:104(415144125ns) {ProcessConfigured:135(35088878ns) {StartInitialHandshake:157(35086702ns) {DropHandshakes:470(2325ns) {DropIncomingHandshake:442(321ns) DropOutgoingHandshake:457(425ns)} GenerateSessionVirtualId:988(837ns) PrepareNewSessionHandshake:477(3980ns) {SwitchToState:179(786ns)}}}}}
2025-11-27T18:18:18.549965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471205876535258:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.621573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:18.643646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471209076729786:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:18.554444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471206884654585:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.546735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471206095580742:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.550006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471205876535258:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.627808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471208622538164:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.644616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:18.637444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471207554835389:2101] [node 6] ICP22 created new session: [4:7577471207554835415:2101]
2025-11-27T18:18:18.554468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471206884654585:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.557677Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471206884654592:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:18.559807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471206884654585:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:18.574837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471206884654606:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23068
2025-11-27T18:18:18.574891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471202589687224:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.574951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:18.575003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471202589687224:2049] [node 2] ICP22 created new session: [5:7577471206884654607:2049]
2025-11-27T18:18:18.575052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471206884654607:2049] [node 2] ICS09 handshake done sender: [5:7577471206884654592:2115] self: [5:7577471206884654601:0] peer: [2:7577471205876535255:0] socket: 432 qp: -1
2025-11-27T18:18:18.575085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471206884654607:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:18.575117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:18.579053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471206884654605:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23068
2025-11-27T18:18:18.595592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471206884654544:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.606252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:18.606307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471206884654544:2071] [node 6] ICP22 created new session: [5:7577471206884654610:2071]
2025-11-27T18:18:18.617863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471206884654590:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:18.626147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471206884654610:2071] [node 6] ICS09 handshake done sender: [5:7577471206884654585:2110] self: [5:7577471206884654583:0] peer: [6:7577471206095580807:0] socket: 434 qp: -1
2025-11-27T18:18:18.642102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471202589687225:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.642178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:18.642230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471202589687225:2050] [node 3] ICP22 created new session: [5:7577471206884654611:2050]
2025-11-27T18:18:18.642285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471206884654611:2050] [node 3] ICS09 handshake done sender: [5:7577471206884654590:2113] self: [5:7577471206884654600:0] peer: [3:7577471209076729769:0] socket: 430 qp: -1
2025-11-27T18:18:18.642322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471206884654611:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:18.642382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:18.644395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471206884654610:2071] [node 6] ICS10 traffic start
2025-11-27T18:18:18.644430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:18.663709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471206884654606:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.666552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471206884654544:2071] [node 6] ICP35 event processing took too much time PendingConnection:279(72690050ns) {HandleHandshakeStatus:329(72685321ns) {DropIncomingHandshake:442(14584ns) DropOutgoingHandshake:457(649ns) SwitchToState:179(465ns) ProcessPendingSessionEvents:495(3361415ns) {ForwardSessionEventToSession:612(3302338ns) ForwardSessionEventToSession:612(12179ns) ForwardSessionEventToSession:612(3189ns) ForwardSessionEventToSession:612(5830ns) ForwardSessionEventToSession:612(5641ns) ForwardSessionEventToSession:612(2989ns) ForwardSessionEventToSession:612(6138ns) ForwardSessionEventToSession:612(3144ns) ForwardSessionEventToSession:612(2405ns) ForwardSessionEventToSession:612(5691ns)}}}
2025-11-27T18:18:18.667141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471202589687223:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.672006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471206884654615:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23068
2025-11-27T18:18:18.683790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471206884654606:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:18.703379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471202589687223:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471206884654606:2120] held: no
2025-11-27T18:18:18.641680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471208622538178:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13610
2025-11-27T18:18:18.554211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471206095580808:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-27T18:18:18.653863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471209076729782:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.550271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471205876535251:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.644126Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471208622538178:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.644512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471207554835402:2102] [node 5] ICP35 event processing took too much time PendingConnection:279(32334579ns) {IncomingHandshake:261(32332888ns) {DropIncomingHandshake:442(313ns) PrepareNewSessionHandshake:477(2385035ns) {SwitchToState:179(2327ns)} GenerateSessionVirtualId:988(347ns) IssueIncomingHandshakeReply:193(8674ns)}}
2025-11-27T18:18:18.559577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471206095580809:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-27T18:18:18.667471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471208622538162:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.655676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471209076729763:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(50241063ns) {HandleHandshakeStatus:329(50236673ns) {DropIncomingHandshake:442(1061ns) DropOutgoingHandshake:457(1949ns) SwitchToState:179(1014ns) ProcessPendingSessionEvents:495(113298ns) {ForwardSessionEventToSession:612(56045ns) ForwardSessionEventToSession:612(3986ns) ForwardSessionEventToSession:612(4999ns) ForwardSessionEventToSession:612(3519ns) ForwardSessionEventToSession:612(4598ns) ForwardSessionEventToSession:612(2749ns) ForwardSessionEventToSession:612(2011ns) ForwardSessionEventToSession:612(13886ns) ForwardSessionEventToSession:612(7630ns) ForwardSessionEventToSession:612(3880ns)}}}
2025-11-27T18:18:18.655973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471209076729763:2108] [node 5] ICP18 handshake (actor [3:7577471209076729782:2128]) is too old
2025-11-27T18:18:18.664049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471204781762392:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.664105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:18.664169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471204781762392:2049] [node 2] ICP22 created new session: [3:7577471209076729789:2049]
2025-11-27T18:18:18.550303Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471205876535251:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.654837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471207554835415:2101] [node 6] ICS09 handshake done sender: [4:7577471207554835401:2111] self: [4:7577471207554835400:0] peer: [6:7577471206095580805:0] socket: 415 qp: -1
2025-11-27T18:18:18.667495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471208622538162:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.664212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471209076729789:2049] [node 2] ICS09 handshake done sender: [3:7577471209076729776:2126] self: [3:7577471209076729777:0] peer: [2:7577471205876535250:0] socket: 429 qp: -1
2025-11-27T18:18:18.559631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471206095580810:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-27T18:18:18.656346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471207554835413:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.664252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471209076729789:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:18.664286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:18.683443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471209076729783:2129] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.719473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471209076729775:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.719499Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471209076729775:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.720131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471204781762391:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.724389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471209076729775:2125] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:18.733268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471209076729762:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.733308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:18.733364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471209076729762:2109] [node 6] ICP22 created new session: [3:7577471209076729795:2109]
2025-11-27T18:18:18.733419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471209076729795:2109] [node 6] ICS09 handshake done sender: [3:7577471209076729775:2125] self: [3:7577471209076729774:0] peer: [6:7577471206095580825:0] socket: 464 qp: -1
2025-11-27T18:18:18.733460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471209076729795:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:18.733502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:18.734167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471209076729794:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31842
2025-11-27T18:18:18.561710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471205876535254:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.745335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471209076729783:2129] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:18.735679Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471202589687223:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(33781915ns) {HandleHandshakeStatus:395(33779331ns) {LogHandshakeFail:472(7689864ns) {UpdateErrorStateLog:545(6254ns)} DropOutgoingHandshake:457(2517ns) ProcessPendingSessionEvents:495(14279297ns) {DropSessionEvent:512(27705ns) DropSessionEvent:512(11505ns) DropSessionEvent:512(6029ns) DropSessionEvent:512(6483ns) DropSessionEvent:512(5272ns) DropSessionEvent:512(6355ns) DropSessionEvent:512(5624ns) DropSessionEvent:512(25853ns) DropSessionEvent:512(9601ns) DropSessionEvent:512(6869ns)}}}
2025-11-27T18:18:18.737808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471202589687223:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.741041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:18.746599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471202589687223:2048] [node 1] ICP22 created new session: [5:7577471206884654616:2048]
2025-11-27T18:18:18.638069Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471201800613429:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10678) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471206095580799:2114] held: no
2025-11-27T18:18:18.670166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471208622538180:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13610
2025-11-27T18:18:18.657334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471207554835415:2101] [node 6] ICS10 traffic start
2025-11-27T18:18:18.746530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471209076729788:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31842
2025-11-27T18:18:18.562463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471205876535258:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.676228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471208622538162:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:18.750871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471206884654616:2048] [node 1] ICS09 handshake done sender: [5:7577471206884654606:2120] self: [5:7577471206884654614:0] peer: [1:7577471208622538161:0] socket: 455 qp: -1
2025-11-27T18:18:18.638419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471206095580798:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:18.676675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471208622538160:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.657382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:18.676703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471208622538160:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.750894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471204781762391:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.643212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471206095580804:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:18.570419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471205876535249:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.756452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471206884654616:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.679124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471208622538158:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.657454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471203259868038:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.756797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.654234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471206095580742:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7258) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471206095580804:2117] held: no
2025-11-27T18:18:18.770969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:18.679185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:18.679228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471208622538158:2108] [node 5] ICP22 created new session: [1:7577471208622538183:2108]
2025-11-27T18:18:18.679290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471208622538183:2108] [node 5] ICS09 handshake done sender: [1:7577471208622538162:2113] self: [1:7577471208622538161:0] peer: [5:7577471206884654614:0] socket: 453 qp: -1
2025-11-27T18:18:18.679324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471208622538183:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:18.679364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:18.679754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471208622538180:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.683266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471208622538160:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:18.695184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471208622538157:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.695261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:18.695316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471208622538157:2109] [node 6] ICP22 created new session: [1:7577471208622538187:2109]
2025-11-27T18:18:18.695360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471208622538187:2109] [node 6] ICS09 handshake done sender: [1:7577471208622538160:2112] self: [1:7577471208622538159:0] peer: [6:7577471206095580821:0] socket: 451 qp: -1
2025-11-27T18:18:18.695388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471208622538187:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:18.695416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:18.715349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471208622538169:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.715377Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471208622538169:2118] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.716557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471208622538169:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:18.716789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471204327570795:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.716841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:18.716891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471204327570795:2048] [node 2] ICP22 created new session: [1:7577471208622538191:2048]
2025-11-27T18:18:18.716956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471208622538191:2048] [node 2] ICS09 handshake done sender: [1:7577471208622538169:2118] self: [1:7577471208622538167:0] peer: [2:7577471205876535284:0] socket: 440 qp: -1
2025-11-27T18:18:18.716988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471208622538191:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:18.717019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:18.725878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471208622538168:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.725907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471208622538168:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.732452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471208622538168:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:18.742061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471204327570796:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.746425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:18.750942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471204327570796:2049] [node 3] ICP22 created new session: [1:7577471208622538194:2049]
2025-11-27T18:18:18.578939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:18.771183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471208622538194:2049] [node 3] ICS09 handshake done sender: [1:7577471208622538168:2117] self: [1:7577471208622538166:0] peer: [3:7577471209076729793:0] socket: 457 qp: -1
2025-11-27T18:18:18.776993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471208622538194:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:18.779589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:18.579000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471205876535249:2108] [node 5] ICP22 created new session: [2:7577471205876535270:2108]
2025-11-27T18:18:18.781409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471208622538174:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.781439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471208622538174:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.782349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471208622538174:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:18.789662Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471204327570796:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(48604282ns) {HandleHandshakeStatus:329(48600440ns) {DropIncomingHandshake:442(31217ns) DropOutgoingHandshake:457(805ns) SwitchToState:179(759ns) ProcessPendingSessionEvents:495(1613876ns) {ForwardSessionEventToSession:612(1559920ns) ForwardSessionEventToSession:612(8283ns) ForwardSessionEventToSession:612(3631ns) ForwardSessionEventToSession:612(2956ns) ForwardSessionEventToSession:612(8617ns) ForwardSessionEventToSession:612(2811ns) ForwardSessionEventToSession:612(4924ns) ForwardSessionEventToSession:612(3198ns) ForwardSessionEventToSession:612(2997ns) ForwardSessionEventToSession:612(6401ns)}}}
2025-11-27T18:18:18.594058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471205876535251:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:18.594189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471205876535248:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.594259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:18.594318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471205876535248:2109] [node 6] ICP22 created new session: [2:7577471205876535274:2109]
2025-11-27T18:18:18.594366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471205876535274:2109] [node 6] ICS09 handshake done sender: [2:7577471205876535258:2116] self: [2:7577471205876535257:0] peer: [6:7577471206095580806:0] socket: 426 qp: -1
2025-11-27T18:18:18.594406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471205876535274:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:18.594449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:18.595434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471205876535270:2108] [node 5] ICS09 handshake done sender: [2:7577471205876535256:2115] self: [2:7577471205876535255:0] peer: [5:7577471206884654601:0] socket: 425 qp: -1
2025-11-27T18:18:18.608051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471205876535270:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:18.608100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:18.644474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471201581567888:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.660297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471205876535249:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(89988808ns) {HandleHandshakeStatus:329(89984347ns) {DropIncomingHandshake:442(974ns) DropOutgoingHandshake:457(1016ns) SwitchToState:179(4835ns) ProcessPendingSessionEvents:495(15697634ns) {ForwardSessionEventToSession:612(15642053ns) ForwardSessionEventToSession:612(6506ns) ForwardSessionEventToSession:612(6329ns) ForwardSessionEventToSession:612(3394ns) ForwardSessionEventToSession:612(5324ns) ForwardSessionEventToSession:612(5074ns) ForwardSessionEventToSession:612(2828ns) ForwardSessionEventToSession:612(2731ns) ForwardSessionEventToSession:612(7953ns) ForwardSessionEventToSession:612(2242ns)}}}
2025-11-27T18:18:18.660849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:18.661524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471205876535264:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.661548Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471205876535264:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.663127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471201581567888:2049] [node 3] ICP22 created new session: [2:7577471205876535277:2049]
2025-11-27T18:18:18.675303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471205876535277:2049] [node 3] ICS09 handshake done sender: [2:7577471205876535251:2111] self: [2:7577471205876535250:0] peer: [3:7577471209076729777:0] socket: 424 qp: -1
2025-11-27T18:18:18.675348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471205876535277:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:18.675808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471205876535264:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:18.677963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:18.683606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471205876535243:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.683656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:18.693713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471205876535243:2106] [node 4] ICP22 created new session: [2:7577471205876535282:2106]
2025-11-27T18:18:18.693764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471205876535282:2106] [node 4] ICS09 handshake done sender: [2:7577471205876535264:2117] self: [2:7577471205876535263:0] peer: [4:7577471207554835417:0] socket: 459 qp: -1
2025-11-27T18:18:18.693801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471205876535282:2106] [node 4] ICS10 traffic start
2025-11-27T18:18:18.693848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:18.694310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471205876535281:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46633
2025-11-27T18:18:18.714681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471205876535281:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.714816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471201581567887:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.732966Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471201581567887:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471205876535281:2119] held: no
2025-11-27T18:18:18.745646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471205876535285:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46633
2025-11-27T18:18:18.755943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471201581567888:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(80981926ns) {HandleHandshakeStatus:329(80979698ns) {DropIncomingHandshake:442(15698ns) DropOutgoingHandshake:457(586ns) SwitchToState:179(1494ns) ProcessPendingSessionEvents:495(39165060ns) {ForwardSessionEventToSession:612(24040870ns) ForwardSessionEventToSession:612(8457ns) ForwardSessionEventToSession:612(8136979ns) ForwardSessionEventToSession:612(3756ns) ForwardSessionEventToSession:612(3418ns) ForwardSessionEventToSession:612(6363259ns) ForwardSessionEventToSession:612(9119ns) ForwardSessionEventToSession:612(4081ns) ForwardSessionEventToSession:612(572329ns) ForwardSessionEventToSession:612(3506ns)}}}
2025-11-27T18:18:18.765657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471205876535281:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:18.778060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471201581567887:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.654399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471206095580799:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:18.778108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:18.761857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471204781762391:2048] [node 1] ICP22 created new session: [3:7577471209076729797:2048]
2025-11-27T18:18:18.777514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471202589687223:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(41608145ns) {HandleHandshakeStatus:329(39588014ns) {DropIncomingHandshake:442(1446ns) DropOutgoingHandshake:457(591ns) SwitchToState:179(850ns) ProcessPendingSessionEvents:495(571ns)}}
2025-11-27T18:18:18.672635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471207554835419:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4396
2025-11-27T18:18:18.763506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471209076729788:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.769548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471209076729797:2048] [node 1] ICS09 handshake done sender: [3:7577471209076729783:2129] self: [3:7577471209076729793:0] peer: [1:7577471208622538166:0] socket: 461 qp: -1
2025-11-27T18:18:18.769611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471209076729797:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:18.769686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:18.770140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471204781762391:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19046186ns) {HandleHandshakeStatus:329(19041853ns) {DropIncomingHandshake:442(1357ns) DropOutgoingHandshake:457(11067ns) SwitchToState:179(505ns) ProcessPendingSessionEvents:495(106532ns) {ForwardSessionEventToSession:612(56436ns) ForwardSessionEventToSession:612(3496ns) ForwardSessionEventToSession:612(10862ns) ForwardSessionEventToSession:612(2297ns) ForwardSessionEventToSession:612(2040ns) ForwardSessionEventToSession:612(7561ns) ForwardSessionEventToSession:612(3938ns) ForwardSessionEventToSession:612(5651ns) ForwardSessionEventToSession:612(2384ns) ForwardSessionEventToSession:612(2758ns)}}}
2025-11-27T18:18:18.774978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471209076729762:2109] [node 6] ICP18 handshake (actor [3:7577471209076729788:2131]) is too old
2025-11-27T18:18:18.797835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471208622538156:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.683935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471207554835413:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:18.697577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471203259868038:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.797915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:18.698221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:18.703607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471203259868038:2049] [node 2] ICP22 created new session: [4:7577471207554835421:2049]
2025-11-27T18:18:18.798398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471208622538156:2107] [node 4] ICP22 created new session: [1:7577471208622538197:2107]
2025-11-27T18:18:18.724152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471207554835421:2049] [node 2] ICS09 handshake done sender: [4:7577471207554835413:2116] self: [4:7577471207554835417:0] peer: [2:7577471205876535263:0] socket: 465 qp: -1
2025-11-27T18:18:18.797489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471206884654617:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23068
2025-11-27T18:18:18.724704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471207554835421:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:18.806776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471206884654617:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.726994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:18.806930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471206884654553:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.747715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471207554835389:2101] [node 6] ICP35 event processing took too much time PendingConnection:279(88201925ns) {HandleHandshakeStatus:329(88198023ns) {DropIncomingHandshake:442(552ns) DropOutgoingHandshake:457(806ns) SwitchToState:179(582ns) ProcessPendingSessionEvents:495(33624653ns) {ForwardSessionEventToSession:612(7353818ns) ForwardSessionEventToSession:612(804810ns) ForwardSessionEventToSession:612(3794ns) ForwardSessionEventToSession:612(2824ns) ForwardSessionEventToSession:612(7491385ns) ForwardSessionEventToSession:612(3029ns) ForwardSessionEventToSession:612(6303548ns) ForwardSessionEventToSession:612(5449242ns) ForwardSessionEventToSession:612(12832ns) ForwardSessionEventToSession:612(6169198ns)}}}
2025-11-27T18:18:18.762647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471207554835420:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4396
2025-11-27T18:18:18.771662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471207554835420:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.773197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471203259868037:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.785648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471207554835425:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4396
2025-11-27T18:18:18.787451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471207554835426:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4396
2025-11-27T18:18:18.788946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471203259868038:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(81814546ns) {HandleHandshakeStatus:329(81811790ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(4564ns) SwitchToState:179(535ns) ProcessPendingSessionEvents:495(25943533ns) {ForwardSessionEventToSession:612(11630742ns) ForwardSessionEventToSession:612(7525ns) ForwardSessionEventToSession:612(5753414ns) ForwardSessionEventToSession:612(3585ns) ForwardSessionEventToSession:612(4865627ns) ForwardSessionEventToSession:612(5179ns) ForwardSessionEventToSession:612(1462212ns) ForwardSessionEventToSession:612(2187780ns) ForwardSessionEventToSession:612(3927ns) ForwardSessionEventToSession:612(3746ns)}}}
2025-11-27T18:18:18.789111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471207554835425:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.797673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471203259868039:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.801981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471207554835420:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:18.806422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471203259868037:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.656630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471206095580796:2087] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.809664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471208622538197:2107] [node 4] ICS09 handshake done sender: [1:7577471208622538174:2119] self: [1:7577471208622538173:0] peer: [4:7577471207554835424:0] socket: 417 qp: -1
2025-11-27T18:18:18.809727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471208622538197:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:18.809826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:18.810927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:18.811264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471207554835412:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.811283Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471207554835412:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.811355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471203259868037:2048] [node 1] ICP22 created new session: [4:7577471207554835428:2048]
2025-11-27T18:18:18.657492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:18.811838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471207554835428:2048] [node 1] ICS09 handshake done sender: [4:7577471207554835420:2118] self: [4:7577471207554835424:0] peer: [1:7577471208622538173:0] socket: 435 qp: -1
2025-11-27T18:18:18.811880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471207554835428:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:18.811922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:18.778152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471201581567887:2048] [node 1] ICP22 created new session: [2:7577471205876535286:2048]
No RDMA context found for GID: 2025-11-27T18:18:18.810421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471209076729779:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:18.778203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471205876535286:2048] [node 1] ICS09 handshake done sender: [2:7577471205876535281:2119] self: [2:7577471205876535284:0] peer: [1:7577471208622538167:0] socket: 411 qp: -1
2025-11-27T18:18:18.778240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471205876535286:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:18.778277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:18.657563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471206095580796:2087] [node 4] ICP22 created new session: [6:7577471206095580813:2087]
2025-11-27T18:18:18.657609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471206095580813:2087] [node 4] ICS09 handshake done sender: [6:7577471206095580798:2113] self: [6:7577471206095580805:0] peer: [4:7577471207554835400:0] socket: 423 qp: -1
2025-11-27T18:18:18.657643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471206095580813:2087] [node 4] ICS10 traffic start
2025-11-27T18:18:18.657672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:18.658071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471206095580742:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.658107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:18.658146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471206095580742:2061] [node 5] ICP22 created new session: [6:7577471206095580815:2061]
2025-11-27T18:18:18.658195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471206095580815:2061] [node 5] ICS09 handshake done sender: [6:7577471206095580804:2117] self: [6:7577471206095580807:0] peer: [5:7577471206884654583:0] socket: 442 qp: -1
2025-11-27T18:18:18.658226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471206095580815:2061] [node 5] ICS10 traffic start
2025-11-27T18:18:18.658271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:18.658517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471206095580811:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-27T18:18:18.658631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471201800613429:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.658676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:18.658731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471201800613429:2049] [node 2] ICP22 created new session: [6:7577471206095580817:2049]
2025-11-27T18:18:18.658776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471206095580817:2049] [node 2] ICS09 handshake done sender: [6:7577471206095580799:2114] self: [6:7577471206095580806:0] peer: [2:7577471205876535257:0] socket: 427 qp: -1
2025-11-27T18:18:18.658805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471206095580817:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:18.658858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:18.659267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471206095580812:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-27T18:18:18.665965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471206095580811:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.666081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471201800613428:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.677858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471206095580812:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:18.682060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471206095580822:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-27T18:18:18.689362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471206095580811:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:18.689615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471201800613428:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.689680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:18.689729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471201800613428:2048] [node 1] ICP22 created new session: [6:7577471206095580823:2048]
2025-11-27T18:18:18.689775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471206095580823:2048] [node 1] ICS09 handshake done sender: [6:7577471206095580811:2121] self: [6:7577471206095580821:0] peer: [1:7577471208622538159:0] socket: 422 qp: -1
2025-11-27T18:18:18.689814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471206095580823:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:18.689847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:18.818603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471207554835412:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:18.697869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471201800613430:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:18.732166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471206095580826:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31784
2025-11-27T18:18:18.747896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471206095580812:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:18.754634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471201800613430:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.754694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:18.754747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471201800613430:2050] [node 3] ICP22 created new session: [6:7577471206095580827:2050]
2025-11-27T18:18:18.754787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471206095580827:2050] [node 3] ICS09 handshake done sender: [6:7577471206095580812:2122] self: [6:7577471206095580825:0] peer: [3:7577471209076729774:0] socket: 431 qp: -1
2025-11-27T18:18:18.754824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471206095580827:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:18.754865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:18.819319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471207554835402:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.819392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:18.820759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471206884654620:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23068
2025-11-27T18:18:18.821574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471206884654553:2074] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11281) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471206884654617:2122] held: no
2025-11-27T18:18:18.822296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471206884654617:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:18.822457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471206884654553:2074] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.822522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:18.822571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471206884654553:2074] [node 4] ICP22 created new session: [5:7577471206884654621:2074]
2025-11-27T18:18:18.822632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471206884654621:2074] [node 4] ICS09 handshake done sender: [5:7577471206884654617:2122] self: [5:7577471206884654619:0] peer: [4:7577471207554835411:0] socket: 438 qp: -1
2025-11-27T18:18:18.822673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471206884654621:2074] [node 4] ICS10 traffic start
2025-11-27T18:18:18.822707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:18.819453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471207554835402:2102] [node 5] ICP22 created new session: [4:7577471207554835431:2102]
2025-11-27T18:18:18.810468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471209076729779:2127] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:18.819501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471207554835431:2102] [node 5] ICS09 handshake done sender: [4:7577471207554835412:2115] self: [4:7577471207554835411:0] peer: [5:7577471206884654619:0] socket: 416 qp: -1
2025-11-27T18:18:18.819533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471207554835431:2102] [node 5] ICS10 traffic start
2025-11-27T18:18:18.819571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:18.826272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471209076729779:2127] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:18.826943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471207554835433:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4396
2025-11-27T18:18:18.827367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471209076729761:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:18.827431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:18.827475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471209076729761:2107] [node 4] ICP22 created new session: [3:7577471209076729800:2107]
2025-11-27T18:18:18.827522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471209076729800:2107] [node 4] ICS09 handshake done sender: [3:7577471209076729779:2127] self: [3:7577471209076729778:0] peer: [4:7577471207554835427:0] socket: 439 qp: -1
2025-11-27T18:18:18.827560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471209076729800:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:18.827596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:18.828954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471207554835425:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:18.829253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471203259868039:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:18.829350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:18.829402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471203259868039:2050] [node 3] ICP22 created new session: [4:7577471207554835434:2050]
2025-11-27T18:18:18.829440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471207554835434:2050] [node 3] ICS09 handshake done sender: [4:7577471207554835425:2119] self: [4:7577471207554835427:0] peer: [3:7577471209076729778:0] socket: 476 qp: -1
2025-11-27T18:18:18.829471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471207554835434:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:18.829505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:18.876064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:18.876251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471205876535286:2048] [node 1] ICS01 socket: 411 reason# EndOfStream
2025-11-27T18:18:18.876281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471201581567887:2048] [node 1] ICP30 unregister session Session# [2:7577471205876535286:2048] VirtualId# [2:7577471205876535284:0]
2025-11-27T18:18:18.876310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471205876535286:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:18.876359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:18.876388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:18.877370Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:18.877593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471209076729797:2048] [node 1] ICS01 socket: 461 reason# EndOfStream
2025-11-27T18:18:18.877637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471204781762391:2048] [node 1] ICP30 unregister session Session# [3:7577471209076729797:2048] VirtualId# [3:7577471209076729793:0]
2025-11-27T18:18:18.877666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471209076729797:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:18.877734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:18.877776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:18.882780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:18.883081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471206095580823:2048] [node 1] ICS01 socket: 422 reason# EndOfStream
2025-11-27T18:18:18.883122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471201800613428:2048] [node 1] ICP30 unregister session Session# [6:7577471206095580823:2048] VirtualId# [6:7577471206095580821:0]
2025-11-27T18:18:18.883162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471206095580823:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:18.883308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:18.883393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:18.883809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:18.884155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471207554835428:2048] [node 1] ICS01 socket: 435 reason# EndOfStream
2025-11-27T18:18:18.884234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471203259868037:2048] [node 1] ICP30 unregister session Session# [4:7577471207554835428:2048] VirtualId# [4:7577471207554835424:0]
2025-11-27T18:18:18.884303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471207554835428:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:18.884441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:18.884521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:18.885525Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:18.885887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471206884654616:2048] [node 1] ICS01 socket: 455 reason# EndOfStream
2025-11-27T18:18:18.885934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471202589687223:2048] [node 1] ICP30 unregister session Session# [5:7577471206884654616:2048] VirtualId# [5:7577471206884654614:0]
2025-11-27T18:18:18.885959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471206884654616:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:18.886021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:18.886059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:18.919796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:18.920027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471209076729789:2049] [node 2] ICS01 socket: 429 reason# EndOfStream
2025-11-27T18:18:18.920065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471204781762392:2049] [node 2] ICP30 unregister session Session# [3:7577471209076729789:2049] VirtualId# [3:7577471209076729777:0]
2025-11-27T18:18:18.920098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471209076729789:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:18.920200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:18.920245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:18.925848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:18.927118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471206884654607:2049] [node 2] ICS01 socket: 432 reason# EndOfStream
2025-11-27T18:18:18.927157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471202589687224:2049] [node 2] ICP30 unregister session Session# [5:7577471206884654607:2049] VirtualId# [5:7577471206884654601:0]
2025-11-27T18:18:18.927188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471206884654607:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:18.927246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:18.927285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:18.927671Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:18.927849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471206095580817:2049] [node 2] ICS01 socket: 427 reason# EndOfStream
2025-11-27T18:18:18.927881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471201800613429:2049] [node 2] ICP30 unregister session Session# [6:7577471206095580817:2049] VirtualId# [6:7577471206095580806:0]
2025-11-27T18:18:18.927906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471206095580817:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:18.927961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:18.927994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:18.925760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:18.926107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471207554835421:2049] [node 2] ICS07 socket disconnect 465 reason# EndOfStream
2025-11-27T18:18:18.926136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471207554835421:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:18.926210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:18.926250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471207554835421:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:18.929229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471203259868038:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10678) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:18.929303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471207554835421:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:18.929339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471203259868038:2049] [node 2] ICP30 unregister session Session# [4:7577471207554835421:2049] VirtualId# [4:7577471207554835417:0]
2025-11-27T18:18:18.929385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:18.929424Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471203259868038:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10678) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:18.929439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10678) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:18.941561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471203259868038:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:19.059688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:19.059927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471206884654611:2050] [node 3] ICS01 socket: 430 reason# EndOfStream
2025-11-27T18:18:19.059965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471202589687225:2050] [node 3] ICP30 unregister session Session# [5:7577471206884654611:2050] VirtualId# [5:7577471206884654600:0]
2025-11-27T18:18:19.060001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471206884654611:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:19.060083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:19.060129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:19.061440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:19.061693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471207554835434:2050] [node 3] ICS01 socket: 476 reason# EndOfStream
2025-11-27T18:18:19.061736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471203259868039:2050] [node 3] ICP30 unregister session Session# [4:7577471207554835434:2050] VirtualId# [4:7577471207554835427:0]
2025-11-27T18:18:19.061767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471207554835434:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:19.061847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:19.061890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:19.062007Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:19.062211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471206095580827:2050] [node 3] ICS07 socket disconnect 431 reason# EndOfStream
2025-11-27T18:18:19.062237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471206095580827:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:19.062289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:19.062329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471206095580827:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:19.065305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471201800613430:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25212) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:19.065385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471206095580827:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:19.065421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471201800613430:2050] [node 3] ICP30 unregister session Session# [6:7577471206095580827:2050] VirtualId# [6:7577471206095580825:0]
2025-11-27T18:18:19.065472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:19.065501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471201800613430:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25212) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:19.065515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25212) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:19.076344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471201800613430:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:19.137672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:19.137927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471206095580813:2087] [node 4] ICS01 socket: 423 reason# EndOfStream
2025-11-27T18:18:19.137979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471206095580796:2087] [node 4] ICP30 unregister session Session# [6:7577471206095580813:2087] VirtualId# [6:7577471206095580805:0]
2025-11-27T18:18:19.138016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471206095580813:2087] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:19.138101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:19.138154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:19.141376Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:19.141689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471206884654621:2074] [node 4] ICS01 socket: 438 reason# EndOfStream
2025-11-27T18:18:19.141736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471206884654553:2074] [node 4] ICP30 unregister session Session# [5:7577471206884654621:2074] VirtualId# [5:7577471206884654619:0]
2025-11-27T18:18:19.141772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471206884654621:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:19.141859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:19.141901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:19.229691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:19.229933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471206095580815:2061] [node 5] ICS01 socket: 442 reason# EndOfStream
2025-11-27T18:18:19.229980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471206095580742:2061] [node 5] ICP30 unregister session Session# [6:7577471206095580815:2061] VirtualId# [6:7577471206095580807:0]
2025-11-27T18:18:19.230018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471206095580815:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:19.230102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:19.230148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:19.404104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471211109376242:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:19.415757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471211109376243:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:19.441524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471211945503764:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:19.457609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471211945503763:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:19.474937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471213204352716:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:19.474980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471213204352717:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:19.503598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471213599688763:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:19.503639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471213599688764:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:19.503664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471213599688765:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:19.534506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471210230531165:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:19.534547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471210230531166:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:19.534939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471210230531164:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:19.609730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471209847761812:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:19.609782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471209847761813:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:19.609813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471209847761814:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:20.076542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471214142729157:2096] [node 5] ICP01 ready to work
2025-11-27T18:18:20.092637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471216240471121:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:20.269427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471217894656123:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:20.102187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471214142729167:2105] [node 4] ICP01 ready to work
2025-11-27T18:18:20.239406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471215404343601:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:20.266204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471217499320076:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.289572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471216240471124:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:20.288958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471214525498525:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:20.269986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471215404343602:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.281771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471217499320077:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:20.298907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471217894656124:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:20.354753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471213599688765:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(212231910ns) {Configure:104(212225608ns) {ProcessConfigured:135(73232ns) {StartInitialHandshake:157(69960ns) {DropHandshakes:470(2734ns) {DropIncomingHandshake:442(660ns) DropOutgoingHandshake:457(586ns)} GenerateSessionVirtualId:988(508ns) PrepareNewSessionHandshake:477(1009ns) {SwitchToState:179(350ns)}}}}}
2025-11-27T18:18:20.303067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471216240471125:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.270461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471215404343603:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:20.303198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471217499320078:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:20.313904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471216240471119:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44578
2025-11-27T18:18:20.369183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471214525498526:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:20.356853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471216240471119:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.369581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471216240471135:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44578
2025-11-27T18:18:20.334958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471211109376243:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(80841898ns) {Configure:104(80834818ns) {ProcessConfigured:135(81260ns) {StartInitialHandshake:157(78070ns) {DropHandshakes:470(3299ns) {DropIncomingHandshake:442(734ns) DropOutgoingHandshake:457(415ns)} GenerateSessionVirtualId:988(877ns) PrepareNewSessionHandshake:477(2446ns) {SwitchToState:179(1112ns)}}}}}
2025-11-27T18:18:20.337402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471215404343610:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28941
2025-11-27T18:18:20.337570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471215404343611:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28941
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.402232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471211945503764:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(304532719ns) {Configure:104(304527401ns) {ProcessConfigured:135(94671ns) {StartInitialHandshake:157(91965ns) {DropHandshakes:470(3654ns) {DropIncomingHandshake:442(1076ns) DropOutgoingHandshake:457(510ns)} GenerateSessionVirtualId:988(989ns) PrepareNewSessionHandshake:477(2271ns) {SwitchToState:179(691ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.342147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471215404343610:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.435155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471216240471135:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.427311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471217499320076:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(88474065ns) {Configure:104(88468844ns) {ProcessConfigured:135(81591ns) {StartInitialHandshake:157(79297ns) {DropHandshakes:470(3339ns) {DropIncomingHandshake:442(776ns) DropOutgoingHandshake:457(437ns)} GenerateSessionVirtualId:988(993ns) PrepareNewSessionHandshake:477(1854ns) {SwitchToState:179(665ns)}}}}}
2025-11-27T18:18:20.350619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471211109376242:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(110841093ns) {Configure:104(96994280ns) {ProcessConfigured:135(71446ns) {StartInitialHandshake:157(68690ns) {DropHandshakes:470(4157ns) {DropIncomingHandshake:442(1136ns) DropOutgoingHandshake:457(480ns)} GenerateSessionVirtualId:988(1089ns) PrepareNewSessionHandshake:477(1670ns) {SwitchToState:179(529ns)}}}}}
2025-11-27T18:18:20.441773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471211945503763:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.377977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471215404343611:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.429065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471215404343603:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(128877201ns) {Configure:104(128872781ns) {ProcessConfigured:135(15247082ns) {StartInitialHandshake:157(15243934ns) {DropHandshakes:470(3560ns) {DropIncomingHandshake:442(787ns) DropOutgoingHandshake:457(454ns)} GenerateSessionVirtualId:988(793ns) PrepareNewSessionHandshake:477(3829ns) {SwitchToState:179(1693ns)}}}}}
2025-11-27T18:18:20.431435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471217894656127:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40262
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.442047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471210230531166:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(264676116ns) {Configure:104(264668526ns) {ProcessConfigured:135(714523ns) {StartInitialHandshake:157(712504ns) {DropHandshakes:470(3878ns) {DropIncomingHandshake:442(700ns) DropOutgoingHandshake:457(376ns)} GenerateSessionVirtualId:988(869ns) PrepareNewSessionHandshake:477(2547ns) {SwitchToState:179(917ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.457085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471215404343615:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.457143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471215404343615:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.443096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471209847761814:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(178280674ns) {Configure:104(178270514ns) {ProcessConfigured:135(12889824ns) {StartInitialHandshake:157(12887766ns) {DropHandshakes:470(812ns) {DropIncomingHandshake:442(203ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(275ns) PrepareNewSessionHandshake:477(4058ns) {SwitchToState:179(2682ns)}}}}}
2025-11-27T18:18:20.432375Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471217894656124:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(127746024ns) {Configure:104(127742129ns) {ProcessConfigured:135(9829447ns) {StartInitialHandshake:157(9827543ns) {DropHandshakes:470(1141ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(154ns)} GenerateSessionVirtualId:988(510ns) PrepareNewSessionHandshake:477(2689ns) {SwitchToState:179(1399ns)}}}}}
2025-11-27T18:18:20.430782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471213204352716:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(148777597ns) {Configure:104(148773569ns) {ProcessConfigured:135(54013ns) {StartInitialHandshake:157(52163ns) {DropHandshakes:470(2986ns) {DropIncomingHandshake:442(746ns) DropOutgoingHandshake:457(370ns)} GenerateSessionVirtualId:988(665ns) PrepareNewSessionHandshake:477(1139ns) {SwitchToState:179(457ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.466368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471215404343623:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28941
2025-11-27T18:18:20.451473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471216240471121:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(253410664ns) {Configure:104(253402827ns) {ProcessConfigured:135(21262598ns) {StartInitialHandshake:157(21259740ns) {DropHandshakes:470(7562012ns) {DropIncomingHandshake:442(684ns) DropOutgoingHandshake:457(2231ns)} GenerateSessionVirtualId:988(2307ns) PrepareNewSessionHandshake:477(3464ns) {SwitchToState:179(1582ns)}}}}}
2025-11-27T18:18:20.431390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471217499320083:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60986
2025-11-27T18:18:20.434348Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471213599688763:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(147889026ns) {Configure:104(147883839ns) {ProcessConfigured:135(57988ns) {StartInitialHandshake:157(56167ns) {DropHandshakes:470(905ns) {DropIncomingHandshake:442(119ns) DropOutgoingHandshake:457(500ns)} GenerateSessionVirtualId:988(283ns) PrepareNewSessionHandshake:477(1162ns) {SwitchToState:179(667ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.449026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471214142729182:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27403
2025-11-27T18:18:20.469327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471215404343615:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:20.462219Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471214525498525:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(84156475ns) {Configure:104(84149344ns) {ProcessConfigured:135(64062ns) {StartInitialHandshake:157(59083ns) {DropHandshakes:470(3801ns) {DropIncomingHandshake:442(1026ns) DropOutgoingHandshake:457(410ns)} GenerateSessionVirtualId:988(1920ns) PrepareNewSessionHandshake:477(629ns) {SwitchToState:179(256ns)}}}}}
2025-11-27T18:18:20.465904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471216240471142:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44578
2025-11-27T18:18:20.438495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471213204352717:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(157337507ns) {Configure:104(157332742ns) {ProcessConfigured:135(45859ns) {StartInitialHandshake:157(44360ns) {DropHandshakes:470(1948ns) {DropIncomingHandshake:442(521ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(638ns) PrepareNewSessionHandshake:477(1367ns) {SwitchToState:179(518ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.469742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471215404343607:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.442413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471217894656127:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.469519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471214142729157:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(322913653ns) {Configure:104(322900987ns) {ProcessConfigured:135(11095945ns) {StartInitialHandshake:157(11093191ns) {DropHandshakes:470(1891ns) {DropIncomingHandshake:442(581ns) DropOutgoingHandshake:457(147ns)} GenerateSessionVirtualId:988(413ns) PrepareNewSessionHandshake:477(2501ns) {SwitchToState:179(1029ns)}}}}}
2025-11-27T18:18:20.470994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471216240471142:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.463030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471214525498533:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33300
2025-11-27T18:18:20.469089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471217894656132:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40262
2025-11-27T18:18:20.469766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471215404343607:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.473340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471216240471143:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44578
2025-11-27T18:18:20.439177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471217499320089:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60986
2025-11-27T18:18:20.471218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471214142729182:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.473528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471215404343621:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28941
2025-11-27T18:18:20.470729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471214525498533:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.482781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471216240471127:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.443978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471217499320077:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(104984964ns) {Configure:104(104980412ns) {ProcessConfigured:135(45912ns) {StartInitialHandshake:157(44547ns) {DropHandshakes:470(6069ns) {DropIncomingHandshake:442(749ns) DropOutgoingHandshake:457(309ns)} GenerateSessionVirtualId:988(1498ns) PrepareNewSessionHandshake:477(1155ns) {SwitchToState:179(432ns)}}}}}
2025-11-27T18:18:20.482226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471215404343621:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.473001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471214142729186:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27403
2025-11-27T18:18:20.472766Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471210230531164:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(169438927ns) {Configure:104(169428100ns) {ProcessConfigured:135(16886873ns) {StartInitialHandshake:157(16883536ns) {DropHandshakes:470(1398ns) {DropIncomingHandshake:442(450ns) DropOutgoingHandshake:457(401ns)} GenerateSessionVirtualId:988(672ns) PrepareNewSessionHandshake:477(5410ns) {SwitchToState:179(2527ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.482824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471216240471127:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.451834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471217499320083:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.474734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471214142729186:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.491172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471217894656132:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.481531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471210230531165:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(169224238ns) {Configure:104(169213152ns) {ProcessConfigured:135(1765540ns) {StartInitialHandshake:157(1763159ns) {DropHandshakes:470(3553ns) {DropIncomingHandshake:442(768ns) DropOutgoingHandshake:457(707ns)} GenerateSessionVirtualId:988(799ns) PrepareNewSessionHandshake:477(1959ns) {SwitchToState:179(444ns)}}}}}
2025-11-27T18:18:20.487125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471216240471143:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.459213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471213204352716:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.483145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471215404343607:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:20.485714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471210230531165:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.474904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471209847761814:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.493558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471213599688763:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.491205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471216240471144:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44578
2025-11-27T18:18:20.493061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471214525498540:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33300
2025-11-27T18:18:20.484553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471211109376242:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.494611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471217894656131:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.467564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471217499320095:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60986
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.478181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471214142729188:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27403
2025-11-27T18:18:20.491294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471216240471131:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.513357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471214525498540:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.484637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.483009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471209847761813:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.485151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471211109376242:2048] [node 2] ICP22 created new session: [1:7577471215404343626:2048]
2025-11-27T18:18:20.494643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471217894656131:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.473908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471217499320096:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60986
2025-11-27T18:18:20.515878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471214525498526:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(46602793ns) {Configure:104(46594899ns) {ProcessConfigured:135(5067673ns) {StartInitialHandshake:157(5064789ns) {DropHandshakes:470(2738ns) {DropIncomingHandshake:442(341ns) DropOutgoingHandshake:457(403ns)} GenerateSessionVirtualId:988(608ns) PrepareNewSessionHandshake:477(3120ns) {SwitchToState:179(1629ns)}}}}}
2025-11-27T18:18:20.491323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471216240471131:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.498162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471217894656141:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40262
2025-11-27T18:18:20.520364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471210230531166:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.491543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471216240471127:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:20.484462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471214142729191:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27403
2025-11-27T18:18:20.486857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471211109376243:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.475290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471217499320080:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.524033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471214525498543:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33300
2025-11-27T18:18:20.497053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471211945503763:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3441) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471216240471135:2114] held: no
2025-11-27T18:18:20.475314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471217499320080:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.484854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471214142729188:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.502566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471217894656131:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:20.486910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:20.488124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471214142729167:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.476392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471217499320080:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:20.502825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471217894656123:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.530076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471214525498545:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33300
2025-11-27T18:18:20.486962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471211109376243:2049] [node 3] ICP22 created new session: [1:7577471215404343627:2049]
2025-11-27T18:18:20.520494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471216240471131:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.502883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:20.500205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471214142729186:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:20.476836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471217499320076:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.526972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471216240471125:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.538016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471214525498533:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:20.487015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471215404343627:2049] [node 3] ICS09 handshake done sender: [1:7577471215404343607:2111] self: [1:7577471215404343606:0] peer: [3:7577471217499320094:0] socket: 425 qp: -1
2025-11-27T18:18:20.506743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471214142729167:2105] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18077) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471214142729188:2117] held: no
2025-11-27T18:18:20.502930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471217894656123:2109] [node 6] ICP22 created new session: [4:7577471217894656145:2109]
2025-11-27T18:18:20.477938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:20.540309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471210230531165:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.527039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.487043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471215404343627:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:20.516402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471209847761814:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.487064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:20.527095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471216240471125:2110] [node 6] ICP22 created new session: [2:7577471216240471149:2110]
2025-11-27T18:18:20.502979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471217894656145:2109] [node 6] ICS09 handshake done sender: [4:7577471217894656131:2114] self: [4:7577471217894656130:0] peer: [6:7577471214142729192:0] socket: 436 qp: -1
2025-11-27T18:18:20.478254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471217499320076:2109] [node 6] ICP22 created new session: [3:7577471217499320099:2109]
2025-11-27T18:18:20.542490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471214525498546:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33300
2025-11-27T18:18:20.482833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471217499320099:2109] [node 6] ICS09 handshake done sender: [3:7577471217499320080:2111] self: [3:7577471217499320079:0] peer: [6:7577471214142729190:0] socket: 434 qp: -1
2025-11-27T18:18:20.527179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471216240471149:2110] [node 6] ICS09 handshake done sender: [2:7577471216240471127:2111] self: [2:7577471216240471126:0] peer: [6:7577471214142729189:0] socket: 427 qp: -1
2025-11-27T18:18:20.503010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471217894656145:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:20.523304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.543437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471214525498547:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33300
2025-11-27T18:18:20.489289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471215404343626:2048] [node 2] ICS09 handshake done sender: [1:7577471215404343615:2115] self: [1:7577471215404343614:0] peer: [2:7577471216240471141:0] socket: 428 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.526601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471214142729196:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27403
2025-11-27T18:18:20.503041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:20.489336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471215404343626:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:20.527215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471216240471149:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:20.482889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471217499320099:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:20.550005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471214525498540:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:20.511604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471217894656141:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.527249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:20.489371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:20.485501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:20.534704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471217894656149:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40262
2025-11-27T18:18:20.539730Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471217894656132:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:20.540023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471217894656150:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40262
2025-11-27T18:18:20.556072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471213599688763:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.566352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471217894656143:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40262
2025-11-27T18:18:20.573415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:20.574080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471217894656150:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.574335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471213599688763:2048] [node 1] ICP22 created new session: [4:7577471217894656151:2048]
2025-11-27T18:18:20.577882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471213599688764:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.580230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471217894656143:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.583531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471217894656151:2048] [node 1] ICS09 handshake done sender: [4:7577471217894656132:2115] self: [4:7577471217894656142:0] peer: [1:7577471215404343608:0] socket: 432 qp: -1
2025-11-27T18:18:20.583575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471217894656151:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:20.583774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471213599688765:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.584314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:20.606859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471217894656134:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.606892Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471217894656134:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.608296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471213599688764:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8878) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471217894656150:2121] held: no
2025-11-27T18:18:20.608565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471217894656156:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40262
2025-11-27T18:18:20.609059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471217894656157:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40262
2025-11-27T18:18:20.609411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471217894656150:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:20.612157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471217894656143:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:20.554862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:20.613958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471213599688765:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.526717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471214142729197:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27403
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:20.554927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471210230531165:2049] [node 2] ICP22 created new session: [5:7577471214525498548:2049]
2025-11-27T18:18:20.494865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471215404343623:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.486135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471217499320089:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.527662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471216240471135:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:20.614005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:20.554980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471214525498548:2049] [node 2] ICS09 handshake done sender: [5:7577471214525498533:2115] self: [5:7577471214525498539:0] peer: [2:7577471216240471130:0] socket: 449 qp: -1
2025-11-27T18:18:20.555025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471214525498548:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:20.555058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471214525498543:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.555071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:20.555232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471210230531164:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.555390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471210230531165:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16044044ns) {HandleHandshakeStatus:329(16042334ns) {DropIncomingHandshake:442(2514ns) DropOutgoingHandshake:457(8008210ns) SwitchToState:179(4341ns) ProcessPendingSessionEvents:495(79646ns) {ForwardSessionEventToSession:612(44147ns) ForwardSessionEventToSession:612(1961ns) ForwardSessionEventToSession:612(4545ns) ForwardSessionEventToSession:612(2409ns) ForwardSessionEventToSession:612(6000ns) ForwardSessionEventToSession:612(3508ns) ForwardSessionEventToSession:612(2453ns) ForwardSessionEventToSession:612(2558ns) ForwardSessionEventToSession:612(4316ns) ForwardSessionEventToSession:612(1902ns)}}}
2025-11-27T18:18:20.569436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471210230531166:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.569493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:20.569549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471210230531166:2050] [node 3] ICP22 created new session: [5:7577471214525498551:2050]
2025-11-27T18:18:20.573575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471214525498551:2050] [node 3] ICS09 handshake done sender: [5:7577471214525498540:2117] self: [5:7577471214525498544:0] peer: [3:7577471217499320092:0] socket: 463 qp: -1
2025-11-27T18:18:20.576951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471214525498551:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:20.577004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:20.581872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471214525498546:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.589427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471214525498552:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33300
2025-11-27T18:18:20.589458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471214525498556:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33300
2025-11-27T18:18:20.593095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471210230531164:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3441) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471214525498543:2119] held: no
2025-11-27T18:18:20.598254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471214525498556:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.601243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471214525498543:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:20.601808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471214525498525:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.605134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471214525498542:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.605174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471214525498542:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.605501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471210230531164:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.605548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:20.605592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471210230531164:2048] [node 1] ICP22 created new session: [5:7577471214525498558:2048]
2025-11-27T18:18:20.605640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471214525498558:2048] [node 1] ICS09 handshake done sender: [5:7577471214525498543:2119] self: [5:7577471214525498550:0] peer: [1:7577471215404343618:0] socket: 467 qp: -1
2025-11-27T18:18:20.605677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471214525498558:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:20.605721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:20.621482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471214525498542:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:20.627172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471214525498561:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33300
2025-11-27T18:18:20.632611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471214525498526:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.637441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:20.528338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471209847761814:2050] [node 3] ICP22 created new session: [6:7577471214142729194:2050]
2025-11-27T18:18:20.614054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471213599688765:2050] [node 3] ICP22 created new session: [4:7577471217894656159:2050]
2025-11-27T18:18:20.537009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471211945503763:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.529578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471214142729194:2050] [node 3] ICS09 handshake done sender: [6:7577471214142729186:2115] self: [6:7577471214142729190:0] peer: [3:7577471217499320079:0] socket: 442 qp: -1
2025-11-27T18:18:20.642523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471214525498556:2124] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:20.642615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471214525498526:2109] [node 6] ICP22 created new session: [5:7577471214525498562:2109]
2025-11-27T18:18:20.645239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471214525498562:2109] [node 6] ICS09 handshake done sender: [5:7577471214525498542:2118] self: [5:7577471214525498541:0] peer: [6:7577471214142729209:0] socket: 469 qp: -1
2025-11-27T18:18:20.645895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471214525498525:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.645951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:20.510780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471215404343632:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28941
2025-11-27T18:18:20.494670Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471213204352716:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3441) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471217499320083:2113] held: no
2025-11-27T18:18:20.537080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:20.648121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471214525498562:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:20.648737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471214525498525:2108] [node 4] ICP22 created new session: [5:7577471214525498563:2108]
2025-11-27T18:18:20.614102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471217894656159:2050] [node 3] ICS09 handshake done sender: [4:7577471217894656143:2119] self: [4:7577471217894656154:0] peer: [3:7577471217499320086:0] socket: 466 qp: -1
2025-11-27T18:18:20.534251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471214142729188:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:20.650150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:20.512068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471215404343609:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.537138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471211945503763:2048] [node 1] ICP22 created new session: [2:7577471216240471153:2048]
2025-11-27T18:18:20.537205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471216240471153:2048] [node 1] ICS09 handshake done sender: [2:7577471216240471135:2114] self: [2:7577471216240471141:0] peer: [1:7577471215404343614:0] socket: 429 qp: -1
2025-11-27T18:18:20.537246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471216240471153:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:20.537304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:20.539198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471216240471124:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.555811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:20.555858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471216240471124:2109] [node 5] ICP22 created new session: [2:7577471216240471155:2109]
2025-11-27T18:18:20.573688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471216240471155:2109] [node 5] ICS09 handshake done sender: [2:7577471216240471131:2113] self: [2:7577471216240471130:0] peer: [5:7577471214525498539:0] socket: 421 qp: -1
2025-11-27T18:18:20.614135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471217894656159:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:20.579799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471216240471155:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:20.614167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:20.579851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:20.621916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471217894656134:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:20.584777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471216240471140:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.584824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471216240471140:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.624901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471213599688764:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.588652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471216240471138:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.627842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:20.588690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471216240471138:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.630494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471213599688764:2049] [node 2] ICP22 created new session: [4:7577471217894656161:2049]
2025-11-27T18:18:20.594555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471216240471140:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:20.601137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471216240471138:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:20.601846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471216240471121:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.630525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471217894656124:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.604995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:20.633446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:20.610248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471216240471124:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(65301935ns) {HandleHandshakeStatus:329(65298731ns) {DropIncomingHandshake:442(1662ns) DropOutgoingHandshake:457(649ns) SwitchToState:179(1795ns) ProcessPendingSessionEvents:495(16500725ns) {ForwardSessionEventToSession:612(5908718ns) ForwardSessionEventToSession:612(4234ns) ForwardSessionEventToSession:612(2944173ns) ForwardSessionEventToSession:612(2471495ns) ForwardSessionEventToSession:612(5039ns) ForwardSessionEventToSession:612(2193799ns) ForwardSessionEventToSession:612(4271ns) ForwardSessionEventToSession:612(2875ns) ForwardSessionEventToSession:612(2939645ns) ForwardSessionEventToSession:612(3183ns)}}}
2025-11-27T18:18:20.634751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471217894656161:2049] [node 2] ICS09 handshake done sender: [4:7577471217894656150:2121] self: [4:7577471217894656153:0] peer: [2:7577471216240471139:0] socket: 476 qp: -1
2025-11-27T18:18:20.610296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471211945503764:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.636226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471217894656161:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:20.610400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471216240471121:2108] [node 4] ICP22 created new session: [2:7577471216240471159:2108]
2025-11-27T18:18:20.636282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:20.621356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471216240471159:2108] [node 4] ICS09 handshake done sender: [2:7577471216240471140:2116] self: [2:7577471216240471139:0] peer: [4:7577471217894656153:0] socket: 417 qp: -1
2025-11-27T18:18:20.636377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471217894656124:2108] [node 5] ICP22 created new session: [4:7577471217894656162:2108]
2025-11-27T18:18:20.622707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471216240471159:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:20.636962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471213599688763:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(62086200ns) {HandleHandshakeStatus:329(62083908ns) {DropIncomingHandshake:442(2221ns) DropOutgoingHandshake:457(4659ns) SwitchToState:179(2589ns) ProcessPendingSessionEvents:495(19600720ns) {ForwardSessionEventToSession:612(9981658ns) ForwardSessionEventToSession:612(5169ns) ForwardSessionEventToSession:612(2021101ns) ForwardSessionEventToSession:612(8418ns) ForwardSessionEventToSession:612(3303ns) ForwardSessionEventToSession:612(753226ns) ForwardSessionEventToSession:612(3604ns) ForwardSessionEventToSession:612(3447ns) ForwardSessionEventToSession:612(6796711ns) ForwardSessionEventToSession:612(4848ns)}}}
2025-11-27T18:18:20.624858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:20.637599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471217894656162:2108] [node 5] ICS09 handshake done sender: [4:7577471217894656134:2116] self: [4:7577471217894656133:0] peer: [5:7577471214525498557:0] socket: 482 qp: -1
2025-11-27T18:18:20.626980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:20.630695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471211945503764:2049] [node 3] ICP22 created new session: [2:7577471216240471161:2049]
2025-11-27T18:18:20.636683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471216240471161:2049] [node 3] ICS09 handshake done sender: [2:7577471216240471138:2115] self: [2:7577471216240471137:0] peer: [3:7577471217499320109:0] socket: 416 qp: -1
2025-11-27T18:18:20.538604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471214142729195:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27403
2025-11-27T18:18:20.636731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471216240471161:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:20.539597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471214142729194:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:20.636777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:20.539637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:20.637675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471213599688763:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:20.641394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471217894656162:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:20.645074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:20.651510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471213599688764:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23126840ns) {HandleHandshakeStatus:329(23123379ns) {DropIncomingHandshake:442(1110ns) DropOutgoingHandshake:457(371ns) SwitchToState:179(696ns) ProcessPendingSessionEvents:495(1314ns)}}
2025-11-27T18:18:20.496470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471217499320083:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:20.498784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471213204352716:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.498835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:20.498882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471213204352716:2048] [node 1] ICP22 created new session: [3:7577471217499320101:2048]
2025-11-27T18:18:20.498929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471217499320101:2048] [node 1] ICS09 handshake done sender: [3:7577471217499320083:2113] self: [3:7577471217499320094:0] peer: [1:7577471215404343606:0] socket: 433 qp: -1
2025-11-27T18:18:20.498959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471217499320101:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:20.498990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:20.518505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471217499320095:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.529766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471217499320106:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60986
2025-11-27T18:18:20.533695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471217499320093:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.533734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471217499320093:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.538949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471217499320107:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60986
2025-11-27T18:18:20.540785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471217499320076:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(35616309ns) {HandleHandshakeStatus:329(35612646ns) {DropIncomingHandshake:442(374ns) DropOutgoingHandshake:457(565ns) SwitchToState:179(1806ns) ProcessPendingSessionEvents:495(19421316ns) {ForwardSessionEventToSession:612(1903419ns) ForwardSessionEventToSession:612(4857ns) ForwardSessionEventToSession:612(5162886ns) ForwardSessionEventToSession:612(4245ns) ForwardSessionEventToSession:612(31482ns) ForwardSessionEventToSession:612(1890444ns) ForwardSessionEventToSession:612(4330ns) ForwardSessionEventToSession:612(3080ns) ForwardSessionEventToSession:612(1140954ns) ForwardSessionEventToSession:612(3012ns)}}}
2025-11-27T18:18:20.651135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471214525498563:2108] [node 4] ICS09 handshake done sender: [5:7577471214525498556:2124] self: [5:7577471214525498557:0] peer: [4:7577471217894656133:0] socket: 483 qp: -1
2025-11-27T18:18:20.545136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471217499320093:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:20.556305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471217499320107:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.651193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471214525498563:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:20.567504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471217499320077:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.651233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:20.567574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:20.567645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471213204352717:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.652035Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471214525498525:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:20.579676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471217499320077:2108] [node 5] ICP22 created new session: [3:7577471217499320110:2108]
2025-11-27T18:18:20.654165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471214525498526:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(21235730ns) {HandleHandshakeStatus:329(21232485ns) {DropIncomingHandshake:442(15002ns) DropOutgoingHandshake:457(579ns) SwitchToState:179(535ns) ProcessPendingSessionEvents:495(660951ns) {ForwardSessionEventToSession:612(581927ns) ForwardSessionEventToSession:612(4527ns) ForwardSessionEventToSession:612(3235ns) ForwardSessionEventToSession:612(7322ns) ForwardSessionEventToSession:612(2681ns) ForwardSessionEventToSession:612(5297ns) ForwardSessionEventToSession:612(5824ns) ForwardSessionEventToSession:612(33908ns) ForwardSessionEventToSession:612(5373ns) ForwardSessionEventToSession:612(2105ns)}}}
2025-11-27T18:18:20.580723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471217499320110:2108] [node 5] ICS09 handshake done sender: [3:7577471217499320093:2117] self: [3:7577471217499320092:0] peer: [5:7577471214525498544:0] socket: 462 qp: -1
2025-11-27T18:18:20.580771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471217499320110:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:20.583611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:20.589337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471217499320091:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.589369Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471217499320091:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.603031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471217499320091:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:20.604260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471217499320077:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(34177993ns) {HandleHandshakeStatus:329(34174334ns) {DropIncomingHandshake:442(18881ns) DropOutgoingHandshake:457(988ns) SwitchToState:179(706ns) ProcessPendingSessionEvents:495(8879486ns) {ForwardSessionEventToSession:612(4412265ns) ForwardSessionEventToSession:612(4685ns) ForwardSessionEventToSession:612(420394ns) ForwardSessionEventToSession:612(3939ns) ForwardSessionEventToSession:612(2735528ns) ForwardSessionEventToSession:612(3835ns) ForwardSessionEventToSession:612(3505ns) ForwardSessionEventToSession:612(1271149ns) ForwardSessionEventToSession:612(5548ns) ForwardSessionEventToSession:612(3082ns)}}}
2025-11-27T18:18:20.616575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471217499320076:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:20.622515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471217499320078:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.627067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:20.629840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471217499320078:2107] [node 4] ICP22 created new session: [3:7577471217499320114:2107]
2025-11-27T18:18:20.632709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471217499320114:2107] [node 4] ICS09 handshake done sender: [3:7577471217499320091:2116] self: [3:7577471217499320086:0] peer: [4:7577471217894656154:0] socket: 465 qp: -1
2025-11-27T18:18:20.632772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471217499320114:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:20.632822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:20.632856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471213204352717:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8878) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471217499320107:2121] held: no
2025-11-27T18:18:20.635969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471217499320113:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60986
2025-11-27T18:18:20.640077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471217499320107:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:20.642112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471213204352717:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.642225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:20.642297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471213204352717:2049] [node 2] ICP22 created new session: [3:7577471217499320116:2049]
2025-11-27T18:18:20.642344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471217499320116:2049] [node 2] ICS09 handshake done sender: [3:7577471217499320107:2121] self: [3:7577471217499320109:0] peer: [2:7577471216240471137:0] socket: 477 qp: -1
2025-11-27T18:18:20.642392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471217499320116:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:20.642603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:20.651965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471217499320078:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(23894680ns) {HandleHandshakeStatus:329(23891641ns) {DropIncomingHandshake:442(25963ns) DropOutgoingHandshake:457(540ns) SwitchToState:179(667ns) ProcessPendingSessionEvents:495(5376594ns) {ForwardSessionEventToSession:612(2504126ns) ForwardSessionEventToSession:612(5699ns) ForwardSessionEventToSession:612(101896ns) ForwardSessionEventToSession:612(3777ns) ForwardSessionEventToSession:612(345188ns) ForwardSessionEventToSession:612(1519683ns) ForwardSessionEventToSession:612(3336ns) ForwardSessionEventToSession:612(870442ns) ForwardSessionEventToSession:612(2587ns) ForwardSessionEventToSession:612(2518ns)}}}
2025-11-27T18:18:20.651191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471211945503764:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(40707411ns) {HandleHandshakeStatus:329(40705827ns) {DropIncomingHandshake:442(14050ns) DropOutgoingHandshake:457(2411ns) SwitchToState:179(647ns) ProcessPendingSessionEvents:495(1285576ns) {ForwardSessionEventToSession:612(911069ns) ForwardSessionEventToSession:612(3881ns) ForwardSessionEventToSession:612(6211ns) ForwardSessionEventToSession:612(2600ns) ForwardSessionEventToSession:612(3215ns) ForwardSessionEventToSession:612(4286ns) ForwardSessionEventToSession:612(3151ns) ForwardSessionEventToSession:612(10726ns) ForwardSessionEventToSession:612(2124ns) ForwardSessionEventToSession:612(3065ns)}}}
2025-11-27T18:18:20.541276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471209847761814:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(24695233ns) {HandleHandshakeStatus:329(24692675ns) {DropIncomingHandshake:442(845ns) DropOutgoingHandshake:457(5059ns) SwitchToState:179(671ns) ProcessPendingSessionEvents:495(107891ns) {ForwardSessionEventToSession:612(61210ns) ForwardSessionEventToSession:612(3561ns) ForwardSessionEventToSession:612(3659ns) ForwardSessionEventToSession:612(4450ns) ForwardSessionEventToSession:612(2766ns) ForwardSessionEventToSession:612(4962ns) ForwardSessionEventToSession:612(3345ns) ForwardSessionEventToSession:612(2624ns) ForwardSessionEventToSession:612(7723ns) ForwardSessionEventToSession:612(3232ns)}}}
2025-11-27T18:18:20.654264Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471216240471121:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(48390776ns) {HandleHandshakeStatus:329(48388150ns) {DropIncomingHandshake:442(14217ns) DropOutgoingHandshake:457(507ns) SwitchToState:179(165ns) ProcessPendingSessionEvents:495(10025934ns) {ForwardSessionEventToSession:612(3802160ns) ForwardSessionEventToSession:612(5341ns) ForwardSessionEventToSession:612(1545309ns) ForwardSessionEventToSession:612(15803ns) ForwardSessionEventToSession:612(3326ns) ForwardSessionEventToSession:612(2151787ns) ForwardSessionEventToSession:612(4288ns) ForwardSessionEventToSession:612(1122920ns) ForwardSessionEventToSession:612(1355372ns) ForwardSessionEventToSession:612(3095ns)}}}
2025-11-27T18:18:20.512102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471215404343609:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.523029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471215404343609:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:20.541523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471209847761813:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8878) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471214142729182:2113] held: no
2025-11-27T18:18:20.539264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471215404343632:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.543810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471215404343601:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.541636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471214142729167:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.543848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:20.543893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471215404343601:2107] [node 4] ICP22 created new session: [1:7577471215404343635:2107]
2025-11-27T18:18:20.541672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:20.541713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471214142729167:2105] [node 4] ICP22 created new session: [6:7577471214142729200:2105]
2025-11-27T18:18:20.543940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471215404343635:2107] [node 4] ICS09 handshake done sender: [1:7577471215404343609:2112] self: [1:7577471215404343608:0] peer: [4:7577471217894656142:0] socket: 426 qp: -1
2025-11-27T18:18:20.541759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471214142729200:2105] [node 4] ICS09 handshake done sender: [6:7577471214142729188:2117] self: [6:7577471214142729192:0] peer: [4:7577471217894656130:0] socket: 444 qp: -1
2025-11-27T18:18:20.543969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471215404343635:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:20.541794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471214142729200:2105] [node 4] ICS10 traffic start
2025-11-27T18:18:20.543999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:20.541847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:20.544461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471215404343601:2107] [node 4] ICP18 handshake (actor [1:7577471215404343632:2119]) is too old
2025-11-27T18:18:20.548960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471214142729182:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:20.556283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471215404343620:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.556305Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471215404343620:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.569469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471209847761813:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.562264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471215404343620:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:20.569509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:20.579993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471215404343603:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.569550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471209847761813:2049] [node 2] ICP22 created new session: [6:7577471214142729202:2049]
2025-11-27T18:18:20.569615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471214142729202:2049] [node 2] ICS09 handshake done sender: [6:7577471214142729182:2113] self: [6:7577471214142729189:0] peer: [2:7577471216240471126:0] socket: 430 qp: -1
2025-11-27T18:18:20.656468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471217894656124:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(25992011ns) {HandleHandshakeStatus:329(25988594ns) {DropIncomingHandshake:442(19075ns) DropOutgoingHandshake:457(550ns) SwitchToState:179(645ns) ProcessPendingSessionEvents:495(244291ns) {ForwardSessionEventToSession:612(197680ns) ForwardSessionEventToSession:612(4465ns) ForwardSessionEventToSession:612(2846ns) ForwardSessionEventToSession:612(6660ns) ForwardSessionEventToSession:612(2912ns) ForwardSessionEventToSession:612(5369ns) ForwardSessionEventToSession:612(5590ns) ForwardSessionEventToSession:612(2512ns) ForwardSessionEventToSession:612(5302ns) ForwardSessionEventToSession:612(2675ns)}}}
2025-11-27T18:18:20.569663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471214142729202:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:20.569703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:20.582454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:20.570799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471214142729196:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.583648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471215404343605:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.574037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471209847761812:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.583678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471215404343605:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:20.584554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471214142729205:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27403
2025-11-27T18:18:20.585041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471215404343603:2109] [node 5] ICP22 created new session: [1:7577471215404343638:2109]
2025-11-27T18:18:20.593982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471214142729206:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27403
2025-11-27T18:18:20.585951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471215404343638:2109] [node 5] ICS09 handshake done sender: [1:7577471215404343620:2116] self: [1:7577471215404343618:0] peer: [5:7577471214525498550:0] socket: 437 qp: -1
2025-11-27T18:18:20.594297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471214142729196:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:20.595552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471209847761812:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.585991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471215404343638:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:20.595613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:20.586052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:20.595668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471209847761812:2048] [node 1] ICP22 created new session: [6:7577471214142729207:2048]
2025-11-27T18:18:20.593046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471215404343605:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:20.595714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471214142729207:2048] [node 1] ICS09 handshake done sender: [6:7577471214142729196:2120] self: [6:7577471214142729204:0] peer: [1:7577471215404343604:0] socket: 473 qp: -1
2025-11-27T18:18:20.595749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471214142729207:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:20.601139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471215404343602:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:20.602315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:20.595801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:20.607200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471215404343602:2108] [node 6] ICP22 created new session: [1:7577471215404343641:2108]
2025-11-27T18:18:20.600497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471214142729205:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:20.610854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471215404343641:2108] [node 6] ICS09 handshake done sender: [1:7577471215404343605:2110] self: [1:7577471215404343604:0] peer: [6:7577471214142729204:0] socket: 424 qp: -1
2025-11-27T18:18:20.603081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471214142729157:2096] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:20.610897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471215404343641:2108] [node 6] ICS10 traffic start
2025-11-27T18:18:20.639998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471214142729210:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27403
2025-11-27T18:18:20.648887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471214142729157:2096] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5250) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471214142729205:2122] held: no
2025-11-27T18:18:20.649440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471214142729205:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:20.650297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471214142729157:2096] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:20.656176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:20.656235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471214142729157:2096] [node 5] ICP22 created new session: [6:7577471214142729211:2096]
2025-11-27T18:18:20.656296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471214142729211:2096] [node 5] ICS09 handshake done sender: [6:7577471214142729205:2122] self: [6:7577471214142729209:0] peer: [5:7577471214525498541:0] socket: 471 qp: -1
2025-11-27T18:18:20.656342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471214142729211:2096] [node 5] ICS10 traffic start
2025-11-27T18:18:20.656416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:20.610946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:20.738144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:20.738394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471217499320101:2048] [node 1] ICS01 socket: 433 reason# EndOfStream
2025-11-27T18:18:20.738454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471213204352716:2048] [node 1] ICP30 unregister session Session# [3:7577471217499320101:2048] VirtualId# [3:7577471217499320094:0]
2025-11-27T18:18:20.738493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471217499320101:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.738579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:20.738632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:20.735270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:20.735554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471216240471153:2048] [node 1] ICS01 socket: 429 reason# EndOfStream
2025-11-27T18:18:20.735607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471211945503763:2048] [node 1] ICP30 unregister session Session# [2:7577471216240471153:2048] VirtualId# [2:7577471216240471141:0]
2025-11-27T18:18:20.735651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471216240471153:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.735745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:20.735793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:20.747635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:20.747928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471217894656151:2048] [node 1] ICS07 socket disconnect 432 reason# EndOfStream
2025-11-27T18:18:20.747961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471217894656151:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.748045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:20.748089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471217894656151:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:20.750732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471213599688763:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3441) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:20.750781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471217894656151:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:20.750806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471213599688763:2048] [node 1] ICP30 unregister session Session# [4:7577471217894656151:2048] VirtualId# [4:7577471217894656142:0]
2025-11-27T18:18:20.750841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:20.750862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471213599688763:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3441) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:20.750874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3441) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:20.753075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:20.753348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471214142729207:2048] [node 1] ICS07 socket disconnect 473 reason# EndOfStream
2025-11-27T18:18:20.753392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471214142729207:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.753482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:20.753539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471214142729207:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:20.755279Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:20.755714Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471209847761812:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3441) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:20.761413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471213599688763:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:20.761203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471214142729207:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:20.761245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471209847761812:2048] [node 1] ICP30 unregister session Session# [6:7577471214142729207:2048] VirtualId# [6:7577471214142729204:0]
2025-11-27T18:18:20.761313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:20.761359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471209847761812:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3441) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:20.761376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3441) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:20.773862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471209847761812:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:20.755433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471214525498558:2048] [node 1] ICS01 socket: 467 reason# EndOfStream
2025-11-27T18:18:20.755464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471210230531164:2048] [node 1] ICP30 unregister session Session# [5:7577471214525498558:2048] VirtualId# [5:7577471214525498550:0]
2025-11-27T18:18:20.755486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471214525498558:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.755536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:20.755569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:20.853494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:20.853764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471217499320116:2049] [node 2] ICS01 socket: 477 reason# EndOfStream
2025-11-27T18:18:20.853809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471213204352717:2049] [node 2] ICP30 unregister session Session# [3:7577471217499320116:2049] VirtualId# [3:7577471217499320109:0]
2025-11-27T18:18:20.853846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471217499320116:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.853949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:20.854005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:20.857114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:20.857358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471217894656161:2049] [node 2] ICS01 socket: 476 reason# EndOfStream
2025-11-27T18:18:20.857394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471213599688764:2049] [node 2] ICP30 unregister session Session# [4:7577471217894656161:2049] VirtualId# [4:7577471217894656153:0]
2025-11-27T18:18:20.857424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471217894656161:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.857501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:20.857547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:20.857451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:20.857627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471214525498548:2049] [node 2] ICS01 socket: 449 reason# EndOfStream
2025-11-27T18:18:20.857657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471210230531165:2049] [node 2] ICP30 unregister session Session# [5:7577471214525498548:2049] VirtualId# [5:7577471214525498539:0]
2025-11-27T18:18:20.857681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471214525498548:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.857744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:20.857779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:20.861554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:20.861742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471214142729202:2049] [node 2] ICS01 socket: 430 reason# EndOfStream
2025-11-27T18:18:20.861787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471209847761813:2049] [node 2] ICP30 unregister session Session# [6:7577471214142729202:2049] VirtualId# [6:7577471214142729189:0]
2025-11-27T18:18:20.861816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471214142729202:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.861896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:20.861932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:20.941509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:20.941935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471217894656159:2050] [node 3] ICS01 socket: 466 reason# EndOfStream
2025-11-27T18:18:20.942026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471213599688765:2050] [node 3] ICP30 unregister session Session# [4:7577471217894656159:2050] VirtualId# [4:7577471217894656154:0]
2025-11-27T18:18:20.942127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471217894656159:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.942305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:20.942409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:20.943616Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:20.943878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471214525498551:2050] [node 3] ICS01 socket: 463 reason# EndOfStream
2025-11-27T18:18:20.943923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471210230531166:2050] [node 3] ICP30 unregister session Session# [5:7577471214525498551:2050] VirtualId# [5:7577471214525498544:0]
2025-11-27T18:18:20.943948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471214525498551:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.944009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:20.944651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:20.942667Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:20.943052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471214142729194:2050] [node 3] ICS01 socket: 442 reason# EndOfStream
2025-11-27T18:18:20.943129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471209847761814:2050] [node 3] ICP30 unregister session Session# [6:7577471214142729194:2050] VirtualId# [6:7577471214142729190:0]
2025-11-27T18:18:20.943194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471214142729194:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:20.943313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:20.943403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:21.049865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:21.049382Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:21.049631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471214142729200:2105] [node 4] ICS01 socket: 444 reason# EndOfStream
2025-11-27T18:18:21.049684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471214142729167:2105] [node 4] ICP30 unregister session Session# [6:7577471214142729200:2105] VirtualId# [6:7577471214142729192:0]
2025-11-27T18:18:21.049717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471214142729200:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:21.049800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:21.049857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:21.050171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471214525498563:2108] [node 4] ICS01 socket: 483 reason# EndOfStream
2025-11-27T18:18:21.050263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471214525498525:2108] [node 4] ICP30 unregister session Session# [5:7577471214525498563:2108] VirtualId# [5:7577471214525498557:0]
2025-11-27T18:18:21.050299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471214525498563:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:21.050415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:21.050481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:21.112020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:21.112321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471214142729211:2096] [node 5] ICS01 socket: 471 reason# EndOfStream
2025-11-27T18:18:21.112383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471214142729157:2096] [node 5] ICP30 unregister session Session# [6:7577471214142729211:2096] VirtualId# [6:7577471214142729209:0]
2025-11-27T18:18:21.112430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471214142729211:2096] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:21.112542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:21.112599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:21.235446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471220944269964:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:21.237443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471220944269965:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:21.260046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471221926621334:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:21.260259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471221926621335:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:21.285685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471222409008829:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:21.286706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471222409008828:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:21.308619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471222094780855:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:21.308649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471222094780856:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:21.308665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471222094780857:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:21.340080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471220999046887:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:21.340111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471220999046888:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:21.340449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471220999046886:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:21.367094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471219797423088:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:21.367176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471219797423090:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:21.367590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471219797423089:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:21.530302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471220944270030:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:21.596463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471220944270031:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:21.622342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471220999046950:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:21.631109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471222094780855:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(69562926ns) {Configure:104(69553498ns) {ProcessConfigured:135(79773ns) {StartInitialHandshake:157(76174ns) {DropHandshakes:470(2791ns) {DropIncomingHandshake:442(640ns) DropOutgoingHandshake:457(257ns)} GenerateSessionVirtualId:988(940ns) PrepareNewSessionHandshake:477(1522ns) {SwitchToState:179(477ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.620230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471219797423135:2094] [node 5] ICP01 ready to work
2025-11-27T18:18:21.614047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471221926621397:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:21.613869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471222409008909:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:21.633683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471222409008910:2116] [node 6] ICP01 ready to work
2025-11-27T18:18:21.653151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471222409008911:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:21.652630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471219797423146:2090] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.621402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471220944270032:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:21.655149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471222094780921:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:21.644677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471221926621398:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:21.633467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471220999046949:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:21.646149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471220944270033:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-27T18:18:21.654187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471222094780922:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:21.663191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471221926621399:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65092
2025-11-27T18:18:21.678692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471219797423088:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(89032857ns) {Configure:104(89023013ns) {ProcessConfigured:135(79414ns) {StartInitialHandshake:157(76674ns) {DropHandshakes:470(2829ns) {DropIncomingHandshake:442(574ns) DropOutgoingHandshake:457(371ns)} GenerateSessionVirtualId:988(1105ns) PrepareNewSessionHandshake:477(1768ns) {SwitchToState:179(687ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.694211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471220944270034:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-27T18:18:21.694275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471220944270035:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-27T18:18:21.672314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471221926621400:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:21.690159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471222409008828:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(226600381ns) {Configure:104(226594570ns) {ProcessConfigured:135(77844ns) {StartInitialHandshake:157(75147ns) {DropHandshakes:470(2924ns) {DropIncomingHandshake:442(874ns) DropOutgoingHandshake:457(273ns)} GenerateSessionVirtualId:988(1072ns) PrepareNewSessionHandshake:477(1830ns) {SwitchToState:179(604ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.673578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471221926621399:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.717652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471222094780856:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(244681616ns) {Configure:104(244673173ns) {ProcessConfigured:135(7925491ns) {StartInitialHandshake:157(7921880ns) {DropHandshakes:470(2270ns) {DropIncomingHandshake:442(497ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(520ns) PrepareNewSessionHandshake:477(3170ns) {SwitchToState:179(1549ns)}}}}}
2025-11-27T18:18:21.722679Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471222409008829:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(101177332ns) {Configure:104(101171112ns) {ProcessConfigured:135(70806ns) {StartInitialHandshake:157(62687ns) {DropHandshakes:470(1981ns) {DropIncomingHandshake:442(534ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(1451ns) {SwitchToState:179(522ns)}}}}}
2025-11-27T18:18:21.723763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471221926621405:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65092
2025-11-27T18:18:21.778267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471222409008911:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(104554787ns) {Configure:104(104544752ns) {ProcessConfigured:135(105385ns) {StartInitialHandshake:157(101782ns) {DropHandshakes:470(3779ns) {DropIncomingHandshake:442(607ns) DropOutgoingHandshake:457(733ns)} GenerateSessionVirtualId:988(1355ns) PrepareNewSessionHandshake:477(2032ns) {SwitchToState:179(611ns)}}}}}
2025-11-27T18:18:21.734420Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471219797423089:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(112988753ns) {Configure:104(112981308ns) {ProcessConfigured:135(86466ns) {StartInitialHandshake:157(83487ns) {DropHandshakes:470(3120ns) {DropIncomingHandshake:442(890ns) DropOutgoingHandshake:457(258ns)} GenerateSessionVirtualId:988(1414ns) PrepareNewSessionHandshake:477(1807ns) {SwitchToState:179(801ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.798222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471220999046949:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(137937819ns) {Configure:104(137928187ns) {ProcessConfigured:135(69933ns) {StartInitialHandshake:157(66717ns) {DropHandshakes:470(5237ns) {DropIncomingHandshake:442(565ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(788ns) PrepareNewSessionHandshake:477(1348ns) {SwitchToState:179(396ns)}}}}}
2025-11-27T18:18:21.798879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471220999046954:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4910
2025-11-27T18:18:21.799034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471220999046955:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4910
2025-11-27T18:18:21.799474Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471220999046887:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(212686486ns) {Configure:104(212677364ns) {ProcessConfigured:135(13440586ns) {StartInitialHandshake:157(13438078ns) {DropHandshakes:470(2733ns) {DropIncomingHandshake:442(811ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(1290ns) PrepareNewSessionHandshake:477(1888ns) {SwitchToState:179(196ns)}}}}}
2025-11-27T18:18:21.729916Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471221926621335:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(271829705ns) {Configure:104(271825181ns) {ProcessConfigured:135(78454ns) {StartInitialHandshake:157(75652ns) {DropHandshakes:470(5887ns) {DropIncomingHandshake:442(550ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(778ns) PrepareNewSessionHandshake:477(2633ns) {SwitchToState:179(823ns)}}}}}
2025-11-27T18:18:21.738099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471219797423135:2094] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(117591653ns) {Configure:104(117585881ns) {ProcessConfigured:135(6757858ns) {StartInitialHandshake:157(6754573ns) {DropHandshakes:470(1425ns) {DropIncomingHandshake:442(310ns) DropOutgoingHandshake:457(265ns)} GenerateSessionVirtualId:988(618ns) PrepareNewSessionHandshake:477(3932ns) {SwitchToState:179(1477ns)}}}}}
2025-11-27T18:18:21.785708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471220944270035:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.800645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471220999046954:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.838195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471222094780921:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(165851657ns) {Configure:104(165846417ns) {ProcessConfigured:135(77090ns) {StartInitialHandshake:157(74325ns) {DropHandshakes:470(2317ns) {DropIncomingHandshake:442(509ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(1003ns) PrepareNewSessionHandshake:477(2257ns) {SwitchToState:179(997ns)}}}}}
2025-11-27T18:18:21.839701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471222094780923:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2552
2025-11-27T18:18:21.763046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471219797423146:2090] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(90832495ns) {Configure:104(90825275ns) {ProcessConfigured:135(79329ns) {StartInitialHandshake:157(76899ns) {DropHandshakes:470(2826ns) {DropIncomingHandshake:442(600ns) DropOutgoingHandshake:457(405ns)} GenerateSessionVirtualId:988(1057ns) PrepareNewSessionHandshake:477(1586ns) {SwitchToState:179(477ns)}}}}}
2025-11-27T18:18:21.790551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471221926621397:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(102438225ns) {Configure:104(102432710ns) {ProcessConfigured:135(43076165ns) {StartInitialHandshake:157(43072933ns) {DropHandshakes:470(6693512ns) {DropIncomingHandshake:442(355ns) DropOutgoingHandshake:457(2658ns)} GenerateSessionVirtualId:988(2070ns) PrepareNewSessionHandshake:477(4003ns) {SwitchToState:179(2559ns)}}}}}
2025-11-27T18:18:21.829700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471222409008910:2116] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(134595528ns) {Configure:104(129156714ns) {ProcessConfigured:135(84296ns) {StartInitialHandshake:157(80846ns) {DropHandshakes:470(4729ns) {DropIncomingHandshake:442(543ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(1374ns) PrepareNewSessionHandshake:477(1528ns) {SwitchToState:179(453ns)}}}}}
2025-11-27T18:18:21.842161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471222409008920:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.842183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471222409008920:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:21.845853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471222409008909:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(177212096ns) {Configure:104(177204095ns) {ProcessConfigured:135(89369ns) {StartInitialHandshake:157(86121ns) {DropHandshakes:470(3088ns) {DropIncomingHandshake:442(606ns) DropOutgoingHandshake:457(329ns)} GenerateSessionVirtualId:988(1031ns) PrepareNewSessionHandshake:477(2470ns) {SwitchToState:179(497ns)}}}}}
2025-11-27T18:18:21.845981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471222409008920:2131] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.818073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471220999046886:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(221949938ns) {Configure:104(221939389ns) {ProcessConfigured:135(15884893ns) {StartInitialHandshake:157(15882244ns) {DropHandshakes:470(2605ns) {DropIncomingHandshake:442(513ns) DropOutgoingHandshake:457(506ns)} GenerateSessionVirtualId:988(564ns) PrepareNewSessionHandshake:477(6073ns) {SwitchToState:179(2771ns)}}}}}
2025-11-27T18:18:21.826934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471220944270034:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.769351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471219797423090:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(112517512ns) {Configure:104(112504699ns) {ProcessConfigured:135(15776505ns) {StartInitialHandshake:157(15773935ns) {DropHandshakes:470(2144ns) {DropIncomingHandshake:442(497ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(539ns) PrepareNewSessionHandshake:477(3491ns) {SwitchToState:179(1871ns)}}}}}
2025-11-27T18:18:21.829512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471220944270031:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(176470587ns) {Configure:104(176460751ns) {ProcessConfigured:135(10512402ns) {StartInitialHandshake:157(10509611ns) {DropHandshakes:470(1363ns) {DropIncomingHandshake:442(385ns) DropOutgoingHandshake:457(180ns)} GenerateSessionVirtualId:988(1048ns) PrepareNewSessionHandshake:477(4030ns) {SwitchToState:179(2441ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.818259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471220999046955:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.790858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471221926621405:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.841276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471219797423166:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54129
2025-11-27T18:18:21.855106Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471222094780857:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(205747674ns) {Configure:104(205741139ns) {ProcessConfigured:135(22060224ns) {StartInitialHandshake:157(22058416ns) {DropHandshakes:470(1755ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(272ns)} GenerateSessionVirtualId:988(544ns) PrepareNewSessionHandshake:477(2518ns) {SwitchToState:179(1352ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.833673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471220944269965:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(229347601ns) {Configure:104(229342234ns) {ProcessConfigured:135(24234740ns) {StartInitialHandshake:157(24232532ns) {DropHandshakes:470(2599ns) {DropIncomingHandshake:442(890ns) DropOutgoingHandshake:457(442ns)} GenerateSessionVirtualId:988(969ns) PrepareNewSessionHandshake:477(1916ns) {SwitchToState:179(611ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.886952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471222409008918:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.800389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471221926621410:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65092
2025-11-27T18:18:21.841618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471220999046888:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(234449989ns) {Configure:104(234443962ns) {ProcessConfigured:135(15189588ns) {StartInitialHandshake:157(15186656ns) {DropHandshakes:470(2763ns) {DropIncomingHandshake:442(488ns) DropOutgoingHandshake:457(330ns)} GenerateSessionVirtualId:988(534ns) PrepareNewSessionHandshake:477(3696ns) {SwitchToState:179(2045ns)}}}}}
2025-11-27T18:18:21.861400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471222094780923:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.841798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471220999046888:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:21.890495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471222094780856:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:21.840912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471221926621414:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65092
2025-11-27T18:18:21.902328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471222409008911:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:21.853966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471220944270042:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-27T18:18:21.902097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471219797423167:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54129
2025-11-27T18:18:21.868496Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471221926621334:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(237681152ns) {Configure:104(237671319ns) {ProcessConfigured:135(29077596ns) {StartInitialHandshake:157(29072640ns) {DropHandshakes:470(2606ns) {DropIncomingHandshake:442(982ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(2333ns) PrepareNewSessionHandshake:477(5007ns) {SwitchToState:179(2266ns)}}}}}
2025-11-27T18:18:21.890096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471220999046965:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4910
2025-11-27T18:18:21.877018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471220944270030:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(257768012ns) {Configure:104(257762318ns) {ProcessConfigured:135(63877056ns) {StartInitialHandshake:157(63874895ns) {DropHandshakes:470(1413ns) {DropIncomingHandshake:442(412ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(364ns) PrepareNewSessionHandshake:477(1396ns) {SwitchToState:179(510ns)}}}}}
2025-11-27T18:18:21.906785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:21.890581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471222094780933:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2552
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.913861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471222409008918:2130] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.916078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471222409008911:2108] [node 5] ICP22 created new session: [3:7577471222409008927:2108]
2025-11-27T18:18:21.922179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471222409008829:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:21.923157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471222409008927:2108] [node 5] ICS09 handshake done sender: [3:7577471222409008920:2131] self: [3:7577471222409008919:0] peer: [5:7577471220999046964:0] socket: 440 qp: -1
2025-11-27T18:18:21.928472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471222409008927:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:21.937494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:21.909400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471219797423168:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54129
2025-11-27T18:18:21.930642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471220999046955:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:21.889502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471220944270025:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.914123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471222094780922:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(206199263ns) {Configure:104(206193037ns) {ProcessConfigured:135(12391305ns) {StartInitialHandshake:157(12389032ns) {DropHandshakes:470(1327ns) {DropIncomingHandshake:442(286ns) DropOutgoingHandshake:457(164ns)} GenerateSessionVirtualId:988(367ns) PrepareNewSessionHandshake:477(2848ns) {SwitchToState:179(1512ns)}}}}}
2025-11-27T18:18:21.933559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471219797423167:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.889530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471220944270025:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:21.873774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471221926621334:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:21.937335Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471220999046950:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(246987124ns) {Configure:104(246980848ns) {ProcessConfigured:135(20179944ns) {StartInitialHandshake:157(20176834ns) {DropHandshakes:470(2996ns) {DropIncomingHandshake:442(590ns) DropOutgoingHandshake:457(533ns)} GenerateSessionVirtualId:988(824ns) PrepareNewSessionHandshake:477(2536ns) {SwitchToState:179(249ns)}}}}}
2025-11-27T18:18:21.955023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471220999046888:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:21.955082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:21.956409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471220999046888:2050] [node 3] ICP22 created new session: [5:7577471220999046969:2050]
2025-11-27T18:18:21.960963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471220999046969:2050] [node 3] ICS09 handshake done sender: [5:7577471220999046955:2112] self: [5:7577471220999046964:0] peer: [3:7577471222409008919:0] socket: 442 qp: -1
2025-11-27T18:18:21.931010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471222094780933:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.933985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471219797423088:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:21.905412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471221926621402:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.954905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471222094780932:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2552
2025-11-27T18:18:21.893414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471220944270033:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.965725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471222409008932:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-27T18:18:21.966228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471222409008911:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(63779525ns) {HandleHandshakeStatus:329(63775590ns) {DropIncomingHandshake:442(466ns) DropOutgoingHandshake:457(441ns) SwitchToState:179(505ns) ProcessPendingSessionEvents:495(5349224ns) {ForwardSessionEventToSession:612(5296579ns) ForwardSessionEventToSession:612(3995ns) ForwardSessionEventToSession:612(2083ns) ForwardSessionEventToSession:612(7270ns) ForwardSessionEventToSession:612(2238ns) ForwardSessionEventToSession:612(2839ns) ForwardSessionEventToSession:612(13097ns) ForwardSessionEventToSession:612(4276ns) ForwardSessionEventToSession:612(4086ns) ForwardSessionEventToSession:612(2651ns)}}}
2025-11-27T18:18:21.945506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471219797423166:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.905451Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471221926621402:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.956587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471222094780939:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2552
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.964205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471220999046969:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.916198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471220944270032:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(230366341ns) {Configure:104(230359869ns) {ProcessConfigured:135(20332327ns) {StartInitialHandshake:157(20329810ns) {DropHandshakes:470(2888ns) {DropIncomingHandshake:442(712ns) DropOutgoingHandshake:457(280ns)} GenerateSessionVirtualId:988(1046ns) PrepareNewSessionHandshake:477(1936ns) {SwitchToState:179(661ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.974202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471222409008928:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-27T18:18:21.913505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471221926621402:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.958295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471219797423146:2090] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:21.917876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471220944270025:2107] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.974419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471222409008918:2130] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:21.984572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471222409008829:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:21.989565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471222409008928:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.964290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:21.961660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471222094780929:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.917505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471221926621400:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(225324816ns) {Configure:104(219776539ns) {ProcessConfigured:135(5581056ns) {StartInitialHandshake:157(5577954ns) {DropHandshakes:470(2404ns) {DropIncomingHandshake:442(702ns) DropOutgoingHandshake:457(334ns)} GenerateSessionVirtualId:988(1312ns) PrepareNewSessionHandshake:477(2416ns) {SwitchToState:179(1417ns)}}}}}
2025-11-27T18:18:21.942348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471220944269964:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:21.978306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471220999046888:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(35481790ns) {HandleHandshakeStatus:329(23074907ns) {DropIncomingHandshake:442(912ns) DropOutgoingHandshake:457(13151ns) SwitchToState:179(511ns) ProcessPendingSessionEvents:495(3794385ns) {ForwardSessionEventToSession:612(3733653ns) ForwardSessionEventToSession:612(6594ns) ForwardSessionEventToSession:612(9623ns) ForwardSessionEventToSession:612(2594ns) ForwardSessionEventToSession:612(8495ns) ForwardSessionEventToSession:612(2524ns) ForwardSessionEventToSession:612(6354ns) ForwardSessionEventToSession:612(5646ns) ForwardSessionEventToSession:612(2830ns) ForwardSessionEventToSession:612(2569ns)}}}
2025-11-27T18:18:21.961690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471222094780929:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:21.997724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471222409008828:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:21.955903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471221926621414:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.964449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471219797423168:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.990645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471220999046973:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4910
2025-11-27T18:18:21.970135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471222094780856:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17662) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471222094780923:2111] held: no
2025-11-27T18:18:21.997776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.942414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:21.956727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471221926621398:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.999005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471220999046972:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4910
2025-11-27T18:18:21.942464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471220944269964:2048] [node 2] ICP22 created new session: [1:7577471220944270052:2048]
2025-11-27T18:18:21.977707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471219797423174:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54129
2025-11-27T18:18:22.006105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471222409008829:2049] [node 2] ICP22 created new session: [3:7577471222409008934:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.956756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:21.970387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471222094780932:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.977848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471219797423175:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54129
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:22.022196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471220999046972:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.974170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471222094780937:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2552
2025-11-27T18:18:21.956807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471221926621398:2107] [node 4] ICP22 created new session: [2:7577471221926621420:2107]
2025-11-27T18:18:21.942497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471220944270042:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.018256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471222409008934:2049] [node 2] ICS09 handshake done sender: [3:7577471222409008918:2130] self: [3:7577471222409008930:0] peer: [2:7577471221926621406:0] socket: 436 qp: -1
2025-11-27T18:18:21.982561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471219797423089:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:22.038441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471220999046887:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:22.022457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471222409008934:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:21.974499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471222094780939:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.956848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471221926621420:2107] [node 4] ICS09 handshake done sender: [2:7577471221926621402:2111] self: [2:7577471221926621401:0] peer: [4:7577471222094780936:0] socket: 425 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.947377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471220944270039:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.989522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471219797423169:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54129
2025-11-27T18:18:21.975151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471222094780855:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:22.070134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471220999046973:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.022505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:21.947410Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471220944270039:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:21.956875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471221926621420:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:21.989870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471219797423174:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.999152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471222094780929:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:22.070287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471220999046967:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:21.959983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471220944270039:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:22.058971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471226703976234:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-27T18:18:21.999345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471219797423088:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15864) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471219797423167:2116] held: no
2025-11-27T18:18:22.013451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471222094780923:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:21.956915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:22.070322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471220999046967:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:22.059175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471226703976232:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-27T18:18:22.005451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471219797423173:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54129
2025-11-27T18:18:21.959072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471221926621398:2107] [node 4] ICP18 handshake (actor [2:7577471221926621414:2117]) is too old
2025-11-27T18:18:21.963700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471220944270052:2048] [node 2] ICS09 handshake done sender: [1:7577471220944270025:2107] self: [1:7577471220944270024:0] peer: [2:7577471221926621415:0] socket: 420 qp: -1
2025-11-27T18:18:22.076196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471225294014270:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4910
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:22.022337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471222094780921:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:21.959841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471221926621407:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.017873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471219797423135:2094] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:22.076224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471220999046886:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:21.967018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471220944270052:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:22.026070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:22.071848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471222409008928:2134] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:22.079691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471220999046967:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:21.959872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471221926621407:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:22.072030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471226703976232:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.053829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471219797423088:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20389161ns) {HandleHandshakeStatus:395(20385787ns) {LogHandshakeFail:472(1595467ns) {UpdateErrorStateLog:545(4826ns)} DropOutgoingHandshake:457(358ns) ProcessPendingSessionEvents:495(18739894ns) {DropSessionEvent:512(5885563ns) DropSessionEvent:512(16383ns) DropSessionEvent:512(5009ns) DropSessionEvent:512(5316ns) DropSessionEvent:512(14258ns) DropSessionEvent:512(4453ns) DropSessionEvent:512(7338ns) DropSessionEvent:512(4465ns) DropSessionEvent:512(3344ns) DropSessionEvent:512(38736ns)}}}
2025-11-27T18:18:21.967503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:22.031749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471222094780856:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:21.960661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471221926621410:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.031850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:22.082478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471225294014270:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.974732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471220944270031:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:22.062189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471219797423169:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.075194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471222409008828:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15864) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471222409008928:2134] held: no
2025-11-27T18:18:22.086343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471220999046949:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:22.081091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471222409008828:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:21.964460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471221926621407:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:21.978961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:22.063950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471219797423146:2090] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63497) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471219797423166:2115] held: no
2025-11-27T18:18:22.031931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471222094780856:2049] [node 2] ICP22 created new session: [4:7577471226389748243:2049]
2025-11-27T18:18:21.974736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471221926621335:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:21.974792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:21.981555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471221926621409:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:21.981578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471221926621409:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:21.982199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471221926621424:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65092
2025-11-27T18:18:21.983238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471221926621399:2110] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:21.989490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471221926621335:2049] [node 3] ICP22 created new session: [2:7577471221926621426:2049]
2025-11-27T18:18:22.082155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471226703976237:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60995
2025-11-27T18:18:22.031987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471226389748243:2049] [node 2] ICS09 handshake done sender: [4:7577471222094780923:2111] self: [4:7577471222094780936:0] peer: [2:7577471221926621401:0] socket: 426 qp: -1
2025-11-27T18:18:22.064119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471219797423166:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:21.982730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471220944270031:2110] [node 6] ICP22 created new session: [1:7577471220944270057:2110]
2025-11-27T18:18:22.086426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471220999046887:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17662) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471220999046972:2118] held: no
2025-11-27T18:18:21.991207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471221926621409:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:21.998197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471221926621334:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:21.998247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:21.983523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471220944270041:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.082363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471222409008923:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.086519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471220999046950:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:22.065116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471219797423167:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:22.032016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471226389748243:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:21.999218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471221926621334:2048] [node 1] ICP22 created new session: [2:7577471221926621429:2048]
2025-11-27T18:18:22.002512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471221926621426:2049] [node 3] ICS09 handshake done sender: [2:7577471221926621407:2113] self: [2:7577471221926621406:0] peer: [3:7577471222409008930:0] socket: 435 qp: -1
2025-11-27T18:18:22.005446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471221926621426:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:22.005464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471221926621397:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:22.005486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:21.983537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471220944270041:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:21.984010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471220944270056:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63549
2025-11-27T18:18:21.991227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471220944270041:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:22.082407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471222409008923:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:21.997890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471220944270057:2110] [node 6] ICS09 handshake done sender: [1:7577471220944270039:2115] self: [1:7577471220944270037:0] peer: [6:7577471219797423170:0] socket: 459 qp: -1
2025-11-27T18:18:22.069810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471219797423090:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:22.082661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:22.082706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471222409008828:2048] [node 1] ICP22 created new session: [3:7577471226703976236:2048]
2025-11-27T18:18:22.002797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471220944269964:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(64324676ns) {HandleHandshakeStatus:329(63899408ns) {DropIncomingHandshake:442(1753ns) DropOutgoingHandshake:457(967ns) SwitchToState:179(445ns) ProcessPendingSessionEvents:495(18236410ns) {ForwardSessionEventToSession:612(9428662ns) ForwardSessionEventToSession:612(4225965ns) ForwardSessionEventToSession:612(3591ns) ForwardSessionEventToSession:612(4529158ns) ForwardSessionEventToSession:612(5529ns) ForwardSessionEventToSession:612(3112ns) ForwardSessionEventToSession:612(5137ns) ForwardSessionEventToSession:612(2473ns) ForwardSessionEventToSession:612(5455ns) ForwardSessionEventToSession:612(5432ns)}}}
2025-11-27T18:18:22.005388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471220944270057:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:22.072657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471219797423088:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:22.006135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:22.009388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:22.018085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471220944270030:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:22.018162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:22.032051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:22.022615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471220944270030:2108] [node 4] ICP22 created new session: [1:7577471225239237357:2108]
2025-11-27T18:18:22.025863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471220944270038:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.033948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471226389748241:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2552
2025-11-27T18:18:22.025905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471220944270038:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:22.041515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471222094780921:2109] [node 6] ICP22 created new session: [4:7577471226389748242:2109]
2025-11-27T18:18:22.032610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471220944269964:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:22.073524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:22.057927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471226389748242:2109] [node 6] ICS09 handshake done sender: [4:7577471222094780929:2114] self: [4:7577471222094780928:0] peer: [6:7577471219797423171:0] socket: 451 qp: -1
2025-11-27T18:18:22.033416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471225239237357:2108] [node 4] ICS09 handshake done sender: [1:7577471220944270041:2116] self: [1:7577471220944270040:0] peer: [4:7577471222094780942:0] socket: 469 qp: -1
2025-11-27T18:18:22.057993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471226389748242:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:22.073980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471219797423146:2090] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:22.036203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471220944270038:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:22.074018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:22.058075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:22.043598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471225239237357:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:22.058108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471222094780939:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:22.043657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:22.075482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471219797423146:2090] [node 4] ICP22 created new session: [6:7577471224092390477:2090]
2025-11-27T18:18:22.053728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471220944269965:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:22.064499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471222094780855:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15864) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471222094780939:2119] held: no
2025-11-27T18:18:22.076837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471219797423088:2048] [node 1] ICP22 created new session: [6:7577471224092390476:2048]
2025-11-27T18:18:22.064780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471226389748246:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2552
2025-11-27T18:18:22.053805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:22.078708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471224092390476:2048] [node 1] ICS09 handshake done sender: [6:7577471219797423167:2116] self: [6:7577471219797423170:0] peer: [1:7577471220944270037:0] socket: 460 qp: -1
2025-11-27T18:18:22.078739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471224092390476:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:22.063147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471220944269965:2049] [node 3] ICP22 created new session: [1:7577471225239237360:2049]
2025-11-27T18:18:22.066718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471222094780855:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:22.066535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471225239237360:2049] [node 3] ICS09 handshake done sender: [1:7577471220944270038:2114] self: [1:7577471220944270036:0] peer: [3:7577471222409008933:0] socket: 457 qp: -1
2025-11-27T18:18:22.078772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:22.066756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:22.079725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471224092390477:2090] [node 4] ICS09 handshake done sender: [6:7577471219797423166:2115] self: [6:7577471219797423171:0] peer: [4:7577471222094780928:0] socket: 453 qp: -1
2025-11-27T18:18:22.070650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471225239237360:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:22.081467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471224092390477:2090] [node 4] ICS10 traffic start
2025-11-27T18:18:22.066804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471222094780855:2048] [node 1] ICP22 created new session: [4:7577471226389748247:2048]
2025-11-27T18:18:22.071321Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471220944270030:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(61696641ns) {HandleHandshakeStatus:329(61694493ns) {DropIncomingHandshake:442(18793ns) DropOutgoingHandshake:457(405ns) SwitchToState:179(480ns) ProcessPendingSessionEvents:495(1559742ns) {ForwardSessionEventToSession:612(1500643ns) ForwardSessionEventToSession:612(4397ns) ForwardSessionEventToSession:612(14282ns) ForwardSessionEventToSession:612(11972ns) ForwardSessionEventToSession:612(7433ns) ForwardSessionEventToSession:612(2823ns) ForwardSessionEventToSession:612(2165ns) ForwardSessionEventToSession:612(4013ns) ForwardSessionEventToSession:612(1697ns) ForwardSessionEventToSession:612(1825ns)}}}
2025-11-27T18:18:22.081547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471219797423176:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54129
2025-11-27T18:18:22.066843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471226389748247:2048] [node 1] ICS09 handshake done sender: [4:7577471222094780939:2119] self: [4:7577471222094780942:0] peer: [1:7577471220944270040:0] socket: 470 qp: -1
2025-11-27T18:18:22.071984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:22.085839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:22.066873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471226389748247:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:22.101782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471219797423168:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:22.066903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:22.077042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471220944270048:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.075986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471226389748246:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.110039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471219797423135:2094] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471219797423174:2120] held: no
2025-11-27T18:18:22.111433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471224092390482:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54129
2025-11-27T18:18:22.077062Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471220944270048:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:22.075986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471222094780921:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(42977755ns) {HandleHandshakeStatus:329(42974205ns) {DropIncomingHandshake:442(15464ns) DropOutgoingHandshake:457(367ns) SwitchToState:179(344ns) ProcessPendingSessionEvents:495(7062637ns) {ForwardSessionEventToSession:612(61988ns) ForwardSessionEventToSession:612(3124ns) ForwardSessionEventToSession:612(2706ns) ForwardSessionEventToSession:612(2405083ns) ForwardSessionEventToSession:612(3011ns) ForwardSessionEventToSession:612(2227945ns) ForwardSessionEventToSession:612(5328ns) ForwardSessionEventToSession:612(2600ns) ForwardSessionEventToSession:612(2317584ns) ForwardSessionEventToSession:612(3401ns)}}}
2025-11-27T18:18:22.114669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471219797423089:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:22.084065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471222094780857:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:22.082487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471220944270048:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:22.114755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471219797423146:2090] [node 4] ICP35 event processing took too much time PendingConnection:279(39155505ns) {HandleHandshakeStatus:329(39152730ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(225ns) SwitchToState:179(372ns) ProcessPendingSessionEvents:495(1510ns)}}
2025-11-27T18:18:22.087375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471222094780935:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.084925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471220944270031:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(103270550ns) {HandleHandshakeStatus:329(103267981ns) {DropIncomingHandshake:442(16081ns) DropOutgoingHandshake:457(387ns) SwitchToState:179(295ns) ProcessPendingSessionEvents:495(40004812ns) {ForwardSessionEventToSession:612(29328428ns) ForwardSessionEventToSession:612(6291ns) ForwardSessionEventToSession:612(7609460ns) ForwardSessionEventToSession:612(7020ns) ForwardSessionEventToSession:612(2530ns) ForwardSessionEventToSession:612(1363534ns) ForwardSessionEventToSession:612(3912ns) ForwardSessionEventToSession:612(3156ns) ForwardSessionEventToSession:612(1657146ns) ForwardSessionEventToSession:612(3367ns)}}}
2025-11-27T18:18:22.127085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:22.087398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471222094780935:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:22.104944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471220944270032:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:22.088147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471222094780935:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:22.018208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471221926621429:2048] [node 1] ICS09 handshake done sender: [2:7577471221926621399:2110] self: [2:7577471221926621415:0] peer: [1:7577471220944270024:0] socket: 423 qp: -1
2025-11-27T18:18:22.112185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:22.088363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471222094780922:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:22.112235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471220944270032:2109] [node 5] ICP22 created new session: [1:7577471225239237363:2109]
2025-11-27T18:18:22.088429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:22.018323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471221926621397:2109] [node 6] ICP22 created new session: [2:7577471226221588727:2109]
2025-11-27T18:18:22.117489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471225239237363:2109] [node 5] ICS09 handshake done sender: [1:7577471220944270048:2118] self: [1:7577471220944270047:0] peer: [5:7577471225294014272:0] socket: 485 qp: -1
2025-11-27T18:18:22.088471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471222094780922:2108] [node 5] ICP22 created new session: [4:7577471226389748251:2108]
2025-11-27T18:18:22.018374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471226221588727:2109] [node 6] ICS09 handshake done sender: [2:7577471221926621409:2114] self: [2:7577471221926621404:0] peer: [6:7577471219797423172:0] socket: 461 qp: -1
2025-11-27T18:18:22.122340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471225239237363:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:22.088516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471226389748251:2108] [node 5] ICS09 handshake done sender: [4:7577471222094780935:2117] self: [4:7577471222094780934:0] peer: [5:7577471225294014275:0] socket: 450 qp: -1
2025-11-27T18:18:22.127293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:22.139325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471220944269965:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(60970232ns) {HandleHandshakeStatus:329(60965819ns) {DropIncomingHandshake:442(19224ns) DropOutgoingHandshake:457(725ns) SwitchToState:179(547ns) ProcessPendingSessionEvents:495(31047977ns) {ForwardSessionEventToSession:612(24437666ns) ForwardSessionEventToSession:612(6770ns) ForwardSessionEventToSession:612(3522327ns) ForwardSessionEventToSession:612(3290ns) ForwardSessionEventToSession:612(552421ns) ForwardSessionEventToSession:612(3490ns) ForwardSessionEventToSession:612(3775ns) ForwardSessionEventToSession:612(1659896ns) ForwardSessionEventToSession:612(827702ns) ForwardSessionEventToSession:612(3095ns)}}}
2025-11-27T18:18:22.088545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471226389748251:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:22.018416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471226221588727:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:22.129776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471219797423089:2049] [node 2] ICP22 created new session: [6:7577471224092390483:2049]
2025-11-27T18:18:22.082749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471226703976236:2048] [node 1] ICS09 handshake done sender: [3:7577471222409008928:2134] self: [3:7577471222409008933:0] peer: [1:7577471220944270036:0] socket: 463 qp: -1
2025-11-27T18:18:22.161694Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471220944270032:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(50552696ns) {HandleHandshakeStatus:329(50549330ns) {DropIncomingHandshake:442(16036ns) DropOutgoingHandshake:457(515ns) SwitchToState:179(958ns) ProcessPendingSessionEvents:495(13995508ns) {ForwardSessionEventToSession:612(13182011ns) ForwardSessionEventToSession:612(5360ns) ForwardSessionEventToSession:612(3258ns) ForwardSessionEventToSession:612(9663ns) ForwardSessionEventToSession:612(3447ns) ForwardSessionEventToSession:612(7884ns) ForwardSessionEventToSession:612(2884ns) ForwardSessionEventToSession:612(2749ns) ForwardSessionEventToSession:612(5388ns) ForwardSessionEventToSession:612(2959ns)}}}
2025-11-27T18:18:22.088573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:22.111686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471226389748253:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2552
2025-11-27T18:18:22.113647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471222094780857:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17390) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471226389748246:2121] held: no
2025-11-27T18:18:22.117954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471226389748246:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:22.137792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471224092390483:2049] [node 2] ICS09 handshake done sender: [6:7577471219797423168:2117] self: [6:7577471219797423172:0] peer: [2:7577471221926621404:0] socket: 462 qp: -1
2025-11-27T18:18:22.130545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471222094780857:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:22.139497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471224092390480:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54129
2025-11-27T18:18:22.145707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471222094780922:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(27393992ns) {HandleHandshakeStatus:329(27384818ns) {DropIncomingHandshake:442(33153ns) DropOutgoingHandshake:457(343ns) SwitchToState:179(158ns) ProcessPendingSessionEvents:495(27057558ns) {ForwardSessionEventToSession:612(15411248ns) ForwardSessionEventToSession:612(7784784ns) ForwardSessionEventToSession:612(3453ns) ForwardSessionEventToSession:612(2497ns) ForwardSessionEventToSession:612(584033ns) ForwardSessionEventToSession:612(3059ns) ForwardSessionEventToSession:612(3291ns) ForwardSessionEventToSession:612(951177ns) ForwardSessionEventToSession:612(3172ns) ForwardSessionEventToSession:612(2293624ns)}}}
2025-11-27T18:18:22.150552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471224092390483:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:22.082780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471226703976236:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:22.150473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:22.153206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471219797423169:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:22.082817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:22.153409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471222094780857:2050] [node 3] ICP22 created new session: [4:7577471226389748254:2050]
2025-11-27T18:18:22.153895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:22.093858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471222409008829:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(90701801ns) {HandleHandshakeStatus:329(90699262ns) {DropIncomingHandshake:442(449ns) DropOutgoingHandshake:457(6637ns) SwitchToState:179(214ns) ProcessPendingSessionEvents:495(15711177ns) {ForwardSessionEventToSession:612(10120100ns) ForwardSessionEventToSession:612(3168ns) ForwardSessionEventToSession:612(937847ns) ForwardSessionEventToSession:612(1318829ns) ForwardSessionEventToSession:612(2946ns) ForwardSessionEventToSession:612(3028ns) ForwardSessionEventToSession:612(1401280ns) ForwardSessionEventToSession:612(2651ns) ForwardSessionEventToSession:612(1904697ns) ForwardSessionEventToSession:612(3182ns)}}}
2025-11-27T18:18:22.086590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:22.018479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:22.159917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471219797423174:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:22.100065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471222409008925:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.163316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471219797423090:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:22.100095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471222409008925:2133] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:22.158766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471226389748254:2050] [node 3] ICS09 handshake done sender: [4:7577471226389748246:2121] self: [4:7577471226389748249:0] peer: [3:7577471222409008924:0] socket: 433 qp: -1
2025-11-27T18:18:22.086653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471220999046950:2109] [node 6] ICP22 created new session: [5:7577471225294014276:2109]
2025-11-27T18:18:22.166186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471219797423135:2094] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:22.022436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471221926621429:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:22.169603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:22.086698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471225294014276:2109] [node 6] ICS09 handshake done sender: [5:7577471220999046967:2117] self: [5:7577471220999046966:0] peer: [6:7577471224092390473:0] socket: 477 qp: -1
2025-11-27T18:18:22.169686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471219797423090:2050] [node 3] ICP22 created new session: [6:7577471224092390485:2050]
2025-11-27T18:18:22.022488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:22.052877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471221926621335:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(77916554ns) {HandleHandshakeStatus:329(77914046ns) {DropIncomingHandshake:442(13502ns) DropOutgoingHandshake:457(210ns) SwitchToState:179(240ns) ProcessPendingSessionEvents:495(19084466ns) {ForwardSessionEventToSession:612(19032195ns) ForwardSessionEventToSession:612(8820ns) ForwardSessionEventToSession:612(2888ns) ForwardSessionEventToSession:612(6151ns) ForwardSessionEventToSession:612(3084ns) ForwardSessionEventToSession:612(2122ns) ForwardSessionEventToSession:612(5528ns) ForwardSessionEventToSession:612(3409ns) ForwardSessionEventToSession:612(4620ns) ForwardSessionEventToSession:612(5162ns)}}}
2025-11-27T18:18:22.101187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471222409008925:2133] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:22.101431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471222409008909:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:22.108269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:22.113245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471222409008923:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:22.113859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471222409008909:2107] [node 4] ICP22 created new session: [3:7577471226703976241:2107]
2025-11-27T18:18:22.115242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471226703976241:2107] [node 4] ICS09 handshake done sender: [3:7577471222409008925:2133] self: [3:7577471222409008924:0] peer: [4:7577471226389748249:0] socket: 488 qp: -1
2025-11-27T18:18:22.115542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471226703976241:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:22.116009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471226703976237:2138] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.117571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471222409008910:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:22.117604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:22.118010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:22.121914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471222409008910:2116] [node 6] ICP22 created new session: [3:7577471226703976242:2116]
2025-11-27T18:18:22.139292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471226703976242:2116] [node 6] ICS09 handshake done sender: [3:7577471222409008923:2132] self: [3:7577471222409008922:0] peer: [6:7577471224092390475:0] socket: 467 qp: -1
2025-11-27T18:18:22.147296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471226703976242:2116] [node 6] ICS10 traffic start
2025-11-27T18:18:22.150301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:22.164709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471222409008909:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(51710562ns) {HandleHandshakeStatus:329(51706032ns) {DropIncomingHandshake:442(14834ns) DropOutgoingHandshake:457(397ns) SwitchToState:179(488ns) ProcessPendingSessionEvents:495(12859660ns) {ForwardSessionEventToSession:612(12804636ns) ForwardSessionEventToSession:612(9051ns) ForwardSessionEventToSession:612(3133ns) ForwardSessionEventToSession:612(2075ns) ForwardSessionEventToSession:612(5989ns) ForwardSessionEventToSession:612(2575ns) ForwardSessionEventToSession:612(8199ns) ForwardSessionEventToSession:612(3155ns) ForwardSessionEventToSession:612(4882ns) ForwardSessionEventToSession:612(6120ns)}}}
2025-11-27T18:18:22.165565Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471222409008910:2116] [node 6] ICP35 event processing took too much time PendingConnection:279(47789488ns) {HandleHandshakeStatus:329(47786850ns) {DropIncomingHandshake:442(359ns) DropOutgoingHandshake:457(239ns) SwitchToState:179(216ns) ProcessPendingSessionEvents:495(3967400ns) {ForwardSessionEventToSession:612(2210734ns) ForwardSessionEventToSession:612(1705851ns) ForwardSessionEventToSession:612(7258ns) ForwardSessionEventToSession:612(2854ns) ForwardSessionEventToSession:612(8305ns) ForwardSessionEventToSession:612(2343ns) ForwardSessionEventToSession:612(1938ns) ForwardSessionEventToSession:612(4429ns) ForwardSessionEventToSession:612(2608ns) ForwardSessionEventToSession:612(7921ns)}}}
2025-11-27T18:18:22.165629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471222409008910:2116] [node 6] ICP18 handshake (actor [3:7577471226703976237:2138]) is too old
2025-11-27T18:18:22.158829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471226389748254:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:22.086741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471225294014276:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:22.169732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471224092390485:2050] [node 3] ICS09 handshake done sender: [6:7577471219797423169:2118] self: [6:7577471224092390475:0] peer: [3:7577471222409008922:0] socket: 468 qp: -1
2025-11-27T18:18:22.171932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471224092390485:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:22.172503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:22.172978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:22.173021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471219797423135:2094] [node 5] ICP22 created new session: [6:7577471224092390487:2094]
2025-11-27T18:18:22.173831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471224092390487:2094] [node 5] ICS09 handshake done sender: [6:7577471219797423174:2120] self: [6:7577471224092390473:0] peer: [5:7577471220999046966:0] socket: 479 qp: -1
2025-11-27T18:18:22.174002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471219797423089:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(60312268ns) {HandleHandshakeStatus:329(60309214ns) {DropIncomingHandshake:442(2073ns) DropOutgoingHandshake:457(12168ns) SwitchToState:179(711ns) ProcessPendingSessionEvents:495(5468385ns) {ForwardSessionEventToSession:612(5089567ns) ForwardSessionEventToSession:612(5215ns) ForwardSessionEventToSession:612(334784ns) ForwardSessionEventToSession:612(3352ns) ForwardSessionEventToSession:612(5223ns) ForwardSessionEventToSession:612(2621ns) ForwardSessionEventToSession:612(3196ns) ForwardSessionEventToSession:612(5749ns) ForwardSessionEventToSession:612(5543ns) ForwardSessionEventToSession:612(2074ns)}}}
2025-11-27T18:18:22.174763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471224092390487:2094] [node 5] ICS10 traffic start
2025-11-27T18:18:22.177819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:22.158893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:22.059339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471221926621419:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.086775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:22.159123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471222094780857:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(28496099ns) {HandleHandshakeStatus:329(28493131ns) {DropIncomingHandshake:442(260ns) DropOutgoingHandshake:457(207ns) SwitchToState:179(149ns) ProcessPendingSessionEvents:495(1161ns)}}
2025-11-27T18:18:22.108833Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471220999046949:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63497) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471225294014270:2120] held: no
2025-11-27T18:18:22.059379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471221926621419:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:22.069472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471221926621419:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:22.076273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471221926621400:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:22.076587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:22.079617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471221926621400:2108] [node 5] ICP22 created new session: [2:7577471226221588731:2108]
2025-11-27T18:18:22.081827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471226221588732:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65092
2025-11-27T18:18:22.083482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471226221588731:2108] [node 5] ICS09 handshake done sender: [2:7577471221926621419:2118] self: [2:7577471221926621417:0] peer: [5:7577471225294014271:0] socket: 481 qp: -1
2025-11-27T18:18:22.083522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471226221588731:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:22.085908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:22.108105Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471221926621334:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(79715895ns) {HandleHandshakeStatus:329(79713292ns) {DropIncomingHandshake:442(600ns) DropOutgoingHandshake:457(4821ns) SwitchToState:179(368ns) ProcessPendingSessionEvents:495(7723513ns) {ForwardSessionEventToSession:612(2583947ns) ForwardSessionEventToSession:612(595579ns) ForwardSessionEventToSession:612(3931ns) ForwardSessionEventToSession:612(2694ns) ForwardSessionEventToSession:612(1945640ns) ForwardSessionEventToSession:612(3045ns) ForwardSessionEventToSession:612(1147362ns) ForwardSessionEventToSession:612(6116ns) ForwardSessionEventToSession:612(3233ns) ForwardSessionEventToSession:612(1400866ns)}}}
2025-11-27T18:18:22.113950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471226221588732:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:22.117688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471221926621397:2109] [node 6] ICP18 handshake (actor [2:7577471226221588732:2120]) is too old
2025-11-27T18:18:22.153454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471221926621400:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(41623409ns) {HandleHandshakeStatus:329(41620989ns) {DropIncomingHandshake:442(17887ns) DropOutgoingHandshake:457(250ns) SwitchToState:179(140ns) ProcessPendingSessionEvents:495(9849454ns) {ForwardSessionEventToSession:612(5933280ns) ForwardSessionEventToSession:612(3664ns) ForwardSessionEventToSession:612(3150ns) ForwardSessionEventToSession:612(793905ns) ForwardSessionEventToSession:612(2915ns) ForwardSessionEventToSession:612(716272ns) ForwardSessionEventToSession:612(8021ns) ForwardSessionEventToSession:612(3534ns) ForwardSessionEventToSession:612(2365534ns) ForwardSessionEventToSession:612(3421ns)}}}
2025-11-27T18:18:22.109271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471225294014278:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4910
2025-11-27T18:18:22.109755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471225294014279:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4910
2025-11-27T18:18:22.113907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471225294014270:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:22.114072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471220999046973:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:22.116787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471220999046949:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:22.116853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:22.116913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471220999046949:2108] [node 4] ICP22 created new session: [5:7577471225294014280:2108]
2025-11-27T18:18:22.116955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471225294014280:2108] [node 4] ICS09 handshake done sender: [5:7577471225294014270:2120] self: [5:7577471225294014275:0] peer: [4:7577471222094780934:0] socket: 475 qp: -1
2025-11-27T18:18:22.116991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471225294014280:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:22.117024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:22.122091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471220999046886:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15864) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471220999046973:2119] held: no
2025-11-27T18:18:22.129686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471225294014273:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4910
2025-11-27T18:18:22.129895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471220999046886:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:22.129944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:22.139960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471220999046886:2048] [node 1] ICP22 created new session: [5:7577471225294014282:2048]
2025-11-27T18:18:22.150559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471220999046972:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:22.153337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471225294014282:2048] [node 1] ICS09 handshake done sender: [5:7577471220999046973:2119] self: [5:7577471225294014272:0] peer: [1:7577471220944270047:0] socket: 486 qp: -1
2025-11-27T18:18:22.154367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471220999046887:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:22.157942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471225294014282:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:22.161537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:22.162664Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471220999046886:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32694035ns) {HandleHandshakeStatus:329(32691605ns) {DropIncomingHandshake:442(758ns) DropOutgoingHandshake:457(263ns) SwitchToState:179(453ns) ProcessPendingSessionEvents:495(745ns)}}
2025-11-27T18:18:22.163139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:22.165185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471220999046887:2049] [node 2] ICP22 created new session: [5:7577471225294014284:2049]
2025-11-27T18:18:22.168093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471225294014284:2049] [node 2] ICS09 handshake done sender: [5:7577471220999046972:2118] self: [5:7577471225294014271:0] peer: [2:7577471221926621417:0] socket: 482 qp: -1
2025-11-27T18:18:22.168133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471225294014284:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:22.171791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:22.174763Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471220999046887:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(18978079ns) {HandleHandshakeStatus:329(18975212ns) {DropIncomingHandshake:442(1101ns) DropOutgoingHandshake:457(549ns) SwitchToState:179(415ns) ProcessPendingSessionEvents:495(784ns)}}
2025-11-27T18:18:22.233817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:22.234040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471221926621429:2048] [node 1] ICS07 socket disconnect 423 reason# EndOfStream
2025-11-27T18:18:22.234070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471221926621429:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.234152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:22.234192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471221926621429:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:22.236103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:22.236327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471226703976236:2048] [node 1] ICS01 socket: 463 reason# EndOfStream
2025-11-27T18:18:22.236409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471222409008828:2048] [node 1] ICP30 unregister session Session# [3:7577471226703976236:2048] VirtualId# [3:7577471222409008933:0]
2025-11-27T18:18:22.236438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471226703976236:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.236503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:22.236630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:22.237684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471221926621334:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15864) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:22.237742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471221926621429:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:22.238801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471221926621334:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15864) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:22.238993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471221926621429:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:22.239037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471221926621334:2048] [node 1] ICP30 unregister session Session# [2:7577471221926621429:2048] VirtualId# [2:7577471221926621415:0]
2025-11-27T18:18:22.239084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:22.239140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471221926621334:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:22.239160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15864) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:22.239219Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:22.239401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471226389748247:2048] [node 1] ICS01 socket: 470 reason# EndOfStream
2025-11-27T18:18:22.239445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471222094780855:2048] [node 1] ICP30 unregister session Session# [4:7577471226389748247:2048] VirtualId# [4:7577471222094780942:0]
2025-11-27T18:18:22.239490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471226389748247:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.239564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:22.239602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:22.240319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:22.240604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471225294014282:2048] [node 1] ICS01 socket: 486 reason# EndOfStream
2025-11-27T18:18:22.240655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471220999046886:2048] [node 1] ICP30 unregister session Session# [5:7577471225294014282:2048] VirtualId# [5:7577471225294014272:0]
2025-11-27T18:18:22.240702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471225294014282:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.240804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:22.240876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:22.241294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:22.241502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471224092390476:2048] [node 1] ICS01 socket: 460 reason# EndOfStream
2025-11-27T18:18:22.241555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471219797423088:2048] [node 1] ICP30 unregister session Session# [6:7577471224092390476:2048] VirtualId# [6:7577471219797423170:0]
2025-11-27T18:18:22.241585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471224092390476:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.241646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:22.241682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:22.283141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:22.283353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471222409008934:2049] [node 2] ICS07 socket disconnect 436 reason# EndOfStream
2025-11-27T18:18:22.283388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471222409008934:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.283462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:22.283499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471222409008934:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:22.286326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471222409008829:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17662) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:22.286377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471222409008934:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:22.287642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471222409008829:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17662) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:22.288764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:22.288981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471226389748243:2049] [node 2] ICS01 socket: 426 reason# EndOfStream
2025-11-27T18:18:22.289016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471222094780856:2049] [node 2] ICP30 unregister session Session# [4:7577471226389748243:2049] VirtualId# [4:7577471222094780936:0]
2025-11-27T18:18:22.289328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471222409008934:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:22.289044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471226389748243:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.289376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471222409008829:2049] [node 2] ICP30 unregister session Session# [3:7577471222409008934:2049] VirtualId# [3:7577471222409008930:0]
2025-11-27T18:18:22.289118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:22.289161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:22.289641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:22.289713Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471222409008829:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17662) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:22.289728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17662) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:22.293496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:22.293768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471225294014284:2049] [node 2] ICS01 socket: 482 reason# EndOfStream
2025-11-27T18:18:22.293843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471220999046887:2049] [node 2] ICP30 unregister session Session# [5:7577471225294014284:2049] VirtualId# [5:7577471225294014271:0]
2025-11-27T18:18:22.293910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471225294014284:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.294008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:22.294061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:22.295583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:22.295914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471224092390483:2049] [node 2] ICS07 socket disconnect 462 reason# EndOfStream
2025-11-27T18:18:22.295969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471224092390483:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.296089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:22.296148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471224092390483:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:22.298890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471219797423089:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17662) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:22.298952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471224092390483:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:22.298991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471219797423089:2049] [node 2] ICP30 unregister session Session# [6:7577471224092390483:2049] VirtualId# [6:7577471219797423172:0]
2025-11-27T18:18:22.299043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:22.299071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471219797423089:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17662) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:22.299084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17662) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:22.301713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471222409008829:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:22.311321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471219797423089:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:22.385739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:22.385915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471220999046969:2050] [node 3] ICS01 socket: 442 reason# EndOfStream
2025-11-27T18:18:22.385960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471220999046888:2050] [node 3] ICP30 unregister session Session# [5:7577471220999046969:2050] VirtualId# [5:7577471220999046964:0]
2025-11-27T18:18:22.386014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471220999046969:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.386080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:22.386110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:22.385936Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:22.386131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471226389748254:2050] [node 3] ICS01 socket: 433 reason# EndOfStream
2025-11-27T18:18:22.386170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471222094780857:2050] [node 3] ICP30 unregister session Session# [4:7577471226389748254:2050] VirtualId# [4:7577471226389748249:0]
2025-11-27T18:18:22.386197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471226389748254:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.386276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:22.386321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:22.387020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:22.387205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471224092390485:2050] [node 3] ICS07 socket disconnect 468 reason# EndOfStream
2025-11-27T18:18:22.387235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471224092390485:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.387307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:22.387345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471224092390485:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:22.388662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471219797423090:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17390) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:22.388783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471224092390485:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:22.388878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471219797423090:2050] [node 3] ICP30 unregister session Session# [6:7577471224092390485:2050] VirtualId# [6:7577471224092390475:0]
2025-11-27T18:18:22.388979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:22.389059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471219797423090:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17390) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:22.389089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17390) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:22.406129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471219797423090:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:22.443941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:22.444190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471225294014280:2108] [node 4] ICS01 socket: 475 reason# EndOfStream
2025-11-27T18:18:22.444236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471220999046949:2108] [node 4] ICP30 unregister session Session# [5:7577471225294014280:2108] VirtualId# [5:7577471225294014275:0]
2025-11-27T18:18:22.444271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471225294014280:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.444346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:22.444392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:22.445537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:22.445727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471224092390477:2090] [node 4] ICS01 socket: 453 reason# EndOfStream
2025-11-27T18:18:22.445760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471219797423146:2090] [node 4] ICP30 unregister session Session# [6:7577471224092390477:2090] VirtualId# [6:7577471219797423171:0]
2025-11-27T18:18:22.445786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471224092390477:2090] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.445850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:22.445886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:22.541411Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:22.541661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471224092390487:2094] [node 5] ICS01 socket: 479 reason# EndOfStream
2025-11-27T18:18:22.541707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471219797423135:2094] [node 5] ICP30 unregister session Session# [6:7577471224092390487:2094] VirtualId# [6:7577471224092390473:0]
2025-11-27T18:18:22.541737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471224092390487:2094] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:22.541843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:22.541889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:22.668308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471224523497704:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:22.668353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471224523497705:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:22.691528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471223103635397:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:22.691586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471223103635398:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:22.714157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471222933303477:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:22.714211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471222933303478:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:22.737941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471222903482951:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:22.738021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471222903482952:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:22.738052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471222903482953:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:22.761309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471222853465833:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:22.761359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471222853465834:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:22.761408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471222853465835:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:22.786515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471223824452413:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:22.786559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471223824452414:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:22.786839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471223824452415:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:23.026881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471228119419770:2105] [node 5] ICP01 ready to work
2025-11-27T18:18:23.071942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471228818465063:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:23.066236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471227398602756:2107] [node 6] ICP01 ready to work
2025-11-27T18:18:23.072294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471228818465064:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:23.085272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471227398602757:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:23.072596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471228818465065:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:23.126355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471227148433192:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:23.144052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471227398602758:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:23.157965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471228119419770:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(130813644ns) {Configure:104(130809376ns) {ProcessConfigured:135(3964731ns) {StartInitialHandshake:157(3962798ns) {DropHandshakes:470(2137ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(840ns) PrepareNewSessionHandshake:477(3038ns) {SwitchToState:179(1429ns)}}}}}
2025-11-27T18:18:23.126658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471227148433193:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:23.154768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471222853465835:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25930021ns) {Configure:104(25925599ns) {ProcessConfigured:135(50287ns) {StartInitialHandshake:157(48174ns) {DropHandshakes:470(636ns) {DropIncomingHandshake:442(237ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(295ns) PrepareNewSessionHandshake:477(1769ns) {SwitchToState:179(803ns)}}}}}
2025-11-27T18:18:23.162474Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471222903482952:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47111662ns) {Configure:104(47100001ns) {ProcessConfigured:135(44735ns) {StartInitialHandshake:157(43263ns) {DropHandshakes:470(620ns) {DropIncomingHandshake:442(112ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:477(852ns) {SwitchToState:179(347ns)}}}}}
2025-11-27T18:18:23.175665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471224523497704:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(90172673ns) {Configure:104(90168549ns) {ProcessConfigured:135(9523826ns) {StartInitialHandshake:157(9522406ns) {DropHandshakes:470(816ns) {DropIncomingHandshake:442(235ns) DropOutgoingHandshake:457(180ns)} GenerateSessionVirtualId:988(296ns) PrepareNewSessionHandshake:477(2653ns) {SwitchToState:179(1192ns)}}}}}
2025-11-27T18:18:23.162989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471228119419775:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:23.171671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471222933303477:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(123098306ns) {Configure:104(123088074ns) {ProcessConfigured:135(1201010ns) {StartInitialHandshake:157(975963ns) {DropHandshakes:470(2599ns) {DropIncomingHandshake:442(650ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(2244ns) PrepareNewSessionHandshake:477(2512ns) {SwitchToState:179(1201ns)}}}}}
2025-11-27T18:18:23.172227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471227198450322:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:23.178314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471227198450323:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:23.189796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471227398602756:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16740650ns) {Configure:104(16734943ns) {ProcessConfigured:135(6272791ns) {StartInitialHandshake:157(6270999ns) {DropHandshakes:470(1485ns) {DropIncomingHandshake:442(238ns) DropOutgoingHandshake:457(174ns)} GenerateSessionVirtualId:988(429ns) PrepareNewSessionHandshake:477(2815ns) {SwitchToState:179(1355ns)}}}}}
2025-11-27T18:18:23.197485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471227228270840:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:23.197882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471227228270841:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:23.198173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471227228270842:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.205816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471228818465065:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17362745ns) {Configure:104(17359607ns) {ProcessConfigured:135(53687ns) {StartInitialHandshake:157(51791ns) {DropHandshakes:470(1435ns) {DropIncomingHandshake:442(256ns) DropOutgoingHandshake:457(531ns)} GenerateSessionVirtualId:988(547ns) PrepareNewSessionHandshake:477(4756ns) {SwitchToState:179(473ns)}}}}}
2025-11-27T18:18:23.206876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471228818465076:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60738
2025-11-27T18:18:23.215085Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471222903482953:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21312349ns) {Configure:104(21307636ns) {ProcessConfigured:135(53383ns) {StartInitialHandshake:157(51683ns) {DropHandshakes:470(2214ns) {DropIncomingHandshake:442(448ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(718ns) PrepareNewSessionHandshake:477(1016ns) {SwitchToState:179(252ns)}}}}}
2025-11-27T18:18:23.217915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471228818465076:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.222252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471227148433208:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47144
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:18:23.226333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471227398602769:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41208
::ffff:127.1.0.0
2025-11-27T18:18:23.226622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471227398602770:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41208
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.227084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471227398602771:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41208
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.230591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471227198450328:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
2025-11-27T18:18:23.230660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471227198450329:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.231056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471228119419787:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11051
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.231281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471227148433208:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.237724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471227228270843:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34312
2025-11-27T18:18:23.228106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471227398602769:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.240516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471227228270852:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34312
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.241093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471227228270854:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34312
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.241349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471228818465080:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60738
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.241682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471227228270855:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34312
2025-11-27T18:18:23.238024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471227398602771:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.246405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471228119419787:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.239218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471227398602770:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.242394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471227398602775:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41208
2025-11-27T18:18:23.246956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471223824452414:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.247923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471227228270843:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.248099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471227228270852:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.247333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471223103635397:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.247020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471228818465081:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60738
2025-11-27T18:18:23.247227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471228818465080:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.247566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471227398602778:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41208
2025-11-27T18:18:23.249383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471222933303477:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.250342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471227198450328:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.250881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471227228270855:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.250801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471227198450329:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.250924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471227198450335:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
2025-11-27T18:18:23.248104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471228818465069:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.248477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471227398602760:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.250995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471227228270854:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.248126Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471228818465069:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:23.251218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471228119419789:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11051
2025-11-27T18:18:23.251304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471222903482951:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.248495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471227398602760:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.251374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471222903482952:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.251331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471228818465081:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.249306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471227398602775:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.251792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471227198450335:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.251568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471228818465083:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60738
2025-11-27T18:18:23.251971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471222903482953:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.250566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471227398602760:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:23.251628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471227398602778:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.251684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471227398602756:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.251762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:23.251943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471227398602756:2107] [node 6] ICP22 created new session: [2:7577471227398602782:2107]
2025-11-27T18:18:23.252059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471227398602782:2107] [node 6] ICS09 handshake done sender: [2:7577471227398602760:2110] self: [2:7577471227398602759:0] peer: [6:7577471228119419788:0] socket: 446 qp: -1
2025-11-27T18:18:23.252101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471228119419787:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:23.252128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471222933303478:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.252079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471228818465067:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.252111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471228818465067:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:23.252105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471227398602782:2107] [node 6] ICS10 traffic start
2025-11-27T18:18:23.252640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471223824452414:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.252689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:23.252563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471227228270845:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.252602Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471227228270845:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:23.252150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:23.255366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471223824452414:2049] [node 2] ICP22 created new session: [6:7577471228119419790:2049]
2025-11-27T18:18:23.253771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471227398602762:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.255575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471227228270845:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:23.259887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471228119419790:2049] [node 2] ICS09 handshake done sender: [6:7577471228119419787:2115] self: [6:7577471228119419788:0] peer: [2:7577471227398602759:0] socket: 451 qp: -1
2025-11-27T18:18:23.259947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471228119419790:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:23.260014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:23.253789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471227398602762:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.263748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471227228270840:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.263155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471227198450339:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
2025-11-27T18:18:23.264234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:23.264232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471228818465071:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.264270Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471228818465071:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:23.264588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471227198450335:2125] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:23.264422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471227228270840:2107] [node 4] ICP22 created new session: [3:7577471227228270863:2107]
2025-11-27T18:18:23.264773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471227398602764:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.264594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471227228270863:2107] [node 4] ICS09 handshake done sender: [3:7577471227228270845:2114] self: [3:7577471227228270844:0] peer: [4:7577471227198450338:0] socket: 462 qp: -1
2025-11-27T18:18:23.264757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471227228270863:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:23.264931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:23.264807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471227398602764:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:23.267573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471228818465083:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.267637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471222903482953:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.267693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:23.267742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471222903482953:2050] [node 3] ICP22 created new session: [4:7577471227198450340:2050]
2025-11-27T18:18:23.267786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471227198450340:2050] [node 3] ICS09 handshake done sender: [4:7577471227198450335:2125] self: [4:7577471227198450338:0] peer: [3:7577471227228270844:0] socket: 467 qp: -1
2025-11-27T18:18:23.267911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471227198450340:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:23.269464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471228818465069:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:23.268188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.270216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471227148433210:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47144
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.271079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471222903482953:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:23.271101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471227228270865:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34312
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.271274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471227228270866:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34312
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.271320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471227148433211:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47144
2025-11-27T18:18:23.271418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471227228270868:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34312
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.272047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471227228270865:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.272048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471227198450342:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
2025-11-27T18:18:23.272132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471227198450343:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
2025-11-27T18:18:23.272782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471227198450329:2123] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.270174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471227398602762:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:23.273316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471222903482952:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.271033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471227398602764:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:23.272078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471227398602787:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41208
2025-11-27T18:18:23.272416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471227398602757:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.272526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:23.272570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471227398602757:2108] [node 4] ICP22 created new session: [2:7577471227398602789:2108]
2025-11-27T18:18:23.272628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471227398602789:2108] [node 4] ICS09 handshake done sender: [2:7577471227398602762:2111] self: [2:7577471227398602761:0] peer: [4:7577471227198450337:0] socket: 447 qp: -1
2025-11-27T18:18:23.272664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471227398602789:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:23.272701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:23.281141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471223103635398:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.281238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:23.281349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471223103635398:2049] [node 3] ICP22 created new session: [2:7577471227398602791:2049]
2025-11-27T18:18:23.281393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471227398602791:2049] [node 3] ICS09 handshake done sender: [2:7577471227398602764:2112] self: [2:7577471227398602763:0] peer: [3:7577471227228270861:0] socket: 448 qp: -1
2025-11-27T18:18:23.281432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471227398602791:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:23.281484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:23.274087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471228119419793:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11051
2025-11-27T18:18:23.283188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471227228270854:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:23.283306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471228818465067:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:23.273369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:23.273525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471222903482952:2049] [node 2] ICP22 created new session: [4:7577471227198450345:2049]
2025-11-27T18:18:23.282210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471227198450345:2049] [node 2] ICS09 handshake done sender: [4:7577471227198450329:2123] self: [4:7577471227198450337:0] peer: [2:7577471227398602761:0] socket: 450 qp: -1
2025-11-27T18:18:23.282491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471227198450345:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:23.282532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:23.284167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471227148433211:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.284704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471222853465835:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.284997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471228818465071:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:23.287632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471227398602770:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:23.287622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471222933303478:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.288754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471227198450328:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:23.287727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:23.291240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471224523497704:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.287777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471222933303478:2049] [node 2] ICP22 created new session: [3:7577471227228270869:2049]
2025-11-27T18:18:23.292946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471228119419793:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.291319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:23.291364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471224523497704:2048] [node 2] ICP22 created new session: [1:7577471228818465090:2048]
2025-11-27T18:18:23.287824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471227228270869:2049] [node 2] ICS09 handshake done sender: [3:7577471227228270854:2119] self: [3:7577471227228270861:0] peer: [2:7577471227398602763:0] socket: 460 qp: -1
2025-11-27T18:18:23.294292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471222903482951:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.294188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471228119419775:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.291410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471228818465090:2048] [node 2] ICS09 handshake done sender: [1:7577471228818465069:2111] self: [1:7577471228818465068:0] peer: [2:7577471227398602779:0] socket: 434 qp: -1
2025-11-27T18:18:23.294357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:23.294400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471222903482951:2048] [node 1] ICP22 created new session: [4:7577471227198450348:2048]
2025-11-27T18:18:23.291442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471228818465090:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:23.294462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471227198450348:2048] [node 1] ICS09 handshake done sender: [4:7577471227198450328:2122] self: [4:7577471227198450336:0] peer: [1:7577471228818465070:0] socket: 449 qp: -1
2025-11-27T18:18:23.291497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:23.294514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471227198450348:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:23.287855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471227228270869:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:23.291926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471224523497705:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.294591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:23.287891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:23.292007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:23.292044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471224523497705:2049] [node 3] ICP22 created new session: [1:7577471228818465092:2049]
2025-11-27T18:18:23.292086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471228818465092:2049] [node 3] ICS09 handshake done sender: [1:7577471228818465067:2110] self: [1:7577471228818465066:0] peer: [3:7577471227228270859:0] socket: 433 qp: -1
2025-11-27T18:18:23.292113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471228818465092:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:23.292147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:23.292450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471228818465063:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.292537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:23.292587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471228818465063:2107] [node 4] ICP22 created new session: [1:7577471228818465094:2107]
2025-11-27T18:18:23.292637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471228818465094:2107] [node 4] ICS09 handshake done sender: [1:7577471228818465071:2112] self: [1:7577471228818465070:0] peer: [4:7577471227198450336:0] socket: 435 qp: -1
2025-11-27T18:18:23.292672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471228818465094:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:23.292699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.296460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471227228270849:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.296497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471227228270849:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:23.296804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471227148433213:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47144
2025-11-27T18:18:23.297620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471222933303478:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:23.297856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471227228270852:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:23.298018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471227148433210:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.298168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471222853465834:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.299064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471227198450327:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.299097Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471227198450327:2121] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.300642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471228119419795:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11051
2025-11-27T18:18:23.300662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471222933303477:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.300706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471227148433215:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47144
2025-11-27T18:18:23.300750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:23.300886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471222933303477:2048] [node 1] ICP22 created new session: [3:7577471227228270873:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.301023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471227228270873:2048] [node 1] ICS09 handshake done sender: [3:7577471227228270852:2118] self: [3:7577471227228270859:0] peer: [1:7577471228818465066:0] socket: 443 qp: -1
2025-11-27T18:18:23.301069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471227228270873:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:23.301248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:23.297824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471223103635397:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.301085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471228818465098:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60738
2025-11-27T18:18:23.297972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:23.298012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471223103635397:2048] [node 1] ICP22 created new session: [2:7577471227398602794:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.298195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471227398602794:2048] [node 1] ICS09 handshake done sender: [2:7577471227398602770:2116] self: [2:7577471227398602779:0] peer: [1:7577471228818465068:0] socket: 444 qp: -1
2025-11-27T18:18:23.298232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471227398602794:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:23.301978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471228119419796:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11051
2025-11-27T18:18:23.298283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:23.300674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471227398602766:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.302417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471222933303477:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:23.300695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471227398602766:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:23.302710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471227198450327:2121] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:23.302706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471227398602766:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:23.303304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471227398602758:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.303298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471228119419793:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:23.303446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.303499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471227398602758:2109] [node 5] ICP22 created new session: [2:7577471227398602797:2109]
2025-11-27T18:18:23.303551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471227398602797:2109] [node 5] ICS09 handshake done sender: [2:7577471227398602766:2113] self: [2:7577471227398602765:0] peer: [5:7577471227148433214:0] socket: 478 qp: -1
2025-11-27T18:18:23.303589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471227398602797:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:23.303679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471227148433216:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47144
2025-11-27T18:18:23.303619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:23.303839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471227198450323:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.303784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471228119419775:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.303954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:23.303997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471227198450323:2110] [node 6] ICP22 created new session: [4:7577471227198450351:2110]
2025-11-27T18:18:23.304049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471227198450351:2110] [node 6] ICS09 handshake done sender: [4:7577471227198450327:2121] self: [4:7577471227198450326:0] peer: [6:7577471228119419794:0] socket: 484 qp: -1
2025-11-27T18:18:23.304039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471227228270849:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:23.304084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471227198450351:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:23.304078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:23.304138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:23.304137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471228119419775:2108] [node 4] ICP22 created new session: [6:7577471228119419797:2108]
2025-11-27T18:18:23.304185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471228119419797:2108] [node 4] ICS09 handshake done sender: [6:7577471228119419793:2117] self: [6:7577471228119419794:0] peer: [4:7577471227198450326:0] socket: 485 qp: -1
2025-11-27T18:18:23.304224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471228119419797:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:23.304845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471227228270841:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.305053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471227148433213:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.304431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:23.306965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471228119419799:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11051
2025-11-27T18:18:23.307189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:23.307359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471227228270841:2108] [node 5] ICP22 created new session: [3:7577471227228270876:2108]
2025-11-27T18:18:23.307583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471227228270876:2108] [node 5] ICS09 handshake done sender: [3:7577471227228270849:2116] self: [3:7577471227228270848:0] peer: [5:7577471227148433212:0] socket: 481 qp: -1
2025-11-27T18:18:23.307619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471227228270876:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:23.308189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:23.308711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471227148433210:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:23.309398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471228119419795:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.309616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471223824452413:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.309624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471222853465834:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471227148433210:2117] held: no
2025-11-27T18:18:23.309744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471222853465834:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.309791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:23.309827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471222853465834:2049] [node 2] ICP22 created new session: [5:7577471227148433218:2049]
2025-11-27T18:18:23.309895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471227148433218:2049] [node 2] ICS09 handshake done sender: [5:7577471227148433210:2117] self: [5:7577471227148433214:0] peer: [2:7577471227398602765:0] socket: 479 qp: -1
2025-11-27T18:18:23.309886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471228818465075:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.309938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471227148433218:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:23.309919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471228818465075:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:23.310062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:23.310644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471227148433215:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.311117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471228119419801:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11051
2025-11-27T18:18:23.310876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471228818465075:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.312440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471228119419795:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:23.312512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471227148433217:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47144
2025-11-27T18:18:23.312563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471228818465065:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.312647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:23.312705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471228818465065:2109] [node 6] ICP22 created new session: [1:7577471228818465100:2109]
2025-11-27T18:18:23.312756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471228818465100:2109] [node 6] ICS09 handshake done sender: [1:7577471228818465075:2114] self: [1:7577471228818465074:0] peer: [6:7577471228119419800:0] socket: 463 qp: -1
2025-11-27T18:18:23.312762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471223824452413:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.312810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:23.312789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471228818465100:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:23.312864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471223824452413:2048] [node 1] ICP22 created new session: [6:7577471228119419802:2048]
2025-11-27T18:18:23.313130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471227148433192:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.313196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471222853465835:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2182) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471227148433211:2118] held: no
2025-11-27T18:18:23.312920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471228119419802:2048] [node 1] ICS09 handshake done sender: [6:7577471228119419795:2118] self: [6:7577471228119419800:0] peer: [1:7577471228818465074:0] socket: 464 qp: -1
2025-11-27T18:18:23.312991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471228119419802:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:23.313656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:23.312830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:23.313646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471227148433211:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:23.313706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471227198450331:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.313762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471227198450331:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:23.314691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471227198450353:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
2025-11-27T18:18:23.315385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471228119419799:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.318467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471227198450331:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:23.318764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471228818465073:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.315073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471222853465833:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.318795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471228818465073:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:23.319460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471228818465073:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:23.319753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471228818465064:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.319805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:23.320333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471228818465064:2108] [node 5] ICP22 created new session: [1:7577471228818465103:2108]
2025-11-27T18:18:23.320383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471228818465103:2108] [node 5] ICS09 handshake done sender: [1:7577471228818465073:2113] self: [1:7577471228818465072:0] peer: [5:7577471227148433222:0] socket: 457 qp: -1
2025-11-27T18:18:23.320414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471228818465103:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:23.320451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:23.321669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471227198450322:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.315151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471222853465835:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.315180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:23.315216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471222853465835:2050] [node 3] ICP22 created new session: [5:7577471227148433223:2050]
2025-11-27T18:18:23.315259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471227148433223:2050] [node 3] ICS09 handshake done sender: [5:7577471227148433211:2118] self: [5:7577471227148433212:0] peer: [3:7577471227228270848:0] socket: 483 qp: -1
2025-11-27T18:18:23.315286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471227148433223:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:23.315312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:23.317592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471227148433225:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47144
2025-11-27T18:18:23.320151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471227148433213:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:23.321733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:23.321782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471227198450322:2108] [node 5] ICP22 created new session: [4:7577471227198450355:2108]
2025-11-27T18:18:23.321832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471227198450355:2108] [node 5] ICS09 handshake done sender: [4:7577471227198450331:2124] self: [4:7577471227198450330:0] peer: [5:7577471227148433221:0] socket: 452 qp: -1
2025-11-27T18:18:23.321864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471227198450355:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:23.321897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:23.323500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471223824452415:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.323700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471228119419804:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11051
2025-11-27T18:18:23.323851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471227228270851:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.323887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471227228270851:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.325552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471228119419806:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11051
2025-11-27T18:18:23.325322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471227228270851:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:23.325968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471228119419799:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:23.326319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471223824452415:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.326365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:23.326414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471223824452415:2050] [node 3] ICP22 created new session: [6:7577471228119419807:2050]
2025-11-27T18:18:23.326457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471228119419807:2050] [node 3] ICS09 handshake done sender: [6:7577471228119419799:2120] self: [6:7577471228119419805:0] peer: [3:7577471227228270850:0] socket: 490 qp: -1
2025-11-27T18:18:23.326485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471228119419807:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:23.326512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:23.326043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471227148433192:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.326087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:23.326138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471227148433192:2108] [node 4] ICP22 created new session: [5:7577471227148433228:2108]
2025-11-27T18:18:23.326195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471227148433228:2108] [node 4] ICS09 handshake done sender: [5:7577471227148433213:2119] self: [5:7577471227148433221:0] peer: [4:7577471227198450330:0] socket: 456 qp: -1
2025-11-27T18:18:23.326227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471227148433228:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:23.328254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471227228270842:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.328308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:23.328349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471227228270842:2109] [node 6] ICP22 created new session: [3:7577471227228270879:2109]
2025-11-27T18:18:23.326261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:23.327583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471227198450353:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.329504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471227198450322:2108] [node 5] ICP18 handshake (actor [4:7577471227198450353:2129]) is too old
2025-11-27T18:18:23.328391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471227228270879:2109] [node 6] ICS09 handshake done sender: [3:7577471227228270851:2117] self: [3:7577471227228270850:0] peer: [6:7577471228119419805:0] socket: 489 qp: -1
2025-11-27T18:18:23.328422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471227228270879:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:23.328445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.331112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471227148433227:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47144
2025-11-27T18:18:23.332622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471222853465833:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17133) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471227148433215:2120] held: no
2025-11-27T18:18:23.333012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471227148433215:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:23.333396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471222853465833:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.334041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471228119419804:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.333459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:23.333511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471222853465833:2048] [node 1] ICP22 created new session: [5:7577471227148433230:2048]
2025-11-27T18:18:23.333552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471227148433230:2048] [node 1] ICS09 handshake done sender: [5:7577471227148433215:2120] self: [5:7577471227148433222:0] peer: [1:7577471228818465072:0] socket: 459 qp: -1
2025-11-27T18:18:23.334173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471228119419770:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:23.333598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471227148433230:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:23.333623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:23.334534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471227148433204:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:23.334567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471227148433204:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:23.335410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471227148433204:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:23.335593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471228119419810:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11051
2025-11-27T18:18:23.335856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471227148433193:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:23.336003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:23.336058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471227148433193:2109] [node 6] ICP22 created new session: [5:7577471227148433233:2109]
2025-11-27T18:18:23.336106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471227148433233:2109] [node 6] ICS09 handshake done sender: [5:7577471227148433204:2115] self: [5:7577471227148433203:0] peer: [6:7577471228119419809:0] socket: 493 qp: -1
2025-11-27T18:18:23.336083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471228119419804:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:23.336137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471227148433233:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:23.336177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:23.336290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471228119419770:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:23.336332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:23.336369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471228119419770:2105] [node 5] ICP22 created new session: [6:7577471228119419811:2105]
2025-11-27T18:18:23.336408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471228119419811:2105] [node 5] ICS09 handshake done sender: [6:7577471228119419804:2122] self: [6:7577471228119419809:0] peer: [5:7577471227148433203:0] socket: 437 qp: -1
2025-11-27T18:18:23.336440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471228119419811:2105] [node 5] ICS10 traffic start
2025-11-27T18:18:23.336470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:23.355479Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:23.355738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471227398602794:2048] [node 1] ICS01 socket: 444 reason# EndOfStream
2025-11-27T18:18:23.355780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471223103635397:2048] [node 1] ICP30 unregister session Session# [2:7577471227398602794:2048] VirtualId# [2:7577471227398602779:0]
2025-11-27T18:18:23.355811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471227398602794:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.355934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:23.356002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:23.356677Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:23.357692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471227228270873:2048] [node 1] ICS01 socket: 443 reason# EndOfStream
2025-11-27T18:18:23.357738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471222933303477:2048] [node 1] ICP30 unregister session Session# [3:7577471227228270873:2048] VirtualId# [3:7577471227228270859:0]
2025-11-27T18:18:23.357770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471227228270873:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.357850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:23.357896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:23.361016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:23.361235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471227198450348:2048] [node 1] ICS01 socket: 449 reason# EndOfStream
2025-11-27T18:18:23.361307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471222903482951:2048] [node 1] ICP30 unregister session Session# [4:7577471227198450348:2048] VirtualId# [4:7577471227198450336:0]
2025-11-27T18:18:23.361340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471227198450348:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.361414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:23.361494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:23.362118Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:23.362260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471228119419802:2048] [node 1] ICS01 socket: 464 reason# EndOfStream
2025-11-27T18:18:23.362283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471223824452413:2048] [node 1] ICP30 unregister session Session# [6:7577471228119419802:2048] VirtualId# [6:7577471228119419800:0]
2025-11-27T18:18:23.362305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471228119419802:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.362378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:23.362433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:23.365426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:23.365754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471227148433230:2048] [node 1] ICS01 socket: 459 reason# EndOfStream
2025-11-27T18:18:23.365845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471222853465833:2048] [node 1] ICP30 unregister session Session# [5:7577471227148433230:2048] VirtualId# [5:7577471227148433222:0]
2025-11-27T18:18:23.365896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471227148433230:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.366019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:23.366122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:23.406320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:23.406552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471227228270869:2049] [node 2] ICS01 socket: 460 reason# EndOfStream
2025-11-27T18:18:23.406600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471222933303478:2049] [node 2] ICP30 unregister session Session# [3:7577471227228270869:2049] VirtualId# [3:7577471227228270861:0]
2025-11-27T18:18:23.406629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471227228270869:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.406702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:23.406745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:23.410969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:23.411212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471228119419790:2049] [node 2] ICS01 socket: 451 reason# EndOfStream
2025-11-27T18:18:23.411263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471223824452414:2049] [node 2] ICP30 unregister session Session# [6:7577471228119419790:2049] VirtualId# [6:7577471228119419788:0]
2025-11-27T18:18:23.411300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471228119419790:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.411398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:23.411461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:23.412046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:23.412245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471227198450345:2049] [node 2] ICS01 socket: 450 reason# EndOfStream
2025-11-27T18:18:23.412278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471222903482952:2049] [node 2] ICP30 unregister session Session# [4:7577471227198450345:2049] VirtualId# [4:7577471227198450337:0]
2025-11-27T18:18:23.412309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471227198450345:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.412381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:23.412429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:23.413037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:23.413253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471227148433218:2049] [node 2] ICS01 socket: 479 reason# EndOfStream
2025-11-27T18:18:23.413312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471222853465834:2049] [node 2] ICP30 unregister session Session# [5:7577471227148433218:2049] VirtualId# [5:7577471227148433214:0]
2025-11-27T18:18:23.413338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471227148433218:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.413404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:23.413439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:23.448008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:23.448233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471227148433223:2050] [node 3] ICS01 socket: 483 reason# EndOfStream
2025-11-27T18:18:23.448287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471222853465835:2050] [node 3] ICP30 unregister session Session# [5:7577471227148433223:2050] VirtualId# [5:7577471227148433212:0]
2025-11-27T18:18:23.448317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471227148433223:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.446945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:23.447176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471227198450340:2050] [node 3] ICS01 socket: 467 reason# EndOfStream
2025-11-27T18:18:23.447218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471222903482953:2050] [node 3] ICP30 unregister session Session# [4:7577471227198450340:2050] VirtualId# [4:7577471227198450338:0]
2025-11-27T18:18:23.447253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471227198450340:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.447330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:23.447372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:23.448379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:23.448442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:23.450177Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:23.450399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471228119419807:2050] [node 3] ICS01 socket: 490 reason# EndOfStream
2025-11-27T18:18:23.450434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471223824452415:2050] [node 3] ICP30 unregister session Session# [6:7577471228119419807:2050] VirtualId# [6:7577471228119419805:0]
2025-11-27T18:18:23.450474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471228119419807:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.450542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:23.450589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:23.524923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:23.525169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471227148433228:2108] [node 4] ICS01 socket: 456 reason# EndOfStream
2025-11-27T18:18:23.525224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471227148433192:2108] [node 4] ICP30 unregister session Session# [5:7577471227148433228:2108] VirtualId# [5:7577471227148433221:0]
2025-11-27T18:18:23.525272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471227148433228:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.525353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:23.525397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:23.526238Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:23.526384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471228119419797:2108] [node 4] ICS07 socket disconnect 485 reason# EndOfStream
2025-11-27T18:18:23.526406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471228119419797:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.526467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:23.526507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471228119419797:2108] [node 4] ICS15 start handshake
2025-11-27T18:18:23.528963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471228119419775:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23136) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:23.529041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471228119419797:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:23.529092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471228119419775:2108] [node 4] ICP30 unregister session Session# [6:7577471228119419797:2108] VirtualId# [6:7577471228119419794:0]
2025-11-27T18:18:23.529151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:23.529185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471228119419775:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23136) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:23.529195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23136) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:23.538929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471228119419775:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:18:23.571354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:23.571600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471228119419811:2105] [node 5] ICS07 socket disconnect 437 reason# EndOfStream
2025-11-27T18:18:23.571630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471228119419811:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:23.571718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:23.571759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471228119419811:2105] [node 5] ICS15 start handshake
2025-11-27T18:18:23.572785Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471228119419770:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10424) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:23.572843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471228119419811:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:23.572874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471228119419770:2105] [node 5] ICP30 unregister session Session# [6:7577471228119419811:2105] VirtualId# [6:7577471228119419809:0]
2025-11-27T18:18:23.572943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:18:23.572992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471228119419770:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10424) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:23.573015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10424) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:23.585377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471228119419770:2105] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:23.660082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471227688134001:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:23.660176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471227688134002:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:23.688393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471227161954883:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:23.688462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471227161954884:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:23.711757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471229243492493:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:23.711784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471229243492492:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:23.733985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471228605006976:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:23.734026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471228605006977:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:23.734053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471228605006978:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:23.766511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471228022104104:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:23.766575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471228022104105:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:23.766610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471228022104106:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:23.790331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471230003188394:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:23.790368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471230003188395:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:23.790394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471230003188396:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:23.880734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471230003188456:2106] [node 5] ICP01 ready to work
2025-11-27T18:18:23.974312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471229243492556:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:24.000883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471227688134002:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(118977836ns) {Configure:104(118974486ns) {ProcessConfigured:135(79707ns) {StartInitialHandshake:157(77130ns) {DropHandshakes:470(5663ns) {DropIncomingHandshake:442(681ns) DropOutgoingHandshake:457(375ns)} GenerateSessionVirtualId:988(913ns) PrepareNewSessionHandshake:477(1384ns) {SwitchToState:179(492ns)}}}}}
2025-11-27T18:18:23.982664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471227161954947:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:23.990284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471228022104168:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:23.989906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471228605007040:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:24.001798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471229243492557:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:24.001985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471227161954948:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:24.001775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471228605007039:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:24.001954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471228022104169:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:24.002605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471227688134066:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:24.005499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471234298155754:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:24.005079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471233538459854:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:24.005048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471231456922245:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:24.021846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471230003188456:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(124271999ns) {Configure:104(124268588ns) {ProcessConfigured:135(2665822ns) {StartInitialHandshake:157(2662897ns) {DropHandshakes:470(1953ns) {DropIncomingHandshake:442(513ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(679ns) PrepareNewSessionHandshake:477(2947ns) {SwitchToState:179(1193ns)}}}}}
2025-11-27T18:18:24.019201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471228022104105:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17243267ns) {Configure:104(17234670ns) {ProcessConfigured:135(71919ns) {StartInitialHandshake:157(69223ns) {DropHandshakes:470(3107ns) {DropIncomingHandshake:442(751ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(1020ns) PrepareNewSessionHandshake:477(1523ns) {SwitchToState:179(456ns)}}}}}
2025-11-27T18:18:24.023241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471228022104106:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21039174ns) {Configure:104(21035192ns) {ProcessConfigured:135(51554ns) {StartInitialHandshake:157(50194ns) {DropHandshakes:470(1233ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(531ns) PrepareNewSessionHandshake:477(1017ns) {SwitchToState:179(482ns)}}}}}
2025-11-27T18:18:24.028133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471231983101365:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:24.035221Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471228605006977:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(32764805ns) {Configure:104(32759442ns) {ProcessConfigured:135(44395ns) {StartInitialHandshake:157(42540ns) {DropHandshakes:470(794ns) {DropIncomingHandshake:442(130ns) DropOutgoingHandshake:457(101ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:477(553ns) {SwitchToState:179(174ns)}}}}}
2025-11-27T18:18:24.029845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471229243492492:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(136194241ns) {Configure:104(136188305ns) {ProcessConfigured:135(40051ns) {StartInitialHandshake:157(38370ns) {DropHandshakes:470(1247ns) {DropIncomingHandshake:442(333ns) DropOutgoingHandshake:457(156ns)} GenerateSessionVirtualId:988(310ns) PrepareNewSessionHandshake:477(3075ns) {SwitchToState:179(322ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.037276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471233538459863:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3326
2025-11-27T18:18:24.028736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471227688134001:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(129530218ns) {Configure:104(129524693ns) {ProcessConfigured:135(6730758ns) {StartInitialHandshake:157(6727886ns) {DropHandshakes:470(1506ns) {DropIncomingHandshake:442(430ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(677ns) PrepareNewSessionHandshake:477(2747ns) {SwitchToState:179(1400ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.039659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471231983101366:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:24.049536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471232317071479:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62011
2025-11-27T18:18:24.050810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471227161954883:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(48593564ns) {Configure:104(48587341ns) {ProcessConfigured:135(33566ns) {StartInitialHandshake:157(32250ns) {DropHandshakes:470(921ns) {DropIncomingHandshake:442(181ns) DropOutgoingHandshake:457(91ns)} GenerateSessionVirtualId:988(542ns) PrepareNewSessionHandshake:477(625ns) {SwitchToState:179(167ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.055443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471231456922254:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58636
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.055950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471231983101367:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56605
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.056816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471228605006976:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28719890ns) {Configure:104(28715487ns) {ProcessConfigured:135(8171349ns) {StartInitialHandshake:157(8169936ns) {DropHandshakes:470(2425ns) {DropIncomingHandshake:442(675ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(812ns) PrepareNewSessionHandshake:477(1906ns) {SwitchToState:179(116ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.060669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471232899974347:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24438
2025-11-27T18:18:24.073674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471233538459867:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3326
2025-11-27T18:18:24.060478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471231983101368:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56605
2025-11-27T18:18:24.085433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471230003188395:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56288244ns) {Configure:104(56282946ns) {ProcessConfigured:135(53572ns) {StartInitialHandshake:157(51069ns) {DropHandshakes:470(2138ns) {DropIncomingHandshake:442(489ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:477(1628ns) {SwitchToState:179(592ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.091548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471232317071480:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62011
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.070369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471231983101378:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56605
2025-11-27T18:18:24.091673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471232317071481:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62011
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.102100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471230003188396:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16454173ns) {Configure:104(16449055ns) {ProcessConfigured:135(59523ns) {StartInitialHandshake:157(57205ns) {DropHandshakes:470(3230ns) {DropIncomingHandshake:442(955ns) DropOutgoingHandshake:457(611ns)} GenerateSessionVirtualId:988(903ns) PrepareNewSessionHandshake:477(1503ns) {SwitchToState:179(402ns)}}}}}
2025-11-27T18:18:24.087096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471232899974347:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.082276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471229243492493:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(67976452ns) {Configure:104(67972349ns) {ProcessConfigured:135(68206ns) {StartInitialHandshake:157(65505ns) {DropHandshakes:470(2890ns) {DropIncomingHandshake:442(925ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(744ns) PrepareNewSessionHandshake:477(4814ns) {SwitchToState:179(406ns)}}}}}
2025-11-27T18:18:24.118347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471228605006978:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.070420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471231983101379:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56605
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.086972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471233538459867:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.114346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471232317071482:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62011
2025-11-27T18:18:24.104360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471234298155765:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1609
2025-11-27T18:18:24.077594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471231983101368:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.123057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471232317071480:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.137604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471231456922258:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58636
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.143980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471232899974352:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24438
2025-11-27T18:18:24.087593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471229243492492:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.121937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471230003188394:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(83318776ns) {Configure:104(83309301ns) {ProcessConfigured:135(10058269ns) {StartInitialHandshake:157(10055456ns) {DropHandshakes:470(1767ns) {DropIncomingHandshake:442(464ns) DropOutgoingHandshake:457(304ns)} GenerateSessionVirtualId:988(974ns) PrepareNewSessionHandshake:477(4194ns) {SwitchToState:179(1424ns)}}}}}
2025-11-27T18:18:24.111272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471233538459863:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.091528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471231983101367:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.123315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471232317071483:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62011
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.163505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471227161954947:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(127606030ns) {Configure:104(127600129ns) {ProcessConfigured:135(39439451ns) {StartInitialHandshake:157(39436567ns) {DropHandshakes:470(2699ns) {DropIncomingHandshake:442(561ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(896ns) PrepareNewSessionHandshake:477(3915ns) {SwitchToState:179(2005ns)}}}}}
2025-11-27T18:18:24.163844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471231456922258:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.123842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471227688134065:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.123885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471227688134065:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.128941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471231983101370:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.128963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471231983101370:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:18:24.123455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471228022104104:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.123887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471232317071479:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.133816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471228022104106:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.149950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471233538459858:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.129631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471231983101378:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.130366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471227688134065:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:24.150001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471233538459858:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.172577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471232317071483:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.130811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471231983101370:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:24.131058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471227688134002:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.131123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:24.131182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471227688134002:2049] [node 3] ICP22 created new session: [1:7577471231983101385:2049]
2025-11-27T18:18:24.131225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471231983101385:2049] [node 3] ICS09 handshake done sender: [1:7577471227688134065:2110] self: [1:7577471227688134064:0] peer: [3:7577471233538459870:0] socket: 447 qp: -1
2025-11-27T18:18:24.131255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471231983101385:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:24.131287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:24.131699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471231983101365:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.131741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:24.131782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471231983101365:2108] [node 5] ICP22 created new session: [1:7577471231983101387:2108]
2025-11-27T18:18:24.131820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471231983101387:2108] [node 5] ICS09 handshake done sender: [1:7577471231983101370:2114] self: [1:7577471231983101369:0] peer: [5:7577471232317071484:0] socket: 448 qp: -1
2025-11-27T18:18:24.131847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471231983101387:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:24.131872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:24.143376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471231983101389:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56605
2025-11-27T18:18:24.164742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471231983101389:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.179350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471231983101374:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.179382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471231983101374:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.169942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471232899974352:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.179238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471234298155769:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1609
2025-11-27T18:18:24.179036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471228605006976:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.185741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471231983101374:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:24.161556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471233538459860:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.186059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471227688134066:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.179249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471232899974353:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24438
2025-11-27T18:18:24.184813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471234298155769:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.180302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471228022104104:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7645) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471232317071480:2117] held: no
2025-11-27T18:18:24.161581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471233538459860:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.186130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:24.186175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471227688134066:2107] [node 4] ICP22 created new session: [1:7577471231983101394:2107]
2025-11-27T18:18:24.162464Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471229243492492:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7645) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471233538459867:2116] held: no
2025-11-27T18:18:24.188514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471228605006976:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7645) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471232899974352:2116] held: no
2025-11-27T18:18:24.186216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471231983101394:2107] [node 4] ICS09 handshake done sender: [1:7577471231983101374:2116] self: [1:7577471231983101373:0] peer: [4:7577471232899974354:0] socket: 475 qp: -1
2025-11-27T18:18:24.163516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471233538459872:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3326
2025-11-27T18:18:24.186252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471231983101394:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:24.185698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471233538459867:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:24.186286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:24.188635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471233538459858:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:24.188970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471233538459860:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.188538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471230003188394:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.189116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471232899974355:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24438
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.189151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471231456922254:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.190006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471231983101372:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.190055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471231983101372:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.192540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471231983101372:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:24.190339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471231456922262:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58636
2025-11-27T18:18:24.191274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471229243492492:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.191370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471231456922259:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58636
2025-11-27T18:18:24.192569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471227161954883:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.192539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:24.192596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471229243492492:2048] [node 1] ICP22 created new session: [3:7577471233538459875:2048]
2025-11-27T18:18:24.193628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471231983101366:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.198345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471231456922262:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.192915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471233538459875:2048] [node 1] ICS09 handshake done sender: [3:7577471233538459867:2116] self: [3:7577471233538459870:0] peer: [1:7577471227688134064:0] socket: 451 qp: -1
2025-11-27T18:18:24.194520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471232899974347:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.199896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471232317071486:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62011
2025-11-27T18:18:24.193355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471229243492557:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.198455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471231456922259:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.194142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:24.200248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471232317071482:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.193469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471233538459875:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:24.199184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471232899974352:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:24.202382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471234298155771:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1609
2025-11-27T18:18:24.193597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:24.194496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471231983101366:2109] [node 6] ICP22 created new session: [1:7577471231983101397:2109]
2025-11-27T18:18:24.202872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471228605006978:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.203164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471232317071487:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62011
2025-11-27T18:18:24.203769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471228605006976:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.195171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471231983101397:2109] [node 6] ICS09 handshake done sender: [1:7577471231983101372:2115] self: [1:7577471231983101371:0] peer: [6:7577471234298155770:0] socket: 474 qp: -1
2025-11-27T18:18:24.204387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471232317071480:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:24.193516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:24.203992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.211863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471230003188394:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7645) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471234298155769:2116] held: no
2025-11-27T18:18:24.204346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471228605006978:2050] [node 3] ICP22 created new session: [4:7577471232899974357:2050]
2025-11-27T18:18:24.204856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:24.193648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471229243492557:2108] [node 5] ICP22 created new session: [3:7577471233538459878:2108]
2025-11-27T18:18:24.204909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471228605006976:2048] [node 1] ICP22 created new session: [4:7577471232899974358:2048]
2025-11-27T18:18:24.201592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471231983101397:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.205219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471232899974358:2048] [node 1] ICS09 handshake done sender: [4:7577471232899974352:2116] self: [4:7577471232899974354:0] peer: [1:7577471231983101373:0] socket: 476 qp: -1
2025-11-27T18:18:24.202219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:24.209940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471228022104168:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.219050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471234298155769:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:24.219600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471231456922267:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58636
2025-11-27T18:18:24.194404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471233538459878:2108] [node 5] ICS09 handshake done sender: [3:7577471233538459860:2113] self: [3:7577471233538459859:0] peer: [5:7577471232317071485:0] socket: 439 qp: -1
2025-11-27T18:18:24.220516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471234298155765:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.205271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471232899974358:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:24.216624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471232317071481:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.205312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:24.222662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471231456922267:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.206500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471232899974357:2050] [node 3] ICS09 handshake done sender: [4:7577471232899974347:2115] self: [4:7577471232899974351:0] peer: [3:7577471233538459857:0] socket: 442 qp: -1
2025-11-27T18:18:24.195143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471233538459878:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:24.207700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471232899974357:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:24.207734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:24.217394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471231983101364:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.222902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471232899974342:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.222924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471232899974342:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.195180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:24.216638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471228022104104:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.223880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471231456922268:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58636
2025-11-27T18:18:24.224098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471230003188394:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.217419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471231983101364:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.224184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:24.218199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471231983101379:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.218660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471231983101364:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:24.224225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471230003188394:2048] [node 1] ICP22 created new session: [6:7577471234298155775:2048]
2025-11-27T18:18:24.220385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471231983101366:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16288009ns) {HandleHandshakeStatus:329(16285675ns) {DropIncomingHandshake:442(15122ns) DropOutgoingHandshake:457(263ns) SwitchToState:179(526ns) ProcessPendingSessionEvents:495(5695087ns) {ForwardSessionEventToSession:612(796539ns) ForwardSessionEventToSession:612(117577ns) ForwardSessionEventToSession:612(3037ns) ForwardSessionEventToSession:612(2539ns) ForwardSessionEventToSession:612(1420102ns) ForwardSessionEventToSession:612(726599ns) ForwardSessionEventToSession:612(325260ns) ForwardSessionEventToSession:612(4831ns) ForwardSessionEventToSession:612(2563ns) ForwardSessionEventToSession:612(2281235ns)}}}
2025-11-27T18:18:24.224265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471234298155775:2048] [node 1] ICS09 handshake done sender: [6:7577471234298155769:2116] self: [6:7577471234298155770:0] peer: [1:7577471231983101371:0] socket: 477 qp: -1
2025-11-27T18:18:24.222305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471227688134001:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.222353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:24.222736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471227688134001:2048] [node 2] ICP22 created new session: [1:7577471231983101401:2048]
2025-11-27T18:18:24.222785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471231983101401:2048] [node 2] ICS09 handshake done sender: [1:7577471231983101364:2111] self: [1:7577471227688134067:0] peer: [2:7577471231456922263:0] socket: 446 qp: -1
2025-11-27T18:18:24.223084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471231983101401:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:24.224300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471234298155775:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:24.223118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:24.200972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471229243492556:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.216716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:24.224329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:24.201647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:24.219195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471228022104104:2048] [node 1] ICP22 created new session: [5:7577471232317071491:2048]
2025-11-27T18:18:24.224855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471232899974342:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:24.222007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471232317071479:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:24.224846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471231456922258:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:24.222423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471232317071491:2048] [node 1] ICS09 handshake done sender: [5:7577471232317071480:2117] self: [5:7577471232317071484:0] peer: [1:7577471231983101369:0] socket: 449 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.202699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471229243492556:2107] [node 4] ICP22 created new session: [3:7577471233538459880:2107]
2025-11-27T18:18:24.226963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471234298155772:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1609
2025-11-27T18:18:24.222703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471228022104106:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65036) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471232317071479:2116] held: no
2025-11-27T18:18:24.227042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471234298155774:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1609
2025-11-27T18:18:24.203608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471233538459880:2107] [node 4] ICS09 handshake done sender: [3:7577471233538459858:2112] self: [3:7577471233538459857:0] peer: [4:7577471232899974351:0] socket: 438 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.227617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471227161954883:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.203658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471233538459880:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:24.227145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471234298155773:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1609
2025-11-27T18:18:24.222809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471228022104106:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.227665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:24.228991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471227161954883:2048] [node 1] ICP22 created new session: [2:7577471231456922270:2048]
2025-11-27T18:18:24.229033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471231456922270:2048] [node 1] ICS09 handshake done sender: [2:7577471231456922258:2116] self: [2:7577471231456922263:0] peer: [1:7577471227688134067:0] socket: 453 qp: -1
2025-11-27T18:18:24.229433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471231456922251:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.229454Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471231456922251:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.231169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471231456922251:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:24.231785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471231456922270:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:24.223048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471232317071491:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:24.242427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471228605007040:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.252968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.227235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471230003188395:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.260193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471232899974364:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24438
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.260342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471232899974365:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24438
2025-11-27T18:18:24.262786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471232899974364:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.262933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471228605006977:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.203770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:24.229302Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471234298155773:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.242531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:24.223093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:24.265686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471228605007040:2108] [node 5] ICP22 created new session: [4:7577471232899974363:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.223108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:24.223155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471228022104106:2050] [node 3] ICP22 created new session: [5:7577471232317071493:2050]
2025-11-27T18:18:24.223321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471232317071493:2050] [node 3] ICS09 handshake done sender: [5:7577471232317071479:2116] self: [5:7577471232317071485:0] peer: [3:7577471233538459859:0] socket: 441 qp: -1
2025-11-27T18:18:24.223634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471232317071493:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:24.223667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:24.229327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471228022104105:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.254630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471232317071497:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62011
2025-11-27T18:18:24.255065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471232317071482:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:24.255246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471228022104168:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.255299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:24.256420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471228022104168:2108] [node 4] ICP22 created new session: [5:7577471232317071498:2108]
2025-11-27T18:18:24.269576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471232317071498:2108] [node 4] ICS09 handshake done sender: [5:7577471232317071482:2119] self: [5:7577471232317071490:0] peer: [4:7577471232899974341:0] socket: 462 qp: -1
2025-11-27T18:18:24.285378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471232317071498:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:24.285481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:24.210434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471229243492557:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17390667ns) {HandleHandshakeStatus:329(17388359ns) {DropIncomingHandshake:442(195256ns) DropOutgoingHandshake:457(840ns) SwitchToState:179(314ns) ProcessPendingSessionEvents:495(6305896ns) {ForwardSessionEventToSession:612(962388ns) ForwardSessionEventToSession:612(4773ns) ForwardSessionEventToSession:612(3333ns) ForwardSessionEventToSession:612(2697531ns) ForwardSessionEventToSession:612(16512ns) ForwardSessionEventToSession:612(2589272ns) ForwardSessionEventToSession:612(4262ns) ForwardSessionEventToSession:612(2650ns) ForwardSessionEventToSession:612(5471ns) ForwardSessionEventToSession:612(2528ns)}}}
2025-11-27T18:18:24.242838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471231456922245:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.269693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471232899974363:2108] [node 5] ICS09 handshake done sender: [4:7577471232899974342:2112] self: [4:7577471232899974341:0] peer: [5:7577471232317071490:0] socket: 459 qp: -1
2025-11-27T18:18:24.242923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:24.231466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471230003188396:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.274452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471232899974363:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:24.277867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:24.297572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471228605007040:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(57849990ns) {HandleHandshakeStatus:329(57846471ns) {DropIncomingHandshake:442(1435ns) DropOutgoingHandshake:457(3158ns) SwitchToState:179(932ns) ProcessPendingSessionEvents:495(6620459ns) {ForwardSessionEventToSession:612(6547149ns) ForwardSessionEventToSession:612(6368ns) ForwardSessionEventToSession:612(6528ns) ForwardSessionEventToSession:612(6025ns) ForwardSessionEventToSession:612(3084ns) ForwardSessionEventToSession:612(4790ns) ForwardSessionEventToSession:612(2943ns) ForwardSessionEventToSession:612(5924ns) ForwardSessionEventToSession:612(5064ns) ForwardSessionEventToSession:612(3420ns)}}}
2025-11-27T18:18:24.211688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471233538459882:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3326
2025-11-27T18:18:24.242982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471231456922245:2109] [node 6] ICP22 created new session: [2:7577471231456922274:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.269537Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471234298155774:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.306149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471228022104168:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(50709292ns) {HandleHandshakeStatus:329(50707189ns) {DropIncomingHandshake:442(832ns) DropOutgoingHandshake:457(4879ns) SwitchToState:179(338ns) ProcessPendingSessionEvents:495(12425728ns) {ForwardSessionEventToSession:612(12364362ns) ForwardSessionEventToSession:612(7504ns) ForwardSessionEventToSession:612(2298ns) ForwardSessionEventToSession:612(14756ns) ForwardSessionEventToSession:612(3259ns) ForwardSessionEventToSession:612(5503ns) ForwardSessionEventToSession:612(6212ns) ForwardSessionEventToSession:612(3243ns) ForwardSessionEventToSession:612(5557ns) ForwardSessionEventToSession:612(3336ns)}}}
2025-11-27T18:18:24.229455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471233538459884:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3326
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.243024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471231456922274:2109] [node 6] ICS09 handshake done sender: [2:7577471231456922251:2113] self: [2:7577471231456922250:0] peer: [6:7577471234298155778:0] socket: 456 qp: -1
2025-11-27T18:18:24.287466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471230003188395:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3301) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471234298155765:2115] held: no
2025-11-27T18:18:24.242498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471233538459884:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.243061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471231456922274:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:24.287740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471234298155754:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.324172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471232317071500:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62011
2025-11-27T18:18:24.330995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471228605007040:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:24.269930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471233538459862:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.243094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:24.257911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471227161954883:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(30127322ns) {HandleHandshakeStatus:329(30125136ns) {DropIncomingHandshake:442(732ns) DropOutgoingHandshake:457(4141ns) SwitchToState:179(167ns) ProcessPendingSessionEvents:495(108765ns) {ForwardSessionEventToSession:612(62297ns) ForwardSessionEventToSession:612(6824ns) ForwardSessionEventToSession:612(2413ns) ForwardSessionEventToSession:612(5978ns) ForwardSessionEventToSession:612(2664ns) ForwardSessionEventToSession:612(2332ns) ForwardSessionEventToSession:612(5300ns) ForwardSessionEventToSession:612(5093ns) ForwardSessionEventToSession:612(3285ns) ForwardSessionEventToSession:612(4876ns)}}}
2025-11-27T18:18:24.264618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471231456922261:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.264644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471231456922261:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.331479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471228605006977:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3301) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471232899974364:2119] held: no
2025-11-27T18:18:24.324584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471232317071481:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:24.269814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471231456922261:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:24.317717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471234298155780:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1609
2025-11-27T18:18:24.269965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471233538459862:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.324747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471228022104105:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.277374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471231456922249:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.277415Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471231456922249:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.280264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471227161954947:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.280332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:24.280377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471227161954947:2107] [node 4] ICP22 created new session: [2:7577471231456922278:2107]
2025-11-27T18:18:24.280431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471231456922278:2107] [node 4] ICS09 handshake done sender: [2:7577471231456922261:2118] self: [2:7577471231456922260:0] peer: [4:7577471232899974366:0] socket: 481 qp: -1
2025-11-27T18:18:24.280464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471231456922278:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:24.293310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471233538459885:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3326
2025-11-27T18:18:24.323305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471234298155772:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.324799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:24.331837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471232899974367:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24438
2025-11-27T18:18:24.305467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471233538459885:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.305622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471229243492493:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.310961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471233538459888:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3326
2025-11-27T18:18:24.326664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471233538459885:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:24.326817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471229243492493:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.326855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:24.326893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471229243492493:2049] [node 2] ICP22 created new session: [3:7577471233538459890:2049]
2025-11-27T18:18:24.326934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471233538459890:2049] [node 2] ICS09 handshake done sender: [3:7577471233538459885:2121] self: [3:7577471233538459887:0] peer: [2:7577471231456922252:0] socket: 493 qp: -1
2025-11-27T18:18:24.326960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471233538459890:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:24.326987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:24.280506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:24.333442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471232899974346:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.323667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471230003188456:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:24.323755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471234298155783:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1609
2025-11-27T18:18:24.324844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471228022104105:2049] [node 2] ICP22 created new session: [5:7577471232317071501:2049]
2025-11-27T18:18:24.329910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471233538459862:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:24.288470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471231456922249:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:24.333946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471233538459854:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.333555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471234298155765:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:24.336464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471230003188395:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.336544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:24.336591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471230003188395:2049] [node 2] ICP22 created new session: [6:7577471234298155785:2049]
2025-11-27T18:18:24.336653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471234298155785:2049] [node 2] ICS09 handshake done sender: [6:7577471234298155765:2115] self: [6:7577471234298155778:0] peer: [2:7577471231456922250:0] socket: 465 qp: -1
2025-11-27T18:18:24.333479Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471232899974346:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.336688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471234298155785:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:24.309558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471231456922253:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.309588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471231456922253:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.325522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471231456922253:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:24.324882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471232317071501:2049] [node 2] ICS09 handshake done sender: [5:7577471232317071481:2118] self: [5:7577471232317071495:0] peer: [2:7577471231456922248:0] socket: 457 qp: -1
2025-11-27T18:18:24.334003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:24.340659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471232899974369:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24438
2025-11-27T18:18:24.334049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471233538459854:2109] [node 6] ICP22 created new session: [3:7577471233538459892:2109]
2025-11-27T18:18:24.340799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471232899974364:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:24.334091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471233538459892:2109] [node 6] ICS09 handshake done sender: [3:7577471233538459862:2114] self: [3:7577471233538459861:0] peer: [6:7577471234298155779:0] socket: 485 qp: -1
2025-11-27T18:18:24.334125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471233538459892:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:24.341530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471228605006977:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.334175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:24.342836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:24.343183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471228605006977:2049] [node 2] ICP22 created new session: [4:7577471232899974371:2049]
2025-11-27T18:18:24.343224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471232899974371:2049] [node 2] ICS09 handshake done sender: [4:7577471232899974364:2119] self: [4:7577471232899974366:0] peer: [2:7577471231456922260:0] socket: 496 qp: -1
2025-11-27T18:18:24.337498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471227161954948:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.336722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:24.343278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471232899974371:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:24.337253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471234298155773:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:24.343326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:24.341505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471230003188396:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.342324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:24.343891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471232899974346:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:24.343013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471230003188396:2050] [node 3] ICP22 created new session: [6:7577471234298155787:2050]
2025-11-27T18:18:24.344212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471228605007039:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.350457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471234298155787:2050] [node 3] ICS09 handshake done sender: [6:7577471234298155773:2119] self: [6:7577471234298155779:0] peer: [3:7577471233538459861:0] socket: 489 qp: -1
2025-11-27T18:18:24.344244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:24.339268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:24.340933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471227161954948:2108] [node 5] ICP22 created new session: [2:7577471231456922282:2108]
2025-11-27T18:18:24.341238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471231456922282:2108] [node 5] ICS09 handshake done sender: [2:7577471231456922249:2112] self: [2:7577471231456922248:0] peer: [5:7577471232317071495:0] socket: 455 qp: -1
2025-11-27T18:18:24.341681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471231456922282:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:24.341719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:24.351781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471227161954884:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.351850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:24.352596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471227161954884:2049] [node 3] ICP22 created new session: [2:7577471231456922284:2049]
2025-11-27T18:18:24.352834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471231456922284:2049] [node 3] ICS09 handshake done sender: [2:7577471231456922253:2114] self: [2:7577471231456922252:0] peer: [3:7577471233538459887:0] socket: 454 qp: -1
2025-11-27T18:18:24.354553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471231456922284:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:24.354603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:24.350811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471234298155787:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:24.324926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471232317071501:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:24.350857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:24.344286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471228605007039:2109] [node 6] ICP22 created new session: [4:7577471232899974373:2109]
2025-11-27T18:18:24.358964Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471230003188396:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:24.344328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471232899974373:2109] [node 6] ICS09 handshake done sender: [4:7577471232899974346:2114] self: [4:7577471232899974345:0] peer: [6:7577471234298155781:0] socket: 436 qp: -1
2025-11-27T18:18:24.324960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:24.335044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471232317071473:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.335081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471232317071473:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:24.343098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471232317071473:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:24.351684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471228022104169:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:24.351739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:24.351785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471228022104169:2109] [node 6] ICP22 created new session: [5:7577471232317071504:2109]
2025-11-27T18:18:24.344358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471232899974373:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:24.351833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471232317071504:2109] [node 6] ICS09 handshake done sender: [5:7577471232317071473:2114] self: [5:7577471232317071472:0] peer: [6:7577471234298155784:0] socket: 471 qp: -1
2025-11-27T18:18:24.351859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471232317071504:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:24.344384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:24.352918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:24.357003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471232899974369:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:24.358864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471228605007039:2109] [node 6] ICP18 handshake (actor [4:7577471232899974369:2122]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.365236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471230003188456:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471234298155772:2118] held: no
2025-11-27T18:18:24.365450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471234298155754:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30303) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471234298155774:2120] held: no
2025-11-27T18:18:24.365669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471234298155789:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1609
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:24.368248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471234298155790:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1609
2025-11-27T18:18:24.368574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471234298155774:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:24.368805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471234298155772:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:24.369117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471230003188456:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.369173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:24.369217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471230003188456:2106] [node 5] ICP22 created new session: [6:7577471234298155791:2106]
2025-11-27T18:18:24.369292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471234298155791:2106] [node 5] ICS09 handshake done sender: [6:7577471234298155772:2118] self: [6:7577471234298155784:0] peer: [5:7577471232317071472:0] socket: 488 qp: -1
2025-11-27T18:18:24.369332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471234298155791:2106] [node 5] ICS10 traffic start
2025-11-27T18:18:24.369368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:24.369934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471234298155754:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:24.370019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:24.370069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471234298155754:2109] [node 4] ICP22 created new session: [6:7577471234298155793:2109]
2025-11-27T18:18:24.370120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471234298155793:2109] [node 4] ICS09 handshake done sender: [6:7577471234298155774:2120] self: [6:7577471234298155781:0] peer: [4:7577471232899974345:0] socket: 490 qp: -1
2025-11-27T18:18:24.370153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471234298155793:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:24.370196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:24.412731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:24.413059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471231456922270:2048] [node 1] ICS01 socket: 453 reason# EndOfStream
2025-11-27T18:18:24.413117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471227161954883:2048] [node 1] ICP30 unregister session Session# [2:7577471231456922270:2048] VirtualId# [2:7577471231456922263:0]
2025-11-27T18:18:24.413163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471231456922270:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.413357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:24.413417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:24.412876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:24.413121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471233538459875:2048] [node 1] ICS01 socket: 451 reason# EndOfStream
2025-11-27T18:18:24.413160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471229243492492:2048] [node 1] ICP30 unregister session Session# [3:7577471233538459875:2048] VirtualId# [3:7577471233538459870:0]
2025-11-27T18:18:24.413193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471233538459875:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.413274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:24.413324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:24.415130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:24.415357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471232899974358:2048] [node 1] ICS01 socket: 476 reason# EndOfStream
2025-11-27T18:18:24.415405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471228605006976:2048] [node 1] ICP30 unregister session Session# [4:7577471232899974358:2048] VirtualId# [4:7577471232899974354:0]
2025-11-27T18:18:24.415439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471232899974358:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.415552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:24.415612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:24.416850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:24.417058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471234298155775:2048] [node 1] ICS01 socket: 477 reason# EndOfStream
2025-11-27T18:18:24.417106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471230003188394:2048] [node 1] ICP30 unregister session Session# [6:7577471234298155775:2048] VirtualId# [6:7577471234298155770:0]
2025-11-27T18:18:24.417132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471234298155775:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.417196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:24.417251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:24.417416Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:24.417662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471232317071491:2048] [node 1] ICS01 socket: 449 reason# EndOfStream
2025-11-27T18:18:24.417699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471228022104104:2048] [node 1] ICP30 unregister session Session# [5:7577471232317071491:2048] VirtualId# [5:7577471232317071484:0]
2025-11-27T18:18:24.417725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471232317071491:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.417818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:24.417861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:24.471828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:24.472083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471233538459890:2049] [node 2] ICS01 socket: 493 reason# EndOfStream
2025-11-27T18:18:24.472122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471229243492493:2049] [node 2] ICP30 unregister session Session# [3:7577471233538459890:2049] VirtualId# [3:7577471233538459887:0]
2025-11-27T18:18:24.472154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471233538459890:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.472230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:24.472271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:24.474909Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:24.475221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471232899974371:2049] [node 2] ICS01 socket: 496 reason# EndOfStream
2025-11-27T18:18:24.475264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471228605006977:2049] [node 2] ICP30 unregister session Session# [4:7577471232899974371:2049] VirtualId# [4:7577471232899974366:0]
2025-11-27T18:18:24.475589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471232899974371:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.475701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:24.475754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:24.481472Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:24.481615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:24.481752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471232317071501:2049] [node 2] ICS01 socket: 457 reason# EndOfStream
2025-11-27T18:18:24.481823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471228022104105:2049] [node 2] ICP30 unregister session Session# [5:7577471232317071501:2049] VirtualId# [5:7577471232317071495:0]
2025-11-27T18:18:24.481862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471232317071501:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.481973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:24.482041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:24.481840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471234298155785:2049] [node 2] ICS01 socket: 465 reason# EndOfStream
2025-11-27T18:18:24.481889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471230003188395:2049] [node 2] ICP30 unregister session Session# [6:7577471234298155785:2049] VirtualId# [6:7577471234298155778:0]
2025-11-27T18:18:24.481920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471234298155785:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.481997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:24.482036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:24.529523Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:24.530057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:24.530230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471232317071493:2050] [node 3] ICS01 socket: 441 reason# EndOfStream
2025-11-27T18:18:24.530261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471228022104106:2050] [node 3] ICP30 unregister session Session# [5:7577471232317071493:2050] VirtualId# [5:7577471232317071485:0]
2025-11-27T18:18:24.530289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471232317071493:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.530336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:24.530375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:24.530447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:24.530617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471234298155787:2050] [node 3] ICS07 socket disconnect 489 reason# EndOfStream
2025-11-27T18:18:24.530639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471234298155787:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.530684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:24.530721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471234298155787:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:24.529758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471232899974357:2050] [node 3] ICS01 socket: 442 reason# EndOfStream
2025-11-27T18:18:24.529807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471228605006978:2050] [node 3] ICP30 unregister session Session# [4:7577471232899974357:2050] VirtualId# [4:7577471232899974351:0]
2025-11-27T18:18:24.529840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471232899974357:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.529921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:24.529969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:24.532161Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471230003188396:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65036) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:24.532215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471234298155787:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:24.532245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471230003188396:2050] [node 3] ICP30 unregister session Session# [6:7577471234298155787:2050] VirtualId# [6:7577471234298155779:0]
2025-11-27T18:18:24.532285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:24.532323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471230003188396:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:24.532339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65036) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:24.541365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471230003188396:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:24.582552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:24.582791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471232317071498:2108] [node 4] ICS01 socket: 462 reason# EndOfStream
2025-11-27T18:18:24.582845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471228022104168:2108] [node 4] ICP30 unregister session Session# [5:7577471232317071498:2108] VirtualId# [5:7577471232317071490:0]
2025-11-27T18:18:24.582877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471232317071498:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.582948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:24.582997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:24.584113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:24.584328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471234298155793:2109] [node 4] ICS01 socket: 490 reason# EndOfStream
2025-11-27T18:18:24.584367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471234298155754:2109] [node 4] ICP30 unregister session Session# [6:7577471234298155793:2109] VirtualId# [6:7577471234298155781:0]
2025-11-27T18:18:24.584416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471234298155793:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.584492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:24.584547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:24.631924Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:24.632154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471234298155791:2106] [node 5] ICS01 socket: 488 reason# EndOfStream
2025-11-27T18:18:24.632204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471230003188456:2106] [node 5] ICP30 unregister session Session# [6:7577471234298155791:2106] VirtualId# [6:7577471234298155784:0]
2025-11-27T18:18:24.632240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471234298155791:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:24.632323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:24.632391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:24.746121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471233999840382:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:24.746180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471233999840383:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:24.769540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471235136130317:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:24.770443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471235136130316:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:24.792084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471232203924028:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:24.792123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471232203924029:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:24.814352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471233607337447:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:24.814351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471233607337446:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:24.814405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471233607337448:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:24.835804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471234445357284:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:24.835843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471234445357285:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:24.835977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471234445357283:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:24.858331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471232960442901:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:24.858383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471232960442902:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:24.858414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471232960442903:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:24.993614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471232960442944:2089] [node 4] ICP01 ready to work
2025-11-27T18:18:25.028768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471239431097677:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:25.029109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471239431097678:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:25.166124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471237902304815:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:25.150128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471236498891389:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:25.137478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471238294807741:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:25.150838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471237255410250:2095] [node 5] ICP01 ready to work
2025-11-27T18:18:25.087136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471239431097680:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:25.150819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471236498891388:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:25.189672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471232960442944:2089] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(195686183ns) {Configure:104(195679053ns) {ProcessConfigured:135(23374078ns) {StartInitialHandshake:157(23370789ns) {DropHandshakes:470(3178ns) {DropIncomingHandshake:442(718ns) DropOutgoingHandshake:457(390ns)} GenerateSessionVirtualId:988(912ns) PrepareNewSessionHandshake:477(6373ns) {SwitchToState:179(3222ns)}}}}}
2025-11-27T18:18:25.213867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471238740324644:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:25.150081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471238294807742:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:25.209508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471238294807743:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:25.249492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471236498891393:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.274825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471239431097677:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(245258534ns) {Configure:104(245253621ns) {ProcessConfigured:135(76862ns) {StartInitialHandshake:157(74411ns) {DropHandshakes:470(3664ns) {DropIncomingHandshake:442(781ns) DropOutgoingHandshake:457(495ns)} GenerateSessionVirtualId:988(1581ns) PrepareNewSessionHandshake:477(4688ns) {SwitchToState:179(623ns)}}}}}
2025-11-27T18:18:25.285745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471239431097686:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21006
2025-11-27T18:18:25.242164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471237902304816:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:25.270342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471238740324645:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:25.302750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471235136130316:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(84630548ns) {Configure:104(84620316ns) {ProcessConfigured:135(91940ns) {StartInitialHandshake:157(88828ns) {DropHandshakes:470(3168ns) {DropIncomingHandshake:442(714ns) DropOutgoingHandshake:457(376ns)} GenerateSessionVirtualId:988(1670ns) PrepareNewSessionHandshake:477(2223ns) {SwitchToState:179(701ns)}}}}}
2025-11-27T18:18:25.289707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471237255410273:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13378
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.310747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471234445357283:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(250939575ns) {Configure:104(250934442ns) {ProcessConfigured:135(69241ns) {StartInitialHandshake:157(66510ns) {DropHandshakes:470(3051ns) {DropIncomingHandshake:442(787ns) DropOutgoingHandshake:457(382ns)} GenerateSessionVirtualId:988(742ns) PrepareNewSessionHandshake:477(1311ns) {SwitchToState:179(470ns)}}}}}
2025-11-27T18:18:25.329049Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471232203924029:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(109376745ns) {Configure:104(109372213ns) {ProcessConfigured:135(44457ns) {StartInitialHandshake:157(41834ns) {DropHandshakes:470(1336ns) {DropIncomingHandshake:442(459ns) DropOutgoingHandshake:457(292ns)} GenerateSessionVirtualId:988(509ns) PrepareNewSessionHandshake:477(1117ns) {SwitchToState:179(663ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.329594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471239431097686:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.333623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471239431097691:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21006
2025-11-27T18:18:25.327052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471233999840383:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(108539413ns) {Configure:104(108531662ns) {ProcessConfigured:135(80423ns) {StartInitialHandshake:157(77439ns) {DropHandshakes:470(3812ns) {DropIncomingHandshake:442(1153ns) DropOutgoingHandshake:457(685ns)} GenerateSessionVirtualId:988(903ns) PrepareNewSessionHandshake:477(2472ns) {SwitchToState:179(923ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.329460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471233999840382:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(339555343ns) {Configure:104(339547524ns) {ProcessConfigured:135(7637302ns) {StartInitialHandshake:157(7633505ns) {DropHandshakes:470(3256ns) {DropIncomingHandshake:442(846ns) DropOutgoingHandshake:457(382ns)} GenerateSessionVirtualId:988(1057ns) PrepareNewSessionHandshake:477(2419ns) {SwitchToState:179(1105ns)}}}}}
2025-11-27T18:18:25.337694Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471233607337446:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(292890452ns) {Configure:104(292884608ns) {ProcessConfigured:135(9769565ns) {StartInitialHandshake:157(9767262ns) {DropHandshakes:470(1622ns) {DropIncomingHandshake:442(336ns) DropOutgoingHandshake:457(294ns)} GenerateSessionVirtualId:988(695ns) PrepareNewSessionHandshake:477(4069ns) {SwitchToState:179(1631ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.342019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471232960442903:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(317106198ns) {Configure:104(317099880ns) {ProcessConfigured:135(11429442ns) {StartInitialHandshake:157(11426839ns) {DropHandshakes:470(1428ns) {DropIncomingHandshake:442(431ns) DropOutgoingHandshake:457(406ns)} GenerateSessionVirtualId:988(353ns) PrepareNewSessionHandshake:477(3527ns) {SwitchToState:179(1533ns)}}}}}
2025-11-27T18:18:25.346540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471234445357284:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(104118944ns) {Configure:104(104110953ns) {ProcessConfigured:135(67782ns) {StartInitialHandshake:157(65665ns) {DropHandshakes:470(3254ns) {DropIncomingHandshake:442(869ns) DropOutgoingHandshake:457(419ns)} GenerateSessionVirtualId:988(841ns) PrepareNewSessionHandshake:477(1844ns) {SwitchToState:179(706ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.338633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471237902304823:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54332
2025-11-27T18:18:25.372002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471239431097678:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(252016110ns) {Configure:104(252008546ns) {ProcessConfigured:135(97212ns) {StartInitialHandshake:157(90988ns) {DropHandshakes:470(2952ns) {DropIncomingHandshake:442(658ns) DropOutgoingHandshake:457(395ns)} GenerateSessionVirtualId:988(831ns) PrepareNewSessionHandshake:477(2236ns) {SwitchToState:179(1013ns)}}}}}
2025-11-27T18:18:25.338687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471237902304824:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54332
2025-11-27T18:18:25.338795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471237902304825:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54332
2025-11-27T18:18:25.351680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471237902304824:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.356957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471236498891389:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(114165656ns) {Configure:104(114152120ns) {ProcessConfigured:135(42263257ns) {StartInitialHandshake:157(42259409ns) {DropHandshakes:470(3742ns) {DropIncomingHandshake:442(835ns) DropOutgoingHandshake:457(465ns)} GenerateSessionVirtualId:988(1102ns) PrepareNewSessionHandshake:477(3634ns) {SwitchToState:179(1824ns)}}}}}
2025-11-27T18:18:25.348958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471238740324648:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51039
2025-11-27T18:18:25.374246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471239431097692:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21006
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.379781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471236498891405:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63583
2025-11-27T18:18:25.349039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471238740324649:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51039
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.384623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471239431097696:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21006
2025-11-27T18:18:25.387518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471233607337448:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(161776433ns) {Configure:104(161770656ns) {ProcessConfigured:135(13008589ns) {StartInitialHandshake:157(13005997ns) {DropHandshakes:470(3609ns) {DropIncomingHandshake:442(996ns) DropOutgoingHandshake:457(346ns)} GenerateSessionVirtualId:988(821ns) PrepareNewSessionHandshake:477(4658ns) {SwitchToState:179(2363ns)}}}}}
2025-11-27T18:18:25.390526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471233607337447:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.391025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471239431097692:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.391153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471239431097685:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.391175Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471239431097685:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.392367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471239431097685:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.369454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471238740324654:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51039
2025-11-27T18:18:25.385220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471238740324649:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.395686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471236498891404:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63583
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.393037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471237255410273:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.394178Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471238294807741:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(161374811ns) {Configure:104(161362287ns) {ProcessConfigured:135(11658091ns) {StartInitialHandshake:157(11654872ns) {DropHandshakes:470(2661ns) {DropIncomingHandshake:442(524ns) DropOutgoingHandshake:457(359ns)} GenerateSessionVirtualId:988(1012ns) PrepareNewSessionHandshake:477(4401ns) {SwitchToState:179(2099ns)}}}}}
2025-11-27T18:18:25.394689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471234445357285:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(82140189ns) {Configure:104(82136912ns) {ProcessConfigured:135(48494ns) {StartInitialHandshake:157(46061ns) {DropHandshakes:470(2774ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(839ns)} GenerateSessionVirtualId:988(636ns) PrepareNewSessionHandshake:477(1135ns) {SwitchToState:179(180ns)}}}}}
2025-11-27T18:18:25.396352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471239431097680:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.396320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471236498891397:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.394057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471232960442903:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.395269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471238294807756:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4948
2025-11-27T18:18:25.394752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471237902304816:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(133102275ns) {Configure:104(133098623ns) {ProcessConfigured:135(6008666ns) {StartInitialHandshake:157(6005003ns) {DropHandshakes:470(2477ns) {DropIncomingHandshake:442(665ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(2672ns) PrepareNewSessionHandshake:477(326511ns) {SwitchToState:179(4218ns)}}}}}
2025-11-27T18:18:25.396419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:25.396354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471236498891397:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.397758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471237902304823:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.396706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471234445357284:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.397231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471239431097680:2107] [node 4] ICP22 created new session: [2:7577471239431097699:2107]
No RDMA context found for GID: 2025-11-27T18:18:25.399744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471236498891404:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:25.395413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471238294807755:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4948
2025-11-27T18:18:25.395529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471238294807757:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4948
2025-11-27T18:18:25.395951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471238294807760:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4948
2025-11-27T18:18:25.396081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471238294807757:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.396295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471238294807756:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.397780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471238294807755:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.397308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471238740324648:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.396182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471237255410277:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13378
2025-11-27T18:18:25.399831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471233607337448:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.399303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471239431097699:2107] [node 4] ICS09 handshake done sender: [2:7577471239431097685:2113] self: [2:7577471239431097684:0] peer: [4:7577471237902304833:0] socket: 449 qp: -1
2025-11-27T18:18:25.408129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471232203924029:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.399341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471239431097699:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:25.398115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471238740324654:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.417862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471233607337447:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3666) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471237902304824:2124] held: no
2025-11-27T18:18:25.436136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471237255410277:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.417384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471236498891397:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:25.442947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471238740324653:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.442980Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471238740324653:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.445326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471238740324653:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:25.445949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471238740324644:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(110421990ns) {Configure:104(110412314ns) {ProcessConfigured:135(25048452ns) {StartInitialHandshake:157(25044424ns) {DropHandshakes:470(2859ns) {DropIncomingHandshake:442(729ns) DropOutgoingHandshake:457(487ns)} GenerateSessionVirtualId:988(1142ns) PrepareNewSessionHandshake:477(3044ns) {SwitchToState:179(1282ns)}}}}}
2025-11-27T18:18:25.400330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:25.426357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471237902304834:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54332
2025-11-27T18:18:25.421319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471236498891395:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.442611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471237255410250:2095] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.446049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471238740324644:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.444590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471238294807760:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.462160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471237902304824:2124] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:25.416568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471239431097683:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.461872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471237255410280:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13378
2025-11-27T18:18:25.457566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471238294807764:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4948
2025-11-27T18:18:25.421366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471236498891395:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.464173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471237902304818:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.464213Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471237902304818:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.466513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471237902304825:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.466567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471233607337447:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.466630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:25.466682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471233607337447:2049] [node 2] ICP22 created new session: [4:7577471237902304838:2049]
2025-11-27T18:18:25.466738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471237902304838:2049] [node 2] ICS09 handshake done sender: [4:7577471237902304824:2124] self: [4:7577471237902304833:0] peer: [2:7577471239431097684:0] socket: 460 qp: -1
2025-11-27T18:18:25.466784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471237902304838:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:25.466845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:25.467363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471237902304837:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54332
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.416628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471239431097683:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.451696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471236498891395:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:25.474330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471233607337446:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.462697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471237255410278:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13378
2025-11-27T18:18:25.439348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471239431097683:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:25.452472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471236498891393:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.474476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471237902304823:2123] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:25.493590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471238294807746:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.489867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471238740324667:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51039
2025-11-27T18:18:25.501964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471237255410273:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:25.441967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471239431097676:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.481216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471233607337448:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.452522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:25.489936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471238740324668:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51039
2025-11-27T18:18:25.493647Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471238294807746:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.442008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471239431097676:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.507059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471237255410277:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:25.453346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471236498891393:2110] [node 6] ICP22 created new session: [3:7577471236498891409:2110]
2025-11-27T18:18:25.521153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471232960442903:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.493860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471238294807764:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.481300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:25.493396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471238740324645:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.481354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471233607337448:2050] [node 3] ICP22 created new session: [4:7577471237902304842:2050]
2025-11-27T18:18:25.462783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471239431097696:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.522151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:25.464335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471236498891409:2110] [node 6] ICS09 handshake done sender: [3:7577471236498891397:2113] self: [3:7577471236498891396:0] peer: [6:7577471237255410276:0] socket: 444 qp: -1
2025-11-27T18:18:25.494025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471238294807749:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.502100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:25.481413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471237902304842:2050] [node 3] ICS09 handshake done sender: [4:7577471237902304823:2123] self: [4:7577471237902304835:0] peer: [3:7577471236498891394:0] socket: 459 qp: -1
2025-11-27T18:18:25.523028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471232960442903:2050] [node 3] ICP22 created new session: [6:7577471237255410283:2050]
2025-11-27T18:18:25.464250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471239431097677:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.468003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471236498891409:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:25.494058Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471238294807749:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.504652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471238740324645:2109] [node 6] ICP22 created new session: [5:7577471238740324669:2109]
2025-11-27T18:18:25.481448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471237902304842:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.523082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471237255410283:2050] [node 3] ICS09 handshake done sender: [6:7577471237255410273:2114] self: [6:7577471237255410276:0] peer: [3:7577471236498891396:0] socket: 447 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.513528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471238294807746:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:25.474757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471236498891405:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.481492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:25.510658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471238740324664:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51039
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.540073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471238294807749:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:25.474840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471236498891388:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.523682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471237255410283:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:25.471272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471239431097680:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(48171731ns) {HandleHandshakeStatus:329(48168540ns) {DropIncomingHandshake:442(12996ns) DropOutgoingHandshake:457(521ns) SwitchToState:179(374ns) ProcessPendingSessionEvents:495(11396512ns) {ForwardSessionEventToSession:612(9120882ns) ForwardSessionEventToSession:612(7444ns) ForwardSessionEventToSession:612(169426ns) ForwardSessionEventToSession:612(3185ns) ForwardSessionEventToSession:612(226993ns) ForwardSessionEventToSession:612(4768ns) ForwardSessionEventToSession:612(3023ns) ForwardSessionEventToSession:612(1829810ns) ForwardSessionEventToSession:612(6832ns) ForwardSessionEventToSession:612(3830ns)}}}
2025-11-27T18:18:25.514131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471238740324669:2109] [node 6] ICS09 handshake done sender: [5:7577471238740324653:2116] self: [5:7577471238740324652:0] peer: [6:7577471237255410279:0] socket: 475 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.479869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471239431097676:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:25.545245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471238294807747:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.503915Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471237902304818:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:25.474886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:25.518633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471238740324669:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:25.524364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.493238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:25.520522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471237902304815:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.537117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471237255410250:2095] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24519) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471237255410277:2116] held: no
2025-11-27T18:18:25.474935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471236498891388:2107] [node 4] ICP22 created new session: [3:7577471236498891412:2107]
2025-11-27T18:18:25.545310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471238294807747:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.518682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:25.537224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471237255410250:2095] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.520583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:25.497370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471239431097691:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.520988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471238740324654:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:25.553572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471233999840383:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.474979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471236498891412:2107] [node 4] ICS09 handshake done sender: [3:7577471236498891395:2112] self: [3:7577471236498891394:0] peer: [4:7577471237902304835:0] socket: 443 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.537283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:25.522228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471237902304815:2108] [node 5] ICP22 created new session: [4:7577471237902304845:2108]
2025-11-27T18:18:25.553655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:25.497430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471235136130317:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.475019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471236498891412:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:25.497887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471239431097677:2108] [node 5] ICP22 created new session: [2:7577471239431097705:2108]
2025-11-27T18:18:25.524676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471238740324644:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.523572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471237902304845:2108] [node 5] ICS09 handshake done sender: [4:7577471237902304818:2120] self: [4:7577471237902304817:0] peer: [5:7577471238740324666:0] socket: 462 qp: -1
2025-11-27T18:18:25.537341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471237255410250:2095] [node 5] ICP22 created new session: [6:7577471237255410285:2095]
2025-11-27T18:18:25.475055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:25.553708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471233999840383:2049] [node 3] ICP22 created new session: [1:7577471238294807769:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.497501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:25.524727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:25.524196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471237902304845:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:25.537384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471237255410285:2095] [node 5] ICS09 handshake done sender: [6:7577471237255410277:2116] self: [6:7577471237255410279:0] peer: [5:7577471238740324652:0] socket: 477 qp: -1
2025-11-27T18:18:25.475886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471232203924028:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.502133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471235136130317:2049] [node 3] ICP22 created new session: [2:7577471239431097706:2049]
2025-11-27T18:18:25.503999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471239431097706:2049] [node 3] ICS09 handshake done sender: [2:7577471239431097676:2110] self: [2:7577471239431097675:0] peer: [3:7577471236498891407:0] socket: 445 qp: -1
2025-11-27T18:18:25.504895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471239431097705:2108] [node 5] ICS09 handshake done sender: [2:7577471239431097683:2112] self: [2:7577471239431097682:0] peer: [5:7577471238740324660:0] socket: 448 qp: -1
2025-11-27T18:18:25.504942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471239431097705:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:25.506381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471239431097706:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:25.506439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:25.510888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:25.524129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471235136130316:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.554839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471239431097710:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21006
2025-11-27T18:18:25.554990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471235136130317:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(23266732ns) {HandleHandshakeStatus:329(23263863ns) {DropIncomingHandshake:442(443ns) DropOutgoingHandshake:457(522ns) SwitchToState:179(413ns) ProcessPendingSessionEvents:495(14048204ns) {ForwardSessionEventToSession:612(4604500ns) ForwardSessionEventToSession:612(3451ns) ForwardSessionEventToSession:612(3885ns) ForwardSessionEventToSession:612(2689788ns) ForwardSessionEventToSession:612(413607ns) ForwardSessionEventToSession:612(6066738ns) ForwardSessionEventToSession:612(4942ns) ForwardSessionEventToSession:612(3429ns) ForwardSessionEventToSession:612(230832ns) ForwardSessionEventToSession:612(4666ns)}}}
2025-11-27T18:18:25.582450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471239431097677:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(82832715ns) {HandleHandshakeStatus:329(82828871ns) {DropIncomingHandshake:442(1757ns) DropOutgoingHandshake:457(1346ns) SwitchToState:179(703ns) ProcessPendingSessionEvents:495(23391212ns) {ForwardSessionEventToSession:612(2791782ns) ForwardSessionEventToSession:612(4450ns) ForwardSessionEventToSession:612(3479ns) ForwardSessionEventToSession:612(2612950ns) ForwardSessionEventToSession:612(3946ns) ForwardSessionEventToSession:612(2790ns) ForwardSessionEventToSession:612(12527491ns) ForwardSessionEventToSession:612(4682853ns) ForwardSessionEventToSession:612(735692ns) ForwardSessionEventToSession:612(3099ns)}}}
2025-11-27T18:18:25.586305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471239431097710:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.586476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471239431097677:2108] [node 5] ICP18 handshake (actor [2:7577471239431097696:2119]) is too old
2025-11-27T18:18:25.594187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471235136130317:2049] [node 3] ICP18 handshake (actor [2:7577471239431097710:2120]) is too old
2025-11-27T18:18:25.594327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471239431097711:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21006
2025-11-27T18:18:25.610589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471239431097694:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.610630Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471239431097694:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.623536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471235136130316:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471239431097691:2116] held: no
2025-11-27T18:18:25.623729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471239431097691:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:25.627069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471235136130316:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.627123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:25.627167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471235136130316:2048] [node 1] ICP22 created new session: [2:7577471239431097713:2048]
2025-11-27T18:18:25.627214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471239431097713:2048] [node 1] ICS09 handshake done sender: [2:7577471239431097691:2116] self: [2:7577471239431097709:0] peer: [1:7577471238294807744:0] socket: 458 qp: -1
2025-11-27T18:18:25.627254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471239431097713:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:25.627284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:25.636127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471239431097694:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:25.644249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471239431097678:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.646907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:25.653023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471239431097678:2109] [node 6] ICP22 created new session: [2:7577471239431097715:2109]
2025-11-27T18:18:25.553754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471238294807769:2049] [node 3] ICS09 handshake done sender: [1:7577471238294807746:2110] self: [1:7577471238294807745:0] peer: [3:7577471236498891414:0] socket: 455 qp: -1
2025-11-27T18:18:25.524780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471238740324644:2108] [node 4] ICP22 created new session: [5:7577471238740324672:2108]
2025-11-27T18:18:25.525025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:25.557648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471237902304846:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54332
2025-11-27T18:18:25.476029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471236498891410:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63583
2025-11-27T18:18:25.537425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471237255410285:2095] [node 5] ICS10 traffic start
2025-11-27T18:18:25.569744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471237902304825:2125] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:25.590967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471237902304832:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.591009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471237902304832:2127] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.597821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471233607337446:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.597873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:25.601629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471237902304848:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54332
2025-11-27T18:18:25.610505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471233607337446:2048] [node 1] ICP22 created new session: [4:7577471237902304849:2048]
2025-11-27T18:18:25.611801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471237902304849:2048] [node 1] ICS09 handshake done sender: [4:7577471237902304825:2125] self: [4:7577471237902304840:0] peer: [1:7577471238294807748:0] socket: 461 qp: -1
2025-11-27T18:18:25.613359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471237902304848:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.617703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471237902304832:2127] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:25.619695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471237902304849:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:25.628006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471237902304816:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.628059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:25.628109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471237902304816:2109] [node 6] ICP22 created new session: [4:7577471237902304852:2109]
2025-11-27T18:18:25.628152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471237902304852:2109] [node 6] ICS09 handshake done sender: [4:7577471237902304832:2127] self: [4:7577471237902304831:0] peer: [6:7577471237255410288:0] socket: 451 qp: -1
2025-11-27T18:18:25.628187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471237902304852:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:25.628225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:25.628714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471237902304816:2109] [node 6] ICP18 handshake (actor [4:7577471237902304848:2131]) is too old
2025-11-27T18:18:25.630036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:25.637610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471237902304815:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(66461855ns) {HandleHandshakeStatus:329(66455540ns) {DropIncomingHandshake:442(702ns) DropOutgoingHandshake:457(988ns) SwitchToState:179(644ns) ProcessPendingSessionEvents:495(41361420ns) {ForwardSessionEventToSession:612(13143112ns) ForwardSessionEventToSession:612(8014453ns) ForwardSessionEventToSession:612(4659ns) ForwardSessionEventToSession:612(2214ns) ForwardSessionEventToSession:612(4711556ns) ForwardSessionEventToSession:612(3467ns) ForwardSessionEventToSession:612(11166998ns) ForwardSessionEventToSession:612(4586ns) ForwardSessionEventToSession:612(3939325ns) ForwardSessionEventToSession:612(346588ns)}}}
2025-11-27T18:18:25.641726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471233607337446:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(43683923ns) {HandleHandshakeStatus:329(43681269ns) {DropIncomingHandshake:442(1157ns) DropOutgoingHandshake:457(4318ns) SwitchToState:179(890ns) ProcessPendingSessionEvents:495(118497ns) {ForwardSessionEventToSession:612(65106ns) ForwardSessionEventToSession:612(6568ns) ForwardSessionEventToSession:612(3515ns) ForwardSessionEventToSession:612(3142ns) ForwardSessionEventToSession:612(6296ns) ForwardSessionEventToSession:612(5490ns) ForwardSessionEventToSession:612(5966ns) ForwardSessionEventToSession:612(5973ns) ForwardSessionEventToSession:612(3023ns) ForwardSessionEventToSession:612(4832ns)}}}
2025-11-27T18:18:25.641794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471233607337446:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:25.553789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471238294807769:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:25.525283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471238740324672:2108] [node 4] ICS09 handshake done sender: [5:7577471238740324654:2117] self: [5:7577471238740324666:0] peer: [4:7577471237902304817:0] socket: 468 qp: -1
2025-11-27T18:18:25.656526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471239431097715:2109] [node 6] ICS09 handshake done sender: [2:7577471239431097694:2118] self: [2:7577471239431097693:0] peer: [6:7577471237255410290:0] socket: 494 qp: -1
2025-11-27T18:18:25.656576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471239431097715:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:25.656617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:25.480225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:25.537461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:25.553831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:25.525327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471238740324672:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:25.497956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471236498891410:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:25.561217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471238294807742:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.561310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:25.561366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471238294807742:2107] [node 4] ICP22 created new session: [1:7577471238294807773:2107]
2025-11-27T18:18:25.561410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471238294807773:2107] [node 4] ICS09 handshake done sender: [1:7577471238294807749:2112] self: [1:7577471238294807748:0] peer: [4:7577471237902304840:0] socket: 457 qp: -1
2025-11-27T18:18:25.561449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471238294807773:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:25.561479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:25.561975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471238294807747:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:25.588406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471233999840382:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.588493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:25.588545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471233999840382:2048] [node 2] ICP22 created new session: [1:7577471238294807776:2048]
2025-11-27T18:18:25.588596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471238294807776:2048] [node 2] ICS09 handshake done sender: [1:7577471238294807747:2111] self: [1:7577471238294807744:0] peer: [2:7577471239431097709:0] socket: 456 qp: -1
2025-11-27T18:18:25.588630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471238294807776:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:25.588673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:25.645072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471238294807751:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.645100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471238294807751:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.646554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471238294807751:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:25.652873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471238294807743:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.653866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:25.655128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471238294807743:2109] [node 6] ICP22 created new session: [1:7577471238294807779:2109]
2025-11-27T18:18:25.656421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471238294807779:2109] [node 6] ICS09 handshake done sender: [1:7577471238294807751:2113] self: [1:7577471238294807750:0] peer: [6:7577471237255410293:0] socket: 465 qp: -1
2025-11-27T18:18:25.658230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471238294807779:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:25.659059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:25.661702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471238294807759:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.661725Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471238294807759:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.538431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471237255410282:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13378
2025-11-27T18:18:25.525358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:25.539718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471236498891388:2107] [node 4] ICP18 handshake (actor [3:7577471236498891410:2118]) is too old
2025-11-27T18:18:25.555027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471237255410282:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.545422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471236498891393:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(68376117ns) {HandleHandshakeStatus:329(68363037ns) {DropIncomingHandshake:442(623ns) DropOutgoingHandshake:457(650ns) SwitchToState:179(413ns) ProcessPendingSessionEvents:495(23125266ns) {ForwardSessionEventToSession:612(6513834ns) ForwardSessionEventToSession:612(2788434ns) ForwardSessionEventToSession:612(6994ns) ForwardSessionEventToSession:612(2975ns) ForwardSessionEventToSession:612(4103083ns) ForwardSessionEventToSession:612(5003ns) ForwardSessionEventToSession:612(3619ns) ForwardSessionEventToSession:612(2481883ns) ForwardSessionEventToSession:612(3012ns) ForwardSessionEventToSession:612(7187299ns)}}}
2025-11-27T18:18:25.552991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471236498891416:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63583
2025-11-27T18:18:25.557802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471236498891418:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63583
2025-11-27T18:18:25.557900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471236498891419:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63583
2025-11-27T18:18:25.591170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471236498891401:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.591194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471236498891401:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:25.594641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471232203924028:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471236498891405:2117] held: no
2025-11-27T18:18:25.594833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471236498891419:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.594894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471236498891404:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:25.597839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471236498891405:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:25.605581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471232203924029:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.605634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:25.605690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471232203924029:2049] [node 2] ICP22 created new session: [3:7577471236498891421:2049]
2025-11-27T18:18:25.605732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471236498891421:2049] [node 2] ICS09 handshake done sender: [3:7577471236498891404:2116] self: [3:7577471236498891407:0] peer: [2:7577471239431097675:0] socket: 453 qp: -1
2025-11-27T18:18:25.605770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471236498891421:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:25.605805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:25.625314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471232203924028:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.625371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:25.625422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471232203924028:2048] [node 1] ICP22 created new session: [3:7577471236498891424:2048]
2025-11-27T18:18:25.633447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471236498891424:2048] [node 1] ICS09 handshake done sender: [3:7577471236498891405:2117] self: [3:7577471236498891414:0] peer: [1:7577471238294807745:0] socket: 469 qp: -1
2025-11-27T18:18:25.633504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471236498891424:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:25.633559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:25.633815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471236498891401:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:25.643147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471236498891389:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.644120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:25.644979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471236498891389:2108] [node 5] ICP22 created new session: [3:7577471236498891426:2108]
2025-11-27T18:18:25.653994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471236498891426:2108] [node 5] ICS09 handshake done sender: [3:7577471236498891401:2114] self: [3:7577471236498891400:0] peer: [5:7577471238740324676:0] socket: 492 qp: -1
2025-11-27T18:18:25.655213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471236498891426:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:25.655265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:25.655951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471236498891389:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(16350713ns) {HandleHandshakeStatus:329(12604883ns) {DropIncomingHandshake:442(21106ns) DropOutgoingHandshake:457(184ns) SwitchToState:179(727ns) ProcessPendingSessionEvents:495(105978ns) {ForwardSessionEventToSession:612(58551ns) ForwardSessionEventToSession:612(3528ns) ForwardSessionEventToSession:612(7708ns) ForwardSessionEventToSession:612(5128ns) ForwardSessionEventToSession:612(2693ns) ForwardSessionEventToSession:612(4641ns) ForwardSessionEventToSession:612(3401ns) ForwardSessionEventToSession:612(5680ns) ForwardSessionEventToSession:612(2891ns) ForwardSessionEventToSession:612(2788ns)}}}
2025-11-27T18:18:25.683366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471238294807759:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:25.525680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471238740324649:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:25.542644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471234445357284:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.542709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:25.542763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471234445357284:2049] [node 2] ICP22 created new session: [5:7577471238740324674:2049]
2025-11-27T18:18:25.542812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471238740324674:2049] [node 2] ICS09 handshake done sender: [5:7577471238740324649:2114] self: [5:7577471238740324660:0] peer: [2:7577471239431097682:0] socket: 452 qp: -1
2025-11-27T18:18:25.542853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471238740324674:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:25.542889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:25.557397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471238740324667:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.566406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471234445357285:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.590596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471238740324677:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51039
2025-11-27T18:18:25.597729Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471238740324645:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(78105693ns) {HandleHandshakeStatus:329(78102125ns) {DropIncomingHandshake:442(17788ns) DropOutgoingHandshake:457(561ns) SwitchToState:179(657ns) ProcessPendingSessionEvents:495(31308658ns) {ForwardSessionEventToSession:612(14848547ns) ForwardSessionEventToSession:612(6172ns) ForwardSessionEventToSession:612(3660ns) ForwardSessionEventToSession:612(987083ns) ForwardSessionEventToSession:612(4752ns) ForwardSessionEventToSession:612(2944ns) ForwardSessionEventToSession:612(1756781ns) ForwardSessionEventToSession:612(8956127ns) ForwardSessionEventToSession:612(4719510ns) ForwardSessionEventToSession:612(3851ns)}}}
2025-11-27T18:18:25.636859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471238740324677:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.637124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471238740324678:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51039
2025-11-27T18:18:25.649378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471238740324667:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:25.659673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471234445357283:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.659737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471234445357285:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.660647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:25.660703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471234445357285:2050] [node 3] ICP22 created new session: [5:7577471238740324680:2050]
2025-11-27T18:18:25.660753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471238740324680:2050] [node 3] ICS09 handshake done sender: [5:7577471238740324667:2121] self: [5:7577471238740324676:0] peer: [3:7577471236498891400:0] socket: 497 qp: -1
2025-11-27T18:18:25.660787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471238740324680:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:25.660834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:25.661085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471234445357285:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:25.681757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471238740324682:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51039
2025-11-27T18:18:25.690362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471238740324677:2123] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:25.691800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471234445357283:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471238740324677:2123] held: no
2025-11-27T18:18:25.684393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471238294807741:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:25.686506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:25.686563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471238294807741:2108] [node 5] ICP22 created new session: [1:7577471238294807782:2108]
2025-11-27T18:18:25.686616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471238294807782:2108] [node 5] ICS09 handshake done sender: [1:7577471238294807759:2117] self: [1:7577471238294807758:0] peer: [5:7577471238740324679:0] socket: 466 qp: -1
2025-11-27T18:18:25.686653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471238294807782:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:25.686691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:25.558749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471232960442944:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.578307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471237255410289:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13378
2025-11-27T18:18:25.582061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471237255410281:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13378
2025-11-27T18:18:25.590006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471237255410281:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.590145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471232960442902:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.602717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471237255410291:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13378
2025-11-27T18:18:25.625580Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471237255410282:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:25.625685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471237255410289:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:25.638853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471232960442901:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:25.638969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471237255410292:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13378
2025-11-27T18:18:25.639247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471232960442944:2089] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15572) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471237255410282:2120] held: no
2025-11-27T18:18:25.643675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471237255410281:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:25.649705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471232960442944:2089] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.649750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:25.651381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471232960442902:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.651478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471232960442944:2089] [node 4] ICP22 created new session: [6:7577471237255410294:2089]
2025-11-27T18:18:25.651549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471237255410294:2089] [node 4] ICS09 handshake done sender: [6:7577471237255410282:2120] self: [6:7577471237255410288:0] peer: [4:7577471237902304831:0] socket: 463 qp: -1
2025-11-27T18:18:25.651597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471237255410294:2089] [node 4] ICS10 traffic start
2025-11-27T18:18:25.651635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:25.652325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471237255410295:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13378
2025-11-27T18:18:25.653202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:25.656850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471232960442902:2049] [node 2] ICP22 created new session: [6:7577471237255410297:2049]
2025-11-27T18:18:25.656949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471237255410297:2049] [node 2] ICS09 handshake done sender: [6:7577471237255410281:2119] self: [6:7577471237255410290:0] peer: [2:7577471239431097693:0] socket: 499 qp: -1
2025-11-27T18:18:25.656984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471237255410297:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:25.657055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:25.657328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471232960442902:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:25.679232Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471232960442901:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471237255410289:2121] held: no
2025-11-27T18:18:25.679461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471237255410289:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:25.679720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471232960442901:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.679764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:25.679816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471232960442901:2048] [node 1] ICP22 created new session: [6:7577471237255410299:2048]
2025-11-27T18:18:25.679860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471237255410299:2048] [node 1] ICS09 handshake done sender: [6:7577471237255410289:2121] self: [6:7577471237255410293:0] peer: [1:7577471238294807750:0] socket: 509 qp: -1
2025-11-27T18:18:25.679896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471237255410299:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:25.679931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:25.694886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471234445357283:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:25.694949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:25.694997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471234445357283:2048] [node 1] ICP22 created new session: [5:7577471238740324683:2048]
2025-11-27T18:18:25.695043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471238740324683:2048] [node 1] ICS09 handshake done sender: [5:7577471238740324677:2123] self: [5:7577471238740324679:0] peer: [1:7577471238294807758:0] socket: 510 qp: -1
2025-11-27T18:18:25.695082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471238740324683:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:25.695118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:25.751042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:25.751336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471239431097713:2048] [node 1] ICS01 socket: 458 reason# EndOfStream
2025-11-27T18:18:25.751389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471235136130316:2048] [node 1] ICP30 unregister session Session# [2:7577471239431097713:2048] VirtualId# [2:7577471239431097709:0]
2025-11-27T18:18:25.751429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471239431097713:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.751516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:25.751569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:25.752517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:25.752711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471236498891424:2048] [node 1] ICS01 socket: 469 reason# EndOfStream
2025-11-27T18:18:25.752748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471232203924028:2048] [node 1] ICP30 unregister session Session# [3:7577471236498891424:2048] VirtualId# [3:7577471236498891414:0]
2025-11-27T18:18:25.752783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471236498891424:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.752844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:25.752892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:25.766972Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:25.767199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471238740324683:2048] [node 1] ICS01 socket: 510 reason# EndOfStream
2025-11-27T18:18:25.767259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471234445357283:2048] [node 1] ICP30 unregister session Session# [5:7577471238740324683:2048] VirtualId# [5:7577471238740324679:0]
2025-11-27T18:18:25.767299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471238740324683:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.767372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:25.767421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:25.768745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:25.768975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471237902304849:2048] [node 1] ICS01 socket: 461 reason# EndOfStream
2025-11-27T18:18:25.769011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471233607337446:2048] [node 1] ICP30 unregister session Session# [4:7577471237902304849:2048] VirtualId# [4:7577471237902304840:0]
2025-11-27T18:18:25.769044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471237902304849:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.769115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:25.769156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:25.769428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:25.789278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471237255410299:2048] [node 1] ICS01 socket: 509 reason# EndOfStream
2025-11-27T18:18:25.789346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471232960442901:2048] [node 1] ICP30 unregister session Session# [6:7577471237255410299:2048] VirtualId# [6:7577471237255410293:0]
2025-11-27T18:18:25.789389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471237255410299:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.789495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:25.789551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:25.807043Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:25.807287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471236498891421:2049] [node 2] ICS01 socket: 453 reason# EndOfStream
2025-11-27T18:18:25.807335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471232203924029:2049] [node 2] ICP30 unregister session Session# [3:7577471236498891421:2049] VirtualId# [3:7577471236498891407:0]
2025-11-27T18:18:25.807385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471236498891421:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.807492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:25.807572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:25.809582Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:25.809983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471237902304838:2049] [node 2] ICS01 socket: 460 reason# EndOfStream
2025-11-27T18:18:25.810033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471233607337447:2049] [node 2] ICP30 unregister session Session# [4:7577471237902304838:2049] VirtualId# [4:7577471237902304833:0]
2025-11-27T18:18:25.810085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471237902304838:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.810191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:25.810244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:25.810309Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:25.810489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471238740324674:2049] [node 2] ICS01 socket: 452 reason# EndOfStream
2025-11-27T18:18:25.810530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471234445357284:2049] [node 2] ICP30 unregister session Session# [5:7577471238740324674:2049] VirtualId# [5:7577471238740324660:0]
2025-11-27T18:18:25.810560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471238740324674:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.810640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:25.810683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:25.811369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:25.811588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471237255410297:2049] [node 2] ICS01 socket: 499 reason# EndOfStream
2025-11-27T18:18:25.811633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471232960442902:2049] [node 2] ICP30 unregister session Session# [6:7577471237255410297:2049] VirtualId# [6:7577471237255410290:0]
2025-11-27T18:18:25.811663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471237255410297:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.811738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:25.811799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:25.919103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:25.919334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471237255410283:2050] [node 3] ICS01 socket: 447 reason# EndOfStream
2025-11-27T18:18:25.919383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471232960442903:2050] [node 3] ICP30 unregister session Session# [6:7577471237255410283:2050] VirtualId# [6:7577471237255410276:0]
2025-11-27T18:18:25.919420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471237255410283:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.919501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:25.919550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:25.911343Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:25.911586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471237902304842:2050] [node 3] ICS07 socket disconnect 459 reason# EndOfStream
2025-11-27T18:18:25.911619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471237902304842:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.911702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:25.911750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471237902304842:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:25.920425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:25.920600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471238740324680:2050] [node 3] ICS01 socket: 497 reason# EndOfStream
2025-11-27T18:18:25.920638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471234445357285:2050] [node 3] ICP30 unregister session Session# [5:7577471238740324680:2050] VirtualId# [5:7577471238740324676:0]
2025-11-27T18:18:25.920665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471238740324680:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.920727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:25.920763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:25.922384Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471233607337448:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24568) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:25.922535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471237902304842:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:25.922573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471233607337448:2050] [node 3] ICP30 unregister session Session# [4:7577471237902304842:2050] VirtualId# [4:7577471237902304835:0]
2025-11-27T18:18:25.922635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:25.922670Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471233607337448:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:25.922683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:25.988127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:25.988358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471238740324672:2108] [node 4] ICS01 socket: 468 reason# EndOfStream
2025-11-27T18:18:25.988404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471238740324644:2108] [node 4] ICP30 unregister session Session# [5:7577471238740324672:2108] VirtualId# [5:7577471238740324666:0]
2025-11-27T18:18:25.988446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471238740324672:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.988529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:25.988589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:25.989193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:25.989423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471237255410294:2089] [node 4] ICS01 socket: 463 reason# EndOfStream
2025-11-27T18:18:25.989469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471232960442944:2089] [node 4] ICP30 unregister session Session# [6:7577471237255410294:2089] VirtualId# [6:7577471237255410288:0]
2025-11-27T18:18:25.989496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471237255410294:2089] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:25.989557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:25.989772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:26.073770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:26.074018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471237255410285:2095] [node 5] ICS01 socket: 477 reason# EndOfStream
2025-11-27T18:18:26.074061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471237255410250:2095] [node 5] ICP30 unregister session Session# [6:7577471237255410285:2095] VirtualId# [6:7577471237255410279:0]
2025-11-27T18:18:26.074100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471237255410285:2095] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:26.074183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:26.074229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:26.232592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471242584456653:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:26.234341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471242584456652:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:26.264063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471240764417013:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:26.264117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471240764417014:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:26.319565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471240394256111:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:26.319610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471240394256112:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:26.356644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471240975745834:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:26.356698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471240975745835:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:26.356726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471240975745836:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:26.398373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471240877541988:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:26.398417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471240877541989:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:26.398444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471240877541990:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:26.429134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471241260938696:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:26.429211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471241260938697:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:26.429245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471241260938698:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:26.694091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471242584456713:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:26.694843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471241260938757:2105] [node 5] ICP01 ready to work
2025-11-27T18:18:26.829650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471241260938769:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: 2025-11-27T18:18:26.771285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471242584456720:2109] [node 5] ICP01 ready to work
::ffff:127.1.0.0
2025-11-27T18:18:26.852207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471240764417073:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32599
2025-11-27T18:18:26.845733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471242584456724:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:26.856845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471240394256187:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:26.861861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471240877542051:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:26.861931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471242584456653:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(115385326ns) {Configure:104(115374877ns) {ProcessConfigured:135(101200539ns) {StartInitialHandshake:157(101197013ns) {DropHandshakes:470(3077ns) {DropIncomingHandshake:442(836ns) DropOutgoingHandshake:457(479ns)} GenerateSessionVirtualId:988(1122ns) PrepareNewSessionHandshake:477(4277ns) {SwitchToState:179(1894ns)}}}}}
2025-11-27T18:18:26.866119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471240764417106:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:26.867219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471240877542052:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:26.866996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471240764417104:2115] [node 5] ICP01 ready to work
2025-11-27T18:18:26.867444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471240764417105:2116] [node 6] ICP01 ready to work
2025-11-27T18:18:26.867134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471240975745906:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:26.861964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471240394256185:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:26.863185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471240394256186:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.869775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471240975745907:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:26.872878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471241260938770:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14713
2025-11-27T18:18:26.873565Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471242584456713:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(176426457ns) {Configure:104(176422412ns) {ProcessConfigured:135(1156705ns) {StartInitialHandshake:157(1153838ns) {DropHandshakes:470(3594ns) {DropIncomingHandshake:442(764ns) DropOutgoingHandshake:457(490ns)} GenerateSessionVirtualId:988(1087ns) PrepareNewSessionHandshake:477(2998ns) {SwitchToState:179(949ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.887033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471240394256188:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40461
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.888755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471240764417104:2115] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18001225ns) {Configure:104(17996516ns) {ProcessConfigured:135(48022ns) {StartInitialHandshake:157(46142ns) {DropHandshakes:470(1387ns) {DropIncomingHandshake:442(302ns) DropOutgoingHandshake:457(258ns)} GenerateSessionVirtualId:988(619ns) PrepareNewSessionHandshake:477(808ns) {SwitchToState:179(211ns)}}}}}
2025-11-27T18:18:26.887945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471240394256189:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40461
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.902821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471242584456731:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14672
2025-11-27T18:18:26.890422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471240764417073:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.893274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471241260938769:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(53688666ns) {Configure:104(53682225ns) {ProcessConfigured:135(9220782ns) {StartInitialHandshake:157(8130394ns) {DropHandshakes:470(3270ns) {DropIncomingHandshake:442(835ns) DropOutgoingHandshake:457(432ns)} GenerateSessionVirtualId:988(1577ns) PrepareNewSessionHandshake:477(4118ns) {SwitchToState:179(2004ns)}}}}}
2025-11-27T18:18:26.899490Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471240877541988:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(55068565ns) {Configure:104(55061066ns) {ProcessConfigured:135(75371ns) {StartInitialHandshake:157(71448ns) {DropHandshakes:470(3101ns) {DropIncomingHandshake:442(651ns) DropOutgoingHandshake:457(445ns)} GenerateSessionVirtualId:988(1011ns) PrepareNewSessionHandshake:477(2555ns) {SwitchToState:179(1036ns)}}}}}
2025-11-27T18:18:26.893878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471241260938696:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(143459633ns) {Configure:104(143453437ns) {ProcessConfigured:135(4452038ns) {StartInitialHandshake:157(4449926ns) {DropHandshakes:470(3384ns) {DropIncomingHandshake:442(989ns) DropOutgoingHandshake:457(646ns)} GenerateSessionVirtualId:988(1145ns) PrepareNewSessionHandshake:477(3045ns) {SwitchToState:179(1591ns)}}}}}
2025-11-27T18:18:26.904196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471240394256189:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.906174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471240975745834:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(162032946ns) {Configure:104(162023003ns) {ProcessConfigured:135(933452ns) {StartInitialHandshake:157(930742ns) {DropHandshakes:470(1321ns) {DropIncomingHandshake:442(442ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(397ns) PrepareNewSessionHandshake:477(3242ns) {SwitchToState:179(1816ns)}}}}}
2025-11-27T18:18:26.890884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471240764417107:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32599
2025-11-27T18:18:26.904197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471241260938770:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.901830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471240877542053:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:109
2025-11-27T18:18:26.907017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471240764417013:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23755353ns) {Configure:104(23750682ns) {ProcessConfigured:135(997395ns) {StartInitialHandshake:157(994737ns) {DropHandshakes:470(1417ns) {DropIncomingHandshake:442(453ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(728ns) PrepareNewSessionHandshake:477(2698ns) {SwitchToState:179(1056ns)}}}}}
2025-11-27T18:18:26.914052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471240975745835:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55259020ns) {Configure:104(55251235ns) {ProcessConfigured:135(12153949ns) {StartInitialHandshake:157(12151853ns) {DropHandshakes:470(630142ns) {DropIncomingHandshake:442(1403ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(745ns) PrepareNewSessionHandshake:477(3403ns) {SwitchToState:179(2110ns)}}}}}
2025-11-27T18:18:26.908781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471240764417014:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33967956ns) {Configure:104(33962414ns) {ProcessConfigured:135(9440334ns) {StartInitialHandshake:157(9437982ns) {DropHandshakes:470(1304ns) {DropIncomingHandshake:442(432ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(514ns) PrepareNewSessionHandshake:477(7679ns) {SwitchToState:179(5322ns)}}}}}
2025-11-27T18:18:26.902102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471240877542058:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:109
2025-11-27T18:18:26.914528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471240764417013:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:26.914894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471240394256112:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(38469180ns) {Configure:104(38462898ns) {ProcessConfigured:135(2561507ns) {StartInitialHandshake:157(2560203ns) {DropHandshakes:470(1173ns) {DropIncomingHandshake:442(192ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(1664ns) {SwitchToState:179(54ns)}}}}}
2025-11-27T18:18:26.919568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471240764417107:2138] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.918209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471242584456731:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.915498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471240877541990:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42678641ns) {Configure:104(42672285ns) {ProcessConfigured:135(2993756ns) {StartInitialHandshake:157(2991799ns) {DropHandshakes:470(1361ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(178ns)} GenerateSessionVirtualId:988(519ns) PrepareNewSessionHandshake:477(1665ns) {SwitchToState:179(564ns)}}}}}
2025-11-27T18:18:26.923768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471240975745906:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(46256947ns) {Configure:104(46251988ns) {ProcessConfigured:135(3759628ns) {StartInitialHandshake:157(3757637ns) {DropHandshakes:470(2591ns) {DropIncomingHandshake:442(268ns) DropOutgoingHandshake:457(466ns)} GenerateSessionVirtualId:988(478ns) PrepareNewSessionHandshake:477(2679ns) {SwitchToState:179(1558ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.919923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471240764417106:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(43575842ns) {Configure:104(43572461ns) {ProcessConfigured:135(4289187ns) {StartInitialHandshake:157(4286167ns) {DropHandshakes:470(1212514ns) {DropIncomingHandshake:442(1827ns) DropOutgoingHandshake:457(702ns)} GenerateSessionVirtualId:988(1895ns) PrepareNewSessionHandshake:477(2535ns) {SwitchToState:179(1466ns)}}}}}
2025-11-27T18:18:26.919702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471240394256111:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(64104494ns) {Configure:104(64097557ns) {ProcessConfigured:135(15262094ns) {StartInitialHandshake:157(15259105ns) {DropHandshakes:470(9637030ns) {DropIncomingHandshake:442(453ns) DropOutgoingHandshake:457(2930ns)} GenerateSessionVirtualId:988(3111ns) PrepareNewSessionHandshake:477(2690ns) {SwitchToState:179(929ns)}}}}}
2025-11-27T18:18:26.921140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471241260938696:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:26.916741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471240877541989:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(62806418ns) {Configure:104(62799821ns) {ProcessConfigured:135(2621794ns) {StartInitialHandshake:157(2619053ns) {DropHandshakes:470(2374ns) {DropIncomingHandshake:442(694ns) DropOutgoingHandshake:457(377ns)} GenerateSessionVirtualId:988(809ns) PrepareNewSessionHandshake:477(2857ns) {SwitchToState:179(1030ns)}}}}}
2025-11-27T18:18:26.925365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471242584456726:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.944618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471240764417121:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32599
2025-11-27T18:18:26.937642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471241260938776:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14713
2025-11-27T18:18:26.921159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471240394256111:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:26.925419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471242584456726:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:26.920607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471240877542053:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.961312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471241260938776:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.935764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471240394256188:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.925642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471242584456721:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.958896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471240877542051:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(55645735ns) {Configure:104(55640647ns) {ProcessConfigured:135(5848609ns) {StartInitialHandshake:157(5845954ns) {DropHandshakes:470(1287765ns) {DropIncomingHandshake:442(467ns) DropOutgoingHandshake:457(1239ns)} GenerateSessionVirtualId:988(994ns) PrepareNewSessionHandshake:477(3073ns) {SwitchToState:179(1304ns)}}}}}
2025-11-27T18:18:26.973663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471240764417121:2144] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.925674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471242584456721:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:26.964801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471241260938757:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:26.964969Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471240975745907:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(62023622ns) {Configure:104(62015240ns) {ProcessConfigured:135(12177772ns) {StartInitialHandshake:157(12174095ns) {DropHandshakes:470(3418ns) {DropIncomingHandshake:442(1223ns) DropOutgoingHandshake:457(369ns)} GenerateSessionVirtualId:988(941ns) PrepareNewSessionHandshake:477(5270ns) {SwitchToState:179(2747ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.952952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471240394256204:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40461
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.963879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471240877542058:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.926905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471242584456715:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.982155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471240764417109:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.974234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471241260938775:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14713
2025-11-27T18:18:26.954823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471240394256186:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(50969316ns) {Configure:104(50962879ns) {ProcessConfigured:135(2662981ns) {StartInitialHandshake:157(2661097ns) {DropHandshakes:470(1733ns) {DropIncomingHandshake:442(453ns) DropOutgoingHandshake:457(428ns)} GenerateSessionVirtualId:988(409ns) PrepareNewSessionHandshake:477(1889ns) {SwitchToState:179(631ns)}}}}}
2025-11-27T18:18:26.965989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471240877542057:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.982214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471240764417109:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:26.983744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471241260938775:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.983795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471241260938777:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14713
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.987663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471240764417109:2139] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:26.969341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471240394256206:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40461
2025-11-27T18:18:26.926939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471242584456715:2108] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.966021Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471240877542057:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:26.983920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471241260938779:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14713
2025-11-27T18:18:26.989387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471240764417111:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.974114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471240394256207:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40461
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.958655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471242584456726:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.967791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471240877542065:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:109
2025-11-27T18:18:26.984481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471241260938770:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:26.989439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471240764417111:2140] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:26.990679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471240394256189:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:26.969823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471242584456721:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.970993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471240877541989:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.984681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471241260938696:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:26.990538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471240764417104:2115] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:26.971798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471242584456736:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14672
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.974438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471240877542057:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:26.993826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471240394256208:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40461
2025-11-27T18:18:26.984743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:27.018835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471245270713218:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45393
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.990576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:26.983645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471240877542070:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:109
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.983777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471242584456715:2108] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:27.020379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471240975745917:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.003310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471240394256204:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.984799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471241260938696:2048] [node 1] ICP22 created new session: [6:7577471241260938782:2048]
2025-11-27T18:18:27.020408Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471240975745917:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:27.020861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471245270713219:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45393
2025-11-27T18:18:27.032700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471240975745921:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45393
2025-11-27T18:18:26.990454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471240877542065:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.993589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471240764417111:2140] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:27.009136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471240394256111:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:26.984845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471241260938782:2048] [node 1] ICS09 handshake done sender: [6:7577471241260938770:2113] self: [6:7577471241260938778:0] peer: [1:7577471242584456725:0] socket: 452 qp: -1
2025-11-27T18:18:26.985475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471242584456737:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14672
2025-11-27T18:18:27.033665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471240975745917:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:26.991065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471240877542052:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:27.009313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471240394256191:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.995609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471240764417124:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32599
2025-11-27T18:18:26.984890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471241260938782:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:27.009353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471240394256191:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:26.991126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:26.995684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471240764417104:2115] [node 5] ICP22 created new session: [2:7577471240764417127:2115]
2025-11-27T18:18:26.986496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471242584456724:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.984942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:27.046736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471245270713220:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45393
2025-11-27T18:18:27.003917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471240764417127:2115] [node 5] ICS09 handshake done sender: [2:7577471240764417109:2139] self: [2:7577471240764417108:0] peer: [5:7577471240877542071:0] socket: 458 qp: -1
2025-11-27T18:18:26.987973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471242584456653:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:27.010616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:26.988920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471241260938777:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.049205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471240975745921:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.991188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471240877542052:2109] [node 6] ICP22 created new session: [5:7577471240877542073:2109]
2025-11-27T18:18:27.009045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471240764417105:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:26.988223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:27.010668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471240394256111:2048] [node 1] ICP22 created new session: [3:7577471244689223505:2048]
2025-11-27T18:18:27.049209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471245270713218:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.991237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471240877542073:2109] [node 6] ICS09 handshake done sender: [5:7577471240877542057:2112] self: [5:7577471240877542056:0] peer: [6:7577471241260938780:0] socket: 464 qp: -1
2025-11-27T18:18:26.989330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471241260938697:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:27.013016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471240394256191:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:26.988349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471242584456724:2110] [node 6] ICP22 created new session: [1:7577471242584456738:2110]
2025-11-27T18:18:26.991286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471240877542073:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:27.009223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471240764417127:2115] [node 5] ICS10 traffic start
2025-11-27T18:18:27.051213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471245270713219:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.990958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471241260938698:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:27.013221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471240394256193:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:27.051502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471240975745907:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:27.009308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471241260938781:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14713
2025-11-27T18:18:26.991320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.989097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471242584456738:2110] [node 6] ICS09 handshake done sender: [1:7577471242584456726:2113] self: [1:7577471242584456725:0] peer: [6:7577471241260938778:0] socket: 451 qp: -1
2025-11-27T18:18:27.010669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:27.013252Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471240394256193:2124] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:26.998327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471240877541990:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:27.051568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:26.990750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:27.016101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471240394256208:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.016070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:27.051624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471240975745907:2108] [node 5] ICP22 created new session: [4:7577471245270713223:2108]
2025-11-27T18:18:27.002880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471240877542070:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:26.995486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471242584456738:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:27.009567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471241260938757:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471241260938776:2117] held: no
2025-11-27T18:18:27.016894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471240764417105:2116] [node 6] ICP22 created new session: [2:7577471245059384428:2116]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:27.015158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471245555906083:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14713
2025-11-27T18:18:27.007550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471240877542075:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:109
2025-11-27T18:18:27.014536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471240877542058:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:27.014772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471240877541989:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.014827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:27.014878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471240877541989:2049] [node 2] ICP22 created new session: [5:7577471245172509376:2049]
2025-11-27T18:18:27.017239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471245172509376:2049] [node 2] ICS09 handshake done sender: [5:7577471240877542058:2113] self: [5:7577471240877542071:0] peer: [2:7577471240764417108:0] socket: 462 qp: -1
2025-11-27T18:18:27.019524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471245172509376:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:27.019560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:27.020006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471240877542051:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:27.020255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471245172509374:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:109
2025-11-27T18:18:27.020749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471245172509375:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:109
2025-11-27T18:18:27.029324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471240877542065:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:27.030199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471245172509374:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.030438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471240877541988:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:27.030525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471245172509379:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:109
2025-11-27T18:18:27.033589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471240877541990:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.040441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:27.042898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471240877541990:2050] [node 3] ICP22 created new session: [5:7577471245172509382:2050]
2025-11-27T18:18:27.044349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471245172509382:2050] [node 3] ICS09 handshake done sender: [5:7577471240877542065:2116] self: [5:7577471240877542077:0] peer: [3:7577471240394256190:0] socket: 473 qp: -1
2025-11-27T18:18:27.045319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471245172509382:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:27.045360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:27.056495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471240877542070:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:27.065205Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471240877541990:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(22549253ns) {HandleHandshakeStatus:329(22547512ns) {DropIncomingHandshake:442(864ns) DropOutgoingHandshake:457(3627253ns) SwitchToState:179(4256ns) ProcessPendingSessionEvents:495(8223080ns) {ForwardSessionEventToSession:612(5543820ns) ForwardSessionEventToSession:612(5036ns) ForwardSessionEventToSession:612(960742ns) ForwardSessionEventToSession:612(3485ns) ForwardSessionEventToSession:612(3673ns) ForwardSessionEventToSession:612(667634ns) ForwardSessionEventToSession:612(3799ns) ForwardSessionEventToSession:612(463927ns) ForwardSessionEventToSession:612(554014ns) ForwardSessionEventToSession:612(3273ns)}}}
2025-11-27T18:18:27.070370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471240877542051:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.070431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:27.075960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471240877541988:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20537) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471245172509374:2120] held: no
2025-11-27T18:18:27.076158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471245172509384:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:109
2025-11-27T18:18:27.077688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471240877542051:2108] [node 4] ICP22 created new session: [5:7577471245172509385:2108]
2025-11-27T18:18:27.078539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471245172509374:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:27.087819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471240877541988:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.087877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:27.087942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471240877541988:2048] [node 1] ICP22 created new session: [5:7577471245172509386:2048]
2025-11-27T18:18:27.087991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471245172509386:2048] [node 1] ICS09 handshake done sender: [5:7577471245172509374:2120] self: [5:7577471245172509380:0] peer: [1:7577471242584456722:0] socket: 506 qp: -1
2025-11-27T18:18:27.088030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471245172509386:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:27.088067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:27.088958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471245172509385:2108] [node 4] ICS09 handshake done sender: [5:7577471240877542070:2118] self: [5:7577471245172509378:0] peer: [4:7577471240975745916:0] socket: 489 qp: -1
2025-11-27T18:18:27.089010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471245172509385:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:27.089062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:27.089469Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471240877542051:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(20362418ns) {HandleHandshakeStatus:329(20360533ns) {DropIncomingHandshake:442(2477ns) DropOutgoingHandshake:457(6057ns) SwitchToState:179(489ns) ProcessPendingSessionEvents:495(96224ns) {ForwardSessionEventToSession:612(48531ns) ForwardSessionEventToSession:612(3490ns) ForwardSessionEventToSession:612(8516ns) ForwardSessionEventToSession:612(4971ns) ForwardSessionEventToSession:612(5699ns) ForwardSessionEventToSession:612(2566ns) ForwardSessionEventToSession:612(3145ns) ForwardSessionEventToSession:612(5306ns) ForwardSessionEventToSession:612(2952ns) ForwardSessionEventToSession:612(2349ns)}}}
2025-11-27T18:18:27.019263Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471240764417073:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:26.995533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471242584456653:2049] [node 3] ICP22 created new session: [1:7577471242584456739:2049]
2025-11-27T18:18:27.051700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471245270713223:2108] [node 5] ICS09 handshake done sender: [4:7577471240975745917:2123] self: [4:7577471240975745916:0] peer: [5:7577471245172509378:0] socket: 483 qp: -1
2025-11-27T18:18:27.016922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471244689223505:2048] [node 1] ICS09 handshake done sender: [3:7577471240394256189:2122] self: [3:7577471240394256201:0] peer: [1:7577471242584456719:0] socket: 457 qp: -1
2025-11-27T18:18:27.025760Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471241260938697:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22399) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471241260938775:2116] held: no
2025-11-27T18:18:27.051751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471245270713223:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:27.023647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471245059384428:2116] [node 6] ICS09 handshake done sender: [2:7577471240764417111:2140] self: [2:7577471240764417110:0] peer: [6:7577471241260938784:0] socket: 459 qp: -1
2025-11-27T18:18:27.019421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471244689223505:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:27.021628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471240394256206:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.025535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:27.027285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471241260938775:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:27.023710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471245059384428:2116] [node 6] ICS10 traffic start
2025-11-27T18:18:27.051805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:27.052365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471240975745834:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:27.025358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471240764417104:2115] [node 5] ICP35 event processing took too much time PendingConnection:279(32770796ns) {HandleHandshakeStatus:329(32768220ns) {DropIncomingHandshake:442(1028ns) DropOutgoingHandshake:457(507ns) SwitchToState:179(306ns) ProcessPendingSessionEvents:495(1707597ns) {ForwardSessionEventToSession:612(1652694ns) ForwardSessionEventToSession:612(8389ns) ForwardSessionEventToSession:612(6044ns) ForwardSessionEventToSession:612(3205ns) ForwardSessionEventToSession:612(5531ns) ForwardSessionEventToSession:612(3411ns) ForwardSessionEventToSession:612(2403ns) ForwardSessionEventToSession:612(5528ns) ForwardSessionEventToSession:612(2341ns) ForwardSessionEventToSession:612(5237ns)}}}
2025-11-27T18:18:27.026686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:27.026869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471240764417013:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20537) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471240764417073:2107] held: no
2025-11-27T18:18:27.027912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471240764417105:2116] [node 6] ICP35 event processing took too much time PendingConnection:279(20191346ns) {HandleHandshakeStatus:329(20188703ns) {DropIncomingHandshake:442(17617ns) DropOutgoingHandshake:457(655ns) SwitchToState:179(624ns) ProcessPendingSessionEvents:495(113610ns) {ForwardSessionEventToSession:612(45698ns) ForwardSessionEventToSession:612(2976ns) ForwardSessionEventToSession:612(22534ns) ForwardSessionEventToSession:612(8914ns) ForwardSessionEventToSession:612(5525ns) ForwardSessionEventToSession:612(3007ns) ForwardSessionEventToSession:612(2842ns) ForwardSessionEventToSession:612(6588ns) ForwardSessionEventToSession:612(2805ns) ForwardSessionEventToSession:612(2940ns)}}}
2025-11-27T18:18:26.995537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:27.028176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471245059384427:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32599
2025-11-27T18:18:27.053595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471240975745836:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:27.031336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471240764417013:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.076396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471245270713220:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.032739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:27.033851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471240764417013:2048] [node 1] ICP22 created new session: [2:7577471245059384431:2048]
2025-11-27T18:18:27.077153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471240975745835:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:27.027189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471240394256112:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:27.027344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471241260938776:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:27.042907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471245059384431:2048] [node 1] ICS09 handshake done sender: [2:7577471240764417073:2107] self: [2:7577471240764417119:0] peer: [1:7577471242584456714:0] socket: 445 qp: -1
2025-11-27T18:18:27.038578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471241260938697:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.077288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471245270713229:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45393
2025-11-27T18:18:27.043491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471245059384431:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:27.038651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:27.080488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471245270713222:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45393
2025-11-27T18:18:27.045428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471240764417104:2115] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:27.038712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471241260938697:2049] [node 2] ICP22 created new session: [6:7577471245555906085:2049]
2025-11-27T18:18:27.090247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471245270713218:2125] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:27.045564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:27.038772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471245555906085:2049] [node 2] ICS09 handshake done sender: [6:7577471241260938775:2116] self: [6:7577471241260938784:0] peer: [2:7577471240764417110:0] socket: 460 qp: -1
2025-11-27T18:18:27.091706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471245270713230:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45393
2025-11-27T18:18:27.038820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471245555906085:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:27.046176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471240764417117:2142] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.099901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471240975745907:2108] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:27.038879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:27.046214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471240764417117:2142] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:27.104017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471240975745836:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3486) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471245270713219:2126] held: no
2025-11-27T18:18:27.059667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471240764417013:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22180416ns) {HandleHandshakeStatus:329(22178672ns) {DropIncomingHandshake:442(507ns) DropOutgoingHandshake:457(326ns) SwitchToState:179(215ns) ProcessPendingSessionEvents:495(1101ns)}}
2025-11-27T18:18:27.104114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471240975745834:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20537) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471245270713218:2125] held: no
2025-11-27T18:18:27.062362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471240764417117:2142] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:27.107290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471245270713219:2126] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:27.066898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471240764417014:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:27.107781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471240975745836:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.066992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:27.067041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471240764417014:2049] [node 3] ICP22 created new session: [2:7577471245059384435:2049]
2025-11-27T18:18:27.107850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:27.067094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471245059384435:2049] [node 3] ICS09 handshake done sender: [2:7577471240764417117:2142] self: [2:7577471240764417115:0] peer: [3:7577471244689223511:0] socket: 499 qp: -1
2025-11-27T18:18:27.107918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471240975745836:2050] [node 3] ICP22 created new session: [4:7577471245270713233:2050]
2025-11-27T18:18:27.039181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471241260938757:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.028183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471240394256193:2124] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:27.067132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471245059384435:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:26.998975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471242584456736:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.039216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:27.030169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471240394256187:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:27.107973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471245270713233:2050] [node 3] ICS09 handshake done sender: [4:7577471245270713219:2126] self: [4:7577471245270713227:0] peer: [3:7577471240394256202:0] socket: 508 qp: -1
2025-11-27T18:18:27.108008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471245270713233:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:27.108044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:27.108394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471240975745834:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.108492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:27.108534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471240975745834:2048] [node 1] ICP22 created new session: [4:7577471245270713236:2048]
2025-11-27T18:18:27.108576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471245270713236:2048] [node 1] ICS09 handshake done sender: [4:7577471245270713218:2125] self: [4:7577471245270713226:0] peer: [1:7577471242584456729:0] socket: 505 qp: -1
2025-11-27T18:18:27.108607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471245270713236:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:27.108675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:27.109094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471245270713232:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45393
2025-11-27T18:18:27.113524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471245270713220:2127] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:27.113753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471240975745835:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.113833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:27.113883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471240975745835:2049] [node 2] ICP22 created new session: [4:7577471245270713238:2049]
2025-11-27T18:18:27.113932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471245270713238:2049] [node 2] ICS09 handshake done sender: [4:7577471245270713220:2127] self: [4:7577471245270713231:0] peer: [2:7577471240764417118:0] socket: 509 qp: -1
2025-11-27T18:18:27.067165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:26.999053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471242584456739:2049] [node 3] ICS09 handshake done sender: [1:7577471242584456721:2111] self: [1:7577471242584456719:0] peer: [3:7577471240394256201:0] socket: 450 qp: -1
2025-11-27T18:18:27.113969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471245270713238:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:27.114017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:27.039258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471241260938757:2105] [node 5] ICP22 created new session: [6:7577471245555906087:2105]
2025-11-27T18:18:27.039325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471245555906087:2105] [node 5] ICS09 handshake done sender: [6:7577471241260938776:2117] self: [6:7577471241260938780:0] peer: [5:7577471240877542056:0] socket: 468 qp: -1
2025-11-27T18:18:27.030224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:27.039359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471245555906087:2105] [node 5] ICS10 traffic start
2025-11-27T18:18:27.031387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471240394256185:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:27.039422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:27.046335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471241260938698:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3486) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471241260938777:2118] held: no
2025-11-27T18:18:27.031435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:27.053383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471245555906084:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14713
2025-11-27T18:18:27.032408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471240394256187:2110] [node 6] ICP22 created new session: [3:7577471244689223513:2110]
2025-11-27T18:18:27.064422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471241260938777:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:27.032929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471240394256185:2108] [node 5] ICP22 created new session: [3:7577471244689223514:2108]
2025-11-27T18:18:27.040690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471244689223514:2108] [node 5] ICS09 handshake done sender: [3:7577471240394256191:2123] self: [3:7577471240394256190:0] peer: [5:7577471240877542077:0] socket: 467 qp: -1
2025-11-27T18:18:27.068851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471241260938698:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.042755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471244689223514:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:27.078035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:27.042896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471244689223513:2110] [node 6] ICS09 handshake done sender: [3:7577471240394256193:2124] self: [3:7577471240394256192:0] peer: [6:7577471241260938786:0] socket: 469 qp: -1
2025-11-27T18:18:27.080740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471241260938698:2050] [node 3] ICP22 created new session: [6:7577471245555906089:2050]
2025-11-27T18:18:27.044671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:27.045558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471244689223513:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:27.045609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:27.049952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471240394256185:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(22409454ns) {HandleHandshakeStatus:329(22407323ns) {DropIncomingHandshake:442(13326ns) DropOutgoingHandshake:457(298ns) SwitchToState:179(55ns) ProcessPendingSessionEvents:495(97279ns) {ForwardSessionEventToSession:612(50780ns) ForwardSessionEventToSession:612(3448ns) ForwardSessionEventToSession:612(6049ns) ForwardSessionEventToSession:612(4933ns) ForwardSessionEventToSession:612(8161ns) ForwardSessionEventToSession:612(3136ns) ForwardSessionEventToSession:612(2660ns) ForwardSessionEventToSession:612(5351ns) ForwardSessionEventToSession:612(2645ns) ForwardSessionEventToSession:612(2320ns)}}}
2025-11-27T18:18:27.052761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471244689223517:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40461
2025-11-27T18:18:27.052924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471240394256111:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(42457961ns) {HandleHandshakeStatus:329(42452454ns) {DropIncomingHandshake:442(2126ns) DropOutgoingHandshake:457(24288ns) SwitchToState:179(616ns) ProcessPendingSessionEvents:495(4918573ns) {ForwardSessionEventToSession:612(3250483ns) ForwardSessionEventToSession:612(6112ns) ForwardSessionEventToSession:612(616717ns) ForwardSessionEventToSession:612(11341ns) ForwardSessionEventToSession:612(971529ns) ForwardSessionEventToSession:612(13915ns) ForwardSessionEventToSession:612(3132ns) ForwardSessionEventToSession:612(21956ns) ForwardSessionEventToSession:612(2835ns) ForwardSessionEventToSession:612(1999ns)}}}
2025-11-27T18:18:27.054022Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471240394256187:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(23657140ns) {HandleHandshakeStatus:329(23654683ns) {DropIncomingHandshake:442(13007ns) DropOutgoingHandshake:457(318ns) SwitchToState:179(307ns) ProcessPendingSessionEvents:495(108547ns) {ForwardSessionEventToSession:612(42998ns) ForwardSessionEventToSession:612(6675ns) ForwardSessionEventToSession:612(6208ns) ForwardSessionEventToSession:612(3733ns) ForwardSessionEventToSession:612(21891ns) ForwardSessionEventToSession:612(2921ns) ForwardSessionEventToSession:612(6199ns) ForwardSessionEventToSession:612(2542ns) ForwardSessionEventToSession:612(2860ns) ForwardSessionEventToSession:612(5690ns)}}}
2025-11-27T18:18:27.055929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471240394256203:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.055962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471240394256203:2127] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:27.088851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471245555906089:2050] [node 3] ICS09 handshake done sender: [6:7577471241260938777:2118] self: [6:7577471241260938786:0] peer: [3:7577471240394256192:0] socket: 472 qp: -1
2025-11-27T18:18:27.065129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471240394256203:2127] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:27.091678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471245555906089:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:27.077808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471240394256208:2131] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:27.094406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:27.078082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471240394256112:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22399) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471240394256208:2131] held: no
2025-11-27T18:18:27.108343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471241260938698:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(38008535ns) {HandleHandshakeStatus:329(38004855ns) {DropIncomingHandshake:442(2875ns) DropOutgoingHandshake:457(508ns) SwitchToState:179(702ns) ProcessPendingSessionEvents:495(968ns)}}
2025-11-27T18:18:27.079264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471240394256186:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:27.109915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471245555906091:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14713
2025-11-27T18:18:27.079345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:27.079401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471240394256186:2107] [node 4] ICP22 created new session: [3:7577471244689223519:2107]
2025-11-27T18:18:27.086479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471240764417120:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.086512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471240764417120:2143] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:27.087543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471240764417120:2143] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:27.088945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471240764417106:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:27.089025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:27.089078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471240764417106:2108] [node 4] ICP22 created new session: [2:7577471245059384438:2108]
2025-11-27T18:18:27.091468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471245059384438:2108] [node 4] ICS09 handshake done sender: [2:7577471240764417120:2143] self: [2:7577471240764417118:0] peer: [4:7577471245270713231:0] socket: 502 qp: -1
2025-11-27T18:18:27.091749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471245059384438:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:27.094410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:27.079443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471244689223519:2107] [node 4] ICS09 handshake done sender: [3:7577471240394256203:2127] self: [3:7577471240394256202:0] peer: [4:7577471245270713227:0] socket: 507 qp: -1
2025-11-27T18:18:27.003038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471242584456739:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:27.103148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471245059384434:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32599
2025-11-27T18:18:27.104606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471245059384434:2147] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.079477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471244689223519:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:27.079511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:27.093062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471240394256112:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.094475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:27.095936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471240394256112:2049] [node 2] ICP22 created new session: [3:7577471244689223521:2049]
2025-11-27T18:18:27.103079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471244689223521:2049] [node 2] ICS09 handshake done sender: [3:7577471240394256208:2131] self: [3:7577471244689223511:0] peer: [2:7577471240764417115:0] socket: 500 qp: -1
2025-11-27T18:18:27.103566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471244689223521:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:27.106678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:27.003091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:27.005642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471242584456652:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:27.005688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:27.005739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471242584456652:2048] [node 2] ICP22 created new session: [1:7577471246879424038:2048]
2025-11-27T18:18:27.005780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471246879424038:2048] [node 2] ICS09 handshake done sender: [1:7577471242584456715:2108] self: [1:7577471242584456714:0] peer: [2:7577471240764417119:0] socket: 444 qp: -1
2025-11-27T18:18:27.005817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471246879424038:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:27.126380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471245555906091:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.005848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:27.006401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471242584456737:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.006707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471242584456652:2048] [node 2] ICP18 handshake (actor [1:7577471242584456736:2116]) is too old
2025-11-27T18:18:27.118457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471240764417106:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(19948879ns) {HandleHandshakeStatus:329(19945910ns) {DropIncomingHandshake:442(274ns) DropOutgoingHandshake:457(401ns) SwitchToState:179(366ns) ProcessPendingSessionEvents:495(5875519ns) {ForwardSessionEventToSession:612(3783541ns) ForwardSessionEventToSession:612(5672ns) ForwardSessionEventToSession:612(404331ns) ForwardSessionEventToSession:612(3870ns) ForwardSessionEventToSession:612(393239ns) ForwardSessionEventToSession:612(2920ns) ForwardSessionEventToSession:612(689439ns) ForwardSessionEventToSession:612(568197ns) ForwardSessionEventToSession:612(3737ns) ForwardSessionEventToSession:612(2350ns)}}}
2025-11-27T18:18:27.031157Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471242584456724:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(30043734ns) {HandleHandshakeStatus:329(30041299ns) {DropIncomingHandshake:442(219ns) DropOutgoingHandshake:457(443ns) SwitchToState:179(227ns) ProcessPendingSessionEvents:495(3151435ns) {ForwardSessionEventToSession:612(3086771ns) ForwardSessionEventToSession:612(8807ns) ForwardSessionEventToSession:612(3318ns) ForwardSessionEventToSession:612(10413ns) ForwardSessionEventToSession:612(3075ns) ForwardSessionEventToSession:612(5406ns) ForwardSessionEventToSession:612(3395ns) ForwardSessionEventToSession:612(3227ns) ForwardSessionEventToSession:612(5402ns) ForwardSessionEventToSession:612(8013ns)}}}
2025-11-27T18:18:27.033380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471242584456723:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.118897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471240764417106:2108] [node 4] ICP18 handshake (actor [2:7577471245059384434:2147]) is too old
2025-11-27T18:18:27.033411Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471242584456723:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:27.038423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471242584456723:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:27.126647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471241260938769:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:27.044340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471242584456720:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:27.045489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:27.045811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471242584456720:2109] [node 5] ICP22 created new session: [1:7577471246879424044:2109]
2025-11-27T18:18:27.047868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471246879424044:2109] [node 5] ICS09 handshake done sender: [1:7577471242584456723:2112] self: [1:7577471242584456722:0] peer: [5:7577471245172509380:0] socket: 503 qp: -1
2025-11-27T18:18:27.049272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471246879424044:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:27.049382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471242584456653:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(45077836ns) {HandleHandshakeStatus:329(45075193ns) {DropIncomingHandshake:442(1146ns) DropOutgoingHandshake:457(438ns) SwitchToState:179(289ns) ProcessPendingSessionEvents:495(17172654ns) {ForwardSessionEventToSession:612(8038126ns) ForwardSessionEventToSession:612(5158ns) ForwardSessionEventToSession:612(3628ns) ForwardSessionEventToSession:612(2087945ns) ForwardSessionEventToSession:612(3020ns) ForwardSessionEventToSession:612(5421442ns) ForwardSessionEventToSession:612(5299ns) ForwardSessionEventToSession:612(3153ns) ForwardSessionEventToSession:612(1578303ns) ForwardSessionEventToSession:612(5089ns)}}}
2025-11-27T18:18:27.051187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:27.060003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471242584456730:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.060032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471242584456730:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:27.063348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471242584456730:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:27.073270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471242584456713:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:27.073329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:27.073383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471242584456713:2107] [node 4] ICP22 created new session: [1:7577471246879424047:2107]
2025-11-27T18:18:27.073426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471246879424047:2107] [node 4] ICS09 handshake done sender: [1:7577471242584456730:2114] self: [1:7577471242584456729:0] peer: [4:7577471245270713226:0] socket: 504 qp: -1
2025-11-27T18:18:27.073462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471246879424047:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:27.073500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:27.127058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471240975745915:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:27.127099Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471240975745915:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:27.088846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471242584456720:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(29338222ns) {HandleHandshakeStatus:329(28692431ns) {DropIncomingHandshake:442(14426ns) DropOutgoingHandshake:457(323ns) SwitchToState:179(340ns) ProcessPendingSessionEvents:495(16984445ns) {ForwardSessionEventToSession:612(7544741ns) ForwardSessionEventToSession:612(7820ns) ForwardSessionEventToSession:612(3317ns) ForwardSessionEventToSession:612(770635ns) ForwardSessionEventToSession:612(3439ns) ForwardSessionEventToSession:612(3221ns) ForwardSessionEventToSession:612(240884ns) ForwardSessionEventToSession:612(534225ns) ForwardSessionEventToSession:612(7847575ns) ForwardSessionEventToSession:612(8158ns)}}}
2025-11-27T18:18:27.128184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471240975745915:2122] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:27.128553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471245555906093:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14713
2025-11-27T18:18:27.129401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471245555906091:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:27.129861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471241260938769:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:27.129938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:27.129985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471241260938769:2110] [node 4] ICP22 created new session: [6:7577471245555906094:2110]
2025-11-27T18:18:27.130044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471245555906094:2110] [node 4] ICS09 handshake done sender: [6:7577471245555906091:2123] self: [6:7577471245555906092:0] peer: [4:7577471240975745914:0] socket: 470 qp: -1
2025-11-27T18:18:27.130156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471245555906094:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:27.130204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:27.129603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471240975745906:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:27.129727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:27.129763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471240975745906:2109] [node 6] ICP22 created new session: [4:7577471245270713241:2109]
2025-11-27T18:18:27.131044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471245270713241:2109] [node 6] ICS09 handshake done sender: [4:7577471240975745915:2122] self: [4:7577471240975745914:0] peer: [6:7577471245555906092:0] socket: 463 qp: -1
2025-11-27T18:18:27.137551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471245270713241:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:27.138475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:27.148411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471240975745906:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18488905ns) {HandleHandshakeStatus:329(18482787ns) {DropIncomingHandshake:442(29434ns) DropOutgoingHandshake:457(360ns) SwitchToState:179(879ns) ProcessPendingSessionEvents:495(3365348ns) {ForwardSessionEventToSession:612(3296976ns) ForwardSessionEventToSession:612(6516ns) ForwardSessionEventToSession:612(5809ns) ForwardSessionEventToSession:612(3127ns) ForwardSessionEventToSession:612(9648ns) ForwardSessionEventToSession:612(3523ns) ForwardSessionEventToSession:612(3453ns) ForwardSessionEventToSession:612(6624ns) ForwardSessionEventToSession:612(5147ns) ForwardSessionEventToSession:612(2246ns)}}}
2025-11-27T18:18:27.205692Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:27.205962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471245059384431:2048] [node 1] ICS01 socket: 445 reason# EndOfStream
2025-11-27T18:18:27.206013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471240764417013:2048] [node 1] ICP30 unregister session Session# [2:7577471245059384431:2048] VirtualId# [2:7577471240764417119:0]
2025-11-27T18:18:27.206049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471245059384431:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.206143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:27.206193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:27.207334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:27.207540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471244689223505:2048] [node 1] ICS07 socket disconnect 457 reason# EndOfStream
2025-11-27T18:18:27.207573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471244689223505:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.207658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:27.207703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471244689223505:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:27.211769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471240394256111:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20537) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:27.211831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471244689223505:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:27.213737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:27.213953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471245270713236:2048] [node 1] ICS01 socket: 505 reason# EndOfStream
2025-11-27T18:18:27.213999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471240975745834:2048] [node 1] ICP30 unregister session Session# [4:7577471245270713236:2048] VirtualId# [4:7577471245270713226:0]
2025-11-27T18:18:27.214038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471245270713236:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.214111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:27.214150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:27.214493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471240394256111:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20537) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:27.214987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471244689223505:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:27.215251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471240394256111:2048] [node 1] ICP30 unregister session Session# [3:7577471244689223505:2048] VirtualId# [3:7577471240394256201:0]
2025-11-27T18:18:27.215304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:27.215355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471240394256111:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20537) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:27.215373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20537) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:27.214957Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:27.215756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471245172509386:2048] [node 1] ICS01 socket: 506 reason# EndOfStream
2025-11-27T18:18:27.215806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471240877541988:2048] [node 1] ICP30 unregister session Session# [5:7577471245172509386:2048] VirtualId# [5:7577471245172509380:0]
2025-11-27T18:18:27.215838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471245172509386:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.215907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:27.215953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:27.219513Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:27.219732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471241260938782:2048] [node 1] ICS01 socket: 452 reason# EndOfStream
2025-11-27T18:18:27.219780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471241260938696:2048] [node 1] ICP30 unregister session Session# [6:7577471241260938782:2048] VirtualId# [6:7577471241260938778:0]
2025-11-27T18:18:27.219817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471241260938782:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.219895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:27.219937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:27.237394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471240394256111:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:27.249946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:27.250205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471244689223521:2049] [node 2] ICS01 socket: 500 reason# EndOfStream
2025-11-27T18:18:27.250247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471240394256112:2049] [node 2] ICP30 unregister session Session# [3:7577471244689223521:2049] VirtualId# [3:7577471244689223511:0]
2025-11-27T18:18:27.250279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471244689223521:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.250369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:27.250414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:27.254796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:27.255037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471245270713238:2049] [node 2] ICS01 socket: 509 reason# EndOfStream
2025-11-27T18:18:27.255077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471240975745835:2049] [node 2] ICP30 unregister session Session# [4:7577471245270713238:2049] VirtualId# [4:7577471245270713231:0]
2025-11-27T18:18:27.255112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471245270713238:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.255187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:27.255248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:27.256048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:27.256243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471245172509376:2049] [node 2] ICS01 socket: 462 reason# EndOfStream
2025-11-27T18:18:27.256270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471240877541989:2049] [node 2] ICP30 unregister session Session# [5:7577471245172509376:2049] VirtualId# [5:7577471240877542071:0]
2025-11-27T18:18:27.256294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471245172509376:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.256367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:27.256436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:27.261583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:27.261838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471245555906085:2049] [node 2] ICS01 socket: 460 reason# EndOfStream
2025-11-27T18:18:27.261891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471241260938697:2049] [node 2] ICP30 unregister session Session# [6:7577471245555906085:2049] VirtualId# [6:7577471241260938784:0]
2025-11-27T18:18:27.261929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471245555906085:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.262040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:27.262096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:27.306856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:27.307089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471245270713233:2050] [node 3] ICS01 socket: 508 reason# EndOfStream
2025-11-27T18:18:27.307134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471240975745836:2050] [node 3] ICP30 unregister session Session# [4:7577471245270713233:2050] VirtualId# [4:7577471245270713227:0]
2025-11-27T18:18:27.307172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471245270713233:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.307252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:27.307302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:27.309552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:27.309802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471245555906089:2050] [node 3] ICS01 socket: 472 reason# EndOfStream
2025-11-27T18:18:27.309831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471241260938698:2050] [node 3] ICP30 unregister session Session# [6:7577471245555906089:2050] VirtualId# [6:7577471241260938786:0]
2025-11-27T18:18:27.309862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471245555906089:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.309937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:27.309995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:27.317400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:27.317666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471245172509382:2050] [node 3] ICS07 socket disconnect 473 reason# EndOfStream
2025-11-27T18:18:27.317698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471245172509382:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.317797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:27.317843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471245172509382:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:27.319689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471240877541990:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3486) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:27.319763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471245172509382:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:27.319799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471240877541990:2050] [node 3] ICP30 unregister session Session# [5:7577471245172509382:2050] VirtualId# [5:7577471240877542077:0]
2025-11-27T18:18:27.319847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:27.319880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471240877541990:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3486) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:27.319894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3486) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:27.329363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471240877541990:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:27.370620Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:27.370914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471245172509385:2108] [node 4] ICS01 socket: 489 reason# EndOfStream
2025-11-27T18:18:27.370964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471240877542051:2108] [node 4] ICP30 unregister session Session# [5:7577471245172509385:2108] VirtualId# [5:7577471245172509378:0]
2025-11-27T18:18:27.371002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471245172509385:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.371088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:27.371147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:27.373387Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:27.373620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471245555906094:2110] [node 4] ICS01 socket: 470 reason# EndOfStream
2025-11-27T18:18:27.373672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471241260938769:2110] [node 4] ICP30 unregister session Session# [6:7577471245555906094:2110] VirtualId# [6:7577471245555906092:0]
2025-11-27T18:18:27.373709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471245555906094:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.373803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:27.373852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:27.403027Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:27.403289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471245555906087:2105] [node 5] ICS01 socket: 468 reason# EndOfStream
2025-11-27T18:18:27.403334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471241260938757:2105] [node 5] ICP30 unregister session Session# [6:7577471245555906087:2105] VirtualId# [6:7577471241260938780:0]
2025-11-27T18:18:27.403368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471245555906087:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:27.403445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:27.403489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:27.547980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471247137007984:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:27.548032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471247137007985:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:27.575141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471248079582031:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:27.575567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471248079582030:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:27.597042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471247766678460:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:27.600763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471247766678459:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:27.626271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471244598184717:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:27.626313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471244598184718:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:27.626347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471244598184719:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:27.651462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471247762677510:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:27.651497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471247762677511:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:27.651564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471247762677512:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:27.678532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471246405051798:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:27.678591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471246405051799:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:27.678629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471246405051800:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:27.821689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471246405051859:2106] [node 5] ICP01 ready to work
2025-11-27T18:18:27.831175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471247137008047:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:27.845806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471247766678523:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:27.863153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471248079582094:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:27.941462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471247766678527:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:27.930604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471247137008048:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:27.946549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471244598184798:2126] [node 6] ICP01 ready to work
2025-11-27T18:18:27.964561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471244598184799:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:27.958573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471247762677574:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:27.962049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471247762677575:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:27.969915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471247766678528:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:27.962120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471248079582095:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:27.930673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471247137008049:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:27.994516Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471247137007984:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(197434834ns) {Configure:104(197428321ns) {ProcessConfigured:135(83342ns) {StartInitialHandshake:157(77469ns) {DropHandshakes:470(2286ns) {DropIncomingHandshake:442(547ns) DropOutgoingHandshake:457(305ns)} GenerateSessionVirtualId:988(1213ns) PrepareNewSessionHandshake:477(1305ns) {SwitchToState:179(461ns)}}}}}
2025-11-27T18:18:27.973658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471248079582096:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:27.990022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471246405051862:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.015875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471246405051859:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(175462427ns) {Configure:104(175451969ns) {ProcessConfigured:135(80631ns) {StartInitialHandshake:157(77524ns) {DropHandshakes:470(2042ns) {DropIncomingHandshake:442(446ns) DropOutgoingHandshake:457(280ns)} GenerateSessionVirtualId:988(834ns) PrepareNewSessionHandshake:477(1520ns) {SwitchToState:179(419ns)}}}}}
2025-11-27T18:18:28.023225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471246405051800:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(81722608ns) {Configure:104(76670595ns) {ProcessConfigured:135(77701ns) {StartInitialHandshake:157(75050ns) {DropHandshakes:470(2574ns) {DropIncomingHandshake:442(517ns) DropOutgoingHandshake:457(390ns)} GenerateSessionVirtualId:988(828ns) PrepareNewSessionHandshake:477(1619ns) {SwitchToState:179(569ns)}}}}}
2025-11-27T18:18:28.028741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471246405051798:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(115100098ns) {Configure:104(115079121ns) {ProcessConfigured:135(88015ns) {StartInitialHandshake:157(86481ns) {DropHandshakes:470(1103ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(387ns) PrepareNewSessionHandshake:477(1163ns) {SwitchToState:179(430ns)}}}}}
2025-11-27T18:18:28.018188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471248079582030:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(204181592ns) {Configure:104(196214852ns) {ProcessConfigured:135(91067ns) {StartInitialHandshake:157(88098ns) {DropHandshakes:470(2848ns) {DropIncomingHandshake:442(571ns) DropOutgoingHandshake:457(939ns)} GenerateSessionVirtualId:988(567ns) PrepareNewSessionHandshake:477(1477ns) {SwitchToState:179(675ns)}}}}}
2025-11-27T18:18:28.026715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471247766678459:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(196921302ns) {Configure:104(196914748ns) {ProcessConfigured:135(2400295ns) {StartInitialHandshake:157(2397348ns) {DropHandshakes:470(1187ns) {DropIncomingHandshake:442(281ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(552ns) PrepareNewSessionHandshake:477(3650ns) {SwitchToState:179(1672ns)}}}}}
2025-11-27T18:18:28.034756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471246405051799:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(173677861ns) {Configure:104(173671088ns) {ProcessConfigured:135(7637060ns) {StartInitialHandshake:157(7634645ns) {DropHandshakes:470(1138ns) {DropIncomingHandshake:442(326ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(367ns) PrepareNewSessionHandshake:477(2197ns) {SwitchToState:179(772ns)}}}}}
2025-11-27T18:18:28.050652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471244598184718:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(103584650ns) {Configure:104(103578308ns) {ProcessConfigured:135(82369ns) {StartInitialHandshake:157(79282ns) {DropHandshakes:470(3216ns) {DropIncomingHandshake:442(749ns) DropOutgoingHandshake:457(534ns)} GenerateSessionVirtualId:988(1183ns) PrepareNewSessionHandshake:477(2181ns) {SwitchToState:179(704ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.018683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471252374549395:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22572
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.089158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471248079582095:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(107156997ns) {Configure:104(107148441ns) {ProcessConfigured:135(107808ns) {StartInitialHandshake:157(104361ns) {DropHandshakes:470(2737ns) {DropIncomingHandshake:442(861ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(1225ns) PrepareNewSessionHandshake:477(1775ns) {SwitchToState:179(540ns)}}}}}
2025-11-27T18:18:28.094541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471247137008049:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(136134851ns) {Configure:104(136124553ns) {ProcessConfigured:135(16226989ns) {StartInitialHandshake:157(16224491ns) {DropHandshakes:470(3352ns) {DropIncomingHandshake:442(574ns) DropOutgoingHandshake:457(678ns)} GenerateSessionVirtualId:988(1369ns) PrepareNewSessionHandshake:477(3141ns) {SwitchToState:179(1817ns)}}}}}
2025-11-27T18:18:28.058311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471252061645829:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47944
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.106020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471252374549399:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22572
2025-11-27T18:18:28.109474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471251431975350:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39228
2025-11-27T18:18:28.111176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471251431975357:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39228
2025-11-27T18:18:28.066105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471250700019170:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21612
2025-11-27T18:18:28.066470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471247766678460:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(206706014ns) {Configure:104(206700967ns) {ProcessConfigured:135(11316918ns) {StartInitialHandshake:157(11314433ns) {DropHandshakes:470(1317ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(287ns) PrepareNewSessionHandshake:477(3834ns) {SwitchToState:179(1688ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.095748Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471244598184798:2126] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(82926006ns) {Configure:104(82915428ns) {ProcessConfigured:135(20857894ns) {StartInitialHandshake:157(16849720ns) {DropHandshakes:470(2250ns) {DropIncomingHandshake:442(410ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(1517ns) PrepareNewSessionHandshake:477(4617ns) {SwitchToState:179(2214ns)}}}}}
2025-11-27T18:18:28.136827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471247762677574:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(160531307ns) {Configure:104(160521600ns) {ProcessConfigured:135(34480793ns) {StartInitialHandshake:157(34476709ns) {DropHandshakes:470(10230246ns) {DropIncomingHandshake:442(3071ns) DropOutgoingHandshake:457(289ns)} GenerateSessionVirtualId:988(6082ns) PrepareNewSessionHandshake:477(4000ns) {SwitchToState:179(2069ns)}}}}}
2025-11-27T18:18:28.093635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471246405051862:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(80530405ns) {Configure:104(80521460ns) {ProcessConfigured:135(104764ns) {StartInitialHandshake:157(102225ns) {DropHandshakes:470(3749ns) {DropIncomingHandshake:442(1062ns) DropOutgoingHandshake:457(454ns)} GenerateSessionVirtualId:988(1028ns) PrepareNewSessionHandshake:477(2191ns) {SwitchToState:179(819ns)}}}}}
2025-11-27T18:18:28.088258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471252061645829:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.112660Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471247137008047:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(153377281ns) {Configure:104(153367272ns) {ProcessConfigured:135(7612084ns) {StartInitialHandshake:157(7609480ns) {DropHandshakes:470(3270ns) {DropIncomingHandshake:442(749ns) DropOutgoingHandshake:457(363ns)} GenerateSessionVirtualId:988(796ns) PrepareNewSessionHandshake:477(2996ns) {SwitchToState:179(1260ns)}}}}}
2025-11-27T18:18:28.140885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471252374549399:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.165709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471247762677510:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(172513133ns) {Configure:104(172507124ns) {ProcessConfigured:135(27985586ns) {StartInitialHandshake:157(27983644ns) {DropHandshakes:470(1713ns) {DropIncomingHandshake:442(390ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:477(1749ns) {SwitchToState:179(395ns)}}}}}
2025-11-27T18:18:28.165802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471247762677575:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(175230356ns) {Configure:104(175217241ns) {ProcessConfigured:135(4579556ns) {StartInitialHandshake:157(4576992ns) {DropHandshakes:470(4507341ns) {DropIncomingHandshake:442(556ns) DropOutgoingHandshake:457(1771ns)} GenerateSessionVirtualId:988(3161ns) PrepareNewSessionHandshake:477(2487ns) {SwitchToState:179(1449ns)}}}}}
2025-11-27T18:18:28.166000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471247762677576:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8457
2025-11-27T18:18:28.118409Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471244598184717:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(156248802ns) {Configure:104(156242170ns) {ProcessConfigured:135(15913049ns) {StartInitialHandshake:157(15910004ns) {DropHandshakes:470(1926ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(923ns) PrepareNewSessionHandshake:477(4129ns) {SwitchToState:179(2199ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.117887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471247137007985:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(262398138ns) {Configure:104(262387482ns) {ProcessConfigured:135(30813407ns) {StartInitialHandshake:157(30810314ns) {DropHandshakes:470(3769ns) {DropIncomingHandshake:442(987ns) DropOutgoingHandshake:457(809ns)} GenerateSessionVirtualId:988(1112ns) PrepareNewSessionHandshake:477(5480ns) {SwitchToState:179(3111ns)}}}}}
2025-11-27T18:18:28.149606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471248079582094:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(155726895ns) {Configure:104(155719944ns) {ProcessConfigured:135(16494632ns) {StartInitialHandshake:157(16490996ns) {DropHandshakes:470(2236ns) {DropIncomingHandshake:442(462ns) DropOutgoingHandshake:457(348ns)} GenerateSessionVirtualId:988(1001ns) PrepareNewSessionHandshake:477(3827ns) {SwitchToState:179(1997ns)}}}}}
2025-11-27T18:18:28.174197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471247762677512:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(203890052ns) {Configure:104(203879324ns) {ProcessConfigured:135(31145832ns) {StartInitialHandshake:157(31142987ns) {DropHandshakes:470(2780ns) {DropIncomingHandshake:442(671ns) DropOutgoingHandshake:457(468ns)} GenerateSessionVirtualId:988(661ns) PrepareNewSessionHandshake:477(4559ns) {SwitchToState:179(2069ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.126984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471244598184799:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(162073263ns) {Configure:104(162067800ns) {ProcessConfigured:135(24647770ns) {StartInitialHandshake:157(24642319ns) {DropHandshakes:470(3015ns) {DropIncomingHandshake:442(761ns) DropOutgoingHandshake:457(453ns)} GenerateSessionVirtualId:988(1187ns) PrepareNewSessionHandshake:477(4336ns) {SwitchToState:179(2399ns)}}}}}
2025-11-27T18:18:28.133604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471251431975357:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.170016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471252061645835:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47944
2025-11-27T18:18:28.140731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471251431975350:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.176747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471250700019170:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.137762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471248893152108:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-11-27T18:18:28.151065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471248079582031:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(274820821ns) {Configure:104(269322665ns) {ProcessConfigured:135(32028397ns) {StartInitialHandshake:157(28083632ns) {DropHandshakes:470(2246ns) {DropIncomingHandshake:442(677ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(1288ns) PrepareNewSessionHandshake:477(4978ns) {SwitchToState:179(2357ns)}}}}}
2025-11-27T18:18:28.185303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471251431975349:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39228
2025-11-27T18:18:28.189047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471247766678528:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(187725930ns) {Configure:104(187717838ns) {ProcessConfigured:135(11325449ns) {StartInitialHandshake:157(11323562ns) {DropHandshakes:470(2784ns) {DropIncomingHandshake:442(723ns) DropOutgoingHandshake:457(796ns)} GenerateSessionVirtualId:988(906ns) PrepareNewSessionHandshake:477(2271ns) {SwitchToState:179(486ns)}}}}}
2025-11-27T18:18:28.185471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471247137008048:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(203767167ns) {Configure:104(203758099ns) {ProcessConfigured:135(47228834ns) {StartInitialHandshake:157(47225588ns) {DropHandshakes:470(2962ns) {DropIncomingHandshake:442(497ns) DropOutgoingHandshake:457(368ns)} GenerateSessionVirtualId:988(693ns) PrepareNewSessionHandshake:477(3130ns) {SwitchToState:179(1242ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.177404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471252374549395:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.186734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471248079582096:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(179619526ns) {Configure:104(179609826ns) {ProcessConfigured:135(8141936ns) {StartInitialHandshake:157(8139060ns) {DropHandshakes:470(6012ns) {DropIncomingHandshake:442(852ns) DropOutgoingHandshake:457(694ns)} GenerateSessionVirtualId:988(1033ns) PrepareNewSessionHandshake:477(2711ns) {SwitchToState:179(1074ns)}}}}}
2025-11-27T18:18:28.213746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471251431975365:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39228
2025-11-27T18:18:28.198799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471252374549397:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22572
2025-11-27T18:18:28.213536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471248079582030:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:28.202853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471246405051862:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:28.194261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471247762677576:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.217890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471251431975349:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.194370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471244598184719:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(204571047ns) {Configure:104(204562315ns) {ProcessConfigured:135(53817346ns) {StartInitialHandshake:157(35367582ns) {DropHandshakes:470(5095ns) {DropIncomingHandshake:442(1838ns) DropOutgoingHandshake:457(412ns)} GenerateSessionVirtualId:988(2277ns) PrepareNewSessionHandshake:477(2719ns) {SwitchToState:179(1455ns)}}}}}
2025-11-27T18:18:28.194669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471252061645832:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47944
2025-11-27T18:18:28.201481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471252057644880:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8457
2025-11-27T18:18:28.221703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471248893152110:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.244366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471250700019175:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21612
2025-11-27T18:18:28.218192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471252374549397:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.233427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471247137008051:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.201696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471252057644879:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8457
2025-11-27T18:18:28.202654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471247766678527:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(212343068ns) {Configure:104(209288582ns) {ProcessConfigured:135(25018001ns) {StartInitialHandshake:157(25014877ns) {DropHandshakes:470(3221ns) {DropIncomingHandshake:442(919ns) DropOutgoingHandshake:457(439ns)} GenerateSessionVirtualId:988(1190ns) PrepareNewSessionHandshake:477(1429ns) {SwitchToState:179(57ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.278616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471250700019175:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.222171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471248893152097:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.203470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471252061645832:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.202134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471247762677512:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.253746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471252374549403:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.233475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471247137008051:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.292976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471250700019177:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21612
2025-11-27T18:18:28.253800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471252374549403:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.222205Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471248893152097:2128] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.241389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471251431975353:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.203629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471247762677511:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(227730578ns) {Configure:104(227719508ns) {ProcessConfigured:135(40827073ns) {StartInitialHandshake:157(40822619ns) {DropHandshakes:470(3873ns) {DropIncomingHandshake:442(1522ns) DropOutgoingHandshake:457(331ns)} GenerateSessionVirtualId:988(2973ns) PrepareNewSessionHandshake:477(3430ns) {SwitchToState:179(1251ns)}}}}}
2025-11-27T18:18:28.214058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471247766678525:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.268440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471252374549403:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:28.214097Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471247766678525:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.213952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471252057644879:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.228287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471248893152108:2132] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.297023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471246405051798:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.241425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471251431975353:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.228459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471244598184717:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:28.235232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471252057644880:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.252595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471252057644891:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8457
2025-11-27T18:18:28.253653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471247762677511:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:28.267878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471252057644892:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8457
2025-11-27T18:18:28.274819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471247762677511:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(18253212ns) {IncomingHandshake:261(18251619ns) {DropIncomingHandshake:442(227ns) PrepareNewSessionHandshake:477(725ns) {SwitchToState:179(345ns)} GenerateSessionVirtualId:988(382ns) IssueIncomingHandshakeReply:193(8110404ns) {SwitchToState:179(1561ns)}}}
2025-11-27T18:18:28.325659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471247762677576:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:28.326538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471252057644891:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.326676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471247762677510:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:28.331089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471247762677512:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.331153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:28.331198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471247762677512:2050] [node 3] ICP22 created new session: [5:7577471252057644894:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.268760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471248079582031:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.217936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471247766678460:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:28.303457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471250700019177:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.242575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471251431975353:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:28.230454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471248893152097:2128] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.230643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471252061645835:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.268876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:28.362559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471252057644894:2050] [node 3] ICS09 handshake done sender: [5:7577471247762677576:2111] self: [5:7577471252057644886:0] peer: [3:7577471247766678524:0] socket: 451 qp: -1
2025-11-27T18:18:28.242904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471247137008047:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.303606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471246405051799:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.234025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471247766678459:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.313530Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471246405051798:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18265566ns) {IncomingHandshake:261(18263221ns) {DropIncomingHandshake:442(814ns) PrepareNewSessionHandshake:477(2854ns) {SwitchToState:179(1113ns)} GenerateSessionVirtualId:988(2047ns) IssueIncomingHandshakeReply:193(7493046ns) {SwitchToState:179(1605ns)}}}
2025-11-27T18:18:28.342008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471250700019174:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21612
2025-11-27T18:18:28.242952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:28.261755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471248893152110:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.268924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471248079582031:2049] [node 3] ICP22 created new session: [2:7577471252374549413:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.373792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471252057644894:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:28.266899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471252061645843:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47944
2025-11-27T18:18:28.268968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471252374549413:2049] [node 3] ICS09 handshake done sender: [2:7577471252374549403:2116] self: [2:7577471252374549398:0] peer: [3:7577471252061645839:0] socket: 473 qp: -1
2025-11-27T18:18:28.270296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471244598184798:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.399180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471250700019179:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21612
2025-11-27T18:18:28.373861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.243001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471247137008047:2107] [node 4] ICP22 created new session: [1:7577471251431975368:2107]
2025-11-27T18:18:28.269830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471247766678525:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:28.268999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471252374549413:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:28.270335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.405636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471252057644895:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8457
2025-11-27T18:18:28.408130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471246405051798:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15513) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471250700019175:2117] held: no
2025-11-27T18:18:28.243047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471251431975368:2107] [node 4] ICS09 handshake done sender: [1:7577471251431975353:2113] self: [1:7577471251431975351:0] peer: [4:7577471248893152111:0] socket: 479 qp: -1
2025-11-27T18:18:28.285379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471247766678459:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15513) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471252061645835:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.277557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471244598184798:2126] [node 6] ICP22 created new session: [4:7577471248893152114:2126]
2025-11-27T18:18:28.412986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471250700019180:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21612
2025-11-27T18:18:28.414524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471247762677510:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15513) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471252057644891:2119] held: no
2025-11-27T18:18:28.269034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.296096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471252061645841:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47944
2025-11-27T18:18:28.243078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471251431975368:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:28.291122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471248893152114:2126] [node 6] ICS09 handshake done sender: [4:7577471248893152097:2128] self: [4:7577471248893152096:0] peer: [6:7577471250700019173:0] socket: 462 qp: -1
2025-11-27T18:18:28.305726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471247766678460:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11352) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471252061645832:2115] held: no
2025-11-27T18:18:28.243130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:28.413676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471250700019170:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:28.273434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471252374549402:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.431416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471252057644880:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:28.417804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471250700019181:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21612
2025-11-27T18:18:28.307986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471247766678523:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.296508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471248893152114:2126] [node 6] ICS10 traffic start
2025-11-27T18:18:28.248955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471251431975365:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.273464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471252374549402:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.431450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471252057644896:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8457
2025-11-27T18:18:28.296584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.249211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471251431975355:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.419206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471246405051862:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.433246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471247762677510:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22911933ns) {HandleHandshakeStatus:395(22908733ns) {LogHandshakeFail:472(11607ns) {UpdateErrorStateLog:545(4349ns)} DropOutgoingHandshake:457(1279ns) ProcessPendingSessionEvents:495(13126280ns) {DropSessionEvent:512(13208ns) DropSessionEvent:512(17412ns) DropSessionEvent:512(5610ns) DropSessionEvent:512(6016852ns) DropSessionEvent:512(14177ns) DropSessionEvent:512(5351ns) DropSessionEvent:512(5232ns) DropSessionEvent:512(4681ns) DropSessionEvent:512(4931ns) DropSessionEvent:512(17711ns)}}}
2025-11-27T18:18:28.434537Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471247762677512:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(102178385ns) {HandleHandshakeStatus:329(98267483ns) {DropIncomingHandshake:442(849ns) DropOutgoingHandshake:457(6312ns) SwitchToState:179(524ns) ProcessPendingSessionEvents:495(21635256ns) {ForwardSessionEventToSession:612(7185283ns) ForwardSessionEventToSession:612(4269704ns) ForwardSessionEventToSession:612(17355ns) ForwardSessionEventToSession:612(7546ns) ForwardSessionEventToSession:612(3046673ns) ForwardSessionEventToSession:612(3437ns) ForwardSessionEventToSession:612(2201486ns) ForwardSessionEventToSession:612(2826ns) ForwardSessionEventToSession:612(2453ns) ForwardSessionEventToSession:612(1588565ns)}}}
2025-11-27T18:18:28.435454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471247762677511:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.439774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:28.444223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471247762677511:2049] [node 2] ICP22 created new session: [5:7577471252057644901:2049]
2025-11-27T18:18:28.447922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471252057644898:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8457
2025-11-27T18:18:28.454759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471252057644896:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.457689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471252057644901:2049] [node 2] ICS09 handshake done sender: [5:7577471252057644880:2115] self: [5:7577471252057644890:0] peer: [2:7577471252374549401:0] socket: 481 qp: -1
2025-11-27T18:18:28.466595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471252057644901:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:28.308080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:28.292067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471252374549402:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:28.421093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471250700019179:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.422164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471246405051800:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:28.422255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471250700019175:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:28.422594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:28.423184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471250700019177:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:28.425693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471246405051862:2109] [node 4] ICP22 created new session: [6:7577471250700019182:2109]
2025-11-27T18:18:28.426714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471246405051798:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.426758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:28.426807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471246405051798:2048] [node 1] ICP22 created new session: [6:7577471250700019184:2048]
2025-11-27T18:18:28.426851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471250700019184:2048] [node 1] ICS09 handshake done sender: [6:7577471250700019175:2117] self: [6:7577471250700019176:0] peer: [1:7577471251431975354:0] socket: 496 qp: -1
2025-11-27T18:18:28.426884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471250700019184:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:28.426922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:28.427484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471250700019186:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21612
2025-11-27T18:18:28.429393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471250700019182:2109] [node 4] ICS09 handshake done sender: [6:7577471250700019170:2114] self: [6:7577471250700019173:0] peer: [4:7577471248893152096:0] socket: 464 qp: -1
2025-11-27T18:18:28.429423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471246405051799:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.429895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:28.430587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471250700019182:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:28.430709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471246405051799:2049] [node 2] ICP22 created new session: [6:7577471250700019188:2049]
2025-11-27T18:18:28.431589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:28.433720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471250700019188:2049] [node 2] ICS09 handshake done sender: [6:7577471250700019177:2118] self: [6:7577471250700019178:0] peer: [2:7577471252374549400:0] socket: 508 qp: -1
2025-11-27T18:18:28.434812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471250700019188:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:28.435367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:28.474765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471246405051799:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(46166059ns) {HandleHandshakeStatus:329(46163517ns) {DropIncomingHandshake:442(1873ns) DropOutgoingHandshake:457(5005ns) SwitchToState:179(269ns) ProcessPendingSessionEvents:495(24403331ns) {ForwardSessionEventToSession:612(16673640ns) ForwardSessionEventToSession:612(5502ns) ForwardSessionEventToSession:612(7295ns) ForwardSessionEventToSession:612(2807ns) ForwardSessionEventToSession:612(2300ns) ForwardSessionEventToSession:612(21576ns) ForwardSessionEventToSession:612(10332ns) ForwardSessionEventToSession:612(2894ns) ForwardSessionEventToSession:612(4836ns) ForwardSessionEventToSession:612(2951ns)}}}
2025-11-27T18:18:28.326062Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471244598184798:2126] [node 6] ICP35 event processing took too much time PendingConnection:279(71945489ns) {HandleHandshakeStatus:329(64020536ns) {DropIncomingHandshake:442(1741ns) DropOutgoingHandshake:457(1334ns) SwitchToState:179(413ns) ProcessPendingSessionEvents:495(7185932ns) {ForwardSessionEventToSession:612(7126938ns) ForwardSessionEventToSession:612(5971ns) ForwardSessionEventToSession:612(4161ns) ForwardSessionEventToSession:612(9074ns) ForwardSessionEventToSession:612(6583ns) ForwardSessionEventToSession:612(2892ns) ForwardSessionEventToSession:612(5385ns) ForwardSessionEventToSession:612(2958ns) ForwardSessionEventToSession:612(5467ns) ForwardSessionEventToSession:612(2831ns)}}}
2025-11-27T18:18:28.486664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471250700019186:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.486836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471246405051859:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:28.471056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:28.298366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471252374549417:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22572
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.489181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471250700019191:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21612
2025-11-27T18:18:28.249234Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471251431975355:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.308129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471247766678523:2108] [node 5] ICP22 created new session: [3:7577471252061645845:2108]
2025-11-27T18:18:28.493312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471250700019179:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:28.338353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471248893152119:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-11-27T18:18:28.250870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471251431975355:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:28.299218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471252374549395:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:28.308185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471252061645845:2108] [node 5] ICS09 handshake done sender: [3:7577471247766678525:2111] self: [3:7577471247766678524:0] peer: [5:7577471252057644886:0] socket: 450 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.338675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471248893152118:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-11-27T18:18:28.480059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471252057644891:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:28.300637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471248079582095:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.494100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471246405051862:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(57263456ns) {HandleHandshakeStatus:329(56540930ns) {DropIncomingHandshake:442(615ns) DropOutgoingHandshake:457(5968ns) SwitchToState:179(492ns) ProcessPendingSessionEvents:495(40423522ns) {ForwardSessionEventToSession:612(12484823ns) ForwardSessionEventToSession:612(5110ns) ForwardSessionEventToSession:612(3435ns) ForwardSessionEventToSession:612(2456140ns) ForwardSessionEventToSession:612(3547ns) ForwardSessionEventToSession:612(7667496ns) ForwardSessionEventToSession:612(12706525ns) ForwardSessionEventToSession:612(8803ns) ForwardSessionEventToSession:612(5060867ns) ForwardSessionEventToSession:612(4538ns)}}}
2025-11-27T18:18:28.251117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471247137007985:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.485503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471247762677574:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:28.308223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471252061645845:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:28.300698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:28.339143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471248893152119:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.496102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471246405051800:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.308276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:28.251181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:28.496662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.485773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471247762677510:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.301830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471248079582030:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.317593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471247766678459:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21330863ns) {HandleHandshakeStatus:395(21327600ns) {LogHandshakeFail:472(7281238ns) {UpdateErrorStateLog:545(5721ns)} DropOutgoingHandshake:457(4801ns) ProcessPendingSessionEvents:495(5996386ns) {DropSessionEvent:512(16776ns) DropSessionEvent:512(6120ns) DropSessionEvent:512(21832ns) DropSessionEvent:512(5687ns) DropSessionEvent:512(5890ns) DropSessionEvent:512(5348ns) DropSessionEvent:512(6183ns) DropSessionEvent:512(5033ns) DropSessionEvent:512(6727ns) DropSessionEvent:512(5640ns)}}}
2025-11-27T18:18:28.339262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471244598184719:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:28.485827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:28.498197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471246405051800:2050] [node 3] ICP22 created new session: [6:7577471250700019193:2050]
2025-11-27T18:18:28.251233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471247137007985:2049] [node 3] ICP22 created new session: [1:7577471251431975373:2049]
2025-11-27T18:18:28.301876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:28.251277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471251431975373:2049] [node 3] ICS09 handshake done sender: [1:7577471251431975355:2114] self: [1:7577471251431975352:0] peer: [3:7577471252061645840:0] socket: 483 qp: -1
2025-11-27T18:18:28.317720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471252061645835:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:28.357975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471244598184717:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15513) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471248893152108:2132] held: no
2025-11-27T18:18:28.499426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471250700019193:2050] [node 3] ICS09 handshake done sender: [6:7577471250700019179:2119] self: [6:7577471250700019183:0] peer: [3:7577471252061645834:0] socket: 513 qp: -1
2025-11-27T18:18:28.499943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471250700019193:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:28.500707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:28.502957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471250700019195:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21612
2025-11-27T18:18:28.503437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471250700019186:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:28.504265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471246405051859:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.511402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:28.511483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471246405051859:2106] [node 5] ICP22 created new session: [6:7577471250700019196:2106]
2025-11-27T18:18:28.511529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471250700019196:2106] [node 5] ICS09 handshake done sender: [6:7577471250700019186:2122] self: [6:7577471250700019192:0] peer: [5:7577471252057644882:0] socket: 485 qp: -1
2025-11-27T18:18:28.511569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471250700019196:2106] [node 5] ICS10 traffic start
2025-11-27T18:18:28.511602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:28.301928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471248079582030:2048] [node 1] ICP22 created new session: [2:7577471252374549420:2048]
2025-11-27T18:18:28.485876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471247762677510:2048] [node 1] ICP22 created new session: [5:7577471252057644904:2048]
2025-11-27T18:18:28.318604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471247766678459:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.485916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471252057644904:2048] [node 1] ICS09 handshake done sender: [5:7577471252057644891:2119] self: [5:7577471252057644893:0] peer: [1:7577471251431975359:0] socket: 499 qp: -1
2025-11-27T18:18:28.374023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471248893152121:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-11-27T18:18:28.251305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471251431975373:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:28.485952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471252057644904:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:28.383229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471248893152118:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.251340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:28.399584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471248893152113:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-11-27T18:18:28.289733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471251431975362:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39228
2025-11-27T18:18:28.404653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471244598184718:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:28.291558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471247137008051:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:28.412014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471248893152119:2137] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:28.304616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471247137007984:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.412154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471244598184717:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18692259ns) {HandleHandshakeStatus:395(18688698ns) {LogHandshakeFail:472(15236ns) {UpdateErrorStateLog:545(5358ns)} DropOutgoingHandshake:457(1968ns) ProcessPendingSessionEvents:495(15860680ns) {DropSessionEvent:512(22958ns) DropSessionEvent:512(5396ns) DropSessionEvent:512(5574ns) DropSessionEvent:512(27869ns) DropSessionEvent:512(5719ns) DropSessionEvent:512(5286ns) DropSessionEvent:512(11121349ns) DropSessionEvent:512(12636ns) DropSessionEvent:512(7062ns) DropSessionEvent:512(5986ns)}}}
2025-11-27T18:18:28.313341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471251431975356:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.417535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471248893152108:2132] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:28.313395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471251431975356:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.422635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471244598184717:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.314720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:28.427679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471244598184719:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.427778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:28.314757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471251431975362:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.428521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471244598184717:2048] [node 1] ICP22 created new session: [4:7577471248893152124:2048]
2025-11-27T18:18:28.318486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471247137007984:2048] [node 2] ICP22 created new session: [1:7577471251431975377:2048]
2025-11-27T18:18:28.429439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471248893152124:2048] [node 1] ICS09 handshake done sender: [4:7577471248893152108:2132] self: [4:7577471248893152111:0] peer: [1:7577471251431975351:0] socket: 482 qp: -1
2025-11-27T18:18:28.321985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471247137008047:2107] [node 4] ICP18 handshake (actor [1:7577471251431975362:2118]) is too old
2025-11-27T18:18:28.429995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471248893152124:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:28.329494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471251431975361:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.430805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:28.329525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471251431975361:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.432458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:28.337380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471251431975377:2048] [node 2] ICS09 handshake done sender: [1:7577471247137008051:2110] self: [1:7577471247137008050:0] peer: [2:7577471252374549410:0] socket: 453 qp: -1
2025-11-27T18:18:28.433190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471248893152126:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-11-27T18:18:28.433300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471244598184719:2050] [node 3] ICP22 created new session: [4:7577471248893152127:2050]
2025-11-27T18:18:28.341486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471251431975356:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:28.435411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471248893152127:2050] [node 3] ICS09 handshake done sender: [4:7577471248893152119:2137] self: [4:7577471248893152120:0] peer: [3:7577471252061645833:0] socket: 516 qp: -1
2025-11-27T18:18:28.327063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:28.345015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471251431975361:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:28.345414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471251431975377:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:28.444038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471248893152127:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:28.331310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471252061645841:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.345477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:28.331819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471247766678459:2048] [node 1] ICP22 created new session: [3:7577471252061645849:2048]
2025-11-27T18:18:28.447367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:28.363014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471247137008049:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.447574Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471244598184717:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17365958ns) {HandleHandshakeStatus:329(17363662ns) {DropIncomingHandshake:442(451ns) DropOutgoingHandshake:457(201ns) SwitchToState:179(387ns) ProcessPendingSessionEvents:495(970ns)}}
2025-11-27T18:18:28.339639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471252061645837:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.453472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471248893152118:2136] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:28.365799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471247137007984:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(60994402ns) {HandleHandshakeStatus:329(60992396ns) {DropIncomingHandshake:442(19605ns) DropOutgoingHandshake:457(635ns) SwitchToState:179(210ns) ProcessPendingSessionEvents:495(144153ns) {ForwardSessionEventToSession:612(91572ns) ForwardSessionEventToSession:612(7664ns) ForwardSessionEventToSession:612(2878ns) ForwardSessionEventToSession:612(3642ns) ForwardSessionEventToSession:612(6335ns) ForwardSessionEventToSession:612(3226ns) ForwardSessionEventToSession:612(8133ns) ForwardSessionEventToSession:612(3329ns) ForwardSessionEventToSession:612(3111ns) ForwardSessionEventToSession:612(5636ns)}}}
2025-11-27T18:18:28.339664Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471252061645837:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.366452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471247137008048:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.345535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471252061645837:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:28.366519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:28.346035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471247766678523:2108] [node 5] ICP18 handshake (actor [3:7577471252061645841:2119]) is too old
2025-11-27T18:18:28.366567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471247137008048:2108] [node 5] ICP22 created new session: [1:7577471251431975381:2108]
2025-11-27T18:18:28.357974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471252061645849:2048] [node 1] ICS09 handshake done sender: [3:7577471252061645835:2116] self: [3:7577471252061645840:0] peer: [1:7577471251431975352:0] socket: 484 qp: -1
2025-11-27T18:18:28.362697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471252061645849:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:28.366614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471251431975381:2108] [node 5] ICS09 handshake done sender: [1:7577471251431975361:2117] self: [1:7577471251431975359:0] peer: [5:7577471252057644893:0] socket: 498 qp: -1
2025-11-27T18:18:28.369573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:28.366649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471251431975381:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:28.382690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471247766678527:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.366683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:28.382730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:28.397839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:28.404820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471247137008049:2109] [node 6] ICP22 created new session: [1:7577471251431975383:2109]
2025-11-27T18:18:28.386092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471247766678527:2107] [node 4] ICP22 created new session: [3:7577471252061645852:2107]
2025-11-27T18:18:28.398479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471252061645848:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47944
2025-11-27T18:18:28.412512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471251431975383:2109] [node 6] ICS09 handshake done sender: [1:7577471251431975356:2115] self: [1:7577471251431975354:0] peer: [6:7577471250700019176:0] socket: 495 qp: -1
2025-11-27T18:18:28.398839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471252061645832:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:28.418280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471251431975383:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:28.399035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471247766678460:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.418334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:28.404480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471252061645852:2107] [node 4] ICS09 handshake done sender: [3:7577471252061645837:2118] self: [3:7577471252061645833:0] peer: [4:7577471248893152120:0] socket: 515 qp: -1
2025-11-27T18:18:28.407365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471252061645852:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:28.475576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471247137008049:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(87149363ns) {HandleHandshakeStatus:329(87144643ns) {DropIncomingHandshake:442(14318ns) DropOutgoingHandshake:457(454ns) SwitchToState:179(408ns) ProcessPendingSessionEvents:495(19483251ns) {ForwardSessionEventToSession:612(3204734ns) ForwardSessionEventToSession:612(1102763ns) ForwardSessionEventToSession:612(5249ns) ForwardSessionEventToSession:612(4894259ns) ForwardSessionEventToSession:612(3577ns) ForwardSessionEventToSession:612(7507664ns) ForwardSessionEventToSession:612(5640ns) ForwardSessionEventToSession:612(3079ns) ForwardSessionEventToSession:612(656044ns) ForwardSessionEventToSession:612(2077251ns)}}}
2025-11-27T18:18:28.414524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:28.418001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:28.419198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471247766678460:2049] [node 2] ICP22 created new session: [3:7577471252061645854:2049]
2025-11-27T18:18:28.422519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471247766678459:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(89447131ns) {HandleHandshakeStatus:329(89445063ns) {DropIncomingHandshake:442(814ns) DropOutgoingHandshake:457(2434ns) SwitchToState:179(2223ns) ProcessPendingSessionEvents:495(1510ns)}}
2025-11-27T18:18:28.428189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471252061645854:2049] [node 2] ICS09 handshake done sender: [3:7577471252061645832:2115] self: [3:7577471252061645839:0] peer: [2:7577471252374549398:0] socket: 476 qp: -1
2025-11-27T18:18:28.428229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471252061645854:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:28.429895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:28.435261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471252061645836:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.435288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471252061645836:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:28.439816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471247766678460:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(34454369ns) {HandleHandshakeStatus:329(34452462ns) {DropIncomingHandshake:442(760ns) DropOutgoingHandshake:457(445ns) SwitchToState:179(512ns) ProcessPendingSessionEvents:495(673ns)}}
2025-11-27T18:18:28.457674Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471252061645836:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:28.458280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471247766678527:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(59424956ns) {HandleHandshakeStatus:329(50852390ns) {DropIncomingHandshake:442(443ns) DropOutgoingHandshake:457(288ns) SwitchToState:179(402ns) ProcessPendingSessionEvents:495(4229974ns) {ForwardSessionEventToSession:612(1236714ns) ForwardSessionEventToSession:612(4925ns) ForwardSessionEventToSession:612(1143132ns) ForwardSessionEventToSession:612(3135ns) ForwardSessionEventToSession:612(808270ns) ForwardSessionEventToSession:612(3694ns) ForwardSessionEventToSession:612(3361ns) ForwardSessionEventToSession:612(1005319ns) ForwardSessionEventToSession:612(2701ns) ForwardSessionEventToSession:612(2804ns)}}}
2025-11-27T18:18:28.458322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471252061645855:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47944
2025-11-27T18:18:28.477577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471247766678527:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:28.489580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471247766678528:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.489662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:28.489734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471247766678528:2109] [node 6] ICP22 created new session: [3:7577471252061645858:2109]
2025-11-27T18:18:28.489948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471252061645858:2109] [node 6] ICS09 handshake done sender: [3:7577471252061645836:2117] self: [3:7577471252061645834:0] peer: [6:7577471250700019183:0] socket: 468 qp: -1
2025-11-27T18:18:28.493394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471252061645858:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:28.493433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:28.523555Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471247766678528:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(20961497ns) {HandleHandshakeStatus:329(20958146ns) {DropIncomingHandshake:442(15953ns) DropOutgoingHandshake:457(327ns) SwitchToState:179(253ns) ProcessPendingSessionEvents:495(13837176ns) {ForwardSessionEventToSession:612(3275291ns) ForwardSessionEventToSession:612(429881ns) ForwardSessionEventToSession:612(3154ns) ForwardSessionEventToSession:612(2318ns) ForwardSessionEventToSession:612(170532ns) ForwardSessionEventToSession:612(2558ns) ForwardSessionEventToSession:612(1462300ns) ForwardSessionEventToSession:612(13527ns) ForwardSessionEventToSession:612(711158ns) ForwardSessionEventToSession:612(6049483ns)}}}
2025-11-27T18:18:28.301972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471252374549420:2048] [node 1] ICS09 handshake done sender: [2:7577471252374549395:2112] self: [2:7577471252374549410:0] peer: [1:7577471247137008050:0] socket: 455 qp: -1
2025-11-27T18:18:28.485986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:28.487925Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471244598184718:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11352) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471248893152118:2136] held: no
2025-11-27T18:18:28.523943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471246405051800:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(23376421ns) {HandleHandshakeStatus:329(22263457ns) {DropIncomingHandshake:442(838ns) DropOutgoingHandshake:457(5299ns) SwitchToState:179(804ns) ProcessPendingSessionEvents:495(17495166ns) {ForwardSessionEventToSession:612(11820193ns) ForwardSessionEventToSession:612(3758ns) ForwardSessionEventToSession:612(1197763ns) ForwardSessionEventToSession:612(3775ns) ForwardSessionEventToSession:612(3433ns) ForwardSessionEventToSession:612(1926961ns) ForwardSessionEventToSession:612(425135ns) ForwardSessionEventToSession:612(5579ns) ForwardSessionEventToSession:612(1629406ns) ForwardSessionEventToSession:612(8702ns)}}}
2025-11-27T18:18:28.495041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471248893152106:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.302009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471252374549420:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:28.494809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471252057644883:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.302044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:28.304030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471248079582095:2108] [node 5] ICP22 created new session: [2:7577471252374549419:2108]
2025-11-27T18:18:28.306388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471252374549404:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.306413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471252374549404:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.308795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471252374549404:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:28.311417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471248079582094:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.311488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:28.314811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471248079582094:2110] [node 6] ICP22 created new session: [2:7577471252374549423:2110]
2025-11-27T18:18:28.318403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471252374549419:2108] [node 5] ICS09 handshake done sender: [2:7577471252374549402:2115] self: [2:7577471252374549401:0] peer: [5:7577471252057644890:0] socket: 472 qp: -1
2025-11-27T18:18:28.325431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471252374549419:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:28.331450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471252374549423:2110] [node 6] ICS09 handshake done sender: [2:7577471252374549404:2117] self: [2:7577471252374549400:0] peer: [6:7577471250700019178:0] socket: 507 qp: -1
2025-11-27T18:18:28.331507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471252374549423:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:28.331545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:28.346864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:28.418358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471252374549409:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:28.418392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471252374549409:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.427883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471248079582094:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(96363020ns) {HandleHandshakeStatus:329(96360132ns) {DropIncomingHandshake:442(738ns) DropOutgoingHandshake:457(757ns) SwitchToState:179(460ns) ProcessPendingSessionEvents:495(49740229ns) {ForwardSessionEventToSession:612(23504470ns) ForwardSessionEventToSession:612(7008ns) ForwardSessionEventToSession:612(17941ns) ForwardSessionEventToSession:612(3032172ns) ForwardSessionEventToSession:612(3529ns) ForwardSessionEventToSession:612(13266959ns) ForwardSessionEventToSession:612(5125ns) ForwardSessionEventToSession:612(6462561ns) ForwardSessionEventToSession:612(1980113ns) ForwardSessionEventToSession:612(3767ns)}}}
2025-11-27T18:18:28.431502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471252374549409:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:28.432259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471248079582095:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(129057979ns) {HandleHandshakeStatus:329(129054079ns) {DropIncomingHandshake:442(1922ns) DropOutgoingHandshake:457(927ns) SwitchToState:179(827ns) ProcessPendingSessionEvents:495(28047146ns) {ForwardSessionEventToSession:612(9864968ns) ForwardSessionEventToSession:612(4931098ns) ForwardSessionEventToSession:612(3325ns) ForwardSessionEventToSession:612(2315961ns) ForwardSessionEventToSession:612(5245ns) ForwardSessionEventToSession:612(2680ns) ForwardSessionEventToSession:612(3485781ns) ForwardSessionEventToSession:612(4176ns) ForwardSessionEventToSession:612(4319029ns) ForwardSessionEventToSession:612(3078737ns)}}}
2025-11-27T18:18:28.447287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471248079582096:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.457544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:28.465974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471252374549428:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22572
2025-11-27T18:18:28.475477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471248079582096:2107] [node 4] ICP22 created new session: [2:7577471252374549429:2107]
2025-11-27T18:18:28.480589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471252374549429:2107] [node 4] ICS09 handshake done sender: [2:7577471252374549409:2118] self: [2:7577471252374549407:0] peer: [4:7577471248893152123:0] socket: 509 qp: -1
2025-11-27T18:18:28.489884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471252374549429:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:28.493156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:28.495509Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471248079582096:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(48040923ns) {HandleHandshakeStatus:329(48037595ns) {DropIncomingHandshake:442(21909ns) DropOutgoingHandshake:457(426ns) SwitchToState:179(467ns) ProcessPendingSessionEvents:495(85634ns) {ForwardSessionEventToSession:612(43477ns) ForwardSessionEventToSession:612(2793ns) ForwardSessionEventToSession:612(6265ns) ForwardSessionEventToSession:612(3031ns) ForwardSessionEventToSession:612(2734ns) ForwardSessionEventToSession:612(7858ns) ForwardSessionEventToSession:612(2978ns) ForwardSessionEventToSession:612(5546ns) ForwardSessionEventToSession:612(2091ns) ForwardSessionEventToSession:612(2501ns)}}}
2025-11-27T18:18:28.498984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471252374549427:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22572
2025-11-27T18:18:28.524026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471246405051800:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:28.494857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471252057644883:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.495066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471248893152106:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:28.499001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471244598184718:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(22863059ns) {HandleHandshakeStatus:395(22860013ns) {LogHandshakeFail:472(5088321ns) {UpdateErrorStateLog:545(8930ns)} DropOutgoingHandshake:457(2333ns) ProcessPendingSessionEvents:495(5616757ns) {DropSessionEvent:512(48927ns) DropSessionEvent:512(5531ns) DropSessionEvent:512(283650ns) DropSessionEvent:512(6435ns) DropSessionEvent:512(4810ns) DropSessionEvent:512(4373ns) DropSessionEvent:512(5015ns) DropSessionEvent:512(4598ns) DropSessionEvent:512(25765ns) DropSessionEvent:512(8376ns)}}}
2025-11-27T18:18:28.499820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471244598184718:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.499860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:28.500188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471244598184718:2049] [node 2] ICP22 created new session: [4:7577471248893152129:2049]
2025-11-27T18:18:28.511265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471248893152129:2049] [node 2] ICS09 handshake done sender: [4:7577471248893152118:2136] self: [4:7577471248893152123:0] peer: [2:7577471252374549407:0] socket: 510 qp: -1
2025-11-27T18:18:28.512545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471244598184719:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(74170893ns) {HandleHandshakeStatus:329(74168412ns) {DropIncomingHandshake:442(2388ns) DropOutgoingHandshake:457(17564ns) SwitchToState:179(453ns) ProcessPendingSessionEvents:495(6510633ns) {ForwardSessionEventToSession:612(3523246ns) ForwardSessionEventToSession:612(4385ns) ForwardSessionEventToSession:612(423459ns) ForwardSessionEventToSession:612(3086ns) ForwardSessionEventToSession:612(1682017ns) ForwardSessionEventToSession:612(4528ns) ForwardSessionEventToSession:612(2927ns) ForwardSessionEventToSession:612(847895ns) ForwardSessionEventToSession:612(3549ns) ForwardSessionEventToSession:612(2555ns)}}}
2025-11-27T18:18:28.512830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471248893152106:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:28.513071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471248893152129:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:28.513109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:28.521390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471244598184718:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20496949ns) {HandleHandshakeStatus:329(20494767ns) {DropIncomingHandshake:442(889ns) DropOutgoingHandshake:457(281ns) SwitchToState:179(536ns) ProcessPendingSessionEvents:495(872ns)}}
2025-11-27T18:18:28.524034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471244598184799:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.524101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:28.524167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471244598184799:2108] [node 5] ICP22 created new session: [4:7577471248893152132:2108]
2025-11-27T18:18:28.524240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471248893152132:2108] [node 5] ICS09 handshake done sender: [4:7577471248893152106:2131] self: [4:7577471248893152102:0] peer: [5:7577471252057644903:0] socket: 505 qp: -1
2025-11-27T18:18:28.524278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471248893152132:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:28.524334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:28.548058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471244598184799:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(19065885ns) {HandleHandshakeStatus:329(19062238ns) {DropIncomingHandshake:442(14828ns) DropOutgoingHandshake:457(427ns) SwitchToState:179(529ns) ProcessPendingSessionEvents:495(18649714ns) {ForwardSessionEventToSession:612(6932039ns) ForwardSessionEventToSession:612(2552959ns) ForwardSessionEventToSession:612(5703ns) ForwardSessionEventToSession:612(2404ns) ForwardSessionEventToSession:612(1042604ns) ForwardSessionEventToSession:612(2990ns) ForwardSessionEventToSession:612(2499ns) ForwardSessionEventToSession:612(2876997ns) ForwardSessionEventToSession:612(3702603ns) ForwardSessionEventToSession:612(1505332ns)}}}
2025-11-27T18:18:28.501487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471252057644883:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:28.511034Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471247762677511:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(64734186ns) {HandleHandshakeStatus:329(64731861ns) {DropIncomingHandshake:442(1155ns) DropOutgoingHandshake:457(4584ns) SwitchToState:179(431ns) ProcessPendingSessionEvents:495(6220031ns) {ForwardSessionEventToSession:612(858592ns) ForwardSessionEventToSession:612(796356ns) ForwardSessionEventToSession:612(3006ns) ForwardSessionEventToSession:612(981612ns) ForwardSessionEventToSession:612(5047ns) ForwardSessionEventToSession:612(838990ns) ForwardSessionEventToSession:612(2293480ns) ForwardSessionEventToSession:612(3595ns) ForwardSessionEventToSession:612(2870ns) ForwardSessionEventToSession:612(415102ns)}}}
2025-11-27T18:18:28.511085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471247762677575:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:28.513012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:28.515758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471247762677575:2110] [node 6] ICP22 created new session: [5:7577471252057644907:2110]
2025-11-27T18:18:28.519964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471252057644907:2110] [node 6] ICS09 handshake done sender: [5:7577471252057644883:2117] self: [5:7577471252057644882:0] peer: [6:7577471250700019192:0] socket: 460 qp: -1
2025-11-27T18:18:28.520016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471252057644907:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:28.520054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:28.523708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471252057644908:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8457
2025-11-27T18:18:28.533846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471252057644896:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:28.540689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471247762677574:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27879) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471252057644896:2122] held: no
2025-11-27T18:18:28.542050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471247762677575:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(30740184ns) {HandleHandshakeStatus:329(30728689ns) {DropIncomingHandshake:442(15816ns) DropOutgoingHandshake:457(695ns) SwitchToState:179(540ns) ProcessPendingSessionEvents:495(16949816ns) {ForwardSessionEventToSession:612(2320333ns) ForwardSessionEventToSession:612(3770485ns) ForwardSessionEventToSession:612(9169ns) ForwardSessionEventToSession:612(3040893ns) ForwardSessionEventToSession:612(2470ns) ForwardSessionEventToSession:612(2294ns) ForwardSessionEventToSession:612(1092274ns) ForwardSessionEventToSession:612(3187ns) ForwardSessionEventToSession:612(2972792ns) ForwardSessionEventToSession:612(3713171ns)}}}
2025-11-27T18:18:28.544207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471247762677574:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:28.544249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:28.547063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471247762677574:2108] [node 4] ICP22 created new session: [5:7577471252057644910:2108]
2025-11-27T18:18:28.547120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471252057644910:2108] [node 4] ICS09 handshake done sender: [5:7577471252057644896:2122] self: [5:7577471252057644903:0] peer: [4:7577471248893152102:0] socket: 519 qp: -1
2025-11-27T18:18:28.547156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471252057644910:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:28.547213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:28.629617Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:28.629888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471252374549420:2048] [node 1] ICS01 socket: 455 reason# EndOfStream
2025-11-27T18:18:28.629930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471248079582030:2048] [node 1] ICP30 unregister session Session# [2:7577471252374549420:2048] VirtualId# [2:7577471252374549410:0]
2025-11-27T18:18:28.629965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471252374549420:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.630047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:28.630101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:28.631114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:28.636963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471252061645849:2048] [node 1] ICS01 socket: 484 reason# EndOfStream
2025-11-27T18:18:28.637026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471247766678459:2048] [node 1] ICP30 unregister session Session# [3:7577471252061645849:2048] VirtualId# [3:7577471252061645840:0]
2025-11-27T18:18:28.637070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471252061645849:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.637187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:28.637242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:28.641697Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:28.641939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471250700019184:2048] [node 1] ICS01 socket: 496 reason# EndOfStream
2025-11-27T18:18:28.642003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471246405051798:2048] [node 1] ICP30 unregister session Session# [6:7577471250700019184:2048] VirtualId# [6:7577471250700019176:0]
2025-11-27T18:18:28.642041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471250700019184:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.642121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:28.642205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:28.642756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:28.642936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471248893152124:2048] [node 1] ICS01 socket: 482 reason# EndOfStream
2025-11-27T18:18:28.642968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471244598184717:2048] [node 1] ICP30 unregister session Session# [4:7577471248893152124:2048] VirtualId# [4:7577471248893152111:0]
2025-11-27T18:18:28.642985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471248893152124:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.643054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:28.643103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:28.645531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:28.645816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471252057644904:2048] [node 1] ICS01 socket: 499 reason# EndOfStream
2025-11-27T18:18:28.645896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471247762677510:2048] [node 1] ICP30 unregister session Session# [5:7577471252057644904:2048] VirtualId# [5:7577471252057644893:0]
2025-11-27T18:18:28.645955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471252057644904:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.646086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:28.646133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:28.711395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:28.711654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471252061645854:2049] [node 2] ICS01 socket: 476 reason# EndOfStream
2025-11-27T18:18:28.711693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471247766678460:2049] [node 2] ICP30 unregister session Session# [3:7577471252061645854:2049] VirtualId# [3:7577471252061645839:0]
2025-11-27T18:18:28.711723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471252061645854:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.711795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:28.711835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:28.715316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:28.715554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471252057644901:2049] [node 2] ICS07 socket disconnect 481 reason# EndOfStream
2025-11-27T18:18:28.715872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:28.715588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471252057644901:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.715677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:28.715713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471252057644901:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:28.716111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471248893152129:2049] [node 2] ICS01 socket: 510 reason# EndOfStream
2025-11-27T18:18:28.716160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471244598184718:2049] [node 2] ICP30 unregister session Session# [4:7577471248893152129:2049] VirtualId# [4:7577471248893152123:0]
2025-11-27T18:18:28.716218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471248893152129:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.716315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:28.716386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:28.720397Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:28.720766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471250700019188:2049] [node 2] ICS07 socket disconnect 508 reason# EndOfStream
2025-11-27T18:18:28.720805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471250700019188:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.720897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:28.720946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471250700019188:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:28.716930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471247762677511:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11352) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:28.717005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471252057644901:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:28.717040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471247762677511:2049] [node 2] ICP30 unregister session Session# [5:7577471252057644901:2049] VirtualId# [5:7577471252057644890:0]
2025-11-27T18:18:28.717078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:28.717456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471247762677511:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11352) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:28.717476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11352) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:28.723352Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471246405051799:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11352) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:28.723463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471250700019188:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:28.723597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471246405051799:2049] [node 2] ICP30 unregister session Session# [6:7577471250700019188:2049] VirtualId# [6:7577471250700019178:0]
2025-11-27T18:18:28.723647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:28.723678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471246405051799:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11352) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:28.723695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11352) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:28.728024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471247762677511:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:28.733816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471246405051799:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:28.792780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:28.793005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471248893152127:2050] [node 3] ICS07 socket disconnect 516 reason# EndOfStream
2025-11-27T18:18:28.793040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471248893152127:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.793123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:28.793176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471248893152127:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:28.794823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:28.795058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471250700019193:2050] [node 3] ICS07 socket disconnect 513 reason# EndOfStream
2025-11-27T18:18:28.795085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471250700019193:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.795176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:28.795274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471250700019193:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:28.796003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471246405051800:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18619) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:28.796109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471250700019193:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:28.796266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471246405051800:2050] [node 3] ICP30 unregister session Session# [6:7577471250700019193:2050] VirtualId# [6:7577471250700019183:0]
2025-11-27T18:18:28.796317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:28.796350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471246405051800:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18619) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:28.796360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18619) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:28.797782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:28.797967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471252057644894:2050] [node 3] ICS07 socket disconnect 451 reason# EndOfStream
2025-11-27T18:18:28.797995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471252057644894:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.798071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:28.798110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471252057644894:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:28.804950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471247762677512:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18619) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:28.805018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471252057644894:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:28.805052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471247762677512:2050] [node 3] ICP30 unregister session Session# [5:7577471252057644894:2050] VirtualId# [5:7577471252057644886:0]
2025-11-27T18:18:28.805097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:28.805134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471247762677512:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18619) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:28.805879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471246405051800:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:28.805148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18619) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:28.815188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471247762677512:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:28.864345Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:28.864617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471250700019182:2109] [node 4] ICS07 socket disconnect 464 reason# EndOfStream
2025-11-27T18:18:28.864646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471250700019182:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.864398Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:28.864605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471252057644910:2108] [node 4] ICS01 socket: 519 reason# EndOfStream
2025-11-27T18:18:28.864652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471247762677574:2108] [node 4] ICP30 unregister session Session# [5:7577471252057644910:2108] VirtualId# [5:7577471252057644903:0]
2025-11-27T18:18:28.864683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471252057644910:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.864765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:28.864855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:28.864745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:28.864792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471250700019182:2109] [node 4] ICS15 start handshake
2025-11-27T18:18:28.870428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471246405051862:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27879) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:28.870501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471250700019182:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:28.870544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471246405051862:2109] [node 4] ICP30 unregister session Session# [6:7577471250700019182:2109] VirtualId# [6:7577471250700019173:0]
2025-11-27T18:18:28.870592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:28.870630Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471246405051862:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27879) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:28.870659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27879) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:28.879703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471246405051862:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:18:28.938032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:28.938271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471250700019196:2106] [node 5] ICS01 socket: 485 reason# EndOfStream
2025-11-27T18:18:28.938319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471246405051859:2106] [node 5] ICP30 unregister session Session# [6:7577471250700019196:2106] VirtualId# [6:7577471250700019192:0]
2025-11-27T18:18:28.938351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471250700019196:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:28.938432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:28.938486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:29.064390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471254474558619:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:29.064447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471254474558620:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:29.115355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471256288661419:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:29.115413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471256288661420:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:29.129588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471253847444482:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:29.129639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471253847444483:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:29.150561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471255608213427:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:29.150597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471255608213428:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:29.150635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471255608213429:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:29.173498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471256035429271:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:29.173555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471256035429272:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:29.173753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471256035429273:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:29.205335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471255184437838:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:29.205395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471255184437839:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:29.205425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471255184437840:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:29.373355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471255184437900:2103] [node 5] ICP01 ready to work
2025-11-27T18:18:29.526835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471255184437901:2104] [node 4] ICP01 ready to work
2025-11-27T18:18:29.545436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471256288661491:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:29.510649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471253847444563:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:29.544365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471255608213427:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(138943320ns) {Configure:104(138938787ns) {ProcessConfigured:135(79532ns) {StartInitialHandshake:157(76485ns) {DropHandshakes:470(2147ns) {DropIncomingHandshake:442(535ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(1218ns) PrepareNewSessionHandshake:477(7023ns) {SwitchToState:179(373ns)}}}}}
2025-11-27T18:18:29.555141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471256288661492:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:29.555656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471256288661493:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:29.557467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471256035429337:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:29.567847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471254474558619:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(200817236ns) {Configure:104(200804977ns) {ProcessConfigured:135(21076886ns) {StartInitialHandshake:157(20366660ns) {DropHandshakes:470(2651ns) {DropIncomingHandshake:442(812ns) DropOutgoingHandshake:457(473ns)} GenerateSessionVirtualId:988(1634ns) PrepareNewSessionHandshake:477(3105ns) {SwitchToState:179(1729ns)}}}}}
2025-11-27T18:18:29.568329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471254474558684:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:29.568620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471254474558685:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:29.568959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471254474558686:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:29.563250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471253847444482:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17100686ns) {Configure:104(16664883ns) {ProcessConfigured:135(78152ns) {StartInitialHandshake:157(75410ns) {DropHandshakes:470(3443ns) {DropIncomingHandshake:442(678ns) DropOutgoingHandshake:457(695ns)} GenerateSessionVirtualId:988(1188ns) PrepareNewSessionHandshake:477(1487ns) {SwitchToState:179(578ns)}}}}}
2025-11-27T18:18:29.576534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471256035429338:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:29.563657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471253847444566:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:29.564004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471253847444567:2117] [node 6] ICP01 ready to work
2025-11-27T18:18:29.599569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471255608213428:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(54980117ns) {Configure:104(54975474ns) {ProcessConfigured:135(54844ns) {StartInitialHandshake:157(52489ns) {DropHandshakes:470(1117ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(357ns) PrepareNewSessionHandshake:477(1219ns) {SwitchToState:179(561ns)}}}}}
2025-11-27T18:18:29.600562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471255608213494:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:29.611003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471253847444567:2117] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22427501ns) {Configure:104(22423602ns) {ProcessConfigured:135(57420ns) {StartInitialHandshake:157(55274ns) {DropHandshakes:470(2080ns) {DropIncomingHandshake:442(422ns) DropOutgoingHandshake:457(312ns)} GenerateSessionVirtualId:988(714ns) PrepareNewSessionHandshake:477(1720ns) {SwitchToState:179(798ns)}}}}}
2025-11-27T18:18:29.600807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471255608213495:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:29.604332Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471254474558686:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28657852ns) {Configure:104(28653775ns) {ProcessConfigured:135(10032592ns) {StartInitialHandshake:157(10030962ns) {DropHandshakes:470(4773208ns) {DropIncomingHandshake:442(612ns) DropOutgoingHandshake:457(1144ns)} GenerateSessionVirtualId:988(1819ns) PrepareNewSessionHandshake:477(2794ns) {SwitchToState:179(1191ns)}}}}}
2025-11-27T18:18:29.630049Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471256288661420:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(38005749ns) {Configure:104(38000436ns) {ProcessConfigured:135(12779334ns) {StartInitialHandshake:157(12775368ns) {DropHandshakes:470(2153ns) {DropIncomingHandshake:442(572ns) DropOutgoingHandshake:457(296ns)} GenerateSessionVirtualId:988(1885ns) PrepareNewSessionHandshake:477(3771ns) {SwitchToState:179(1423ns)}}}}}
2025-11-27T18:18:29.677886Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471255184437900:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(104570379ns) {Configure:104(104564433ns) {ProcessConfigured:135(3854278ns) {StartInitialHandshake:157(3851568ns) {DropHandshakes:470(663ns) {DropIncomingHandshake:442(140ns) DropOutgoingHandshake:457(112ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:477(4050ns) {SwitchToState:179(1941ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.685664Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471256288661492:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(42208997ns) {Configure:104(42202025ns) {ProcessConfigured:135(67008ns) {StartInitialHandshake:157(61281ns) {DropHandshakes:470(1993ns) {DropIncomingHandshake:442(486ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(958ns) PrepareNewSessionHandshake:477(1359ns) {SwitchToState:179(543ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.696479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471256035429349:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50754
2025-11-27T18:18:29.690739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471255608213495:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(63565252ns) {Configure:104(63560369ns) {ProcessConfigured:135(4261307ns) {StartInitialHandshake:157(4259228ns) {DropHandshakes:470(4150ns) {DropIncomingHandshake:442(612ns) DropOutgoingHandshake:457(1023ns)} GenerateSessionVirtualId:988(911ns) PrepareNewSessionHandshake:477(2757ns) {SwitchToState:179(1423ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.699125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471255608213502:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14864
2025-11-27T18:18:29.699208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471255608213503:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14864
2025-11-27T18:18:29.699337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471255608213504:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14864
2025-11-27T18:18:29.699390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471255608213505:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14864
2025-11-27T18:18:29.688143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471256288661500:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34030
2025-11-27T18:18:29.688549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471256288661505:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34030
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.713668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471255608213503:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.710186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471256288661505:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.718647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471254474558687:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65365
2025-11-27T18:18:29.722921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471255608213502:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.726415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471255608213429:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.737205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471255608213505:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.730713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471254474558687:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.739262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471254474558697:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65365
2025-11-27T18:18:29.739393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471254474558698:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65365
2025-11-27T18:18:29.739458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471254474558699:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65365
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.753746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471256288661509:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34030
2025-11-27T18:18:29.753911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471256288661510:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34030
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.737523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471256035429349:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.761539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471253847444579:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-27T18:18:29.761358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471256288661509:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.754747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471255608213512:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14864
2025-11-27T18:18:29.764503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471255608213504:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.764644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471255608213428:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.774646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471256288661499:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.774682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471256288661499:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.775181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471256288661510:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.775476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471256288661419:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.775789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471255608213514:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14864
2025-11-27T18:18:29.757206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471254474558699:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.770203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471254474558698:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.777005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471253847444569:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.777442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471254474558683:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.777474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471254474558683:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.777048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471253847444569:2129] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.778016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471256288661499:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.779091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471253847444579:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.779326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471253847444580:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-27T18:18:29.779447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471253847444581:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-27T18:18:29.779741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471253847444483:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.779818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471253847444582:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-27T18:18:29.777471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471255608213512:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.778067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471255608213427:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.780503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471253847444580:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.780604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471253847444569:2129] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:29.780748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471256288661491:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.780865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:29.780916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471256288661491:2107] [node 4] ICP22 created new session: [2:7577471256288661515:2107]
2025-11-27T18:18:29.781009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471256288661515:2107] [node 4] ICS09 handshake done sender: [2:7577471256288661499:2121] self: [2:7577471256288661498:0] peer: [4:7577471255608213513:0] socket: 469 qp: -1
2025-11-27T18:18:29.781035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471256288661515:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:29.781842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471255608213504:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:29.781993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:29.782546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471254474558691:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.782597Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471254474558691:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.782656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471253847444581:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.782711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471253847444563:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.783020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:29.783241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471256288661500:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.783209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471253847444563:2107] [node 4] ICP22 created new session: [3:7577471253847444585:2107]
2025-11-27T18:18:29.782610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471255608213428:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.782668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:29.784026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471256288661495:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.783916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471253847444585:2107] [node 4] ICS09 handshake done sender: [3:7577471253847444569:2129] self: [3:7577471253847444568:0] peer: [4:7577471255608213510:0] socket: 460 qp: -1
2025-11-27T18:18:29.782719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471255608213428:2049] [node 2] ICP22 created new session: [4:7577471255608213517:2049]
2025-11-27T18:18:29.783019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471255608213517:2049] [node 2] ICS09 handshake done sender: [4:7577471255608213504:2117] self: [4:7577471255608213513:0] peer: [2:7577471256288661498:0] socket: 478 qp: -1
2025-11-27T18:18:29.783957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471253847444585:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:29.783063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471255608213517:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:29.783998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:29.783097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:29.784048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471256288661495:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.786206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471255608213516:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14864
2025-11-27T18:18:29.786530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471254474558697:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.787569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471253847444582:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.787666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471254474558683:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:29.787834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471253847444563:2107] [node 4] ICP18 handshake (actor [3:7577471253847444581:2135]) is too old
2025-11-27T18:18:29.788323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471256288661495:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.788893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471253847444482:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.789251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471253847444589:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-27T18:18:29.790663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471254474558691:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.791112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471256288661420:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.791168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:29.791205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471256288661420:2049] [node 3] ICP22 created new session: [2:7577471256288661521:2049]
2025-11-27T18:18:29.791247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471256288661521:2049] [node 3] ICS09 handshake done sender: [2:7577471256288661495:2119] self: [2:7577471256288661494:0] peer: [3:7577471253847444584:0] socket: 467 qp: -1
2025-11-27T18:18:29.791274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471256288661521:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:29.791321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:29.791756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471256288661520:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34030
2025-11-27T18:18:29.788753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471256035429351:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50754
2025-11-27T18:18:29.801493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471254474558619:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.801616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:29.801674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471254474558619:2048] [node 2] ICP22 created new session: [1:7577471254474558708:2048]
2025-11-27T18:18:29.801716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471254474558708:2048] [node 2] ICS09 handshake done sender: [1:7577471254474558683:2110] self: [1:7577471254474558682:0] peer: [2:7577471256288661514:0] socket: 491 qp: -1
2025-11-27T18:18:29.801752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471254474558708:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:29.801801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:29.802823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471255608213502:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:29.804798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471254474558689:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.804831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471254474558689:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.805770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471254474558684:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.805847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:29.805894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471254474558684:2107] [node 4] ICP22 created new session: [1:7577471254474558710:2107]
2025-11-27T18:18:29.805940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471254474558710:2107] [node 4] ICS09 handshake done sender: [1:7577471254474558691:2113] self: [1:7577471254474558690:0] peer: [4:7577471255608213515:0] socket: 493 qp: -1
2025-11-27T18:18:29.805974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471254474558710:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:29.806003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.808768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471256035429352:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50754
2025-11-27T18:18:29.804468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471253847444579:2133] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:29.809501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471253847444483:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61060) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471253847444579:2133] held: no
2025-11-27T18:18:29.809555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471255608213429:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25778) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471255608213502:2115] held: no
2025-11-27T18:18:29.809612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471253847444483:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.822682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471256035429352:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.822845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471256035429353:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50754
2025-11-27T18:18:29.822964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471256035429273:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.810637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471254474558689:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.825832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471255184437916:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13094
2025-11-27T18:18:29.809646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.809653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471255608213429:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.809711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471253847444483:2049] [node 2] ICP22 created new session: [3:7577471253847444591:2049]
2025-11-27T18:18:29.809692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:29.809857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471255608213429:2050] [node 3] ICP22 created new session: [4:7577471255608213520:2050]
2025-11-27T18:18:29.809907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471255608213520:2050] [node 3] ICS09 handshake done sender: [4:7577471255608213502:2115] self: [4:7577471255608213510:0] peer: [3:7577471253847444568:0] socket: 476 qp: -1
2025-11-27T18:18:29.809941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471255608213520:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:29.809972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:29.833243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471255608213427:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22015) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471255608213512:2119] held: no
2025-11-27T18:18:29.838702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471255608213519:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14864
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.845357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471255184437917:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13094
2025-11-27T18:18:29.837716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471254474558620:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.837793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:29.855366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471255184437916:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.855353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471256035429353:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.853370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471256288661510:2127] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:29.854137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471256288661419:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22015) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471256288661510:2127] held: no
2025-11-27T18:18:29.854230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471256288661419:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.854292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:29.854341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471256288661419:2048] [node 1] ICP22 created new session: [2:7577471256288661524:2048]
2025-11-27T18:18:29.854387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471256288661524:2048] [node 1] ICS09 handshake done sender: [2:7577471256288661510:2127] self: [2:7577471256288661514:0] peer: [1:7577471254474558682:0] socket: 495 qp: -1
2025-11-27T18:18:29.854420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471256288661524:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:29.854469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:29.855480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471256035429351:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.837839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471254474558620:2049] [node 3] ICP22 created new session: [1:7577471254474558715:2049]
2025-11-27T18:18:29.837884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471254474558715:2049] [node 3] ICS09 handshake done sender: [1:7577471254474558689:2112] self: [1:7577471254474558688:0] peer: [3:7577471253847444590:0] socket: 492 qp: -1
2025-11-27T18:18:29.837917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471254474558715:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:29.837951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:29.817441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471253847444591:2049] [node 2] ICS09 handshake done sender: [3:7577471253847444579:2133] self: [3:7577471253847444584:0] peer: [2:7577471256288661494:0] socket: 470 qp: -1
2025-11-27T18:18:29.817499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471253847444591:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:29.817576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:29.818947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471253847444593:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45668
2025-11-27T18:18:29.865957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471253847444573:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.865982Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471253847444573:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.855713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471255184437838:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.869412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471254474558695:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.869457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471254474558695:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.855799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471255184437918:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13094
2025-11-27T18:18:29.869325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471255184437918:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.873445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471255184437839:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.873885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471255184437921:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13094
2025-11-27T18:18:29.875696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471256288661504:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.875726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471256288661504:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.875302Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471254474558695:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:29.870554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471255608213512:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:29.870762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471256035429271:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.876804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471255608213427:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.876881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:29.876930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471255608213427:2048] [node 1] ICP22 created new session: [4:7577471255608213524:2048]
2025-11-27T18:18:29.876974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471255608213524:2048] [node 1] ICS09 handshake done sender: [4:7577471255608213512:2119] self: [4:7577471255608213515:0] peer: [1:7577471254474558690:0] socket: 494 qp: -1
2025-11-27T18:18:29.877001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471255608213524:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:29.877038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:29.881655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471254474558693:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.881691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471254474558693:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.882461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471254474558686:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.884602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471256288661504:2124] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:29.882501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:29.882549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471254474558686:2109] [node 6] ICP22 created new session: [1:7577471254474558718:2109]
2025-11-27T18:18:29.882587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471254474558718:2109] [node 6] ICS09 handshake done sender: [1:7577471254474558695:2115] self: [1:7577471254474558694:0] peer: [6:7577471255184437919:0] socket: 482 qp: -1
2025-11-27T18:18:29.882621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471254474558718:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:29.882674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:29.885001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471254474558693:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:29.886438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471254474558685:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.886513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:29.886555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471254474558685:2108] [node 5] ICP22 created new session: [1:7577471254474558721:2108]
2025-11-27T18:18:29.886595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471254474558721:2108] [node 5] ICS09 handshake done sender: [1:7577471254474558693:2114] self: [1:7577471254474558692:0] peer: [5:7577471256035429355:0] socket: 463 qp: -1
2025-11-27T18:18:29.874721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471256035429272:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.886642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471253847444582:2136] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:29.886628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471254474558721:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:29.886674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:29.885878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471256288661502:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.889523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471253847444482:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22015) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471253847444582:2136] held: no
2025-11-27T18:18:29.889626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471253847444482:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.889681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:29.889739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471253847444482:2048] [node 1] ICP22 created new session: [3:7577471253847444596:2048]
2025-11-27T18:18:29.889793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471253847444596:2048] [node 1] ICS09 handshake done sender: [3:7577471253847444582:2136] self: [3:7577471253847444590:0] peer: [1:7577471254474558688:0] socket: 499 qp: -1
2025-11-27T18:18:29.889828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471253847444596:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:29.889875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.893012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471255184437921:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.893195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471255184437922:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13094
2025-11-27T18:18:29.885902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471256288661502:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.894542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471253847444573:2131] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.895030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471255184437923:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13094
2025-11-27T18:18:29.888835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471256288661493:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.889050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:29.889185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471256288661493:2109] [node 6] ICP22 created new session: [2:7577471256288661527:2109]
2025-11-27T18:18:29.893718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471256288661527:2109] [node 6] ICS09 handshake done sender: [2:7577471256288661504:2124] self: [2:7577471256288661503:0] peer: [6:7577471255184437920:0] socket: 468 qp: -1
2025-11-27T18:18:29.893760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471256288661527:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:29.893801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:29.897457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471256288661502:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:29.897766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471256288661492:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.897824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:29.897871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471256288661492:2108] [node 5] ICP22 created new session: [2:7577471256288661530:2108]
2025-11-27T18:18:29.897909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471256288661530:2108] [node 5] ICS09 handshake done sender: [2:7577471256288661502:2123] self: [2:7577471256288661501:0] peer: [5:7577471256035429356:0] socket: 504 qp: -1
2025-11-27T18:18:29.897954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471256288661530:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:29.897982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:29.899148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471255184437916:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:29.899739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471256035429271:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22015) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471256035429353:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.899982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471256035429357:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50754
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.900319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471256035429358:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50754
2025-11-27T18:18:29.901534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471255184437918:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:29.901902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471253847444566:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.901952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:29.902007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471253847444566:2108] [node 5] ICP22 created new session: [3:7577471253847444598:2108]
2025-11-27T18:18:29.902050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471253847444598:2108] [node 5] ICS09 handshake done sender: [3:7577471253847444573:2131] self: [3:7577471253847444572:0] peer: [5:7577471256035429354:0] socket: 486 qp: -1
2025-11-27T18:18:29.902079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471253847444598:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:29.902115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:29.903326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471255184437901:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.903381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471255184437838:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.903419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:29.903457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471255184437838:2048] [node 1] ICP22 created new session: [6:7577471255184437925:2048]
2025-11-27T18:18:29.903498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471255184437925:2048] [node 1] ICS09 handshake done sender: [6:7577471255184437916:2115] self: [6:7577471255184437919:0] peer: [1:7577471254474558694:0] socket: 490 qp: -1
2025-11-27T18:18:29.903527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471255184437925:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:29.905752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471253847444575:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.905786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471253847444575:2132] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.903551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:29.909025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471256035429359:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50754
2025-11-27T18:18:29.907591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471255608213501:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.909774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471256035429357:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.904108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471255184437917:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.904142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471255184437839:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.904173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:29.904209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471255184437839:2049] [node 2] ICP22 created new session: [6:7577471255184437927:2049]
2025-11-27T18:18:29.904239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471255184437927:2049] [node 2] ICS09 handshake done sender: [6:7577471255184437918:2117] self: [6:7577471255184437920:0] peer: [2:7577471256288661503:0] socket: 480 qp: -1
2025-11-27T18:18:29.904264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471255184437927:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:29.904288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:29.905478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471255184437840:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.909446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471255184437930:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13094
2025-11-27T18:18:29.911594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471253847444575:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:29.912311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471253847444567:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.912411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:29.912454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471253847444567:2117] [node 6] ICP22 created new session: [3:7577471253847444601:2117]
2025-11-27T18:18:29.912495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471253847444601:2117] [node 6] ICS09 handshake done sender: [3:7577471253847444575:2132] self: [3:7577471253847444574:0] peer: [6:7577471255184437929:0] socket: 510 qp: -1
2025-11-27T18:18:29.912525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471253847444601:2117] [node 6] ICS10 traffic start
2025-11-27T18:18:29.912553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:29.913774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471255184437921:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:29.907638Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471255608213501:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.909077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471255608213501:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.914097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471256035429360:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50754
2025-11-27T18:18:29.914600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471256035429353:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:29.914927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471255608213495:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.915046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:29.915099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471255608213495:2109] [node 6] ICP22 created new session: [4:7577471255608213527:2109]
2025-11-27T18:18:29.915139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471255608213527:2109] [node 6] ICS09 handshake done sender: [4:7577471255608213501:2114] self: [4:7577471255608213500:0] peer: [6:7577471255184437924:0] socket: 512 qp: -1
2025-11-27T18:18:29.915171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471255608213527:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:29.915220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:29.915762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471256035429272:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61060) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471256035429351:2116] held: no
2025-11-27T18:18:29.915940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471256035429352:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:29.916694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471256035429271:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.916733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:29.916771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471256035429271:2048] [node 1] ICP22 created new session: [5:7577471256035429362:2048]
2025-11-27T18:18:29.916809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471256035429362:2048] [node 1] ICS09 handshake done sender: [5:7577471256035429353:2118] self: [5:7577471256035429355:0] peer: [1:7577471254474558692:0] socket: 489 qp: -1
2025-11-27T18:18:29.916835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471256035429362:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:29.916874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:29.917711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471255184437901:2104] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.917761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:29.917804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471255184437901:2104] [node 4] ICP22 created new session: [6:7577471255184437933:2104]
2025-11-27T18:18:29.917844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471255184437933:2104] [node 4] ICS09 handshake done sender: [6:7577471255184437921:2118] self: [6:7577471255184437924:0] peer: [4:7577471255608213500:0] socket: 513 qp: -1
2025-11-27T18:18:29.917898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471255184437933:2104] [node 4] ICS10 traffic start
2025-11-27T18:18:29.917928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:29.918143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471255184437901:2104] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.919487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471255184437931:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13094
2025-11-27T18:18:29.922017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471255184437840:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25778) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471255184437917:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.922283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471255184437932:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13094
2025-11-27T18:18:29.922088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471256035429351:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:29.922394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471255184437917:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:29.923160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471255184437840:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.923210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:29.923253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471255184437840:2050] [node 3] ICP22 created new session: [6:7577471255184437935:2050]
2025-11-27T18:18:29.923294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471255184437935:2050] [node 3] ICS09 handshake done sender: [6:7577471255184437917:2116] self: [6:7577471255184437929:0] peer: [3:7577471253847444574:0] socket: 511 qp: -1
2025-11-27T18:18:29.923326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471255184437935:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:29.923358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:29.924267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471256035429273:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.924300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:29.924321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471256035429273:2050] [node 3] ICP22 created new session: [5:7577471256035429364:2050]
2025-11-27T18:18:29.924347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471256035429364:2050] [node 3] ICS09 handshake done sender: [5:7577471256035429352:2117] self: [5:7577471256035429354:0] peer: [3:7577471253847444572:0] socket: 508 qp: -1
2025-11-27T18:18:29.924369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471256035429364:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:29.924385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:29.924809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471256035429337:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.924956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471256035429272:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.924977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:29.925001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471256035429272:2049] [node 2] ICP22 created new session: [5:7577471256035429367:2049]
2025-11-27T18:18:29.925026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471256035429367:2049] [node 2] ICS09 handshake done sender: [5:7577471256035429351:2116] self: [5:7577471256035429356:0] peer: [2:7577471256288661501:0] socket: 505 qp: -1
2025-11-27T18:18:29.925043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471256035429367:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:29.925058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:29.931989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471255184437932:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.931639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471255608213499:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.931678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471255608213499:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:29.932136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471255184437900:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:29.932933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471255608213499:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:29.934416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471255608213494:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.934494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:29.934555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471255608213494:2108] [node 5] ICP22 created new session: [4:7577471255608213530:2108]
2025-11-27T18:18:29.934603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471255608213530:2108] [node 5] ICS09 handshake done sender: [4:7577471255608213499:2113] self: [4:7577471255608213498:0] peer: [5:7577471256035429366:0] socket: 488 qp: -1
2025-11-27T18:18:29.934648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471255608213530:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:29.934740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:29.934563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471256035429344:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:29.934600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471256035429344:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.939142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471256035429337:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4154) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471256035429357:2119] held: no
2025-11-27T18:18:29.939443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471256035429370:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50754
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:29.940963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471255184437938:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13094
2025-11-27T18:18:29.941476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471256035429344:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:29.941899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471256035429338:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:29.941990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:29.942041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471256035429338:2110] [node 6] ICP22 created new session: [5:7577471256035429372:2110]
2025-11-27T18:18:29.942080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471256035429372:2110] [node 6] ICS09 handshake done sender: [5:7577471256035429344:2113] self: [5:7577471256035429343:0] peer: [6:7577471255184437937:0] socket: 524 qp: -1
2025-11-27T18:18:29.942113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471256035429372:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:29.942146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:29.942041Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471255184437932:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:29.942621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471255184437900:2103] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.942669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:29.942706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471255184437900:2103] [node 5] ICP22 created new session: [6:7577471255184437939:2103]
2025-11-27T18:18:29.942748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471255184437939:2103] [node 5] ICS09 handshake done sender: [6:7577471255184437932:2123] self: [6:7577471255184437937:0] peer: [5:7577471256035429343:0] socket: 479 qp: -1
2025-11-27T18:18:29.942785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471255184437939:2103] [node 5] ICS10 traffic start
2025-11-27T18:18:29.942812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:29.942676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471256035429357:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:29.943481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471256035429337:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:29.943515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:29.943557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471256035429337:2109] [node 4] ICP22 created new session: [5:7577471256035429374:2109]
2025-11-27T18:18:29.943595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471256035429374:2109] [node 4] ICS09 handshake done sender: [5:7577471256035429357:2119] self: [5:7577471256035429366:0] peer: [4:7577471255608213498:0] socket: 516 qp: -1
2025-11-27T18:18:29.943621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471256035429374:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:29.943649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:29.972307Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:29.972568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471256288661524:2048] [node 1] ICS01 socket: 495 reason# EndOfStream
2025-11-27T18:18:29.972619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471256288661419:2048] [node 1] ICP30 unregister session Session# [2:7577471256288661524:2048] VirtualId# [2:7577471256288661514:0]
2025-11-27T18:18:29.972652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471256288661524:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:29.972749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:29.972803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:29.973821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:29.974022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471253847444596:2048] [node 1] ICS01 socket: 499 reason# EndOfStream
2025-11-27T18:18:29.974073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471253847444482:2048] [node 1] ICP30 unregister session Session# [3:7577471253847444596:2048] VirtualId# [3:7577471253847444590:0]
2025-11-27T18:18:29.974101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471253847444596:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:29.974165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:29.974211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:29.976018Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:29.976214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471255608213524:2048] [node 1] ICS01 socket: 494 reason# EndOfStream
2025-11-27T18:18:29.976248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471255608213427:2048] [node 1] ICP30 unregister session Session# [4:7577471255608213524:2048] VirtualId# [4:7577471255608213515:0]
2025-11-27T18:18:29.976275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471255608213524:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:29.976336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:29.976376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:29.980572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:29.980562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:29.980770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471255184437925:2048] [node 1] ICS01 socket: 490 reason# EndOfStream
2025-11-27T18:18:29.980771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471256035429362:2048] [node 1] ICS01 socket: 489 reason# EndOfStream
2025-11-27T18:18:29.980819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471255184437838:2048] [node 1] ICP30 unregister session Session# [6:7577471255184437925:2048] VirtualId# [6:7577471255184437919:0]
2025-11-27T18:18:29.980808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471256035429271:2048] [node 1] ICP30 unregister session Session# [5:7577471256035429362:2048] VirtualId# [5:7577471256035429355:0]
2025-11-27T18:18:29.980843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471255184437925:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:29.980840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471256035429362:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:29.980928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:29.980936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:29.980968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:29.980976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:30.003924Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:30.004131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471253847444591:2049] [node 2] ICS01 socket: 470 reason# EndOfStream
2025-11-27T18:18:30.004171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471253847444483:2049] [node 2] ICP30 unregister session Session# [3:7577471253847444591:2049] VirtualId# [3:7577471253847444584:0]
2025-11-27T18:18:30.004200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471253847444591:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:30.004262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:30.004301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:30.006968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:30.007171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471255608213517:2049] [node 2] ICS01 socket: 478 reason# EndOfStream
2025-11-27T18:18:30.007211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471255608213428:2049] [node 2] ICP30 unregister session Session# [4:7577471255608213517:2049] VirtualId# [4:7577471255608213513:0]
2025-11-27T18:18:30.007239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471255608213517:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:30.007307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:30.007348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:30.009131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:30.009363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471256035429367:2049] [node 2] ICS01 socket: 505 reason# EndOfStream
2025-11-27T18:18:30.009410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471256035429272:2049] [node 2] ICP30 unregister session Session# [5:7577471256035429367:2049] VirtualId# [5:7577471256035429356:0]
2025-11-27T18:18:30.009439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471256035429367:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:30.009505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:30.009547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:30.009969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:30.010144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471255184437927:2049] [node 2] ICS01 socket: 480 reason# EndOfStream
2025-11-27T18:18:30.010181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471255184437839:2049] [node 2] ICP30 unregister session Session# [6:7577471255184437927:2049] VirtualId# [6:7577471255184437920:0]
2025-11-27T18:18:30.010207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471255184437927:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:30.010318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:30.010426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:30.061861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:30.062139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471255608213520:2050] [node 3] ICS01 socket: 476 reason# EndOfStream
2025-11-27T18:18:30.062193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471255608213429:2050] [node 3] ICP30 unregister session Session# [4:7577471255608213520:2050] VirtualId# [4:7577471255608213510:0]
2025-11-27T18:18:30.062234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471255608213520:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:30.062298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:30.062334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:30.062850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:30.063046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471256035429364:2050] [node 3] ICS07 socket disconnect 508 reason# EndOfStream
2025-11-27T18:18:30.063071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471256035429364:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:30.063127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:30.063165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471256035429364:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:30.064045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471256035429273:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25778) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:30.064101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471256035429364:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:30.064133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471256035429273:2050] [node 3] ICP30 unregister session Session# [5:7577471256035429364:2050] VirtualId# [5:7577471256035429354:0]
2025-11-27T18:18:30.064172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:30.064227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471256035429273:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:30.064247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:30.069848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:30.070064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471255184437935:2050] [node 3] ICS01 socket: 511 reason# EndOfStream
2025-11-27T18:18:30.070309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471255184437840:2050] [node 3] ICP30 unregister session Session# [6:7577471255184437935:2050] VirtualId# [6:7577471255184437929:0]
2025-11-27T18:18:30.070376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471255184437935:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:30.070481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:30.070537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:30.075945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471256035429273:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:30.092232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:30.092494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471256035429374:2109] [node 4] ICS01 socket: 516 reason# EndOfStream
2025-11-27T18:18:30.092547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471256035429337:2109] [node 4] ICP30 unregister session Session# [5:7577471256035429374:2109] VirtualId# [5:7577471256035429366:0]
2025-11-27T18:18:30.092577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471256035429374:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:30.092638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:30.092691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:30.097474Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:30.097776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471255184437933:2104] [node 4] ICS01 socket: 513 reason# EndOfStream
2025-11-27T18:18:30.097836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471255184437901:2104] [node 4] ICP30 unregister session Session# [6:7577471255184437933:2104] VirtualId# [6:7577471255184437924:0]
2025-11-27T18:18:30.097921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471255184437933:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:30.098124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:30.098259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:30.156122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:30.156362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471255184437939:2103] [node 5] ICS01 socket: 479 reason# EndOfStream
2025-11-27T18:18:30.156447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471255184437900:2103] [node 5] ICP30 unregister session Session# [6:7577471255184437939:2103] VirtualId# [6:7577471255184437937:0]
2025-11-27T18:18:30.156492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471255184437939:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:30.156579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:30.156628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:30.248942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471259184379454:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:30.250937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471259184379453:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:30.275443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471258736245045:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:30.275515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471258736245046:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:30.299898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471260829372928:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:30.299928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471260829372929:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:30.326917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471258114708228:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:30.326966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471258114708229:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:30.326988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471258114708230:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:30.352420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471259748660262:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:30.352502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471259748660263:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:30.352599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471259748660264:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:30.375074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471257335368968:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:30.375117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471257335368969:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:30.375147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471257335368970:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:30.588418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471257335369029:2105] [node 5] ICP01 ready to work
2025-11-27T18:18:30.636375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471260829372993:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:30.731643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471259748660326:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:30.735136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471259184379545:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:30.809759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471259184379547:2136] [node 6] ICP01 ready to work
2025-11-27T18:18:30.731943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471259748660327:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:30.826937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471258736245045:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(228339146ns) {Configure:104(228334293ns) {ProcessConfigured:135(58508ns) {StartInitialHandshake:157(56519ns) {DropHandshakes:470(3885ns) {DropIncomingHandshake:442(266ns) DropOutgoingHandshake:457(175ns)} GenerateSessionVirtualId:988(848ns) PrepareNewSessionHandshake:477(1338ns) {SwitchToState:179(380ns)}}}}}
2025-11-27T18:18:30.827393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471259748660262:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(95227999ns) {Configure:104(95222964ns) {ProcessConfigured:135(5303030ns) {StartInitialHandshake:157(5300503ns) {DropHandshakes:470(1947ns) {DropIncomingHandshake:442(531ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(923ns) PrepareNewSessionHandshake:477(2170ns) {SwitchToState:179(1106ns)}}}}}
2025-11-27T18:18:30.806038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471257335369035:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:30.793629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471260829372993:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(156883672ns) {Configure:104(156877958ns) {ProcessConfigured:135(7517687ns) {StartInitialHandshake:157(7513293ns) {DropHandshakes:470(7427179ns) {DropIncomingHandshake:442(3151ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(2637ns) PrepareNewSessionHandshake:477(2484ns) {SwitchToState:179(1557ns)}}}}}
2025-11-27T18:18:30.834183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471258114708291:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.842625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471259748660323:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47417
2025-11-27T18:18:30.813498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471259184379546:2112] [node 5] ICP01 ready to work
2025-11-27T18:18:30.845966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471258736245111:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.854721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471258114708292:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:30.813325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471260829373001:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.833841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471260829373002:2112] [node 6] ICP01 ready to work
2025-11-27T18:18:30.846406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471258736245112:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:30.870368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471258114708295:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12030
2025-11-27T18:18:30.895359Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471258114708228:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35386275ns) {Configure:104(35381363ns) {ProcessConfigured:135(93896ns) {StartInitialHandshake:157(88069ns) {DropHandshakes:470(3456ns) {DropIncomingHandshake:442(980ns) DropOutgoingHandshake:457(280ns)} GenerateSessionVirtualId:988(1103ns) PrepareNewSessionHandshake:477(1523ns) {SwitchToState:179(548ns)}}}}}
2025-11-27T18:18:30.873535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471258736245114:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:30.873811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471258736245105:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64306
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.917732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471259184379546:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(48237823ns) {Configure:104(48229832ns) {ProcessConfigured:135(28337388ns) {StartInitialHandshake:157(28334281ns) {DropHandshakes:470(2984ns) {DropIncomingHandshake:442(640ns) DropOutgoingHandshake:457(654ns)} GenerateSessionVirtualId:988(1067ns) PrepareNewSessionHandshake:477(3913ns) {SwitchToState:179(2116ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.921853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471259748660339:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47417
2025-11-27T18:18:30.929793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471259748660339:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.929940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471259748660326:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:30.931588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471260829373001:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(85725524ns) {Configure:104(85720807ns) {ProcessConfigured:135(12680778ns) {StartInitialHandshake:157(12677225ns) {DropHandshakes:470(2524ns) {DropIncomingHandshake:442(535ns) DropOutgoingHandshake:457(419ns)} GenerateSessionVirtualId:988(1068ns) PrepareNewSessionHandshake:477(2037ns) {SwitchToState:179(298ns)}}}}}
2025-11-27T18:18:30.939485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471257335368969:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(114310338ns) {Configure:104(114303039ns) {ProcessConfigured:135(58453ns) {StartInitialHandshake:157(55210ns) {DropHandshakes:470(3598ns) {DropIncomingHandshake:442(199ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(457ns) PrepareNewSessionHandshake:477(2009ns) {SwitchToState:179(698ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.931129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471259184379554:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6264
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.952167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471259748660263:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(162251465ns) {Configure:104(162244875ns) {ProcessConfigured:135(48687ns) {StartInitialHandshake:157(46670ns) {DropHandshakes:470(2707ns) {DropIncomingHandshake:442(635ns) DropOutgoingHandshake:457(334ns)} GenerateSessionVirtualId:988(447ns) PrepareNewSessionHandshake:477(977ns) {SwitchToState:179(268ns)}}}}}
2025-11-27T18:18:30.952306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471259748660323:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.947478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471258736245111:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(94033422ns) {Configure:104(94024868ns) {ProcessConfigured:135(9515382ns) {StartInitialHandshake:157(9512416ns) {DropHandshakes:470(2266ns) {DropIncomingHandshake:442(558ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(1143ns) PrepareNewSessionHandshake:477(2116ns) {SwitchToState:179(1224ns)}}}}}
2025-11-27T18:18:30.937497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471258114708297:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.950106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471257335369042:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17758
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.942896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471259184379554:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.965470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471260829373002:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(26521257ns) {Configure:104(26515598ns) {ProcessConfigured:135(25067468ns) {StartInitialHandshake:157(25065318ns) {DropHandshakes:470(2115ns) {DropIncomingHandshake:442(447ns) DropOutgoingHandshake:457(416ns)} GenerateSessionVirtualId:988(1006ns) PrepareNewSessionHandshake:477(2955ns) {SwitchToState:179(115ns)}}}}}
2025-11-27T18:18:30.969075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471259748660343:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47417
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.937542Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471258114708297:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:30.953953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471258736245112:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(78237707ns) {Configure:104(78233575ns) {ProcessConfigured:135(49327ns) {StartInitialHandshake:157(41773ns) {DropHandshakes:470(1902ns) {DropIncomingHandshake:442(718ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(714ns) PrepareNewSessionHandshake:477(842ns) {SwitchToState:179(214ns)}}}}}
2025-11-27T18:18:30.970657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471259748660344:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47417
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.961725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471259184379514:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.937606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471258114708295:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.955788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471258736245105:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.986124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471257335368968:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(139186603ns) {Configure:104(139180351ns) {ProcessConfigured:135(7324163ns) {StartInitialHandshake:157(7320812ns) {DropHandshakes:470(2158ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(299ns)} GenerateSessionVirtualId:988(867ns) PrepareNewSessionHandshake:477(4684ns) {SwitchToState:179(2528ns)}}}}}
2025-11-27T18:18:30.961750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471259184379514:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:30.987276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471257335369042:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.947169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471258114708297:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:30.956130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471258736245113:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64306
2025-11-27T18:18:30.956625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471258736245045:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.964903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471259184379558:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6264
2025-11-27T18:18:30.952611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471258114708309:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12030
2025-11-27T18:18:30.989076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471257335369035:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.018610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471261630336344:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17758
2025-11-27T18:18:30.975942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471258736245127:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64306
2025-11-27T18:18:30.969540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471258114708309:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.969246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471259184379514:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:31.009468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471259748660326:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65070) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471259748660339:2115] held: no
2025-11-27T18:18:30.976086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471258736245128:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64306
2025-11-27T18:18:30.969601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471258114708292:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.021078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471259748660339:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:31.023574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471259748660326:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.023625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:31.023668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471259748660326:2109] [node 4] ICP22 created new session: [5:7577471264043627644:2109]
2025-11-27T18:18:31.023709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471264043627644:2109] [node 4] ICS09 handshake done sender: [5:7577471259748660339:2115] self: [5:7577471259748660340:0] peer: [4:7577471258114708296:0] socket: 479 qp: -1
2025-11-27T18:18:31.023746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471264043627644:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:31.023792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:31.006656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471259184379547:2136] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(182980674ns) {Configure:104(182973532ns) {ProcessConfigured:135(24890565ns) {StartInitialHandshake:157(16768556ns) {DropHandshakes:470(2781ns) {DropIncomingHandshake:442(423ns) DropOutgoingHandshake:457(532ns)} GenerateSessionVirtualId:988(1860ns) PrepareNewSessionHandshake:477(3218ns) {SwitchToState:179(1717ns)}}}}}
2025-11-27T18:18:30.969662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:30.969714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471258114708292:2108] [node 5] ICP22 created new session: [4:7577471258114708310:2108]
2025-11-27T18:18:31.024069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471261630336344:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.976144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471258736245129:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64306
2025-11-27T18:18:31.006767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471259184379551:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.979955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471258736245127:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.987989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471258736245105:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:31.015324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471258736245045:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.015378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:31.015426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471258736245045:2048] [node 1] ICP22 created new session: [2:7577471263031212426:2048]
2025-11-27T18:18:31.015470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471263031212426:2048] [node 1] ICS09 handshake done sender: [2:7577471258736245105:2107] self: [2:7577471258736245126:0] peer: [1:7577471259184379513:0] socket: 464 qp: -1
2025-11-27T18:18:31.015500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471263031212426:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:31.015534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:31.024470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471258736245128:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.024771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471258736245113:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.969768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471258114708310:2108] [node 5] ICS09 handshake done sender: [4:7577471258114708297:2112] self: [4:7577471258114708296:0] peer: [5:7577471259748660340:0] socket: 477 qp: -1
2025-11-27T18:18:31.036346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471259748660344:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.036922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471259748660263:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:30.969801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471258114708310:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:31.025994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471257335368968:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.042555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471264043627642:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47417
2025-11-27T18:18:31.047769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471258736245118:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.047811Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471258736245118:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:31.048549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471258736245118:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:31.006784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471259184379551:2138] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.049099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471260829373008:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59713
2025-11-27T18:18:31.034027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471261630336347:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17758
2025-11-27T18:18:30.969857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.050178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471260829373006:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59713
2025-11-27T18:18:31.007149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471259184379453:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.053710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471265124340307:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59713
2025-11-27T18:18:31.007204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:31.038416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471261630336348:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17758
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:30.973271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471258114708228:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:31.050370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471258736245111:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.007245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471259184379453:2048] [node 2] ICP22 created new session: [1:7577471263479346858:2048]
2025-11-27T18:18:31.007285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471263479346858:2048] [node 2] ICS09 handshake done sender: [1:7577471259184379514:2107] self: [1:7577471259184379513:0] peer: [2:7577471258736245126:0] socket: 462 qp: -1
2025-11-27T18:18:31.007332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471263479346858:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:31.007370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:31.008453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471259184379551:2138] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:31.014730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471259184379545:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.014780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:31.020487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471259184379545:2108] [node 4] ICP22 created new session: [1:7577471263479346862:2108]
2025-11-27T18:18:31.021402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471263479346862:2108] [node 4] ICS09 handshake done sender: [1:7577471259184379551:2138] self: [1:7577471259184379550:0] peer: [4:7577471258114708314:0] socket: 483 qp: -1
2025-11-27T18:18:31.021733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471263479346862:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:31.024140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:31.025404Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471259184379453:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:31.034534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471259184379545:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(19610258ns) {HandleHandshakeStatus:329(19606628ns) {DropIncomingHandshake:442(627ns) DropOutgoingHandshake:457(330ns) SwitchToState:179(506ns) ProcessPendingSessionEvents:495(6886560ns) {ForwardSessionEventToSession:612(6802902ns) ForwardSessionEventToSession:612(9641ns) ForwardSessionEventToSession:612(6273ns) ForwardSessionEventToSession:612(2864ns) ForwardSessionEventToSession:612(24173ns) ForwardSessionEventToSession:612(3045ns) ForwardSessionEventToSession:612(2768ns) ForwardSessionEventToSession:612(5431ns) ForwardSessionEventToSession:612(2664ns) ForwardSessionEventToSession:612(13385ns)}}}
2025-11-27T18:18:31.045460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471259184379561:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.045504Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471259184379561:2142] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.055429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471264043627642:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:30.973659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471258114708312:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12030
2025-11-27T18:18:31.055878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471265124340308:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59713
2025-11-27T18:18:31.061789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471261630336347:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.055598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471258736245045:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(26782658ns) {HandleHandshakeStatus:329(26780454ns) {DropIncomingHandshake:442(478ns) DropOutgoingHandshake:457(5211ns) SwitchToState:179(628ns) ProcessPendingSessionEvents:495(20302814ns) {ForwardSessionEventToSession:612(3708770ns) ForwardSessionEventToSession:612(402489ns) ForwardSessionEventToSession:612(2943ns) ForwardSessionEventToSession:612(415734ns) ForwardSessionEventToSession:612(277785ns) ForwardSessionEventToSession:612(12511ns) ForwardSessionEventToSession:612(9246353ns) ForwardSessionEventToSession:612(5426ns) ForwardSessionEventToSession:612(4557ns) ForwardSessionEventToSession:612(6198540ns)}}}
2025-11-27T18:18:31.005461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471258114708312:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.056075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471259748660262:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.069313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471257335369035:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65070) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471257335369042:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.074466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471260829373006:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.061602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:31.006089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471258114708299:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.068767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471258736245111:2109] [node 5] ICP22 created new session: [2:7577471263031212433:2109]
2025-11-27T18:18:31.074879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471265124340307:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.006139Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471258114708299:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:31.089998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471261630336345:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17758
2025-11-27T18:18:31.094594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471261630336348:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.094811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471257335368969:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:31.093772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471264043627647:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47417
2025-11-27T18:18:31.014733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471262409675612:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12030
2025-11-27T18:18:31.022689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471258114708309:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:31.024045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471258114708228:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.025773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:31.025836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471258114708228:2048] [node 1] ICP22 created new session: [4:7577471262409675614:2048]
2025-11-27T18:18:31.025877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471262409675614:2048] [node 1] ICS09 handshake done sender: [4:7577471258114708309:2116] self: [4:7577471258114708314:0] peer: [1:7577471259184379550:0] socket: 486 qp: -1
2025-11-27T18:18:31.026072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471262409675614:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:31.026656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:31.034783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471258114708299:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:31.046335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471258114708291:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.046390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:31.046427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471258114708291:2109] [node 6] ICP22 created new session: [4:7577471262409675617:2109]
2025-11-27T18:18:31.046456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471262409675617:2109] [node 6] ICS09 handshake done sender: [4:7577471258114708299:2113] self: [4:7577471258114708298:0] peer: [6:7577471257335369047:0] socket: 478 qp: -1
2025-11-27T18:18:31.046480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471262409675617:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:31.046519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:31.055878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471258114708228:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27125402ns) {HandleHandshakeStatus:329(27122739ns) {DropIncomingHandshake:442(1453ns) DropOutgoingHandshake:457(14130ns) SwitchToState:179(519ns) ProcessPendingSessionEvents:495(2993188ns) {ForwardSessionEventToSession:612(2272446ns) ForwardSessionEventToSession:612(3627ns) ForwardSessionEventToSession:612(5469ns) ForwardSessionEventToSession:612(3134ns) ForwardSessionEventToSession:612(4375ns) ForwardSessionEventToSession:612(2767ns) ForwardSessionEventToSession:612(2612ns) ForwardSessionEventToSession:612(368061ns) ForwardSessionEventToSession:612(312793ns) ForwardSessionEventToSession:612(3324ns)}}}
2025-11-27T18:18:31.065166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471262409675616:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12030
2025-11-27T18:18:31.081482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471262409675619:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12030
2025-11-27T18:18:31.090269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471262409675616:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.090414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471262409675619:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.090566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471258114708230:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:31.090619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471258114708229:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:31.094872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471257335369042:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:31.069837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471263031212433:2109] [node 5] ICS09 handshake done sender: [2:7577471258736245118:2114] self: [2:7577471258736245117:0] peer: [5:7577471264043627646:0] socket: 495 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.091034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471259184379553:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.097861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471257335369035:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.081330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471265124340308:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.091078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471259184379553:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:31.085768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471260829373008:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.099378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471262409675622:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12030
2025-11-27T18:18:31.069877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471263031212433:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:31.100118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471259748660335:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.097941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:31.106276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471262409675616:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:31.086000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471260829372929:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:31.100149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471259748660335:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:31.101223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471259184379549:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.069948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:31.097991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471257335369035:2110] [node 4] ICP22 created new session: [6:7577471261630336351:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.113249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471258114708230:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.086230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471260829372928:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.101242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471259184379549:2137] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:31.104308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471264043627650:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47417
2025-11-27T18:18:31.098046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471261630336351:2110] [node 4] ICS09 handshake done sender: [6:7577471257335369042:2114] self: [6:7577471257335369047:0] peer: [4:7577471258114708298:0] socket: 490 qp: -1
2025-11-27T18:18:31.070336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471258736245111:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(19809874ns) {HandleHandshakeStatus:329(19807813ns) {DropIncomingHandshake:442(492ns) DropOutgoingHandshake:457(440ns) SwitchToState:179(1521ns) ProcessPendingSessionEvents:495(101689ns) {ForwardSessionEventToSession:612(54001ns) ForwardSessionEventToSession:612(3802ns) ForwardSessionEventToSession:612(8446ns) ForwardSessionEventToSession:612(3018ns) ForwardSessionEventToSession:612(2733ns) ForwardSessionEventToSession:612(5318ns) ForwardSessionEventToSession:612(2738ns) ForwardSessionEventToSession:612(2031ns) ForwardSessionEventToSession:612(7281ns) ForwardSessionEventToSession:612(2785ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.092889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471260829372998:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.104492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471259748660344:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:31.134147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:31.101881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471263479346865:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6264
2025-11-27T18:18:31.098082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471261630336351:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:31.098139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:31.098587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471257335369029:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:31.102991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471261630336354:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17758
2025-11-27T18:18:31.107028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471261630336355:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17758
2025-11-27T18:18:31.128067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471261630336344:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:31.137439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471261630336348:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:31.145335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471257335368968:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.145404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:31.145451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471257335368968:2048] [node 1] ICP22 created new session: [6:7577471261630336357:2048]
2025-11-27T18:18:31.145485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471261630336357:2048] [node 1] ICS09 handshake done sender: [6:7577471261630336344:2116] self: [6:7577471261630336346:0] peer: [1:7577471259184379560:0] socket: 507 qp: -1
2025-11-27T18:18:31.145519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471261630336357:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:31.145551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:31.146183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471257335369029:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14777) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471261630336348:2119] held: no
2025-11-27T18:18:31.146278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471257335369029:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.146341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:31.146383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471257335369029:2105] [node 5] ICP22 created new session: [6:7577471261630336359:2105]
2025-11-27T18:18:31.146423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471261630336359:2105] [node 5] ICS09 handshake done sender: [6:7577471261630336348:2119] self: [6:7577471261630336353:0] peer: [5:7577471259748660334:0] socket: 516 qp: -1
2025-11-27T18:18:31.146456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471261630336359:2105] [node 5] ICS10 traffic start
2025-11-27T18:18:31.146482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:31.147742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471261630336356:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17758
2025-11-27T18:18:31.149985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471261630336361:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17758
2025-11-27T18:18:31.134206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471258114708230:2050] [node 3] ICP22 created new session: [4:7577471262409675624:2050]
2025-11-27T18:18:31.134743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471262409675623:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12030
2025-11-27T18:18:31.140409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471262409675619:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:31.148809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471262409675624:2050] [node 3] ICS09 handshake done sender: [4:7577471262409675616:2119] self: [4:7577471262409675620:0] peer: [3:7577471260829372997:0] socket: 517 qp: -1
2025-11-27T18:18:31.150115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471258114708229:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.150145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:31.150181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471258114708229:2049] [node 2] ICP22 created new session: [4:7577471262409675625:2049]
2025-11-27T18:18:31.150207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471262409675625:2049] [node 2] ICS09 handshake done sender: [4:7577471262409675619:2120] self: [4:7577471262409675621:0] peer: [2:7577471258736245121:0] socket: 520 qp: -1
2025-11-27T18:18:31.150226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471262409675625:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:31.150247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:31.153471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471262409675624:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:31.157067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:31.092915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471260829372998:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:31.088617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471258736245116:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.108032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471259748660335:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:31.103843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471259184379561:2142] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:31.159525Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471258114708230:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(50333841ns) {HandleHandshakeStatus:329(50332107ns) {DropIncomingHandshake:442(1820ns) DropOutgoingHandshake:457(5546ns) SwitchToState:179(277ns) ProcessPendingSessionEvents:495(95929ns) {ForwardSessionEventToSession:612(47237ns) ForwardSessionEventToSession:612(12013ns) ForwardSessionEventToSession:612(2049ns) ForwardSessionEventToSession:612(2227ns) ForwardSessionEventToSession:612(5111ns) ForwardSessionEventToSession:612(5060ns) ForwardSessionEventToSession:612(5548ns) ForwardSessionEventToSession:612(2327ns) ForwardSessionEventToSession:612(2282ns) ForwardSessionEventToSession:612(5547ns)}}}
2025-11-27T18:18:31.159709Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471258114708230:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:31.124520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471259184379553:2139] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:31.134401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471259184379549:2137] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:31.137632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471259184379547:2136] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.137671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:31.137717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471259184379547:2136] [node 6] ICP22 created new session: [1:7577471263479346869:2136]
2025-11-27T18:18:31.137771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471263479346869:2136] [node 6] ICS09 handshake done sender: [1:7577471259184379561:2142] self: [1:7577471259184379560:0] peer: [6:7577471261630336346:0] socket: 506 qp: -1
2025-11-27T18:18:31.137804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471263479346869:2136] [node 6] ICS10 traffic start
2025-11-27T18:18:31.137838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:31.138308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471263479346865:2143] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.138497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471259184379546:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.138552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:31.138597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471259184379546:2112] [node 5] ICP22 created new session: [1:7577471263479346871:2112]
2025-11-27T18:18:31.138635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471263479346871:2112] [node 5] ICS09 handshake done sender: [1:7577471259184379553:2139] self: [1:7577471259184379552:0] peer: [5:7577471264043627648:0] socket: 503 qp: -1
2025-11-27T18:18:31.138667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471263479346871:2112] [node 5] ICS10 traffic start
2025-11-27T18:18:31.138694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:31.154089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471259184379454:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.154125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:31.154202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471259184379454:2049] [node 3] ICP22 created new session: [1:7577471263479346873:2049]
2025-11-27T18:18:31.154242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471263479346873:2049] [node 3] ICS09 handshake done sender: [1:7577471259184379549:2137] self: [1:7577471259184379548:0] peer: [3:7577471265124340312:0] socket: 482 qp: -1
2025-11-27T18:18:31.154270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471263479346873:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:31.154304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:31.154507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471259184379454:2049] [node 3] ICP18 handshake (actor [1:7577471263479346865:2143]) is too old
2025-11-27T18:18:31.094051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471265124340313:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59713
2025-11-27T18:18:31.094221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471260829372998:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:31.097343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471260829372993:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.097432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:31.097479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471260829372993:2108] [node 4] ICP22 created new session: [3:7577471265124340315:2108]
2025-11-27T18:18:31.099215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471265124340315:2108] [node 4] ICS09 handshake done sender: [3:7577471260829372998:2113] self: [3:7577471260829372997:0] peer: [4:7577471262409675620:0] socket: 512 qp: -1
2025-11-27T18:18:31.103506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471265124340315:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:31.104726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:31.113096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471265124340307:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:31.132438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471260829372929:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.132499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:31.132548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471260829372929:2049] [node 2] ICP22 created new session: [3:7577471265124340320:2049]
2025-11-27T18:18:31.132591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471265124340320:2049] [node 2] ICS09 handshake done sender: [3:7577471265124340307:2119] self: [3:7577471265124340310:0] peer: [2:7577471258736245115:0] socket: 497 qp: -1
2025-11-27T18:18:31.132623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471265124340320:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:31.132654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:31.132879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471260829372929:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:31.133185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471265124340317:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59713
2025-11-27T18:18:31.133298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471265124340318:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59713
2025-11-27T18:18:31.141557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471260829373008:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:31.143237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471260829372928:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.143307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:31.143354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471260829372928:2048] [node 1] ICP22 created new session: [3:7577471265124340323:2048]
2025-11-27T18:18:31.143401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471265124340323:2048] [node 1] ICS09 handshake done sender: [3:7577471260829373008:2118] self: [3:7577471265124340312:0] peer: [1:7577471259184379548:0] socket: 485 qp: -1
2025-11-27T18:18:31.143431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471265124340323:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:31.143472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:31.149496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471265124340317:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.088641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471258736245116:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:31.096604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471258736245122:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.096633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471258736245122:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:31.097440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471258736245116:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:31.103559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471258736245120:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.103591Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471258736245120:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:31.108360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471258736245046:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.108461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:31.108517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471258736245046:2049] [node 3] ICP22 created new session: [2:7577471263031212438:2049]
2025-11-27T18:18:31.108566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471263031212438:2049] [node 3] ICS09 handshake done sender: [2:7577471258736245116:2113] self: [2:7577471258736245115:0] peer: [3:7577471265124340310:0] socket: 494 qp: -1
2025-11-27T18:18:31.108595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471263031212438:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:31.108658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:31.114444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471258736245122:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:31.134306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471258736245120:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:31.147911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471258736245114:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.147952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:31.147981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471258736245114:2108] [node 4] ICP22 created new session: [2:7577471263031212441:2108]
2025-11-27T18:18:31.148010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471263031212441:2108] [node 4] ICS09 handshake done sender: [2:7577471258736245122:2116] self: [2:7577471258736245121:0] peer: [4:7577471262409675621:0] socket: 499 qp: -1
2025-11-27T18:18:31.148032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471263031212441:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:31.148054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:31.152346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471258736245112:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.152395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:31.152441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471258736245112:2110] [node 6] ICP22 created new session: [2:7577471263031212443:2110]
2025-11-27T18:18:31.152478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471263031212443:2110] [node 6] ICS09 handshake done sender: [2:7577471258736245120:2115] self: [2:7577471258736245119:0] peer: [6:7577471261630336349:0] socket: 509 qp: -1
2025-11-27T18:18:31.152505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471263031212443:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:31.152535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:31.109416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471259748660263:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.109510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:31.109559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471259748660263:2049] [node 2] ICP22 created new session: [5:7577471264043627651:2049]
2025-11-27T18:18:31.109600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471264043627651:2049] [node 2] ICS09 handshake done sender: [5:7577471259748660344:2118] self: [5:7577471264043627646:0] peer: [2:7577471258736245117:0] socket: 496 qp: -1
2025-11-27T18:18:31.109630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471264043627651:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:31.109689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:31.123280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471259748660327:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.123376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:31.123425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471259748660327:2110] [node 6] ICP22 created new session: [5:7577471264043627653:2110]
2025-11-27T18:18:31.123469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471264043627653:2110] [node 6] ICS09 handshake done sender: [5:7577471259748660335:2114] self: [5:7577471259748660334:0] peer: [6:7577471261630336353:0] socket: 515 qp: -1
2025-11-27T18:18:31.123508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471264043627653:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:31.123539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:31.131419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471264043627642:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:31.131753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471264043627655:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47417
2025-11-27T18:18:31.133877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471259748660262:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.133921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:31.136097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471259748660262:2048] [node 1] ICP22 created new session: [5:7577471264043627656:2048]
2025-11-27T18:18:31.153190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471264043627656:2048] [node 1] ICS09 handshake done sender: [5:7577471264043627642:2119] self: [5:7577471264043627648:0] peer: [1:7577471259184379552:0] socket: 505 qp: -1
2025-11-27T18:18:31.153238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471264043627656:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:31.157148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:31.158142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471259748660262:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24077700ns) {HandleHandshakeStatus:329(24075755ns) {DropIncomingHandshake:442(983ns) DropOutgoingHandshake:457(4977ns) SwitchToState:179(359ns) ProcessPendingSessionEvents:495(104980ns) {ForwardSessionEventToSession:612(53869ns) ForwardSessionEventToSession:612(8445ns) ForwardSessionEventToSession:612(7883ns) ForwardSessionEventToSession:612(2444ns) ForwardSessionEventToSession:612(5387ns) ForwardSessionEventToSession:612(2223ns) ForwardSessionEventToSession:612(5845ns) ForwardSessionEventToSession:612(2140ns) ForwardSessionEventToSession:612(2316ns) ForwardSessionEventToSession:612(5193ns)}}}
2025-11-27T18:18:31.166476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471257335368969:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13051) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471261630336347:2118] held: no
2025-11-27T18:18:31.166662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471261630336347:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:31.164702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471264043627655:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.169075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471259748660264:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:31.170054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471260829373005:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.170091Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471260829373005:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.171842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471260829373005:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:31.171889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471264043627659:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47417
2025-11-27T18:18:31.172576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471264043627655:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:31.174397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471259748660264:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.172085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471261630336361:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.172144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471257335368969:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.172817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:31.172884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471257335368969:2049] [node 2] ICP22 created new session: [6:7577471261630336362:2049]
2025-11-27T18:18:31.172925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471261630336362:2049] [node 2] ICS09 handshake done sender: [6:7577471261630336347:2118] self: [6:7577471261630336349:0] peer: [2:7577471258736245119:0] socket: 511 qp: -1
2025-11-27T18:18:31.172958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471261630336362:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:31.172993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:31.173677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471257335368970:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:31.176081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:31.176226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471259748660264:2050] [node 3] ICP22 created new session: [5:7577471264043627660:2050]
2025-11-27T18:18:31.176454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471264043627660:2050] [node 3] ICS09 handshake done sender: [5:7577471264043627655:2122] self: [5:7577471264043627658:0] peer: [3:7577471260829373003:0] socket: 465 qp: -1
2025-11-27T18:18:31.176550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471264043627660:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:31.176263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471260829373007:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:31.176904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:31.176300Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471260829373007:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:31.176498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471260829373001:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.176818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:31.176878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471260829373001:2111] [node 5] ICP22 created new session: [3:7577471265124340327:2111]
2025-11-27T18:18:31.177135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471265124340327:2111] [node 5] ICS09 handshake done sender: [3:7577471260829373005:2115] self: [3:7577471260829373003:0] peer: [5:7577471264043627658:0] socket: 502 qp: -1
2025-11-27T18:18:31.177237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471265124340327:2111] [node 5] ICS10 traffic start
2025-11-27T18:18:31.177286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:31.180027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471260829373007:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:31.180350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471261630336365:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17758
2025-11-27T18:18:31.180781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471261630336361:2123] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:31.181000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471257335368970:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:31.180288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471260829373002:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:31.182343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:31.181050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:31.181078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471257335368970:2050] [node 3] ICP22 created new session: [6:7577471261630336366:2050]
2025-11-27T18:18:31.181107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471261630336366:2050] [node 3] ICS09 handshake done sender: [6:7577471261630336361:2123] self: [6:7577471261630336364:0] peer: [3:7577471260829373004:0] socket: 467 qp: -1
2025-11-27T18:18:31.181125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471261630336366:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:31.181143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:31.182377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471260829373002:2112] [node 6] ICP22 created new session: [3:7577471265124340330:2112]
2025-11-27T18:18:31.182410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471265124340330:2112] [node 6] ICS09 handshake done sender: [3:7577471260829373007:2117] self: [3:7577471260829373004:0] peer: [6:7577471261630336364:0] socket: 463 qp: -1
2025-11-27T18:18:31.182432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471265124340330:2112] [node 6] ICS10 traffic start
2025-11-27T18:18:31.182465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:31.218572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:31.218918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471263031212426:2048] [node 1] ICS07 socket disconnect 464 reason# EndOfStream
2025-11-27T18:18:31.218940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471263031212426:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.219009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:31.219036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471263031212426:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:31.219699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:31.219941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471265124340323:2048] [node 1] ICS01 socket: 485 reason# EndOfStream
2025-11-27T18:18:31.219973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471260829372928:2048] [node 1] ICP30 unregister session Session# [3:7577471265124340323:2048] VirtualId# [3:7577471265124340312:0]
2025-11-27T18:18:31.219999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471265124340323:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.220070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:31.220115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:31.225523Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:31.225731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471264043627656:2048] [node 1] ICS01 socket: 505 reason# EndOfStream
2025-11-27T18:18:31.225772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471259748660262:2048] [node 1] ICP30 unregister session Session# [5:7577471264043627656:2048] VirtualId# [5:7577471264043627648:0]
2025-11-27T18:18:31.225811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471264043627656:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.225886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:31.225929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:31.229738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:31.222325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471258736245045:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30744) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:31.230430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471262409675614:2048] [node 1] ICS01 socket: 486 reason# EndOfStream
2025-11-27T18:18:31.230461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471258114708228:2048] [node 1] ICP30 unregister session Session# [4:7577471262409675614:2048] VirtualId# [4:7577471258114708314:0]
2025-11-27T18:18:31.230477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471262409675614:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.230548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:31.230602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:31.222367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471263031212426:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:31.229630Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:31.229821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471261630336357:2048] [node 1] ICS01 socket: 507 reason# EndOfStream
2025-11-27T18:18:31.230355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471257335368968:2048] [node 1] ICP30 unregister session Session# [6:7577471261630336357:2048] VirtualId# [6:7577471261630336346:0]
2025-11-27T18:18:31.230676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471261630336357:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.230725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:31.230780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:31.270715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:31.270939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471265124340320:2049] [node 2] ICS01 socket: 497 reason# EndOfStream
2025-11-27T18:18:31.270984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471260829372929:2049] [node 2] ICP30 unregister session Session# [3:7577471265124340320:2049] VirtualId# [3:7577471265124340310:0]
2025-11-27T18:18:31.271011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471265124340320:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.271085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:31.271123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:31.275533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:31.274920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:31.275174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471261630336362:2049] [node 2] ICS01 socket: 511 reason# EndOfStream
2025-11-27T18:18:31.276648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471262409675625:2049] [node 2] ICS01 socket: 520 reason# EndOfStream
2025-11-27T18:18:31.276701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471258114708229:2049] [node 2] ICP30 unregister session Session# [4:7577471262409675625:2049] VirtualId# [4:7577471262409675621:0]
2025-11-27T18:18:31.276732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471262409675625:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.275209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471257335368969:2049] [node 2] ICP30 unregister session Session# [6:7577471261630336362:2049] VirtualId# [6:7577471261630336349:0]
2025-11-27T18:18:31.275242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471261630336362:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.275531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:31.275594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:31.275732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:31.275894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471264043627651:2049] [node 2] ICS01 socket: 496 reason# EndOfStream
2025-11-27T18:18:31.276196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471259748660263:2049] [node 2] ICP30 unregister session Session# [5:7577471264043627651:2049] VirtualId# [5:7577471264043627646:0]
2025-11-27T18:18:31.276224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471264043627651:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.276283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:31.276325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:31.276817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:31.276892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:31.345440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:31.345442Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:31.345641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471261630336366:2050] [node 3] ICS07 socket disconnect 467 reason# EndOfStream
2025-11-27T18:18:31.345676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471261630336366:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.345653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471264043627660:2050] [node 3] ICS01 socket: 465 reason# EndOfStream
2025-11-27T18:18:31.345717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471259748660264:2050] [node 3] ICP30 unregister session Session# [5:7577471264043627660:2050] VirtualId# [5:7577471264043627658:0]
2025-11-27T18:18:31.345751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471264043627660:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.345769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:31.345820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471261630336366:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:31.345842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:31.345903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:31.345440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:31.345689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471262409675624:2050] [node 3] ICS01 socket: 517 reason# EndOfStream
2025-11-27T18:18:31.345735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471258114708230:2050] [node 3] ICP30 unregister session Session# [4:7577471262409675624:2050] VirtualId# [4:7577471262409675620:0]
2025-11-27T18:18:31.345762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471262409675624:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.345847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:31.345902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:31.346775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471257335368970:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16873) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:31.346844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471261630336366:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:31.346889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471257335368970:2050] [node 3] ICP30 unregister session Session# [6:7577471261630336366:2050] VirtualId# [6:7577471261630336364:0]
2025-11-27T18:18:31.346939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:31.346973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471257335368970:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16873) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:31.347017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16873) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:31.356879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471257335368970:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:31.388895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:31.389145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471264043627644:2109] [node 4] ICS01 socket: 479 reason# EndOfStream
2025-11-27T18:18:31.389189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471259748660326:2109] [node 4] ICP30 unregister session Session# [5:7577471264043627644:2109] VirtualId# [5:7577471259748660340:0]
2025-11-27T18:18:31.389236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471264043627644:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.389326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:31.389376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:31.393458Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:31.393869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471261630336351:2110] [node 4] ICS01 socket: 490 reason# EndOfStream
2025-11-27T18:18:31.393968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471257335369035:2110] [node 4] ICP30 unregister session Session# [6:7577471261630336351:2110] VirtualId# [6:7577471257335369047:0]
2025-11-27T18:18:31.394022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471261630336351:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.394125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:31.394198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:31.474084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:31.474356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471261630336359:2105] [node 5] ICS01 socket: 516 reason# EndOfStream
2025-11-27T18:18:31.474405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471257335369029:2105] [node 5] ICP30 unregister session Session# [6:7577471261630336359:2105] VirtualId# [6:7577471261630336353:0]
2025-11-27T18:18:31.474451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471261630336359:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:31.474563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:31.474618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:31.585639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471265210303944:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:31.585690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471265210303945:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:31.618723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471263599638854:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:31.618771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471263599638855:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:31.642371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471263245186468:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:31.642417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471263245186469:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:31.669423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471261450267106:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:31.669473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471261450267107:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:31.669510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471261450267108:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:31.694696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471261354972950:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:31.694759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471261354972951:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:31.694809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471261354972952:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:31.727057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471261760841383:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:31.727111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471261760841384:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:31.727213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471261760841385:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:32.005867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471266055808741:2106] [node 5] ICP01 ready to work
2025-11-27T18:18:32.012565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471269505271301:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:32.160911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471266055808741:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(154620758ns) {Configure:104(154616090ns) {ProcessConfigured:135(91818ns) {StartInitialHandshake:157(88550ns) {DropHandshakes:470(3096ns) {DropIncomingHandshake:442(737ns) DropOutgoingHandshake:457(440ns)} GenerateSessionVirtualId:988(1293ns) PrepareNewSessionHandshake:477(2030ns) {SwitchToState:179(664ns)}}}}}
2025-11-27T18:18:32.161404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471266055808745:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:32.183582Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471269505271301:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(170684906ns) {Configure:104(170680890ns) {ProcessConfigured:135(85909ns) {StartInitialHandshake:157(83786ns) {DropHandshakes:470(6029ns) {DropIncomingHandshake:442(976ns) DropOutgoingHandshake:457(391ns)} GenerateSessionVirtualId:988(1073ns) PrepareNewSessionHandshake:477(1935ns) {SwitchToState:179(821ns)}}}}}
2025-11-27T18:18:32.184045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471269505271306:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:32.149470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471267894606222:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:32.201884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471265745234474:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:32.184404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471269505271307:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:32.185703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471267540153854:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:32.207570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471267540153855:2117] [node 5] ICP01 ready to work
2025-11-27T18:18:32.181891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471267894606223:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:32.223548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471266055808745:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(60026297ns) {Configure:104(60022179ns) {ProcessConfigured:135(81213ns) {StartInitialHandshake:157(78806ns) {DropHandshakes:470(2951ns) {DropIncomingHandshake:442(696ns) DropOutgoingHandshake:457(634ns)} GenerateSessionVirtualId:988(973ns) PrepareNewSessionHandshake:477(3051ns) {SwitchToState:179(702ns)}}}}}
2025-11-27T18:18:32.231396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471261760841384:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49775950ns) {Configure:104(49758148ns) {ProcessConfigured:135(86229ns) {StartInitialHandshake:157(84621ns) {DropHandshakes:470(2939ns) {DropIncomingHandshake:442(777ns) DropOutgoingHandshake:457(370ns)} GenerateSessionVirtualId:988(795ns) PrepareNewSessionHandshake:477(1838ns) {SwitchToState:179(636ns)}}}}}
2025-11-27T18:18:32.224524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471265649940310:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:32.213407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471267894606228:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:32.227251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471265745234477:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.240798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471267540153856:2116] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.237456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471265649940309:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:32.265772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471265210303945:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(259398139ns) {Configure:104(259392734ns) {ProcessConfigured:135(58710ns) {StartInitialHandshake:157(56364ns) {DropHandshakes:470(2853ns) {DropIncomingHandshake:442(732ns) DropOutgoingHandshake:457(417ns)} GenerateSessionVirtualId:988(952ns) PrepareNewSessionHandshake:477(2296ns) {SwitchToState:179(812ns)}}}}}
2025-11-27T18:18:32.268490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471269505271314:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51453
2025-11-27T18:18:32.268552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471269505271315:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51453
2025-11-27T18:18:32.285653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471265210303944:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(267700945ns) {Configure:104(267695877ns) {ProcessConfigured:135(41346079ns) {StartInitialHandshake:157(41343961ns) {DropHandshakes:470(1272ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(177ns)} GenerateSessionVirtualId:988(304ns) PrepareNewSessionHandshake:477(4477ns) {SwitchToState:179(1863ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.245661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471261450267107:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41431232ns) {Configure:104(41428240ns) {ProcessConfigured:135(7829137ns) {StartInitialHandshake:157(7826767ns) {DropHandshakes:470(2338ns) {DropIncomingHandshake:442(616ns) DropOutgoingHandshake:457(349ns)} GenerateSessionVirtualId:988(938ns) PrepareNewSessionHandshake:477(5252ns) {SwitchToState:179(2892ns)}}}}}
2025-11-27T18:18:32.278591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471267540153854:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(37496476ns) {Configure:104(37491091ns) {ProcessConfigured:135(75583ns) {StartInitialHandshake:157(73455ns) {DropHandshakes:470(2517ns) {DropIncomingHandshake:442(576ns) DropOutgoingHandshake:457(385ns)} GenerateSessionVirtualId:988(898ns) PrepareNewSessionHandshake:477(1807ns) {SwitchToState:179(664ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.258919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471261450267106:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(101084335ns) {Configure:104(101077965ns) {ProcessConfigured:135(89947ns) {StartInitialHandshake:157(86204ns) {DropHandshakes:470(3876ns) {DropIncomingHandshake:442(1183ns) DropOutgoingHandshake:457(561ns)} GenerateSessionVirtualId:988(1709ns) PrepareNewSessionHandshake:477(3026ns) {SwitchToState:179(1483ns)}}}}}
2025-11-27T18:18:32.261244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471265745234484:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29801
2025-11-27T18:18:32.294945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471266055808757:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64805
2025-11-27T18:18:32.309614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471267540153861:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50803
2025-11-27T18:18:32.310557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471269505271306:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(109746346ns) {Configure:104(109740869ns) {ProcessConfigured:135(20483327ns) {StartInitialHandshake:157(20480667ns) {DropHandshakes:470(2760ns) {DropIncomingHandshake:442(678ns) DropOutgoingHandshake:457(483ns)} GenerateSessionVirtualId:988(797ns) PrepareNewSessionHandshake:477(3330ns) {SwitchToState:179(1384ns)}}}}}
2025-11-27T18:18:32.309713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471267540153862:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50803
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.338435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471265745234488:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29801
2025-11-27T18:18:32.335083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471267540153862:2140] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.344064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471263245186469:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(108762086ns) {Configure:104(108755924ns) {ProcessConfigured:135(64010ns) {StartInitialHandshake:157(59383ns) {DropHandshakes:470(2730ns) {DropIncomingHandshake:442(746ns) DropOutgoingHandshake:457(369ns)} GenerateSessionVirtualId:988(917ns) PrepareNewSessionHandshake:477(1814ns) {SwitchToState:179(653ns)}}}}}
2025-11-27T18:18:32.334940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471269505271314:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.361746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471263245186468:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(137253873ns) {Configure:104(131944578ns) {ProcessConfigured:135(8731558ns) {StartInitialHandshake:157(8728216ns) {DropHandshakes:470(2745ns) {DropIncomingHandshake:442(543ns) DropOutgoingHandshake:457(484ns)} GenerateSessionVirtualId:988(831ns) PrepareNewSessionHandshake:477(4979ns) {SwitchToState:179(2926ns)}}}}}
2025-11-27T18:18:32.341795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471266055808759:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64805
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.349796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471265745234488:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.350799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471269505271305:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.350301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471265745234489:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29801
2025-11-27T18:18:32.342487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471266055808757:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.350841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471269505271305:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.368010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471267894606236:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5243
2025-11-27T18:18:32.350436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471261450267106:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.368218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471261354972950:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(108055430ns) {Configure:104(108051077ns) {ProcessConfigured:135(8956156ns) {StartInitialHandshake:157(8953036ns) {DropHandshakes:470(2731ns) {DropIncomingHandshake:442(402ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(721ns) PrepareNewSessionHandshake:477(4906ns) {SwitchToState:179(1965ns)}}}}}
2025-11-27T18:18:32.364075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471266055808758:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64805
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.352807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471269505271305:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:32.369051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471263599638854:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(195785676ns) {Configure:104(195776207ns) {ProcessConfigured:135(21354461ns) {StartInitialHandshake:157(21351626ns) {DropHandshakes:470(2335ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(400ns)} GenerateSessionVirtualId:988(539ns) PrepareNewSessionHandshake:477(4447ns) {SwitchToState:179(2136ns)}}}}}
2025-11-27T18:18:32.365332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471265745234484:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.371343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471267540153861:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.367839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471266055808759:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.370164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471267894606230:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.369480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471261450267107:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.369816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471269505271313:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.370100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471261354972952:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(194536432ns) {Configure:104(194530334ns) {ProcessConfigured:135(9840491ns) {StartInitialHandshake:157(9836323ns) {DropHandshakes:470(2849ns) {DropIncomingHandshake:442(794ns) DropOutgoingHandshake:457(454ns)} GenerateSessionVirtualId:988(1005ns) PrepareNewSessionHandshake:477(3529ns) {SwitchToState:179(1408ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.368079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471261760841383:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.370185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471267894606230:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.369841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471269505271313:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.376713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471263245186468:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.370042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471265745234491:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29801
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.371165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471267894606236:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.371106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471269505271301:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.369662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471266055808760:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64805
2025-11-27T18:18:32.397457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471261354972951:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(129040126ns) {Configure:104(129032577ns) {ProcessConfigured:135(6660382ns) {StartInitialHandshake:157(6658094ns) {DropHandshakes:470(1677ns) {DropIncomingHandshake:442(525ns) DropOutgoingHandshake:457(365ns)} GenerateSessionVirtualId:988(735ns) PrepareNewSessionHandshake:477(5135ns) {SwitchToState:179(2523ns)}}}}}
2025-11-27T18:18:32.376639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471265745234474:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(138239788ns) {Configure:104(138232848ns) {ProcessConfigured:135(1791862ns) {StartInitialHandshake:157(1286320ns) {DropHandshakes:470(2472ns) {DropIncomingHandshake:442(760ns) DropOutgoingHandshake:457(475ns)} GenerateSessionVirtualId:988(1529ns) PrepareNewSessionHandshake:477(1181ns) {SwitchToState:179(263ns)}}}}}
2025-11-27T18:18:32.400629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471267540153874:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50803
2025-11-27T18:18:32.375723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:32.369785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471266055808758:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.379371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471269505271301:2107] [node 4] ICP22 created new session: [1:7577471269505271323:2107]
2025-11-27T18:18:32.405094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471267540153864:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.405125Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471267540153864:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.371458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471267894606230:2121] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:32.381748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471265745234483:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.369913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471261760841384:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.393157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471267894606232:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.381789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471265745234483:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.369945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471266055808745:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.385581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471269505271323:2107] [node 4] ICS09 handshake done sender: [1:7577471269505271305:2110] self: [1:7577471269505271304:0] peer: [4:7577471265745234490:0] socket: 479 qp: -1
2025-11-27T18:18:32.393202Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471267894606232:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.405327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471265745234483:2122] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:32.430738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471265649940310:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(60785452ns) {Configure:104(60777545ns) {ProcessConfigured:135(4606567ns) {StartInitialHandshake:157(4603518ns) {DropHandshakes:470(3149ns) {DropIncomingHandshake:442(765ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(888ns) PrepareNewSessionHandshake:477(4689ns) {SwitchToState:179(2556ns)}}}}}
2025-11-27T18:18:32.426243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471267540153864:2141] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.385639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471269505271323:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:32.406322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471265745234488:2124] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:32.405968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471267894606223:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.387788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471266055808760:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.429476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471267540153858:2137] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.385704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:32.394120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471261760841385:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.410183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471265745234477:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.406030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:32.386150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471269505271311:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.429525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471267540153858:2137] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.410382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471266055808766:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64805
2025-11-27T18:18:32.410232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:32.406088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471267894606223:2109] [node 6] ICP22 created new session: [2:7577471267894606242:2109]
2025-11-27T18:18:32.431102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471267540153862:2140] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:32.386177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471269505271311:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.448968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471265649940309:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(64847753ns) {Configure:104(62963950ns) {ProcessConfigured:135(70625ns) {StartInitialHandshake:157(68746ns) {DropHandshakes:470(2502ns) {DropIncomingHandshake:442(824ns) DropOutgoingHandshake:457(374ns)} GenerateSessionVirtualId:988(653ns) PrepareNewSessionHandshake:477(1721ns) {SwitchToState:179(620ns)}}}}}
2025-11-27T18:18:32.409884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471267894606242:2109] [node 6] ICS09 handshake done sender: [2:7577471267894606230:2121] self: [2:7577471267894606229:0] peer: [6:7577471266055808763:0] socket: 482 qp: -1
2025-11-27T18:18:32.429663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471266055808764:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64805
2025-11-27T18:18:32.410385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471265745234489:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.396980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471269505271311:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:32.436708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471263245186468:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.411610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471267894606242:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:32.411858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471265745234477:2109] [node 6] ICP22 created new session: [4:7577471265745234496:2109]
2025-11-27T18:18:32.439077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471266055808758:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:32.436770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.406134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471269505271315:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.414847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:32.439441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471266055808759:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.411912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471265745234496:2109] [node 6] ICS09 handshake done sender: [4:7577471265745234483:2122] self: [4:7577471265745234482:0] peer: [6:7577471266055808762:0] socket: 483 qp: -1
2025-11-27T18:18:32.409613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471265210303945:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.436816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471263245186468:2048] [node 1] ICP22 created new session: [3:7577471267540153877:2048]
2025-11-27T18:18:32.442015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471261760841383:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64968) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471266055808757:2115] held: no
2025-11-27T18:18:32.411950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471265745234496:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:32.425176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471267894606223:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(25155353ns) {HandleHandshakeStatus:329(24032142ns) {DropIncomingHandshake:442(1467ns) DropOutgoingHandshake:457(1390ns) SwitchToState:179(862ns) ProcessPendingSessionEvents:495(174882ns) {ForwardSessionEventToSession:612(68105ns) ForwardSessionEventToSession:612(7212ns) ForwardSessionEventToSession:612(3597ns) ForwardSessionEventToSession:612(2861ns) ForwardSessionEventToSession:612(5039ns) ForwardSessionEventToSession:612(6003ns) ForwardSessionEventToSession:612(2826ns) ForwardSessionEventToSession:612(5845ns) ForwardSessionEventToSession:612(2999ns) ForwardSessionEventToSession:612(57205ns)}}}
2025-11-27T18:18:32.414914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:32.416999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471269505271301:2107] [node 4] ICP18 handshake (actor [1:7577471269505271315:2115]) is too old
2025-11-27T18:18:32.417837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471269505271313:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:32.418388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471265210303945:2049] [node 3] ICP22 created new session: [1:7577471269505271328:2049]
2025-11-27T18:18:32.424301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471269505271328:2049] [node 3] ICS09 handshake done sender: [1:7577471269505271311:2112] self: [1:7577471269505271310:0] peer: [3:7577471267540153872:0] socket: 480 qp: -1
2025-11-27T18:18:32.429625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471269505271328:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:32.431776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471269505271307:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.431853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:32.431907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471269505271307:2109] [node 6] ICP22 created new session: [1:7577471269505271330:2109]
2025-11-27T18:18:32.431953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471269505271330:2109] [node 6] ICS09 handshake done sender: [1:7577471269505271313:2113] self: [1:7577471269505271312:0] peer: [6:7577471266055808761:0] socket: 481 qp: -1
2025-11-27T18:18:32.431993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471269505271330:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:32.432036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:32.435773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:32.451859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471265210303945:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(32549502ns) {HandleHandshakeStatus:329(32547057ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(844ns) SwitchToState:179(904ns) ProcessPendingSessionEvents:495(6268198ns) {ForwardSessionEventToSession:612(50588ns) ForwardSessionEventToSession:612(12753ns) ForwardSessionEventToSession:612(2992ns) ForwardSessionEventToSession:612(2643ns) ForwardSessionEventToSession:612(3562745ns) ForwardSessionEventToSession:612(4090ns) ForwardSessionEventToSession:612(2741ns) ForwardSessionEventToSession:612(241041ns) ForwardSessionEventToSession:612(1817ns) ForwardSessionEventToSession:612(2371745ns)}}}
2025-11-27T18:18:32.457894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471269505271309:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.457917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471269505271309:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.458882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471269505271309:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:32.436872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471267540153877:2048] [node 1] ICS09 handshake done sender: [3:7577471267540153862:2140] self: [3:7577471267540153872:0] peer: [1:7577471269505271310:0] socket: 488 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.436902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471267540153877:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:32.436926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:32.433107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471267894606232:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:32.445399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471266055808767:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64805
2025-11-27T18:18:32.439475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471267894606228:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.439619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471267540153856:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.412004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.468944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471265210303944:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.412429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471261450267106:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.439772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:32.445545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471261760841384:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.440424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.445619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:32.440491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471267894606241:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5243
2025-11-27T18:18:32.485791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:32.412471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.439820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471267540153856:2116] [node 6] ICP22 created new session: [3:7577471267540153881:2116]
2025-11-27T18:18:32.488013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471265210303944:2048] [node 2] ICP22 created new session: [1:7577471269505271335:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.412516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471261450267106:2048] [node 1] ICP22 created new session: [4:7577471265745234498:2048]
2025-11-27T18:18:32.440639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471267540153881:2116] [node 6] ICS09 handshake done sender: [3:7577471267540153864:2141] self: [3:7577471267540153863:0] peer: [6:7577471266055808765:0] socket: 495 qp: -1
2025-11-27T18:18:32.412557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471265745234498:2048] [node 1] ICS09 handshake done sender: [4:7577471265745234488:2124] self: [4:7577471265745234490:0] peer: [1:7577471269505271304:0] socket: 486 qp: -1
2025-11-27T18:18:32.488141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471265649940316:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22058
2025-11-27T18:18:32.488382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471265649940317:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22058
2025-11-27T18:18:32.489946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471265649940318:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22058
2025-11-27T18:18:32.490142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471265649940324:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22058
2025-11-27T18:18:32.412588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471265745234498:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:32.488601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471269505271335:2048] [node 2] ICS09 handshake done sender: [1:7577471269505271309:2111] self: [1:7577471269505271308:0] peer: [2:7577471267894606250:0] socket: 508 qp: -1
2025-11-27T18:18:32.440679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471267540153881:2116] [node 6] ICS10 traffic start
2025-11-27T18:18:32.442105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471267894606228:2107] [node 4] ICP22 created new session: [2:7577471267894606246:2107]
2025-11-27T18:18:32.445667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471261760841384:2049] [node 2] ICP22 created new session: [6:7577471266055808769:2049]
2025-11-27T18:18:32.440716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:32.441469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471267540153876:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50803
2025-11-27T18:18:32.445707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471266055808769:2049] [node 2] ICS09 handshake done sender: [6:7577471266055808758:2116] self: [6:7577471266055808763:0] peer: [2:7577471267894606229:0] socket: 489 qp: -1
2025-11-27T18:18:32.445155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471267540153876:2145] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.445737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471266055808769:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:32.445763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:32.448468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471267540153879:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50803
2025-11-27T18:18:32.412641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.497814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471265649940324:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.446268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471266055808745:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.488656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471269505271334:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51453
2025-11-27T18:18:32.449341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471267540153858:2137] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:32.443857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471267894606246:2107] [node 4] ICS09 handshake done sender: [2:7577471267894606232:2122] self: [2:7577471267894606231:0] peer: [4:7577471265745234492:0] socket: 472 qp: -1
2025-11-27T18:18:32.413208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471261450267108:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.450986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471267540153854:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.499459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471265649940318:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.446361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:32.449729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471265745234504:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29801
2025-11-27T18:18:32.446392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471266055808745:2109] [node 4] ICP22 created new session: [6:7577471266055808771:2109]
2025-11-27T18:18:32.450315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471265745234503:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29801
2025-11-27T18:18:32.446422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471266055808771:2109] [node 4] ICS09 handshake done sender: [6:7577471266055808759:2117] self: [6:7577471266055808762:0] peer: [4:7577471265745234482:0] socket: 490 qp: -1
2025-11-27T18:18:32.453475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471261450267107:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31508) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471265745234484:2123] held: no
2025-11-27T18:18:32.446444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471266055808771:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:32.454016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471265745234484:2123] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:32.446464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:32.456414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471265745234489:2125] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:32.455572Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471261760841385:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29638) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471266055808760:2118] held: no
2025-11-27T18:18:32.464437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471261450267108:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.455744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471266055808768:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64805
2025-11-27T18:18:32.451033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:32.501776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471265649940317:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.451213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471267540153854:2107] [node 4] ICP22 created new session: [3:7577471267540153884:2107]
2025-11-27T18:18:32.444530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471267894606246:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:32.469102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471266055808760:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:32.488768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471269505271335:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.444587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:32.502103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471265649940325:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22058
2025-11-27T18:18:32.452265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471267540153884:2107] [node 4] ICS09 handshake done sender: [3:7577471267540153858:2137] self: [3:7577471267540153857:0] peer: [4:7577471265745234500:0] socket: 493 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.483363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471266055808757:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:32.489881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:32.464522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:32.502163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471265649940316:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.448025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471267894606241:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.502350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471261354972950:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.452717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471267540153884:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:32.457290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471263599638854:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.485528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471261760841385:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.464590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471261450267108:2050] [node 3] ICP22 created new session: [4:7577471265745234505:2050]
2025-11-27T18:18:32.485587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:32.502353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471261354972952:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.453836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.457623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471267894606249:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5243
2025-11-27T18:18:32.503828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471265210303944:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(33476563ns) {HandleHandshakeStatus:329(33472692ns) {DropIncomingHandshake:442(424ns) DropOutgoingHandshake:457(290ns) SwitchToState:179(394ns) ProcessPendingSessionEvents:495(3550299ns) {ForwardSessionEventToSession:612(3492101ns) ForwardSessionEventToSession:612(4574ns) ForwardSessionEventToSession:612(3138ns) ForwardSessionEventToSession:612(6276ns) ForwardSessionEventToSession:612(10280ns) ForwardSessionEventToSession:612(4893ns) ForwardSessionEventToSession:612(2372ns) ForwardSessionEventToSession:612(2697ns) ForwardSessionEventToSession:612(8988ns) ForwardSessionEventToSession:612(2623ns)}}}
2025-11-27T18:18:32.464640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471265745234505:2050] [node 3] ICS09 handshake done sender: [4:7577471265745234489:2125] self: [4:7577471265745234500:0] peer: [3:7577471267540153857:0] socket: 499 qp: -1
2025-11-27T18:18:32.485640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471261760841385:2050] [node 3] ICP22 created new session: [6:7577471266055808774:2050]
2025-11-27T18:18:32.465558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471267894606225:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.502420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471261354972951:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.456452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471267540153879:2146] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.507286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471269505271320:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.464686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471265745234505:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.486840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471261760841383:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.456815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471263245186469:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.465585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471267894606225:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.487731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:32.507313Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471269505271320:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.507299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471265649940325:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.464721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:32.468051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471267894606249:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.468664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471267540153887:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50803
2025-11-27T18:18:32.508464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471265649940310:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.488231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471261760841383:2048] [node 1] ICP22 created new session: [6:7577471266055808775:2048]
2025-11-27T18:18:32.508245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471269505271334:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.466788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471261450267107:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.487946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471267540153854:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17969337ns) {HandleHandshakeStatus:329(17954341ns) {DropIncomingHandshake:442(483ns) DropOutgoingHandshake:457(345ns) SwitchToState:179(679ns) ProcessPendingSessionEvents:495(11991502ns) {ForwardSessionEventToSession:612(2856801ns) ForwardSessionEventToSession:612(5252ns) ForwardSessionEventToSession:612(1986051ns) ForwardSessionEventToSession:612(3687ns) ForwardSessionEventToSession:612(4944589ns) ForwardSessionEventToSession:612(5334ns) ForwardSessionEventToSession:612(3338ns) ForwardSessionEventToSession:612(2157413ns) ForwardSessionEventToSession:612(4535ns) ForwardSessionEventToSession:612(3021ns)}}}
2025-11-27T18:18:32.523926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471265649940333:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22058
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.488420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471266055808774:2050] [node 3] ICS09 handshake done sender: [6:7577471266055808760:2118] self: [6:7577471266055808765:0] peer: [3:7577471267540153863:0] socket: 498 qp: -1
2025-11-27T18:18:32.473773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:32.474517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471267894606225:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:32.475887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471261450267107:2049] [node 2] ICP22 created new session: [4:7577471265745234507:2049]
2025-11-27T18:18:32.516477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471269505271337:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51453
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.484434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471265745234507:2049] [node 2] ICS09 handshake done sender: [4:7577471265745234484:2123] self: [4:7577471265745234492:0] peer: [2:7577471267894606231:0] socket: 474 qp: -1
2025-11-27T18:18:32.488089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471267540153879:2146] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:32.534001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471265649940335:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22058
2025-11-27T18:18:32.474894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471263599638855:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.488695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471266055808774:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:32.484982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471265745234507:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:32.534729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471265649940318:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:32.516491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471269505271320:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:32.488183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471267540153854:2107] [node 4] ICP18 handshake (actor [3:7577471267540153876:2145]) is too old
2025-11-27T18:18:32.488829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471266055808775:2048] [node 1] ICS09 handshake done sender: [6:7577471266055808757:2115] self: [6:7577471266055808761:0] peer: [1:7577471269505271312:0] socket: 484 qp: -1
2025-11-27T18:18:32.485925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.474935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:32.535247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471265649940337:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22058
2025-11-27T18:18:32.541681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471261354972952:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.542399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471265649940336:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22058
2025-11-27T18:18:32.488870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471266055808775:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:32.486151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471261450267107:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(24341804ns) {HandleHandshakeStatus:329(24338374ns) {DropIncomingHandshake:442(3737ns) DropOutgoingHandshake:457(1913ns) SwitchToState:179(2847ns) ProcessPendingSessionEvents:495(1000ns)}}
2025-11-27T18:18:32.490512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471263245186469:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.521407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471265210303944:2048] [node 2] ICP18 handshake (actor [1:7577471269505271334:2117]) is too old
2025-11-27T18:18:32.474982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471263599638855:2049] [node 3] ICP22 created new session: [2:7577471267894606253:2049]
2025-11-27T18:18:32.488915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:32.543997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:32.521509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471265745234494:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.523737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471269505271306:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.475018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471267894606253:2049] [node 3] ICS09 handshake done sender: [2:7577471267894606225:2119] self: [2:7577471267894606224:0] peer: [3:7577471267540153886:0] socket: 520 qp: -1
2025-11-27T18:18:32.475045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471267894606253:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:32.475069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:32.485083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471267894606251:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5243
2025-11-27T18:18:32.488298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471263599638855:2049] [node 3] ICP18 handshake (actor [2:7577471267894606249:2126]) is too old
2025-11-27T18:18:32.502200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471267894606241:2125] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:32.507264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471263599638854:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.507804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471267894606255:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5243
2025-11-27T18:18:32.508418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471267894606227:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.508449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471267894606227:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.512187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:32.512245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471263599638854:2048] [node 1] ICP22 created new session: [2:7577471267894606256:2048]
2025-11-27T18:18:32.514370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471267894606256:2048] [node 1] ICS09 handshake done sender: [2:7577471267894606241:2125] self: [2:7577471267894606250:0] peer: [1:7577471269505271308:0] socket: 510 qp: -1
2025-11-27T18:18:32.515027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471267894606256:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:32.516597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471267894606227:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:32.518228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:32.523749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471267894606222:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.523787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:32.523840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471267894606222:2108] [node 5] ICP22 created new session: [2:7577471267894606259:2108]
2025-11-27T18:18:32.524130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471267894606259:2108] [node 5] ICS09 handshake done sender: [2:7577471267894606227:2120] self: [2:7577471267894606226:0] peer: [5:7577471265649940331:0] socket: 471 qp: -1
2025-11-27T18:18:32.530510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471267894606259:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:32.530831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:32.532391Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471263599638854:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27851758ns) {HandleHandshakeStatus:329(27105040ns) {DropIncomingHandshake:442(2529ns) DropOutgoingHandshake:457(631944ns) SwitchToState:179(1674ns) ProcessPendingSessionEvents:495(8113321ns) {ForwardSessionEventToSession:612(7356481ns) ForwardSessionEventToSession:612(4314ns) ForwardSessionEventToSession:612(2634ns) ForwardSessionEventToSession:612(708452ns) ForwardSessionEventToSession:612(4009ns) ForwardSessionEventToSession:612(11549ns) ForwardSessionEventToSession:612(2806ns) ForwardSessionEventToSession:612(2076ns) ForwardSessionEventToSession:612(6085ns) ForwardSessionEventToSession:612(2828ns)}}}
2025-11-27T18:18:32.534721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471267894606255:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.490575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:32.497191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:32.544198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471261354972952:2050] [node 3] ICP22 created new session: [5:7577471265649940338:2050]
2025-11-27T18:18:32.523783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:32.521541Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471265745234494:2127] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.547650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471267894606222:2108] [node 5] ICP18 handshake (actor [2:7577471267894606255:2128]) is too old
2025-11-27T18:18:32.544734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471265649940338:2050] [node 3] ICS09 handshake done sender: [5:7577471265649940318:2114] self: [5:7577471265649940329:0] peer: [3:7577471267540153859:0] socket: 504 qp: -1
2025-11-27T18:18:32.523845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471269505271306:2108] [node 5] ICP22 created new session: [1:7577471269505271339:2108]
2025-11-27T18:18:32.490612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471263245186469:2049] [node 2] ICP22 created new session: [3:7577471267540153888:2049]
2025-11-27T18:18:32.490646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471267540153888:2049] [node 2] ICS09 handshake done sender: [3:7577471267540153879:2146] self: [3:7577471267540153886:0] peer: [2:7577471267894606224:0] socket: 470 qp: -1
2025-11-27T18:18:32.545021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471265649940338:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:32.531119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471265745234494:2127] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:32.523988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471269505271339:2108] [node 5] ICS09 handshake done sender: [1:7577471269505271320:2116] self: [1:7577471269505271319:0] peer: [5:7577471265649940330:0] socket: 478 qp: -1
2025-11-27T18:18:32.527325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471269505271339:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:32.530758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:32.532237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471269505271337:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.542003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471269505271306:2108] [node 5] ICP18 handshake (actor [1:7577471269505271337:2118]) is too old
2025-11-27T18:18:32.533330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471265745234474:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.533402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:32.534036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471265745234474:2108] [node 5] ICP22 created new session: [4:7577471265745234510:2108]
2025-11-27T18:18:32.534899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471265745234510:2108] [node 5] ICS09 handshake done sender: [4:7577471265745234494:2127] self: [4:7577471265745234493:0] peer: [5:7577471265649940334:0] socket: 515 qp: -1
2025-11-27T18:18:32.535230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471265745234510:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:32.535612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:32.490672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471267540153888:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:32.490701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:32.505963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471267540153860:2138] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.505994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471267540153860:2138] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:32.507167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471267540153890:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50803
2025-11-27T18:18:32.508264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471267540153860:2138] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:32.514693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471267540153855:2117] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.515054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:32.515262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471267540153855:2117] [node 5] ICP22 created new session: [3:7577471267540153892:2117]
2025-11-27T18:18:32.521429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471267540153892:2117] [node 5] ICS09 handshake done sender: [3:7577471267540153860:2138] self: [3:7577471267540153859:0] peer: [5:7577471265649940329:0] socket: 494 qp: -1
2025-11-27T18:18:32.521780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471267540153892:2117] [node 5] ICS10 traffic start
2025-11-27T18:18:32.521829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:32.530379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471267540153890:2148] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.544660Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471267540153855:2117] [node 5] ICP35 event processing took too much time PendingConnection:279(20267394ns) {HandleHandshakeStatus:329(20264516ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(583ns) SwitchToState:179(3932ns) ProcessPendingSessionEvents:495(4197305ns) {ForwardSessionEventToSession:612(2580411ns) ForwardSessionEventToSession:612(205904ns) ForwardSessionEventToSession:612(296837ns) ForwardSessionEventToSession:612(5453ns) ForwardSessionEventToSession:612(284126ns) ForwardSessionEventToSession:612(3847ns) ForwardSessionEventToSession:612(786656ns) ForwardSessionEventToSession:612(2799ns) ForwardSessionEventToSession:612(2652ns) ForwardSessionEventToSession:612(9459ns)}}}
2025-11-27T18:18:32.548171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471267540153855:2117] [node 5] ICP18 handshake (actor [3:7577471267540153890:2148]) is too old
2025-11-27T18:18:32.547361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:32.549710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471265649940324:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:32.550404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471265649940325:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:32.550958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471265649940317:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:32.551158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471261354972950:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64968) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471265649940324:2118] held: no
2025-11-27T18:18:32.551246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471261354972950:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.551325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:32.551372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471261354972950:2048] [node 1] ICP22 created new session: [5:7577471265649940340:2048]
2025-11-27T18:18:32.551416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471265649940340:2048] [node 1] ICS09 handshake done sender: [5:7577471265649940324:2118] self: [5:7577471265649940330:0] peer: [1:7577471269505271319:0] socket: 469 qp: -1
2025-11-27T18:18:32.551455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471265649940340:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:32.551504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:32.553814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471265649940310:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.553924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:32.553974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471265649940310:2108] [node 4] ICP22 created new session: [5:7577471265649940342:2108]
2025-11-27T18:18:32.554046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471265649940342:2108] [node 4] ICS09 handshake done sender: [5:7577471265649940325:2119] self: [5:7577471265649940334:0] peer: [4:7577471265745234493:0] socket: 491 qp: -1
2025-11-27T18:18:32.554100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471265649940342:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:32.554140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:32.554175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471261354972951:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.554259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:32.554309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471261354972951:2049] [node 2] ICP22 created new session: [5:7577471265649940344:2049]
2025-11-27T18:18:32.554805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471265649940344:2049] [node 2] ICS09 handshake done sender: [5:7577471265649940317:2113] self: [5:7577471265649940331:0] peer: [2:7577471267894606226:0] socket: 503 qp: -1
2025-11-27T18:18:32.556179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471265649940344:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:32.556321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:32.557740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471261354972951:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.560824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471266055808778:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64805
2025-11-27T18:18:32.561507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471266055808778:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.561622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471266055808741:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:32.561905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471265649940323:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:32.561942Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471265649940323:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:32.563418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471265649940323:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:32.563489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471266055808780:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64805
2025-11-27T18:18:32.563708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471265649940309:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:32.563807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:32.563871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471265649940309:2109] [node 6] ICP22 created new session: [5:7577471265649940349:2109]
2025-11-27T18:18:32.563935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471265649940349:2109] [node 6] ICS09 handshake done sender: [5:7577471265649940323:2117] self: [5:7577471265649940322:0] peer: [6:7577471266055808779:0] socket: 519 qp: -1
2025-11-27T18:18:32.563971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471265649940349:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:32.563996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471266055808778:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:32.564032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:32.564124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471266055808741:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:32.564200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:32.564251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471266055808741:2106] [node 5] ICP22 created new session: [6:7577471266055808781:2106]
2025-11-27T18:18:32.564540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471266055808781:2106] [node 5] ICS09 handshake done sender: [6:7577471266055808778:2123] self: [6:7577471266055808779:0] peer: [5:7577471265649940322:0] socket: 514 qp: -1
2025-11-27T18:18:32.564576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471266055808781:2106] [node 5] ICS10 traffic start
2025-11-27T18:18:32.564604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:32.600266Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:32.600536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471267894606256:2048] [node 1] ICS07 socket disconnect 510 reason# EndOfStream
2025-11-27T18:18:32.600598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471267894606256:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.600698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:32.600760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471267894606256:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:32.601602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:32.601785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471267540153877:2048] [node 1] ICS01 socket: 488 reason# EndOfStream
2025-11-27T18:18:32.601829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471263245186468:2048] [node 1] ICP30 unregister session Session# [3:7577471267540153877:2048] VirtualId# [3:7577471267540153872:0]
2025-11-27T18:18:32.601862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471267540153877:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.601925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:32.601964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:32.602992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471263599638854:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64968) Socket error# Connection reset by peer state# SendExRequestHeader processed# 0 remain# 8 incoming: [0:0:0] held: no
2025-11-27T18:18:32.603037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471267894606256:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:32.604290Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471263599638854:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64968) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:32.604344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471267894606256:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:32.604728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471263599638854:2048] [node 1] ICP30 unregister session Session# [2:7577471267894606256:2048] VirtualId# [2:7577471267894606250:0]
2025-11-27T18:18:32.604992Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:32.605175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471265745234498:2048] [node 1] ICS01 socket: 486 reason# EndOfStream
2025-11-27T18:18:32.605227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471261450267106:2048] [node 1] ICP30 unregister session Session# [4:7577471265745234498:2048] VirtualId# [4:7577471265745234490:0]
2025-11-27T18:18:32.605291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471265745234498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.605342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:32.605381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:32.604868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:32.604917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471263599638854:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:32.604933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:32.606121Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:32.606311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471265649940340:2048] [node 1] ICS01 socket: 469 reason# EndOfStream
2025-11-27T18:18:32.606354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471261354972950:2048] [node 1] ICP30 unregister session Session# [5:7577471265649940340:2048] VirtualId# [5:7577471265649940330:0]
2025-11-27T18:18:32.606383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471265649940340:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.606446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:32.606488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:32.607031Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:32.607217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471266055808775:2048] [node 1] ICS01 socket: 484 reason# EndOfStream
2025-11-27T18:18:32.607275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471261760841383:2048] [node 1] ICP30 unregister session Session# [6:7577471266055808775:2048] VirtualId# [6:7577471266055808761:0]
2025-11-27T18:18:32.607308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471266055808775:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.607402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:32.607451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:32.668648Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:32.668902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471267540153888:2049] [node 2] ICS01 socket: 470 reason# EndOfStream
2025-11-27T18:18:32.668951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471263245186469:2049] [node 2] ICP30 unregister session Session# [3:7577471267540153888:2049] VirtualId# [3:7577471267540153886:0]
2025-11-27T18:18:32.669001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471267540153888:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.669087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:32.669150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:32.672535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:32.672798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471265649940344:2049] [node 2] ICS01 socket: 503 reason# EndOfStream
2025-11-27T18:18:32.672877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471261354972951:2049] [node 2] ICP30 unregister session Session# [5:7577471265649940344:2049] VirtualId# [5:7577471265649940331:0]
2025-11-27T18:18:32.672928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471265649940344:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.673024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:32.673082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:32.673529Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:32.673709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471266055808769:2049] [node 2] ICS07 socket disconnect 489 reason# EndOfStream
2025-11-27T18:18:32.673740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471266055808769:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.673811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:32.673869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471266055808769:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:32.674574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471261760841384:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31508) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:32.674635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471266055808769:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:32.674683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471261760841384:2049] [node 2] ICP30 unregister session Session# [6:7577471266055808769:2049] VirtualId# [6:7577471266055808763:0]
2025-11-27T18:18:32.674737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:32.674768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471261760841384:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31508) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:32.674786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31508) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:32.676365Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:32.676703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471265745234507:2049] [node 2] ICS01 socket: 474 reason# EndOfStream
2025-11-27T18:18:32.676819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471261450267107:2049] [node 2] ICP30 unregister session Session# [4:7577471265745234507:2049] VirtualId# [4:7577471265745234492:0]
2025-11-27T18:18:32.676896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471265745234507:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.677006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:32.677073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:32.684871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471261760841384:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:32.754651Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:32.754893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471265745234505:2050] [node 3] ICS01 socket: 499 reason# EndOfStream
2025-11-27T18:18:32.754942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471261450267108:2050] [node 3] ICP30 unregister session Session# [4:7577471265745234505:2050] VirtualId# [4:7577471265745234500:0]
2025-11-27T18:18:32.754982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471265745234505:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.755090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:32.755172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:32.756697Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:32.756935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471265649940338:2050] [node 3] ICS07 socket disconnect 504 reason# EndOfStream
2025-11-27T18:18:32.756980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471265649940338:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.757049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:32.757116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471265649940338:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:32.757349Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:32.757577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471266055808774:2050] [node 3] ICS01 socket: 498 reason# EndOfStream
2025-11-27T18:18:32.757627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471261760841385:2050] [node 3] ICP30 unregister session Session# [6:7577471266055808774:2050] VirtualId# [6:7577471266055808765:0]
2025-11-27T18:18:32.757659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471266055808774:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.757730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:32.757783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:32.758215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471261354972952:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29638) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:32.758338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471265649940338:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:32.758388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471261354972952:2050] [node 3] ICP30 unregister session Session# [5:7577471265649940338:2050] VirtualId# [5:7577471265649940329:0]
2025-11-27T18:18:32.758444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:32.758476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471261354972952:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29638) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:32.758492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29638) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:32.768864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471261354972952:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:32.818317Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:32.818568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471265649940342:2108] [node 4] ICS07 socket disconnect 491 reason# EndOfStream
2025-11-27T18:18:32.818609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471265649940342:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.818687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:32.818743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471265649940342:2108] [node 4] ICS15 start handshake
2025-11-27T18:18:32.819358Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:32.819426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471265649940310:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26996) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:32.819623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471266055808771:2109] [node 4] ICS07 socket disconnect 490 reason# EndOfStream
2025-11-27T18:18:32.819486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471265649940342:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:32.819660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471266055808771:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.819575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471265649940310:2108] [node 4] ICP30 unregister session Session# [5:7577471265649940342:2108] VirtualId# [5:7577471265649940334:0]
2025-11-27T18:18:32.819734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:32.819651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:32.819769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471266055808771:2109] [node 4] ICS15 start handshake
2025-11-27T18:18:32.819687Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471265649940310:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26996) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:32.819708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26996) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:32.820401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471266055808745:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26996) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:32.820526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471266055808771:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:32.820588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471266055808745:2109] [node 4] ICP30 unregister session Session# [6:7577471266055808771:2109] VirtualId# [6:7577471266055808762:0]
2025-11-27T18:18:32.820637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:32.820677Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471266055808745:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26996) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:32.820700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26996) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:32.829209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471265649940310:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:18:32.833497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471266055808745:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:18:32.899577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:32.899825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471266055808781:2106] [node 5] ICS07 socket disconnect 514 reason# EndOfStream
2025-11-27T18:18:32.899862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471266055808781:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:32.899949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:32.899999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471266055808781:2106] [node 5] ICS15 start handshake
2025-11-27T18:18:32.900911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471266055808741:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10838) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:32.900981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471266055808781:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:32.901020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471266055808741:2106] [node 5] ICP30 unregister session Session# [6:7577471266055808781:2106] VirtualId# [6:7577471266055808779:0]
2025-11-27T18:18:32.901075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:18:32.901113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471266055808741:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10838) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:32.901135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10838) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:32.991902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471267955503484:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:32.992162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471267955503483:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:33.013463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471270565243396:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:33.013729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471270565243395:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:33.035244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471270556187445:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:33.035376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471270556187446:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:33.058725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471269786024152:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:33.058868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471269786024154:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:33.058908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471269786024153:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:33.081720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471270434694566:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:33.081875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471270434694567:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:33.082243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471270434694565:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:33.105372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471271017049005:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:33.105432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471271017049006:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:33.105462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471271017049007:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:33.230606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471271017049067:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:33.262983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471272250470843:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:33.277677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471270565243468:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:33.269883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471272250470844:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:33.277517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471272250470845:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:33.279632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471269786024215:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:33.279532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471270565243470:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:33.279691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471270565243469:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:33.279182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471270434694646:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:33.279161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471271017049069:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:33.279602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471270556187509:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:33.281349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471269786024216:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:33.279793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471270556187508:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:33.282344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471270434694647:2118] [node 6] ICP01 ready to work
2025-11-27T18:18:33.281277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471270556187510:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:33.290742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471269786024152:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(93569624ns) {Configure:104(93561573ns) {ProcessConfigured:135(499091ns) {StartInitialHandshake:157(496673ns) {DropHandshakes:470(1752ns) {DropIncomingHandshake:442(501ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(669ns) PrepareNewSessionHandshake:477(1992ns) {SwitchToState:179(691ns)}}}}}
2025-11-27T18:18:33.291011Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471269786024154:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(101007384ns) {Configure:104(92899948ns) {ProcessConfigured:135(498435ns) {StartInitialHandshake:157(495619ns) {DropHandshakes:470(2996ns) {DropIncomingHandshake:442(725ns) DropOutgoingHandshake:457(571ns)} GenerateSessionVirtualId:988(1222ns) PrepareNewSessionHandshake:477(2093ns) {SwitchToState:179(823ns)}}}}}
2025-11-27T18:18:33.291238Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471269786024153:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(100826480ns) {Configure:104(100822337ns) {ProcessConfigured:135(639007ns) {StartInitialHandshake:157(636178ns) {DropHandshakes:470(4609ns) {DropIncomingHandshake:442(452ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(547ns) PrepareNewSessionHandshake:477(2095ns) {SwitchToState:179(850ns)}}}}}
2025-11-27T18:18:33.292518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471267955503483:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(107440208ns) {Configure:104(107432106ns) {ProcessConfigured:135(454516ns) {StartInitialHandshake:157(452306ns) {DropHandshakes:470(1246ns) {DropIncomingHandshake:442(348ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(1447ns) {SwitchToState:179(540ns)}}}}}
2025-11-27T18:18:33.298504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471270556187446:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(106162832ns) {Configure:104(106152892ns) {ProcessConfigured:135(534503ns) {StartInitialHandshake:157(532501ns) {DropHandshakes:470(1463ns) {DropIncomingHandshake:442(353ns) DropOutgoingHandshake:457(231ns)} GenerateSessionVirtualId:988(502ns) PrepareNewSessionHandshake:477(1386ns) {SwitchToState:179(679ns)}}}}}
2025-11-27T18:18:33.301370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471270434694566:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(100575375ns) {Configure:104(100571003ns) {ProcessConfigured:135(1344526ns) {StartInitialHandshake:157(1343442ns) {DropHandshakes:470(1049ns) {DropIncomingHandshake:442(353ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(760ns) PrepareNewSessionHandshake:477(1425ns) {SwitchToState:179(119ns)}}}}}
2025-11-27T18:18:33.303386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471270565243396:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25264713ns) {Configure:104(25257725ns) {ProcessConfigured:135(901460ns) {StartInitialHandshake:157(899850ns) {DropHandshakes:470(11825ns) {DropIncomingHandshake:442(276ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(929ns) PrepareNewSessionHandshake:477(1442ns) {SwitchToState:179(640ns)}}}}}
2025-11-27T18:18:33.304413Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471270434694567:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(104598347ns) {Configure:104(104594399ns) {ProcessConfigured:135(4932836ns) {StartInitialHandshake:157(4930808ns) {DropHandshakes:470(977ns) {DropIncomingHandshake:442(276ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(419ns) PrepareNewSessionHandshake:477(1895ns) {SwitchToState:179(977ns)}}}}}
2025-11-27T18:18:33.304632Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471270565243468:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18909498ns) {Configure:104(18891162ns) {ProcessConfigured:135(1064558ns) {StartInitialHandshake:157(1061651ns) {DropHandshakes:470(1316ns) {DropIncomingHandshake:442(275ns) DropOutgoingHandshake:457(137ns)} GenerateSessionVirtualId:988(1029ns) PrepareNewSessionHandshake:477(1730ns) {SwitchToState:179(689ns)}}}}}
2025-11-27T18:18:33.304970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471270434694565:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18325311ns) {Configure:104(18321238ns) {ProcessConfigured:135(5303728ns) {StartInitialHandshake:157(5301737ns) {DropHandshakes:470(984ns) {DropIncomingHandshake:442(348ns) DropOutgoingHandshake:457(231ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:477(1664ns) {SwitchToState:179(406ns)}}}}}
2025-11-27T18:18:33.317406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471271017049006:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40930433ns) {Configure:104(40922455ns) {ProcessConfigured:135(5643921ns) {StartInitialHandshake:157(5641831ns) {DropHandshakes:470(395917ns) {DropIncomingHandshake:442(956ns) DropOutgoingHandshake:457(425ns)} GenerateSessionVirtualId:988(1115ns) PrepareNewSessionHandshake:477(2941ns) {SwitchToState:179(1348ns)}}}}}
2025-11-27T18:18:33.304668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471270565243470:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23331839ns) {Configure:104(23326175ns) {ProcessConfigured:135(5666534ns) {StartInitialHandshake:157(5664141ns) {DropHandshakes:470(1334ns) {DropIncomingHandshake:442(376ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(562ns) PrepareNewSessionHandshake:477(3237ns) {SwitchToState:179(1252ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.321168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471270434694647:2118] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19619271ns) {Configure:104(17604694ns) {ProcessConfigured:135(64926ns) {StartInitialHandshake:157(62376ns) {DropHandshakes:470(2501ns) {DropIncomingHandshake:442(477ns) DropOutgoingHandshake:457(380ns)} GenerateSessionVirtualId:988(906ns) PrepareNewSessionHandshake:477(1491ns) {SwitchToState:179(570ns)}}}}}
2025-11-27T18:18:33.317430Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471271017049007:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21345992ns) {Configure:104(21342009ns) {ProcessConfigured:135(2918686ns) {StartInitialHandshake:157(2916693ns) {DropHandshakes:470(659ns) {DropIncomingHandshake:442(244ns) DropOutgoingHandshake:457(138ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:477(2691ns) {SwitchToState:179(1036ns)}}}}}
2025-11-27T18:18:33.305434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471270565243395:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24539598ns) {Configure:104(24534695ns) {ProcessConfigured:135(5581959ns) {StartInitialHandshake:157(5580142ns) {DropHandshakes:470(1249ns) {DropIncomingHandshake:442(265ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(440ns) PrepareNewSessionHandshake:477(295096ns) {SwitchToState:179(1553ns)}}}}}
2025-11-27T18:18:33.323497Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471271017049067:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(30707817ns) {Configure:104(30697593ns) {ProcessConfigured:135(5535502ns) {StartInitialHandshake:157(5533498ns) {DropHandshakes:470(927ns) {DropIncomingHandshake:442(280ns) DropOutgoingHandshake:457(134ns)} GenerateSessionVirtualId:988(334ns) PrepareNewSessionHandshake:477(1793ns) {SwitchToState:179(743ns)}}}}}
2025-11-27T18:18:33.336051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471269786024216:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36735587ns) {Configure:104(36729979ns) {ProcessConfigured:135(3372326ns) {StartInitialHandshake:157(3369135ns) {DropHandshakes:470(2722ns) {DropIncomingHandshake:442(597ns) DropOutgoingHandshake:457(653ns)} GenerateSessionVirtualId:988(813ns) PrepareNewSessionHandshake:477(2256ns) {SwitchToState:179(1013ns)}}}}}
2025-11-27T18:18:33.323574Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471271017049005:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(41302506ns) {Configure:104(41293215ns) {ProcessConfigured:135(3247948ns) {StartInitialHandshake:157(3246497ns) {DropHandshakes:470(1390ns) {DropIncomingHandshake:442(371ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(491ns) PrepareNewSessionHandshake:477(1010ns) {SwitchToState:179(119ns)}}}}}
2025-11-27T18:18:33.326505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471271017049078:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3434
2025-11-27T18:18:33.338597Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471272250470845:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(43880736ns) {Configure:104(43874911ns) {ProcessConfigured:135(10113544ns) {StartInitialHandshake:157(10110810ns) {DropHandshakes:470(4713ns) {DropIncomingHandshake:442(568ns) DropOutgoingHandshake:457(351ns)} GenerateSessionVirtualId:988(663ns) PrepareNewSessionHandshake:477(3987ns) {SwitchToState:179(1979ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.339691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471271017049078:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.340437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471271017049067:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.341520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471270434694657:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.342661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471272250470855:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20279
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.341559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471270434694657:2132] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.343987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471270556187520:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38708
2025-11-27T18:18:33.346265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471270565243469:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39151047ns) {Configure:104(39143823ns) {ProcessConfigured:135(2546611ns) {StartInitialHandshake:157(2293539ns) {DropHandshakes:470(3067ns) {DropIncomingHandshake:442(778ns) DropOutgoingHandshake:457(296ns)} GenerateSessionVirtualId:988(2620ns) PrepareNewSessionHandshake:477(2324ns) {SwitchToState:179(856ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:18:33.347230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471272250470855:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:33.341859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471270434694661:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64775
2025-11-27T18:18:33.344172Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471270556187510:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(47609572ns) {Configure:104(47604930ns) {ProcessConfigured:135(1616202ns) {StartInitialHandshake:157(1613942ns) {DropHandshakes:470(341089ns) {DropIncomingHandshake:442(481ns) DropOutgoingHandshake:457(920ns)} GenerateSessionVirtualId:988(1180ns) PrepareNewSessionHandshake:477(1906ns) {SwitchToState:179(1092ns)}}}}}
2025-11-27T18:18:33.346850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471270434694657:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:33.347927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471270556187522:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38708
2025-11-27T18:18:33.348310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471272250470860:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20279
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.350101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471271017049069:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22641707ns) {Configure:104(22632752ns) {ProcessConfigured:135(2657329ns) {StartInitialHandshake:157(2654697ns) {DropHandshakes:470(2020ns) {DropIncomingHandshake:442(443ns) DropOutgoingHandshake:457(280ns)} GenerateSessionVirtualId:988(712ns) PrepareNewSessionHandshake:477(2235ns) {SwitchToState:179(1092ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.353539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471270565243479:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44661
2025-11-27T18:18:33.348877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471270556187520:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.348229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471270434694647:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.348607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471272250470861:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20279
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:18:33.350111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471271017049083:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3434
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.356370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471270565243479:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.355215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471272250470852:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.353307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471270556187523:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38708
2025-11-27T18:18:33.348330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471270434694661:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.348891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:33.355262Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471272250470852:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-27T18:18:33.356451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471270565243481:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44661
2025-11-27T18:18:33.354694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471270556187522:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:33.350058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471270434694647:2118] [node 6] ICP22 created new session: [5:7577471270434694663:2118]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.362517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471271017049083:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.357779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471272250470861:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.358557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471270565243480:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44661
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.356351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471270556187445:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.353573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471270434694565:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.369252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471271017049007:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.369954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471269786024230:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5912
2025-11-27T18:18:33.361388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471272250470851:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.357378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471270556187523:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.358560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471270565243482:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44661
2025-11-27T18:18:33.353824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471270434694663:2118] [node 6] ICS09 handshake done sender: [5:7577471270434694657:2132] self: [5:7577471270434694656:0] peer: [6:7577471271017049082:0] socket: 482 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.372245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471269786024231:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5912
2025-11-27T18:18:33.369896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471271017049086:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3434
2025-11-27T18:18:33.378299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471269786024230:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.361426Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471272250470851:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:33.361395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471270565243487:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44661
2025-11-27T18:18:33.356437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471270434694663:2118] [node 6] ICS10 traffic start
2025-11-27T18:18:33.357778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471270556187527:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38708
2025-11-27T18:18:33.373674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471271017049087:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3434
2025-11-27T18:18:33.361576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471272250470863:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20279
2025-11-27T18:18:33.362320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471270565243481:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.365391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471270556187527:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.358642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:33.362296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471272250470860:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.378706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471271017049087:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.370459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471270556187516:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.369693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471270565243480:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.370486Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471270556187516:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:33.379301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471270434694667:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64775
2025-11-27T18:18:33.380615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471269786024231:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.380445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471271017049007:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13463) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471271017049083:2115] held: no
2025-11-27T18:18:33.368584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471272250470852:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:33.369941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471270565243482:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:18:33.381290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471269786024154:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-27T18:18:33.371304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471270556187516:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:33.378117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471272250470851:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:33.370357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471270565243487:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.372971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471270556187528:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38708
2025-11-27T18:18:33.380051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471270434694647:2118] [node 6] ICP35 event processing took too much time PendingConnection:279(29822404ns) {HandleHandshakeStatus:329(29819565ns) {DropIncomingHandshake:442(271ns) DropOutgoingHandshake:457(499ns) SwitchToState:179(553ns) ProcessPendingSessionEvents:495(8058609ns) {ForwardSessionEventToSession:612(68935ns) ForwardSessionEventToSession:612(8550ns) ForwardSessionEventToSession:612(3313ns) ForwardSessionEventToSession:612(3231ns) ForwardSessionEventToSession:612(75384ns) ForwardSessionEventToSession:612(2639ns) ForwardSessionEventToSession:612(3291517ns) ForwardSessionEventToSession:612(3760384ns) ForwardSessionEventToSession:612(3457ns) ForwardSessionEventToSession:612(514231ns)}}}
2025-11-27T18:18:33.380625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471271017049078:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.377944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471270565243395:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.386616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471270434694661:2133] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:33.386944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471271017049067:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.378278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471272250470844:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.373004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471270556187509:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.388382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471270434694565:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.386581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471270565243475:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.378378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:33.387420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471271017049006:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.373668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:33.388635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:33.387716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:33.378386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471272250470850:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.386621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471270565243475:2120] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:33.392195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471270434694565:2048] [node 1] ICP22 created new session: [5:7577471270434694668:2048]
2025-11-27T18:18:33.377555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471270556187509:2109] [node 6] ICP22 created new session: [3:7577471270556187535:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.378412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471272250470850:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:33.392698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471270565243476:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.378440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471270556187535:2109] [node 6] ICS09 handshake done sender: [3:7577471270556187516:2111] self: [3:7577471270556187512:0] peer: [6:7577471271017049088:0] socket: 502 qp: -1
2025-11-27T18:18:33.388368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471271017049067:2107] [node 5] ICP22 created new session: [6:7577471271017049091:2107]
2025-11-27T18:18:33.392740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471270434694668:2048] [node 1] ICS09 handshake done sender: [5:7577471270434694661:2133] self: [5:7577471270434694664:0] peer: [1:7577471272250470847:0] socket: 495 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.392729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471270565243476:2121] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.379035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471272250470844:2108] [node 5] ICP22 created new session: [1:7577471272250470868:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.392415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471271017049091:2107] [node 5] ICS09 handshake done sender: [6:7577471271017049078:2114] self: [6:7577471271017049082:0] peer: [5:7577471270434694656:0] socket: 487 qp: -1
No RDMA context found for GID: 2025-11-27T18:18:33.379043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471270556187528:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:33.393812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471270434694668:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:33.395686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471270565243475:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:33.414629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471269786024234:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5912
2025-11-27T18:18:33.380132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471272250470868:2108] [node 5] ICS09 handshake done sender: [1:7577471272250470852:2113] self: [1:7577471272250470847:0] peer: [5:7577471270434694664:0] socket: 485 qp: -1
2025-11-27T18:18:33.392737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471271017049091:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:33.379995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471270556187535:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.420714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471269786024234:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.393860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:33.380049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:33.415890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471270565243494:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44661
2025-11-27T18:18:33.380258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471272250470863:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.392782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471271017049089:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3434
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.380591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471270556187446:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.420877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471269786024152:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.420435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471270565243476:2121] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:33.380287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471272250470868:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:33.393933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:33.386558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471270556187518:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.421787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471269786024236:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5912
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.425304Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471270434694565:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32343268ns) {HandleHandshakeStatus:329(32341172ns) {DropIncomingHandshake:442(495ns) DropOutgoingHandshake:457(12526ns) SwitchToState:179(362ns) ProcessPendingSessionEvents:495(19229956ns) {ForwardSessionEventToSession:612(17817961ns) ForwardSessionEventToSession:612(6637ns) ForwardSessionEventToSession:612(2356ns) ForwardSessionEventToSession:612(378585ns) ForwardSessionEventToSession:612(7666ns) ForwardSessionEventToSession:612(4716ns) ForwardSessionEventToSession:612(640003ns) ForwardSessionEventToSession:612(4287ns) ForwardSessionEventToSession:612(338723ns) ForwardSessionEventToSession:612(4769ns)}}}
2025-11-27T18:18:33.380326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:33.401546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471271017049083:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:33.386605Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471270556187518:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:33.422150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471270565243395:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471270565243482:2127] held: no
2025-11-27T18:18:33.424603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471269786024235:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5912
2025-11-27T18:18:33.425444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471270434694670:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64775
2025-11-27T18:18:33.380417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471267955503484:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.427161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471269786024235:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.424330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471270565243482:2127] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:33.388610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471270556187536:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38708
2025-11-27T18:18:33.406512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471271017049067:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(19370991ns) {HandleHandshakeStatus:329(19368397ns) {DropIncomingHandshake:442(739ns) DropOutgoingHandshake:457(5709ns) SwitchToState:179(449ns) ProcessPendingSessionEvents:495(4842177ns) {ForwardSessionEventToSession:612(4786659ns) ForwardSessionEventToSession:612(11254ns) ForwardSessionEventToSession:612(2476ns) ForwardSessionEventToSession:612(2286ns) ForwardSessionEventToSession:612(5090ns) ForwardSessionEventToSession:612(3565ns) ForwardSessionEventToSession:612(8818ns) ForwardSessionEventToSession:612(2354ns) ForwardSessionEventToSession:612(2462ns) ForwardSessionEventToSession:612(8078ns)}}}
2025-11-27T18:18:33.381779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:33.424587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471270565243468:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.431706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471270434694670:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.416288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471271017049007:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.427674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471269786024154:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13463) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471269786024231:2116] held: no
2025-11-27T18:18:33.382827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471267955503484:2049] [node 3] ICP22 created new session: [1:7577471272250470873:2049]
2025-11-27T18:18:33.389180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471270556187445:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471270556187522:2115] held: no
2025-11-27T18:18:33.424636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471270565243396:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.419498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:33.394493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471270556187522:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:33.431381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471269786024231:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:33.432352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471270434694567:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.395139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471270556187509:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(17083338ns) {HandleHandshakeStatus:329(17081465ns) {DropIncomingHandshake:442(14824ns) DropOutgoingHandshake:457(530ns) SwitchToState:179(382ns) ProcessPendingSessionEvents:495(7850486ns) {ForwardSessionEventToSession:612(1434052ns) ForwardSessionEventToSession:612(10342ns) ForwardSessionEventToSession:612(3391ns) ForwardSessionEventToSession:612(607767ns) ForwardSessionEventToSession:612(740117ns) ForwardSessionEventToSession:612(10964ns) ForwardSessionEventToSession:612(234513ns) ForwardSessionEventToSession:612(3225ns) ForwardSessionEventToSession:612(985774ns) ForwardSessionEventToSession:612(4600ns)}}}
2025-11-27T18:18:33.386598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471272250470873:2049] [node 3] ICS09 handshake done sender: [1:7577471272250470851:2112] self: [1:7577471272250470846:0] peer: [3:7577471270556187530:0] socket: 486 qp: -1
2025-11-27T18:18:33.420476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471271017049007:2050] [node 3] ICP22 created new session: [6:7577471271017049096:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.432251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471269786024154:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.424693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:33.387636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471272250470873:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.425284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471270565243396:2049] [node 3] ICP22 created new session: [2:7577471270565243497:2049]
2025-11-27T18:18:33.432312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:33.420746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471271017049092:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3434
2025-11-27T18:18:33.405712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471270556187445:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.387689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:33.426462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:33.432374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471269786024154:2050] [node 3] ICP22 created new session: [4:7577471269786024240:2050]
2025-11-27T18:18:33.421603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471271017049096:2050] [node 3] ICS09 handshake done sender: [6:7577471271017049083:2115] self: [6:7577471271017049088:0] peer: [3:7577471270556187512:0] socket: 504 qp: -1
2025-11-27T18:18:33.405769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:33.433774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471269786024240:2050] [node 3] ICS09 handshake done sender: [4:7577471269786024231:2116] self: [4:7577471269786024233:0] peer: [3:7577471270556187513:0] socket: 526 qp: -1
2025-11-27T18:18:33.426729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471270565243497:2049] [node 3] ICS09 handshake done sender: [2:7577471270565243475:2120] self: [2:7577471270565243471:0] peer: [3:7577471270556187538:0] socket: 509 qp: -1
2025-11-27T18:18:33.389141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471272250470850:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:33.434061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471269786024240:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:33.421661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471271017049096:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:33.426765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471270565243497:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:33.405833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471270556187445:2048] [node 1] ICP22 created new session: [3:7577471270556187542:2048]
2025-11-27T18:18:33.434100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:33.405460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471267955503483:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.421707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.405873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471270556187542:2048] [node 1] ICS09 handshake done sender: [3:7577471270556187522:2115] self: [3:7577471270556187530:0] peer: [1:7577471272250470846:0] socket: 494 qp: -1
2025-11-27T18:18:33.405556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:33.427129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471270565243468:2110] [node 6] ICP22 created new session: [2:7577471270565243498:2110]
2025-11-27T18:18:33.424000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471271017049092:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.434212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471269786024152:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471269786024234:2117] held: no
2025-11-27T18:18:33.405906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471270556187542:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:33.460889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471270434694672:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64775
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.440272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471269786024238:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5912
No RDMA context found for GID: 2025-11-27T18:18:33.424418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471271017049005:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.407307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471267955503483:2048] [node 2] ICP22 created new session: [1:7577471272250470876:2048]
::ffff:127.1.0.0
2025-11-27T18:18:33.427771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471270565243498:2110] [node 6] ICS09 handshake done sender: [2:7577471270565243476:2121] self: [2:7577471270565243473:0] peer: [6:7577471271017049090:0] socket: 512 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.405954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:33.427790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:33.407152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471270556187518:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:33.414661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471272250470876:2048] [node 2] ICS09 handshake done sender: [1:7577471272250470850:2111] self: [1:7577471272250470849:0] peer: [2:7577471270565243492:0] socket: 484 qp: -1
2025-11-27T18:18:33.462490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471270434694672:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.427596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471271017049095:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3434
2025-11-27T18:18:33.420691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471270556187508:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.451394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471269786024234:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:33.428103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471270565243498:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:33.434015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471271017049087:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:33.462752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471270434694674:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64775
2025-11-27T18:18:33.419382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471272250470876:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:33.421014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471270556187540:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38708
2025-11-27T18:18:33.428136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:33.460575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471269786024152:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.420643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:33.434025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471271017049006:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30126) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471271017049087:2118] held: no
2025-11-27T18:18:33.421844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:33.463303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471270434694673:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64775
2025-11-27T18:18:33.461891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:33.435194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471271017049099:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3434
2025-11-27T18:18:33.422125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471270556187508:2107] [node 4] ICP22 created new session: [3:7577471270556187544:2107]
2025-11-27T18:18:33.421311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471267955503484:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(29541051ns) {HandleHandshakeStatus:329(29224171ns) {DropIncomingHandshake:442(23524ns) DropOutgoingHandshake:457(199ns) SwitchToState:179(400ns) ProcessPendingSessionEvents:495(15476877ns) {ForwardSessionEventToSession:612(1273605ns) ForwardSessionEventToSession:612(7776ns) ForwardSessionEventToSession:612(208714ns) ForwardSessionEventToSession:612(3131ns) ForwardSessionEventToSession:612(2625ns) ForwardSessionEventToSession:612(5762071ns) ForwardSessionEventToSession:612(3877ns) ForwardSessionEventToSession:612(246222ns) ForwardSessionEventToSession:612(7941026ns) ForwardSessionEventToSession:612(7487ns)}}}
2025-11-27T18:18:33.466269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471270434694566:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.431268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471270565243395:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.462246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471269786024152:2048] [node 1] ICP22 created new session: [4:7577471269786024245:2048]
2025-11-27T18:18:33.439717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471271017049006:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.424765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471272250470853:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.466395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471270434694567:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13463) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471270434694670:2135] held: no
2025-11-27T18:18:33.431965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:33.424008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471270556187544:2107] [node 4] ICS09 handshake done sender: [3:7577471270556187518:2113] self: [3:7577471270556187513:0] peer: [4:7577471269786024233:0] socket: 525 qp: -1
2025-11-27T18:18:33.462420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471269786024243:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5912
2025-11-27T18:18:33.424792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471272250470853:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:33.469633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471270434694670:2135] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:33.445549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:33.466203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471269786024245:2048] [node 1] ICS09 handshake done sender: [4:7577471269786024234:2117] self: [4:7577471269786024237:0] peer: [1:7577471272250470848:0] socket: 529 qp: -1
2025-11-27T18:18:33.432297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471270565243395:2048] [node 1] ICP22 created new session: [2:7577471270565243501:2048]
2025-11-27T18:18:33.424614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471270556187544:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:33.471062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471270434694567:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.424776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:33.450368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471271017049006:2049] [node 2] ICP22 created new session: [6:7577471271017049100:2049]
2025-11-27T18:18:33.432909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471270565243501:2048] [node 1] ICS09 handshake done sender: [2:7577471270565243482:2127] self: [2:7577471270565243492:0] peer: [1:7577471272250470849:0] socket: 505 qp: -1
2025-11-27T18:18:33.425746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471272250470859:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.466244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471269786024245:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:33.450405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471271017049092:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:33.426496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471270556187528:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:33.468890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471269786024243:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.425779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471272250470859:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:33.471402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:33.432955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471270565243501:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:33.457373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471271017049100:2049] [node 2] ICS09 handshake done sender: [6:7577471271017049087:2118] self: [6:7577471271017049090:0] peer: [2:7577471270565243473:0] socket: 515 qp: -1
2025-11-27T18:18:33.469378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471269786024153:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.469412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:33.471473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471270434694673:2137] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.431447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471270556187446:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.427258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471272250470853:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.433816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:33.458201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471271017049100:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:18:33.482671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471269786024224:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:33.427335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471272250470859:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:33.472147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471270434694646:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.470013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471270565243488:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.432210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:33.482703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471269786024224:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:33.433171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471270556187446:2049] [node 2] ICP22 created new session: [3:7577471270556187546:2049]
2025-11-27T18:18:33.459151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471271017049005:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471271017049092:2120] held: no
2025-11-27T18:18:33.472171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471270434694567:2050] [node 3] ICP22 created new session: [5:7577471270434694676:2050]
2025-11-27T18:18:33.470048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471270565243488:2129] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:33.482887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471269786024229:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.459272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:33.427868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471267955503483:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(21046508ns) {HandleHandshakeStatus:329(21036180ns) {DropIncomingHandshake:442(14314ns) DropOutgoingHandshake:457(528ns) SwitchToState:179(1084ns) ProcessPendingSessionEvents:495(4340613ns) {ForwardSessionEventToSession:612(2259285ns) ForwardSessionEventToSession:612(4928ns) ForwardSessionEventToSession:612(351584ns) ForwardSessionEventToSession:612(2185ns) ForwardSessionEventToSession:612(503802ns) ForwardSessionEventToSession:612(6997ns) ForwardSessionEventToSession:612(4710ns) ForwardSessionEventToSession:612(1135878ns) ForwardSessionEventToSession:612(9670ns) ForwardSessionEventToSession:612(2754ns)}}}
2025-11-27T18:18:33.472387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471270434694676:2050] [node 3] ICS09 handshake done sender: [5:7577471270434694670:2135] self: [5:7577471270434694671:0] peer: [3:7577471270556187525:0] socket: 538 qp: -1
2025-11-27T18:18:33.472108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471270565243477:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.482930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471269786024229:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:33.434245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471270556187546:2049] [node 2] ICS09 handshake done sender: [3:7577471270556187528:2119] self: [3:7577471270556187538:0] peer: [2:7577471270565243471:0] socket: 516 qp: -1
2025-11-27T18:18:33.472143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471270565243477:2122] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:18:33.434540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471270556187546:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:33.477481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471270434694676:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:33.429324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471272250470843:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:33.460668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471271017049005:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.490405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471269786024224:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:33.481793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:33.429373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:33.460720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:33.490464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471269786024229:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:33.434592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:33.482417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471270565243488:2129] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:33.429730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471272250470843:2107] [node 4] ICP22 created new session: [1:7577471272250470880:2107]
2025-11-27T18:18:33.439776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471270556187526:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.460764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471271017049005:2048] [node 1] ICP22 created new session: [6:7577471271017049102:2048]
2025-11-27T18:18:33.491203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471270434694679:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64775
2025-11-27T18:18:33.493516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471269786024249:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5912
2025-11-27T18:18:33.489721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471270565243477:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:33.431096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471272250470845:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.431417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:33.461465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471271017049102:2048] [node 1] ICS09 handshake done sender: [6:7577471271017049092:2120] self: [6:7577471271017049098:0] peer: [1:7577471272250470858:0] socket: 490 qp: -1
2025-11-27T18:18:33.494265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471270434694566:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30126) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471270434694672:2136] held: no
2025-11-27T18:18:33.439808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471270556187526:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:33.493884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471269786024215:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.431674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471272250470880:2107] [node 4] ICS09 handshake done sender: [1:7577471272250470853:2114] self: [1:7577471272250470848:0] peer: [4:7577471269786024237:0] socket: 507 qp: -1
2025-11-27T18:18:33.489745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471270565243469:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.461807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471271017049006:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(21137224ns) {HandleHandshakeStatus:329(21134814ns) {DropIncomingHandshake:442(659ns) DropOutgoingHandshake:457(191ns) SwitchToState:179(825ns) ProcessPendingSessionEvents:495(710ns)}}
2025-11-27T18:18:33.440476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471270556187446:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:33.495662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471270434694672:2136] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:33.494866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471269786024216:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.490117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:33.458395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471270556187526:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:33.431707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471272250470880:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:33.495596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:33.490354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471270565243469:2108] [node 5] ICP22 created new session: [2:7577471270565243507:2108]
2025-11-27T18:18:33.461942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471271017049102:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:33.460631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471270556187510:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.497150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471270434694566:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.431746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:33.496068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471269786024215:2108] [node 5] ICP22 created new session: [4:7577471269786024251:2108]
2025-11-27T18:18:33.462506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:33.490416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471270565243507:2108] [node 5] ICS09 handshake done sender: [2:7577471270565243488:2129] self: [2:7577471270565243486:0] peer: [5:7577471270434694675:0] socket: 478 qp: -1
2025-11-27T18:18:33.461235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:33.469082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471271017049103:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3434
2025-11-27T18:18:33.497160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471269786024251:2108] [node 5] ICS09 handshake done sender: [4:7577471269786024224:2113] self: [4:7577471269786024223:0] peer: [5:7577471270434694677:0] socket: 492 qp: -1
2025-11-27T18:18:33.497843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:33.461503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471270556187510:2108] [node 5] ICP22 created new session: [3:7577471270556187549:2108]
2025-11-27T18:18:33.431866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471272250470845:2109] [node 6] ICP22 created new session: [1:7577471272250470881:2109]
2025-11-27T18:18:33.498462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471270434694566:2049] [node 2] ICP22 created new session: [5:7577471270434694681:2049]
2025-11-27T18:18:33.497396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:33.432258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471272250470881:2109] [node 6] ICS09 handshake done sender: [1:7577471272250470859:2116] self: [1:7577471272250470858:0] peer: [6:7577471271017049098:0] socket: 501 qp: -1
2025-11-27T18:18:33.472401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471271017049103:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:33.490589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471270565243507:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:33.497665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471269786024251:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:33.477663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471271017049069:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:33.432908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471272250470881:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:33.462762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471270556187549:2108] [node 5] ICS09 handshake done sender: [3:7577471270556187526:2117] self: [3:7577471270556187525:0] peer: [5:7577471270434694671:0] socket: 537 qp: -1
2025-11-27T18:18:33.490773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471270565243470:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:33.497794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471269786024216:2109] [node 6] ICP22 created new session: [4:7577471269786024252:2109]
2025-11-27T18:18:33.432941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:33.501312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471271017049106:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3434
2025-11-27T18:18:33.462809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471270556187549:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:33.498413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:33.490918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:33.500007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471270434694680:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64775
2025-11-27T18:18:33.505685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471271017049103:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:33.465003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:33.501378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471269786024252:2109] [node 6] ICS09 handshake done sender: [4:7577471269786024229:2114] self: [4:7577471269786024228:0] peer: [6:7577471271017049105:0] socket: 541 qp: -1
2025-11-27T18:18:33.492144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:33.505953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471271017049069:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.501430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471270434694681:2049] [node 2] ICS09 handshake done sender: [5:7577471270434694672:2136] self: [5:7577471270434694675:0] peer: [2:7577471270565243486:0] socket: 480 qp: -1
2025-11-27T18:18:33.501410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471269786024252:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:33.493875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471270565243470:2107] [node 4] ICP22 created new session: [2:7577471270565243509:2107]
2025-11-27T18:18:33.501575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471270434694681:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:33.503086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:33.494836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471270565243509:2107] [node 4] ICS09 handshake done sender: [2:7577471270565243477:2122] self: [2:7577471270565243472:0] peer: [4:7577471269786024247:0] socket: 475 qp: -1
2025-11-27T18:18:33.501599Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471270434694646:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6167) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471270434694673:2137] held: no
2025-11-27T18:18:33.503564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471269786024153:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30126) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471269786024243:2121] held: no
2025-11-27T18:18:33.494888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471270565243509:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:33.506441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:33.501616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:33.496047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:33.504849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471269786024243:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:33.506498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471271017049069:2108] [node 4] ICP22 created new session: [6:7577471271017049107:2108]
2025-11-27T18:18:33.503112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471270434694673:2137] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:33.505414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471269786024153:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.504183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471270434694646:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:33.505694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:33.505741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471269786024153:2049] [node 2] ICP22 created new session: [4:7577471269786024255:2049]
2025-11-27T18:18:33.504751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:33.506547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471271017049107:2108] [node 4] ICS09 handshake done sender: [6:7577471271017049103:2123] self: [6:7577471271017049105:0] peer: [4:7577471269786024228:0] socket: 491 qp: -1
2025-11-27T18:18:33.505948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471269786024255:2049] [node 2] ICS09 handshake done sender: [4:7577471269786024243:2121] self: [4:7577471269786024247:0] peer: [2:7577471270565243472:0] socket: 474 qp: -1
2025-11-27T18:18:33.504795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471270434694646:2108] [node 4] ICP22 created new session: [5:7577471270434694683:2108]
2025-11-27T18:18:33.505981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471269786024255:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:33.505133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471270434694683:2108] [node 4] ICS09 handshake done sender: [5:7577471270434694673:2137] self: [5:7577471270434694677:0] peer: [4:7577471269786024223:0] socket: 496 qp: -1
2025-11-27T18:18:33.506599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471271017049107:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:33.505458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471270434694683:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:33.506685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:33.506425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:33.505506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:33.507094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471271017049069:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:33.543993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:33.544294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471270565243501:2048] [node 1] ICS01 socket: 505 reason# EndOfStream
2025-11-27T18:18:33.544342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471270565243395:2048] [node 1] ICP30 unregister session Session# [2:7577471270565243501:2048] VirtualId# [2:7577471270565243492:0]
2025-11-27T18:18:33.544375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471270565243501:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.544458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:33.544541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:33.545318Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:33.545513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471270556187542:2048] [node 1] ICS01 socket: 494 reason# EndOfStream
2025-11-27T18:18:33.545563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471270556187445:2048] [node 1] ICP30 unregister session Session# [3:7577471270556187542:2048] VirtualId# [3:7577471270556187530:0]
2025-11-27T18:18:33.545588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471270556187542:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.545661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:33.545721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:33.547423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:33.547632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471269786024245:2048] [node 1] ICS01 socket: 529 reason# EndOfStream
2025-11-27T18:18:33.547673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471269786024152:2048] [node 1] ICP30 unregister session Session# [4:7577471269786024245:2048] VirtualId# [4:7577471269786024237:0]
2025-11-27T18:18:33.547698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471269786024245:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.547757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:33.547811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:33.548219Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:33.548474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471270434694668:2048] [node 1] ICS07 socket disconnect 495 reason# EndOfStream
2025-11-27T18:18:33.548511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471270434694668:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.548579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:33.548630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471270434694668:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:33.549181Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:33.549402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471271017049102:2048] [node 1] ICS01 socket: 490 reason# EndOfStream
2025-11-27T18:18:33.549434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471271017049005:2048] [node 1] ICP30 unregister session Session# [6:7577471271017049102:2048] VirtualId# [6:7577471271017049098:0]
2025-11-27T18:18:33.549515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471270434694565:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14159) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:33.549450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471271017049102:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.549510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:33.549647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:33.549723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471270434694668:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:33.549799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471270434694565:2048] [node 1] ICP30 unregister session Session# [5:7577471270434694668:2048] VirtualId# [5:7577471270434694664:0]
2025-11-27T18:18:33.549841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:33.549884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471270434694565:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14159) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:33.549917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14159) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:33.561407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471270434694565:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:33.638135Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:33.638390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471270556187546:2049] [node 2] ICS07 socket disconnect 516 reason# EndOfStream
2025-11-27T18:18:33.638424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471270556187546:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.638506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:33.638559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471270556187546:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:33.640597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:33.641403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471269786024255:2049] [node 2] ICS01 socket: 474 reason# EndOfStream
2025-11-27T18:18:33.641459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471269786024153:2049] [node 2] ICP30 unregister session Session# [4:7577471269786024255:2049] VirtualId# [4:7577471269786024247:0]
2025-11-27T18:18:33.641507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471269786024255:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.641605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:33.641589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471270556187446:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30126) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:33.641674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:33.641660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471270556187546:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:33.642163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:33.642450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471270434694681:2049] [node 2] ICS01 socket: 480 reason# EndOfStream
2025-11-27T18:18:33.642496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471270434694566:2049] [node 2] ICP30 unregister session Session# [5:7577471270434694681:2049] VirtualId# [5:7577471270434694675:0]
2025-11-27T18:18:33.642523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471270434694681:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.642583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:33.642630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:33.642731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471270556187446:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30126) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:33.642799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471270556187546:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:33.642844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471270556187446:2049] [node 2] ICP30 unregister session Session# [3:7577471270556187546:2049] VirtualId# [3:7577471270556187538:0]
2025-11-27T18:18:33.642918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:33.643059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471270556187446:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30126) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:33.643073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30126) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:33.643478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:33.643671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471271017049100:2049] [node 2] ICS01 socket: 515 reason# EndOfStream
2025-11-27T18:18:33.643706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471271017049006:2049] [node 2] ICP30 unregister session Session# [6:7577471271017049100:2049] VirtualId# [6:7577471271017049090:0]
2025-11-27T18:18:33.643733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471271017049100:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.643792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:33.643835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:33.684451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:33.684641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471269786024240:2050] [node 3] ICS01 socket: 526 reason# EndOfStream
2025-11-27T18:18:33.684676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471269786024154:2050] [node 3] ICP30 unregister session Session# [4:7577471269786024240:2050] VirtualId# [4:7577471269786024233:0]
2025-11-27T18:18:33.684702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471269786024240:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.684761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:33.684797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:33.685526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:33.685717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471270434694676:2050] [node 3] ICS01 socket: 538 reason# EndOfStream
2025-11-27T18:18:33.685740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471270434694567:2050] [node 3] ICP30 unregister session Session# [5:7577471270434694676:2050] VirtualId# [5:7577471270434694671:0]
2025-11-27T18:18:33.685756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471270434694676:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.685808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:33.685842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:33.686480Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:33.686688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471271017049096:2050] [node 3] ICS01 socket: 504 reason# EndOfStream
2025-11-27T18:18:33.686731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471271017049007:2050] [node 3] ICP30 unregister session Session# [6:7577471271017049096:2050] VirtualId# [6:7577471271017049088:0]
2025-11-27T18:18:33.686757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471271017049096:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.686822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:33.686861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:33.750630Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:33.750844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471270434694683:2108] [node 4] ICS01 socket: 496 reason# EndOfStream
2025-11-27T18:18:33.750875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471270434694646:2108] [node 4] ICP30 unregister session Session# [5:7577471270434694683:2108] VirtualId# [5:7577471270434694677:0]
2025-11-27T18:18:33.750912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471270434694683:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.750982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:33.751057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:33.751469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:33.751644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471271017049107:2108] [node 4] ICS07 socket disconnect 491 reason# EndOfStream
2025-11-27T18:18:33.751677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471271017049107:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.751744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:33.751808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471271017049107:2108] [node 4] ICS15 start handshake
2025-11-27T18:18:33.752462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471271017049069:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6167) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:33.752514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471271017049107:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:33.752611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471271017049069:2108] [node 4] ICP30 unregister session Session# [6:7577471271017049107:2108] VirtualId# [6:7577471271017049105:0]
2025-11-27T18:18:33.752671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:33.752710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471271017049069:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6167) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:33.752727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6167) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:33.762110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471271017049069:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:18:33.792569Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:33.792806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471271017049091:2107] [node 5] ICS01 socket: 487 reason# EndOfStream
2025-11-27T18:18:33.792864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471271017049067:2107] [node 5] ICP30 unregister session Session# [6:7577471271017049091:2107] VirtualId# [6:7577471271017049082:0]
2025-11-27T18:18:33.792907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471271017049091:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:33.792981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:33.793031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:33.919886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471271020074014:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:33.919886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471271020074015:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:33.940981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471271625402899:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:33.941030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471271625402900:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:33.965447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471273792368594:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:33.965500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471273792368595:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:33.986764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471270968042074:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:33.986814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471270968042075:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:33.986984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471270968042076:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:34.008074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471276190613416:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:34.008111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471276190613417:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:34.008133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471276190613418:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:34.033784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471275121942052:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:34.033843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471275121942053:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:34.033899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471275121942054:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:34.109738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471275121942104:2099] [node 5] ICP01 ready to work
2025-11-27T18:18:34.161546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471275315041373:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:34.181625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471276190613480:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:34.162341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471275121942110:2102] [node 4] ICP01 ready to work
2025-11-27T18:18:34.212883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471278087335955:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:34.212520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471275920370259:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:34.215882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471275315041374:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:34.223310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471275263009434:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:34.223410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471278087335954:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:34.215877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471275920370260:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:34.223348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471275263009435:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:34.215973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471275920370261:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:34.227468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471275315041375:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:34.224033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471278087335956:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:34.227607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471276190613481:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:34.234508Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471275121942104:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(105044993ns) {Configure:104(105034058ns) {ProcessConfigured:135(74261ns) {StartInitialHandshake:157(71191ns) {DropHandshakes:470(2155ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(810ns) PrepareNewSessionHandshake:477(1919ns) {SwitchToState:179(764ns)}}}}}
2025-11-27T18:18:34.247324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471276190613480:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(33036381ns) {Configure:104(33029520ns) {ProcessConfigured:135(100009ns) {StartInitialHandshake:157(96773ns) {DropHandshakes:470(3529ns) {DropIncomingHandshake:442(1019ns) DropOutgoingHandshake:457(399ns)} GenerateSessionVirtualId:988(1117ns) PrepareNewSessionHandshake:477(4982ns) {SwitchToState:179(993ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.257871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471276190613485:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30994
2025-11-27T18:18:34.266345Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471275315041374:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(42036135ns) {Configure:104(42029996ns) {ProcessConfigured:135(91917ns) {StartInitialHandshake:157(86820ns) {DropHandshakes:470(3527ns) {DropIncomingHandshake:442(927ns) DropOutgoingHandshake:457(439ns)} GenerateSessionVirtualId:988(1392ns) PrepareNewSessionHandshake:477(1704ns) {SwitchToState:179(710ns)}}}}}
2025-11-27T18:18:34.282317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471276190613417:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(87460454ns) {Configure:104(87449921ns) {ProcessConfigured:135(87647ns) {StartInitialHandshake:157(83573ns) {DropHandshakes:470(3516ns) {DropIncomingHandshake:442(1045ns) DropOutgoingHandshake:457(456ns)} GenerateSessionVirtualId:988(980ns) PrepareNewSessionHandshake:477(2051ns) {SwitchToState:179(814ns)}}}}}
2025-11-27T18:18:34.292557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471270968042075:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(107740041ns) {Configure:104(107730707ns) {ProcessConfigured:135(15516144ns) {StartInitialHandshake:157(15512416ns) {DropHandshakes:470(2330ns) {DropIncomingHandshake:442(420ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(1006ns) PrepareNewSessionHandshake:477(3744ns) {SwitchToState:179(1468ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.297662Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471275263009434:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(68286802ns) {Configure:104(68280876ns) {ProcessConfigured:135(52562ns) {StartInitialHandshake:157(50317ns) {DropHandshakes:470(2429ns) {DropIncomingHandshake:442(524ns) DropOutgoingHandshake:457(402ns)} GenerateSessionVirtualId:988(1037ns) PrepareNewSessionHandshake:477(1431ns) {SwitchToState:179(599ns)}}}}}
2025-11-27T18:18:34.304544Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471271020074014:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(197709695ns) {Configure:104(197704490ns) {ProcessConfigured:135(16898534ns) {StartInitialHandshake:157(16894849ns) {DropHandshakes:470(3393ns) {DropIncomingHandshake:442(1020ns) DropOutgoingHandshake:457(384ns)} GenerateSessionVirtualId:988(695ns) PrepareNewSessionHandshake:477(3081ns) {SwitchToState:179(979ns)}}}}}
2025-11-27T18:18:34.308902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471275920370259:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(82154849ns) {Configure:104(82149505ns) {ProcessConfigured:135(80961ns) {StartInitialHandshake:157(78247ns) {DropHandshakes:470(3002ns) {DropIncomingHandshake:442(691ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(890ns) PrepareNewSessionHandshake:477(2723ns) {SwitchToState:179(962ns)}}}}}
2025-11-27T18:18:34.302473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471275263009444:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27681
2025-11-27T18:18:34.313545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471271020074015:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(209546501ns) {Configure:104(209543569ns) {ProcessConfigured:135(18363136ns) {StartInitialHandshake:157(18360219ns) {DropHandshakes:470(3190ns) {DropIncomingHandshake:442(1004ns) DropOutgoingHandshake:457(379ns)} GenerateSessionVirtualId:988(697ns) PrepareNewSessionHandshake:477(2486ns) {SwitchToState:179(1393ns)}}}}}
2025-11-27T18:18:34.313380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471275121942052:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(177904320ns) {Configure:104(177898494ns) {ProcessConfigured:135(8940099ns) {StartInitialHandshake:157(8937713ns) {DropHandshakes:470(2827ns) {DropIncomingHandshake:442(807ns) DropOutgoingHandshake:457(393ns)} GenerateSessionVirtualId:988(745ns) PrepareNewSessionHandshake:477(1585724ns) {SwitchToState:179(3307ns)}}}}}
2025-11-27T18:18:34.304816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471270968042076:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(156819506ns) {Configure:104(156815057ns) {ProcessConfigured:135(19720254ns) {StartInitialHandshake:157(19717874ns) {DropHandshakes:470(2029ns) {DropIncomingHandshake:442(636ns) DropOutgoingHandshake:457(395ns)} GenerateSessionVirtualId:988(596ns) PrepareNewSessionHandshake:477(4192ns) {SwitchToState:179(2351ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.324799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471278087335956:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(89495686ns) {Configure:104(89489241ns) {ProcessConfigured:135(12566876ns) {StartInitialHandshake:157(12565004ns) {DropHandshakes:470(1014ns) {DropIncomingHandshake:442(337ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(444ns) PrepareNewSessionHandshake:477(3424ns) {SwitchToState:179(2029ns)}}}}}
2025-11-27T18:18:34.317711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471275121942110:2102] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(92814625ns) {Configure:104(92806689ns) {ProcessConfigured:135(2188250ns) {StartInitialHandshake:157(897928ns) {DropHandshakes:470(1837ns) {DropIncomingHandshake:442(211ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(815ns) PrepareNewSessionHandshake:477(1439ns) {SwitchToState:179(503ns)}}}}}
2025-11-27T18:18:34.317817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471275315041373:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(114212473ns) {Configure:104(98101023ns) {ProcessConfigured:135(7539472ns) {StartInitialHandshake:157(5441605ns) {DropHandshakes:470(3256ns) {DropIncomingHandshake:442(356ns) DropOutgoingHandshake:457(445ns)} GenerateSessionVirtualId:988(1065ns) PrepareNewSessionHandshake:477(3485ns) {SwitchToState:179(1196ns)}}}}}
2025-11-27T18:18:34.323777Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471276190613416:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(102707639ns) {Configure:104(102701570ns) {ProcessConfigured:135(7818376ns) {StartInitialHandshake:157(7816601ns) {DropHandshakes:470(1034ns) {DropIncomingHandshake:442(547ns) DropOutgoingHandshake:457(38ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(5134ns) {SwitchToState:179(3177ns)}}}}}
2025-11-27T18:18:34.313355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471275263009444:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.329602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471275920370261:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(99091986ns) {Configure:104(99082766ns) {ProcessConfigured:135(15275798ns) {StartInitialHandshake:157(15272400ns) {DropHandshakes:470(4833310ns) {DropIncomingHandshake:442(2252ns) DropOutgoingHandshake:457(289ns)} GenerateSessionVirtualId:988(13903ns) PrepareNewSessionHandshake:477(3575ns) {SwitchToState:179(1943ns)}}}}}
2025-11-27T18:18:34.341658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471276190613418:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(99883172ns) {Configure:104(99875595ns) {ProcessConfigured:135(7881508ns) {StartInitialHandshake:157(7879699ns) {DropHandshakes:470(1017ns) {DropIncomingHandshake:442(269ns) DropOutgoingHandshake:457(88ns)} GenerateSessionVirtualId:988(380ns) PrepareNewSessionHandshake:477(1921ns) {SwitchToState:179(454ns)}}}}}
2025-11-27T18:18:34.324950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471275121942054:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(98610270ns) {Configure:104(98603837ns) {ProcessConfigured:135(8536194ns) {StartInitialHandshake:157(8533922ns) {DropHandshakes:470(2969ns) {DropIncomingHandshake:442(537ns) DropOutgoingHandshake:457(709ns)} GenerateSessionVirtualId:988(510ns) PrepareNewSessionHandshake:477(3418ns) {SwitchToState:179(1391ns)}}}}}
2025-11-27T18:18:34.333599Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471270968042074:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(103229675ns) {Configure:104(103218247ns) {ProcessConfigured:135(8523189ns) {StartInitialHandshake:157(8521421ns) {DropHandshakes:470(1201ns) {DropIncomingHandshake:442(305ns) DropOutgoingHandshake:457(244ns)} GenerateSessionVirtualId:988(362ns) PrepareNewSessionHandshake:477(1359ns) {SwitchToState:179(88ns)}}}}}
2025-11-27T18:18:34.349324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471273792368594:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(128405541ns) {Configure:104(128399498ns) {ProcessConfigured:135(4213060ns) {StartInitialHandshake:157(4210836ns) {DropHandshakes:470(1500ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(571ns)} GenerateSessionVirtualId:988(662ns) PrepareNewSessionHandshake:477(1788ns) {SwitchToState:179(656ns)}}}}}
2025-11-27T18:18:34.338173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471275263009450:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27681
2025-11-27T18:18:34.352619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471275315041377:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.341523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471271625402899:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(165784675ns) {Configure:104(155999880ns) {ProcessConfigured:135(4156086ns) {StartInitialHandshake:157(4153674ns) {DropHandshakes:470(2580ns) {DropIncomingHandshake:442(525ns) DropOutgoingHandshake:457(399ns)} GenerateSessionVirtualId:988(788ns) PrepareNewSessionHandshake:477(4198ns) {SwitchToState:179(2383ns)}}}}}
2025-11-27T18:18:34.329760Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471275121942053:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(98034642ns) {Configure:104(98029706ns) {ProcessConfigured:135(4539967ns) {StartInitialHandshake:157(4538067ns) {DropHandshakes:470(766ns) {DropIncomingHandshake:442(202ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(2853ns) {SwitchToState:179(1497ns)}}}}}
2025-11-27T18:18:34.349342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471273792368595:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(111366390ns) {Configure:104(111358840ns) {ProcessConfigured:135(4071805ns) {StartInitialHandshake:157(4069774ns) {DropHandshakes:470(1554ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(372ns)} GenerateSessionVirtualId:988(836ns) PrepareNewSessionHandshake:477(2077ns) {SwitchToState:179(852ns)}}}}}
2025-11-27T18:18:34.353076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471276190613481:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(102198365ns) {Configure:104(102192908ns) {ProcessConfigured:135(19320749ns) {StartInitialHandshake:157(19317984ns) {DropHandshakes:470(1100ns) {DropIncomingHandshake:442(281ns) DropOutgoingHandshake:457(185ns)} GenerateSessionVirtualId:988(413ns) PrepareNewSessionHandshake:477(4305ns) {SwitchToState:179(2328ns)}}}}}
2025-11-27T18:18:34.339750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471275263009450:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.346076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471270968042074:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.349573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471275263009435:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(96468332ns) {Configure:104(96464346ns) {ProcessConfigured:135(7097946ns) {StartInitialHandshake:157(7095836ns) {DropHandshakes:470(1760ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(496ns)} GenerateSessionVirtualId:988(499ns) PrepareNewSessionHandshake:477(3344ns) {SwitchToState:179(1790ns)}}}}}
2025-11-27T18:18:34.352680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471275315041375:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(110282943ns) {Configure:104(110277862ns) {ProcessConfigured:135(28233486ns) {StartInitialHandshake:157(28231116ns) {DropHandshakes:470(688ns) {DropIncomingHandshake:442(86ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(2623ns) {SwitchToState:179(362ns)}}}}}
2025-11-27T18:18:34.341724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471271625402900:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(104131287ns) {Configure:104(104112723ns) {ProcessConfigured:135(8809274ns) {StartInitialHandshake:157(8807436ns) {DropHandshakes:470(758ns) {DropIncomingHandshake:442(247ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:477(2656ns) {SwitchToState:179(968ns)}}}}}
2025-11-27T18:18:34.351137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471275121942121:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36904
2025-11-27T18:18:34.351281Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471278087335954:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(95887852ns) {Configure:104(95881385ns) {ProcessConfigured:135(18887383ns) {StartInitialHandshake:157(18884317ns) {DropHandshakes:470(296752ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(825ns)} GenerateSessionVirtualId:988(816ns) PrepareNewSessionHandshake:477(4825ns) {SwitchToState:179(2732ns)}}}}}
2025-11-27T18:18:34.355098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471276190613485:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.352692Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471275315041377:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:18:34.351402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471275920370260:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(105903749ns) {Configure:104(105892170ns) {ProcessConfigured:135(16072877ns) {StartInitialHandshake:157(16069421ns) {DropHandshakes:470(1398ns) {DropIncomingHandshake:442(367ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(619ns) PrepareNewSessionHandshake:477(2869ns) {SwitchToState:179(791ns)}}}}}
2025-11-27T18:18:34.354432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471275121942121:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.354215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471278087335955:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(118171032ns) {Configure:104(118160859ns) {ProcessConfigured:135(20361889ns) {StartInitialHandshake:157(20358870ns) {DropHandshakes:470(1856ns) {DropIncomingHandshake:442(424ns) DropOutgoingHandshake:457(367ns)} GenerateSessionVirtualId:988(17368ns) PrepareNewSessionHandshake:477(4144ns) {SwitchToState:179(2522ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.360515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471275920370274:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40457
2025-11-27T18:18:34.358015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471276190613484:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.361918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471275263009454:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27681
2025-11-27T18:18:34.364361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471275263009441:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.361911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471275315041377:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.354709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471275121942124:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36904
2025-11-27T18:18:34.358048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471276190613484:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.365830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471278087335968:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:18:34.361603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471275920370273:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40457
2025-11-27T18:18:34.368161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471275315041389:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32530
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.364404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471275263009441:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:34.355783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471275121942104:2099] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.364576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471275920370270:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40457
2025-11-27T18:18:34.370364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471276190613484:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:34.370826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471275315041374:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.371506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471275263009453:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27681
2025-11-27T18:18:34.357694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471275121942124:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.370776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471275920370270:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.371941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471278087335967:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:18:34.373027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471276190613499:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30994
2025-11-27T18:18:34.371290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:34.374941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471275263009453:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.358802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471275121942110:2102] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.373016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471278087335971:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:18:34.370801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471275920370274:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.380004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471276190613480:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.371935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471275315041374:2107] [node 4] ICP22 created new session: [1:7577471275315041392:2107]
2025-11-27T18:18:34.373904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471278087335968:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.379411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471275263009441:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:34.380299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.371391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471275920370273:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.383361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471275121942135:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36904
2025-11-27T18:18:34.379496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471275263009454:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.373045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471275315041392:2107] [node 4] ICS09 handshake done sender: [1:7577471275315041377:2110] self: [1:7577471275315041376:0] peer: [4:7577471275263009451:0] socket: 484 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.375040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471273792368594:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.373962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471275315041389:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.382934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471276190613480:2109] [node 6] ICP22 created new session: [5:7577471276190613500:2109]
2025-11-27T18:18:34.385386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471275121942134:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36904
2025-11-27T18:18:34.373086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471271625402899:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.384230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471270968042075:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.377623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471278087335967:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.374798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471275315041392:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:34.382990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471276190613500:2109] [node 6] ICS09 handshake done sender: [5:7577471276190613484:2112] self: [5:7577471276190613483:0] peer: [6:7577471275121942132:0] socket: 482 qp: -1
2025-11-27T18:18:34.385338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471275263009455:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27681
2025-11-27T18:18:34.383411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471275920370267:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.386220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471275121942104:2099] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4729) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471275121942121:2111] held: no
2025-11-27T18:18:34.379924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471278087335971:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.374819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471275315041391:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32530
2025-11-27T18:18:34.388538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471275121942136:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36904
2025-11-27T18:18:34.383442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471275920370267:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:34.383397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471276190613500:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:34.388891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471275263009444:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:34.380616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471273792368595:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.377584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.385745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471275920370272:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.383941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471276190613501:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30994
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.391238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471275263009434:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.390702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471278087335963:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.388693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471275121942135:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.379402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471275315041381:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.385782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471275920370272:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.390736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471278087335963:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:34.389231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471275121942134:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.392224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471270968042074:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.384286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:34.379439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471275315041381:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.396991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.398272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471278087335965:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.388321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471275920370267:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:34.384382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471276190613499:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.391289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471275121942054:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.380264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471275315041382:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.391540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471275121942053:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.398309Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471278087335965:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:34.385248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471276190613501:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.397153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.393376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471271625402900:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.380287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471275315041382:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:34.398327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471270968042074:2048] [node 1] ICP22 created new session: [4:7577471275263009461:2048]
2025-11-27T18:18:34.386225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471276190613418:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.391726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471275121942137:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36904
2025-11-27T18:18:34.393656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:34.400248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471278087335976:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:18:34.388338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471275315041382:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.387416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471276190613416:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.398318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471271625402900:2049] [node 3] ICP22 created new session: [2:7577471275920370284:2049]
2025-11-27T18:18:34.407878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471273792368594:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4735) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471278087335968:2117] held: no
2025-11-27T18:18:34.398335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471275121942121:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:34.388818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471275315041381:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:34.398410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471275263009434:2110] [node 6] ICP22 created new session: [4:7577471275263009460:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.399156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471275263009460:2110] [node 6] ICS09 handshake done sender: [4:7577471275263009441:2113] self: [4:7577471275263009440:0] peer: [6:7577471275121942133:0] socket: 489 qp: -1
2025-11-27T18:18:34.415770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471278087335978:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:18:34.421247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471276190613480:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18371009ns) {HandleHandshakeStatus:329(18367825ns) {DropIncomingHandshake:442(16372ns) DropOutgoingHandshake:457(1832ns) SwitchToState:179(771ns) ProcessPendingSessionEvents:495(9063253ns) {ForwardSessionEventToSession:612(2132964ns) ForwardSessionEventToSession:612(5101ns) ForwardSessionEventToSession:612(453460ns) ForwardSessionEventToSession:612(3921ns) ForwardSessionEventToSession:612(3278ns) ForwardSessionEventToSession:612(1728259ns) ForwardSessionEventToSession:612(283341ns) ForwardSessionEventToSession:612(3469484ns) ForwardSessionEventToSession:612(11095ns) ForwardSessionEventToSession:612(3178ns)}}}
2025-11-27T18:18:34.399124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471275121942124:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:34.398502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471275920370272:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.389133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471275315041391:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.400308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471275263009460:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:34.420782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471278087335977:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:18:34.399723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471275920370271:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.400948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471275121942110:2102] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8556) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471275121942124:2113] held: no
2025-11-27T18:18:34.391569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471275315041384:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.426644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471276190613506:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30994
2025-11-27T18:18:34.400898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471275263009461:2048] [node 1] ICS09 handshake done sender: [4:7577471275263009444:2114] self: [4:7577471275263009451:0] peer: [1:7577471275315041376:0] socket: 491 qp: -1
2025-11-27T18:18:34.404971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471275121942104:2099] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.410864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:34.410932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471275121942104:2099] [node 5] ICP22 created new session: [6:7577471275121942141:2099]
2025-11-27T18:18:34.419038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471275121942110:2102] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.419096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:34.419146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471275121942110:2102] [node 4] ICP22 created new session: [6:7577471275121942142:2102]
2025-11-27T18:18:34.419201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471275121942142:2102] [node 4] ICS09 handshake done sender: [6:7577471275121942124:2113] self: [6:7577471275121942133:0] peer: [4:7577471275263009440:0] socket: 490 qp: -1
2025-11-27T18:18:34.419239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471275121942142:2102] [node 4] ICS10 traffic start
2025-11-27T18:18:34.419273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:34.422585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471275121942141:2099] [node 5] ICS09 handshake done sender: [6:7577471275121942121:2111] self: [6:7577471275121942132:0] peer: [5:7577471276190613483:0] socket: 486 qp: -1
2025-11-27T18:18:34.428855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471275121942141:2099] [node 5] ICS10 traffic start
2025-11-27T18:18:34.430589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:34.435468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471275121942145:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36904
2025-11-27T18:18:34.435570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471275121942148:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36904
2025-11-27T18:18:34.442496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471275121942134:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:34.453517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471275121942053:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.453640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:34.453700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471275121942053:2049] [node 2] ICP22 created new session: [6:7577471275121942149:2049]
2025-11-27T18:18:34.453732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471275121942149:2049] [node 2] ICS09 handshake done sender: [6:7577471275121942134:2117] self: [6:7577471275121942139:0] peer: [2:7577471275920370268:0] socket: 519 qp: -1
2025-11-27T18:18:34.453760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471275121942149:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:34.456865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:34.464424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471275121942104:2099] [node 5] ICP35 event processing took too much time PendingConnection:279(59004871ns) {HandleHandshakeStatus:329(59002866ns) {DropIncomingHandshake:442(1051ns) DropOutgoingHandshake:457(261ns) SwitchToState:179(574ns) ProcessPendingSessionEvents:495(1208ns)}}
2025-11-27T18:18:34.478099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471275121942145:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.485339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471275121942147:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36904
2025-11-27T18:18:34.491271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471275121942052:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.518029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471275121942135:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:34.399753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471275920370271:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:34.422372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471278087335976:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.391611Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471275315041384:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:34.428844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471276190613509:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30994
2025-11-27T18:18:34.450899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471276190613506:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.457078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471276190613499:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:34.462475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471276190613508:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30994
2025-11-27T18:18:34.464872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471276190613417:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.470755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471276190613418:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.475294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471276190613501:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:34.477544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:34.477853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471276190613507:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30994
2025-11-27T18:18:34.482602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471276190613416:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.483353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471276190613418:2050] [node 3] ICP22 created new session: [5:7577471276190613511:2050]
2025-11-27T18:18:34.485467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471276190613511:2050] [node 3] ICS09 handshake done sender: [5:7577471276190613499:2118] self: [5:7577471276190613505:0] peer: [3:7577471278087335960:0] socket: 516 qp: -1
2025-11-27T18:18:34.485519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471276190613511:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:34.485522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:34.486208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471276190613507:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.493891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471276190613416:2048] [node 1] ICP22 created new session: [5:7577471276190613512:2048]
2025-11-27T18:18:34.493984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471276190613512:2048] [node 1] ICS09 handshake done sender: [5:7577471276190613501:2119] self: [5:7577471276190613504:0] peer: [1:7577471275315041383:0] socket: 529 qp: -1
2025-11-27T18:18:34.494024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471276190613512:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:34.494112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:34.495063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471276190613481:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.499616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:34.529833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471276190613416:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18211808ns) {HandleHandshakeStatus:329(18207832ns) {DropIncomingHandshake:442(681ns) DropOutgoingHandshake:457(9743ns) SwitchToState:179(822ns) ProcessPendingSessionEvents:495(6497143ns) {ForwardSessionEventToSession:612(53738ns) ForwardSessionEventToSession:612(4562ns) ForwardSessionEventToSession:612(6911ns) ForwardSessionEventToSession:612(5798ns) ForwardSessionEventToSession:612(3212ns) ForwardSessionEventToSession:612(3579572ns) ForwardSessionEventToSession:612(3239ns) ForwardSessionEventToSession:612(2816850ns) ForwardSessionEventToSession:612(5700ns) ForwardSessionEventToSession:612(3398ns)}}}
2025-11-27T18:18:34.392905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471271020074014:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.401431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:34.425678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471278087335963:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:34.392976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:34.393021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471271020074014:2048] [node 2] ICP22 created new session: [1:7577471275315041398:2048]
2025-11-27T18:18:34.393075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471275315041398:2048] [node 2] ICS09 handshake done sender: [1:7577471275315041381:2111] self: [1:7577471275315041379:0] peer: [2:7577471275920370278:0] socket: 497 qp: -1
2025-11-27T18:18:34.393117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471275315041398:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:34.393154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:34.393817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471271020074015:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.393868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:34.393958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471275315041374:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(18178945ns) {HandleHandshakeStatus:329(18174716ns) {DropIncomingHandshake:442(1386ns) DropOutgoingHandshake:457(2433ns) SwitchToState:179(911ns) ProcessPendingSessionEvents:495(4236780ns) {ForwardSessionEventToSession:612(1566268ns) ForwardSessionEventToSession:612(8154ns) ForwardSessionEventToSession:612(495937ns) ForwardSessionEventToSession:612(1014684ns) ForwardSessionEventToSession:612(11971ns) ForwardSessionEventToSession:612(796584ns) ForwardSessionEventToSession:612(3352ns) ForwardSessionEventToSession:612(306809ns) ForwardSessionEventToSession:612(3986ns) ForwardSessionEventToSession:612(3458ns)}}}
2025-11-27T18:18:34.397135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471271020074015:2049] [node 3] ICP22 created new session: [1:7577471275315041401:2049]
2025-11-27T18:18:34.400057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471275315041401:2049] [node 3] ICS09 handshake done sender: [1:7577471275315041382:2112] self: [1:7577471275315041380:0] peer: [3:7577471278087335973:0] socket: 493 qp: -1
2025-11-27T18:18:34.403584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471275920370260:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.403498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471275263009461:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:34.400986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471275920370284:2049] [node 3] ICS09 handshake done sender: [2:7577471275920370267:2112] self: [2:7577471275920370262:0] peer: [3:7577471278087335975:0] socket: 510 qp: -1
2025-11-27T18:18:34.535827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471275121942053:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(31981370ns) {HandleHandshakeStatus:329(31973975ns) {DropIncomingHandshake:442(1344ns) DropOutgoingHandshake:457(23578ns) SwitchToState:179(722ns) ProcessPendingSessionEvents:495(10162345ns) {ForwardSessionEventToSession:612(2153795ns) ForwardSessionEventToSession:612(5684ns) ForwardSessionEventToSession:612(3294ns) ForwardSessionEventToSession:612(5800422ns) ForwardSessionEventToSession:612(4782ns) ForwardSessionEventToSession:612(1529963ns) ForwardSessionEventToSession:612(10302ns) ForwardSessionEventToSession:612(3239ns) ForwardSessionEventToSession:612(629028ns) ForwardSessionEventToSession:612(3011ns)}}}
2025-11-27T18:18:34.400866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471275315041401:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:34.429469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471278087335956:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.545997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471275121942054:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.400926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:34.415413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471275315041384:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:34.429533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:34.429147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471271020074015:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(33743306ns) {HandleHandshakeStatus:329(33741561ns) {DropIncomingHandshake:442(12104ns) DropOutgoingHandshake:457(216ns) SwitchToState:179(221ns) ProcessPendingSessionEvents:495(4650637ns) {ForwardSessionEventToSession:612(4586648ns) ForwardSessionEventToSession:612(18747ns) ForwardSessionEventToSession:612(5271ns) ForwardSessionEventToSession:612(3501ns) ForwardSessionEventToSession:612(5529ns) ForwardSessionEventToSession:612(3216ns) ForwardSessionEventToSession:612(5673ns) ForwardSessionEventToSession:612(3070ns) ForwardSessionEventToSession:612(2447ns) ForwardSessionEventToSession:612(5372ns)}}}
2025-11-27T18:18:34.407884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471275920370284:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:34.415725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:34.442860Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471271020074014:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(17692889ns) {HandleHandshakeStatus:329(17686631ns) {DropIncomingHandshake:442(298ns) DropOutgoingHandshake:457(474ns) SwitchToState:179(164ns) ProcessPendingSessionEvents:495(16613904ns) {ForwardSessionEventToSession:612(5977970ns) ForwardSessionEventToSession:612(351314ns) ForwardSessionEventToSession:612(4285ns) ForwardSessionEventToSession:612(3202ns) ForwardSessionEventToSession:612(705734ns) ForwardSessionEventToSession:612(3712ns) ForwardSessionEventToSession:612(3083ns) ForwardSessionEventToSession:612(3864337ns) ForwardSessionEventToSession:612(2972410ns) ForwardSessionEventToSession:612(2701462ns)}}}
2025-11-27T18:18:34.444597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471275315041373:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.461404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:34.462772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471275315041373:2108] [node 5] ICP22 created new session: [1:7577471275315041406:2108]
2025-11-27T18:18:34.466011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471275315041402:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32530
2025-11-27T18:18:34.469494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471275315041406:2108] [node 5] ICS09 handshake done sender: [1:7577471275315041384:2113] self: [1:7577471275315041383:0] peer: [5:7577471276190613504:0] socket: 528 qp: -1
2025-11-27T18:18:34.469554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471275315041406:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:34.469618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:34.470079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471275315041373:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(26947496ns) {HandleHandshakeStatus:329(26943596ns) {DropIncomingHandshake:442(1756ns) DropOutgoingHandshake:457(1165ns) SwitchToState:179(876ns) ProcessPendingSessionEvents:495(127886ns) {ForwardSessionEventToSession:612(68685ns) ForwardSessionEventToSession:612(8558ns) ForwardSessionEventToSession:612(3415ns) ForwardSessionEventToSession:612(3149ns) ForwardSessionEventToSession:612(6207ns) ForwardSessionEventToSession:612(5622ns) ForwardSessionEventToSession:612(6198ns) ForwardSessionEventToSession:612(2960ns) ForwardSessionEventToSession:612(6054ns) ForwardSessionEventToSession:612(5589ns)}}}
2025-11-27T18:18:34.498513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471275315041407:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32530
2025-11-27T18:18:34.511283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471275315041388:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.511328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471275315041388:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:34.523361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471275315041388:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:34.545431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471275315041375:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.545521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:34.545594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471275315041375:2109] [node 6] ICP22 created new session: [1:7577471275315041410:2109]
2025-11-27T18:18:34.545647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471275315041410:2109] [node 6] ICS09 handshake done sender: [1:7577471275315041388:2114] self: [1:7577471275315041386:0] peer: [6:7577471275121942151:0] socket: 541 qp: -1
2025-11-27T18:18:34.546935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471275315041410:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:34.547917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:34.550765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471271020074014:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:34.407938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:34.429867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471278087335965:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.548043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:34.430598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471270968042075:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471275263009454:2119] held: no
2025-11-27T18:18:34.417425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:34.558058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471276190613418:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(77761957ns) {HandleHandshakeStatus:329(77760173ns) {DropIncomingHandshake:442(1103ns) DropOutgoingHandshake:457(4685ns) SwitchToState:179(597ns) ProcessPendingSessionEvents:495(21182828ns) {ForwardSessionEventToSession:612(4103684ns) ForwardSessionEventToSession:612(3324522ns) ForwardSessionEventToSession:612(7386ns) ForwardSessionEventToSession:612(2892ns) ForwardSessionEventToSession:612(7609247ns) ForwardSessionEventToSession:612(3448803ns) ForwardSessionEventToSession:612(11316ns) ForwardSessionEventToSession:612(1878202ns) ForwardSessionEventToSession:612(3102ns) ForwardSessionEventToSession:612(765198ns)}}}
2025-11-27T18:18:34.436301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471278087335956:2109] [node 5] ICP22 created new session: [3:7577471278087335982:2109]
2025-11-27T18:18:34.550103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471275121942054:2050] [node 3] ICP22 created new session: [6:7577471275121942153:2050]
2025-11-27T18:18:34.459588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471275263009464:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27681
2025-11-27T18:18:34.422752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471275920370260:2107] [node 4] ICP22 created new session: [2:7577471275920370287:2107]
2025-11-27T18:18:34.558837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471276190613481:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8556) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471276190613507:2121] held: no
2025-11-27T18:18:34.428838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471275920370287:2107] [node 4] ICS09 handshake done sender: [2:7577471275920370272:2116] self: [2:7577471275920370266:0] peer: [4:7577471275263009458:0] socket: 508 qp: -1
2025-11-27T18:18:34.444276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471278087335983:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:18:34.470705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471275263009434:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(56371657ns) {HandleHandshakeStatus:329(56354261ns) {DropIncomingHandshake:442(12518ns) DropOutgoingHandshake:457(3024ns) SwitchToState:179(430ns) ProcessPendingSessionEvents:495(23768199ns) {ForwardSessionEventToSession:612(7670478ns) ForwardSessionEventToSession:612(11114ns) ForwardSessionEventToSession:612(1794190ns) ForwardSessionEventToSession:612(4235ns) ForwardSessionEventToSession:612(3125ns) ForwardSessionEventToSession:612(5786357ns) ForwardSessionEventToSession:612(3314ns) ForwardSessionEventToSession:612(6809721ns) ForwardSessionEventToSession:612(1642356ns) ForwardSessionEventToSession:612(6905ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.428886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471275920370287:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:34.559406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471276190613417:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471276190613506:2120] held: no
2025-11-27T18:18:34.433964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471275920370271:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:34.434523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471275920370289:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40457
2025-11-27T18:18:34.561587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471276190613516:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30994
2025-11-27T18:18:34.473577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471275263009454:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:34.434586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471275920370290:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40457
2025-11-27T18:18:34.552975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471275121942153:2050] [node 3] ICS09 handshake done sender: [6:7577471275121942135:2118] self: [6:7577471275121942138:0] peer: [3:7577471278087335959:0] socket: 522 qp: -1
2025-11-27T18:18:34.477927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471270968042075:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.434628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471275920370261:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.553009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471275121942153:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:34.434673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:34.478171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471270968042074:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(70096556ns) {HandleHandshakeStatus:329(70093610ns) {DropIncomingHandshake:442(1059ns) DropOutgoingHandshake:457(8864ns) SwitchToState:179(340ns) ProcessPendingSessionEvents:495(26070985ns) {ForwardSessionEventToSession:612(7572439ns) ForwardSessionEventToSession:612(6357ns) ForwardSessionEventToSession:612(6814098ns) ForwardSessionEventToSession:612(13508ns) ForwardSessionEventToSession:612(3515ns) ForwardSessionEventToSession:612(6272233ns) ForwardSessionEventToSession:612(4416ns) ForwardSessionEventToSession:612(5333271ns) ForwardSessionEventToSession:612(6412ns) ForwardSessionEventToSession:612(3594ns)}}}
2025-11-27T18:18:34.434724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471275920370261:2110] [node 6] ICP22 created new session: [2:7577471275920370291:2110]
2025-11-27T18:18:34.553268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471275121942052:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4735) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471275121942145:2121] held: no
2025-11-27T18:18:34.483308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471275263009466:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27681
2025-11-27T18:18:34.445190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471278087335982:2109] [node 5] ICS09 handshake done sender: [3:7577471278087335963:2113] self: [3:7577471278087335960:0] peer: [5:7577471276190613505:0] socket: 514 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:34.562515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471276190613506:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:34.434763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471275920370291:2110] [node 6] ICS09 handshake done sender: [2:7577471275920370271:2115] self: [2:7577471275920370268:0] peer: [6:7577471275121942139:0] socket: 517 qp: -1
2025-11-27T18:18:34.557734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:34.484306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:34.445319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471273792368594:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17268243ns) {HandleHandshakeStatus:395(17265577ns) {LogHandshakeFail:472(1191621ns) {UpdateErrorStateLog:545(9952ns)} DropOutgoingHandshake:457(1982ns) ProcessPendingSessionEvents:495(12945289ns) {DropSessionEvent:512(2651137ns) DropSessionEvent:512(7858979ns) DropSessionEvent:512(5809ns) DropSessionEvent:512(5274ns) DropSessionEvent:512(4819ns) DropSessionEvent:512(6060ns) DropSessionEvent:512(6403ns) DropSessionEvent:512(4803ns) DropSessionEvent:512(5672ns) DropSessionEvent:512(2382296ns)}}}
2025-11-27T18:18:34.485381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471270968042075:2049] [node 2] ICP22 created new session: [4:7577471275263009467:2049]
2025-11-27T18:18:34.490270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471275263009466:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.490471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471270968042076:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:34.490908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471275263009467:2049] [node 2] ICS09 handshake done sender: [4:7577471275263009454:2119] self: [4:7577471275263009458:0] peer: [2:7577471275920370266:0] socket: 509 qp: -1
2025-11-27T18:18:34.499900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471275263009449:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.499936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471275263009449:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:34.509325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471275263009467:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:34.510974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471275263009449:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:34.511691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471275263009435:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.512749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:34.529774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:34.541851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471275263009435:2108] [node 5] ICP22 created new session: [4:7577471275263009472:2108]
2025-11-27T18:18:34.548144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471275263009472:2108] [node 5] ICS09 handshake done sender: [4:7577471275263009449:2116] self: [4:7577471275263009448:0] peer: [5:7577471276190613514:0] socket: 539 qp: -1
2025-11-27T18:18:34.548185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471275263009472:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:34.549047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:34.550830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471270968042075:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(69598132ns) {HandleHandshakeStatus:329(69594479ns) {DropIncomingHandshake:442(457ns) DropOutgoingHandshake:457(603ns) SwitchToState:179(545ns) ProcessPendingSessionEvents:495(421ns)}}
2025-11-27T18:18:34.434798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471275920370291:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:34.559362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471275121942152:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36904
2025-11-27T18:18:34.563350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471276190613417:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.561360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471275121942145:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:34.552214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471270968042076:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9276) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471275263009466:2122] held: no
2025-11-27T18:18:34.434833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:34.456791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:34.552400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471275263009469:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27681
2025-11-27T18:18:34.460451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471275920370282:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40457
2025-11-27T18:18:34.464318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471275920370274:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:34.562267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471275121942052:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.476610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471271625402899:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.562298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:34.476685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:34.563042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471275121942052:2048] [node 1] ICP22 created new session: [6:7577471275121942155:2048]
2025-11-27T18:18:34.556328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471275263009466:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:34.476736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471271625402899:2048] [node 1] ICP22 created new session: [2:7577471275920370295:2048]
2025-11-27T18:18:34.563854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471275121942155:2048] [node 1] ICS09 handshake done sender: [6:7577471275121942145:2121] self: [6:7577471275121942151:0] peer: [1:7577471275315041386:0] socket: 543 qp: -1
2025-11-27T18:18:34.563751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:34.477594Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471271625402900:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(69667838ns) {HandleHandshakeStatus:329(69664737ns) {DropIncomingHandshake:442(1589ns) DropOutgoingHandshake:457(644ns) SwitchToState:179(690ns) ProcessPendingSessionEvents:495(36232267ns) {ForwardSessionEventToSession:612(10779539ns) ForwardSessionEventToSession:612(5825ns) ForwardSessionEventToSession:612(3539ns) ForwardSessionEventToSession:612(1799700ns) ForwardSessionEventToSession:612(6332075ns) ForwardSessionEventToSession:612(8578936ns) ForwardSessionEventToSession:612(16764ns) ForwardSessionEventToSession:612(4043ns) ForwardSessionEventToSession:612(2186425ns) ForwardSessionEventToSession:612(5083ns)}}}
2025-11-27T18:18:34.563892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471275121942155:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:34.478235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471275920370295:2048] [node 1] ICS09 handshake done sender: [2:7577471275920370274:2118] self: [2:7577471275920370278:0] peer: [1:7577471275315041379:0] socket: 502 qp: -1
2025-11-27T18:18:34.563971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:34.564170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471276190613517:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30994
2025-11-27T18:18:34.483539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471275920370264:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.567698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471275121942054:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17667642ns) {HandleHandshakeStatus:329(17663708ns) {DropIncomingHandshake:442(413ns) DropOutgoingHandshake:457(5913ns) SwitchToState:179(348ns) ProcessPendingSessionEvents:495(2980833ns) {ForwardSessionEventToSession:612(1375464ns) ForwardSessionEventToSession:612(4570ns) ForwardSessionEventToSession:612(212090ns) ForwardSessionEventToSession:612(2427ns) ForwardSessionEventToSession:612(529391ns) ForwardSessionEventToSession:612(2435ns) ForwardSessionEventToSession:612(3346ns) ForwardSessionEventToSession:612(357125ns) ForwardSessionEventToSession:612(477591ns) ForwardSessionEventToSession:612(4674ns)}}}
2025-11-27T18:18:34.558904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471275263009435:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(40339086ns) {HandleHandshakeStatus:329(40335842ns) {DropIncomingHandshake:442(14266ns) DropOutgoingHandshake:457(2796ns) SwitchToState:179(898ns) ProcessPendingSessionEvents:495(2851537ns) {ForwardSessionEventToSession:612(55795ns) ForwardSessionEventToSession:612(2697ns) ForwardSessionEventToSession:612(1307819ns) ForwardSessionEventToSession:612(4032ns) ForwardSessionEventToSession:612(2374ns) ForwardSessionEventToSession:612(299907ns) ForwardSessionEventToSession:612(3796ns) ForwardSessionEventToSession:612(1150018ns) ForwardSessionEventToSession:612(7461ns) ForwardSessionEventToSession:612(2905ns)}}}
2025-11-27T18:18:34.483577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471275920370264:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:34.567763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471275121942054:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:34.564485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471276190613417:2049] [node 2] ICP22 created new session: [5:7577471276190613518:2049]
2025-11-27T18:18:34.484295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471275920370295:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:34.559844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471270968042076:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.566151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471276190613518:2049] [node 2] ICS09 handshake done sender: [5:7577471276190613506:2120] self: [5:7577471276190613510:0] peer: [2:7577471275920370263:0] socket: 537 qp: -1
2025-11-27T18:18:34.484892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:34.559895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:34.566186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471276190613518:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:34.559929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471270968042076:2050] [node 3] ICP22 created new session: [4:7577471275263009474:2050]
2025-11-27T18:18:34.496924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471275920370260:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(74551141ns) {HandleHandshakeStatus:329(74547970ns) {DropIncomingHandshake:442(16209ns) DropOutgoingHandshake:457(545ns) SwitchToState:179(525ns) ProcessPendingSessionEvents:495(13794067ns) {ForwardSessionEventToSession:612(9272427ns) ForwardSessionEventToSession:612(7859ns) ForwardSessionEventToSession:612(3739ns) ForwardSessionEventToSession:612(3207575ns) ForwardSessionEventToSession:612(18138ns) ForwardSessionEventToSession:612(3012ns) ForwardSessionEventToSession:612(445114ns) ForwardSessionEventToSession:612(348150ns) ForwardSessionEventToSession:612(468874ns) ForwardSessionEventToSession:612(3695ns)}}}
2025-11-27T18:18:34.566234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:34.560256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471275263009474:2050] [node 3] ICS09 handshake done sender: [4:7577471275263009466:2122] self: [4:7577471275263009468:0] peer: [3:7577471278087335964:0] socket: 555 qp: -1
2025-11-27T18:18:34.509887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471275920370261:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:34.567285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471276190613507:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:34.560799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471275263009474:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:34.510172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471271625402900:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:34.560855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:34.517907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471275920370264:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:34.537464Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471271625402899:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(37770346ns) {HandleHandshakeStatus:329(34813488ns) {DropIncomingHandshake:442(1035ns) DropOutgoingHandshake:457(17866ns) SwitchToState:179(675ns) ProcessPendingSessionEvents:495(14399417ns) {ForwardSessionEventToSession:612(14340095ns) ForwardSessionEventToSession:612(10273ns) ForwardSessionEventToSession:612(8334ns) ForwardSessionEventToSession:612(3373ns) ForwardSessionEventToSession:612(5715ns) ForwardSessionEventToSession:612(2472ns) ForwardSessionEventToSession:612(3266ns) ForwardSessionEventToSession:612(5647ns) ForwardSessionEventToSession:612(3278ns) ForwardSessionEventToSession:612(4995ns)}}}
2025-11-27T18:18:34.540752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471275920370259:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.540842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:34.547768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471275920370259:2109] [node 5] ICP22 created new session: [2:7577471275920370298:2109]
2025-11-27T18:18:34.549865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471275920370298:2109] [node 5] ICS09 handshake done sender: [2:7577471275920370264:2111] self: [2:7577471275920370263:0] peer: [5:7577471276190613510:0] socket: 518 qp: -1
2025-11-27T18:18:34.550783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471275920370298:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:34.552060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:34.567260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471275920370259:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(29965277ns) {HandleHandshakeStatus:329(29962221ns) {DropIncomingHandshake:442(25838ns) DropOutgoingHandshake:457(890ns) SwitchToState:179(897ns) ProcessPendingSessionEvents:495(6185475ns) {ForwardSessionEventToSession:612(3589808ns) ForwardSessionEventToSession:612(532367ns) ForwardSessionEventToSession:612(4027ns) ForwardSessionEventToSession:612(3102ns) ForwardSessionEventToSession:612(738350ns) ForwardSessionEventToSession:612(3304ns) ForwardSessionEventToSession:612(473739ns) ForwardSessionEventToSession:612(542245ns) ForwardSessionEventToSession:612(3510ns) ForwardSessionEventToSession:612(279328ns)}}}
2025-11-27T18:18:34.453523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471278087335971:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:34.450792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471278087335982:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:34.457233Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471278087335968:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:34.462791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:34.463066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471278087335954:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.463135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:34.463201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471278087335954:2110] [node 6] ICP22 created new session: [3:7577471278087335986:2110]
2025-11-27T18:18:34.463248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471278087335986:2110] [node 6] ICS09 handshake done sender: [3:7577471278087335965:2114] self: [3:7577471278087335959:0] peer: [6:7577471275121942138:0] socket: 520 qp: -1
2025-11-27T18:18:34.463294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471278087335986:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:34.463326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:34.463843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471273792368595:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.463926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:34.464090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471273792368594:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.465082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:34.470617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471273792368595:2049] [node 2] ICP22 created new session: [3:7577471278087335988:2049]
2025-11-27T18:18:34.475218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471278087335988:2049] [node 2] ICS09 handshake done sender: [3:7577471278087335971:2118] self: [3:7577471278087335975:0] peer: [2:7577471275920370262:0] socket: 511 qp: -1
2025-11-27T18:18:34.476742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471273792368594:2048] [node 1] ICP22 created new session: [3:7577471278087335989:2048]
2025-11-27T18:18:34.477185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471278087335988:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:34.477207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471278087335989:2048] [node 1] ICS09 handshake done sender: [3:7577471278087335968:2117] self: [3:7577471278087335973:0] peer: [1:7577471275315041380:0] socket: 503 qp: -1
2025-11-27T18:18:34.477546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471278087335989:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:34.478131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:34.483444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471278087335983:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.484251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:34.505473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471278087335966:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:34.505514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471278087335966:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:34.507132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471278087335966:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:34.507161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471278087335954:2110] [node 6] ICP18 handshake (actor [3:7577471278087335983:2122]) is too old
2025-11-27T18:18:34.569341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471276190613481:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:34.527402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471273792368595:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(30840792ns) {HandleHandshakeStatus:329(30838252ns) {DropIncomingHandshake:442(639ns) DropOutgoingHandshake:457(3700ns) SwitchToState:179(375ns) ProcessPendingSessionEvents:495(8602061ns) {ForwardSessionEventToSession:612(8543379ns) ForwardSessionEventToSession:612(8572ns) ForwardSessionEventToSession:612(3160ns) ForwardSessionEventToSession:612(5141ns) ForwardSessionEventToSession:612(2988ns) ForwardSessionEventToSession:612(3316ns) ForwardSessionEventToSession:612(9144ns) ForwardSessionEventToSession:612(3601ns) ForwardSessionEventToSession:612(5425ns) ForwardSessionEventToSession:612(6091ns)}}}
2025-11-27T18:18:34.529758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471273792368594:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(53687130ns) {HandleHandshakeStatus:329(53683468ns) {DropIncomingHandshake:442(511ns) DropOutgoingHandshake:457(232ns) SwitchToState:179(295ns) ProcessPendingSessionEvents:495(961ns)}}
2025-11-27T18:18:34.529889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471278087335956:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(61670286ns) {HandleHandshakeStatus:329(61668083ns) {DropIncomingHandshake:442(517ns) DropOutgoingHandshake:457(476ns) SwitchToState:179(462ns) ProcessPendingSessionEvents:495(14454788ns) {ForwardSessionEventToSession:612(5865399ns) ForwardSessionEventToSession:612(1120618ns) ForwardSessionEventToSession:612(4093ns) ForwardSessionEventToSession:612(2399ns) ForwardSessionEventToSession:612(551607ns) ForwardSessionEventToSession:612(3133ns) ForwardSessionEventToSession:612(598494ns) ForwardSessionEventToSession:612(675339ns) ForwardSessionEventToSession:612(8811ns) ForwardSessionEventToSession:612(4938238ns)}}}
2025-11-27T18:18:34.531005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471278087335956:2109] [node 5] ICP18 handshake (actor [3:7577471278087335976:2119]) is too old
2025-11-27T18:18:34.569676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:34.529901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471278087335955:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:34.549743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:34.549793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471278087335955:2107] [node 4] ICP22 created new session: [3:7577471278087335993:2107]
2025-11-27T18:18:34.569753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471276190613481:2107] [node 4] ICP22 created new session: [5:7577471276190613520:2107]
2025-11-27T18:18:34.551431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471278087335993:2107] [node 4] ICS09 handshake done sender: [3:7577471278087335966:2115] self: [3:7577471278087335964:0] peer: [4:7577471275263009468:0] socket: 554 qp: -1
2025-11-27T18:18:34.553811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471278087335993:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:34.569821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471276190613520:2107] [node 4] ICS09 handshake done sender: [5:7577471276190613507:2121] self: [5:7577471276190613514:0] peer: [4:7577471275263009448:0] socket: 542 qp: -1
2025-11-27T18:18:34.553849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:34.569860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471276190613520:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:34.569900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:34.567104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471278087335955:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(33823657ns) {HandleHandshakeStatus:329(33820247ns) {DropIncomingHandshake:442(6982582ns) DropOutgoingHandshake:457(1398ns) SwitchToState:179(813ns) ProcessPendingSessionEvents:495(3255406ns) {ForwardSessionEventToSession:612(2015152ns) ForwardSessionEventToSession:612(4069ns) ForwardSessionEventToSession:612(566887ns) ForwardSessionEventToSession:612(3469ns) ForwardSessionEventToSession:612(1862ns) ForwardSessionEventToSession:612(611534ns) ForwardSessionEventToSession:612(8111ns) ForwardSessionEventToSession:612(3335ns) ForwardSessionEventToSession:612(7333ns) ForwardSessionEventToSession:612(21443ns)}}}
2025-11-27T18:18:34.611047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:34.611290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471275920370295:2048] [node 1] ICS07 socket disconnect 502 reason# EndOfStream
2025-11-27T18:18:34.611318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471275920370295:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.611409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:34.611345Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:34.611462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471275920370295:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:34.611627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471278087335989:2048] [node 1] ICS01 socket: 503 reason# EndOfStream
2025-11-27T18:18:34.611671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471273792368594:2048] [node 1] ICP30 unregister session Session# [3:7577471278087335989:2048] VirtualId# [3:7577471278087335973:0]
2025-11-27T18:18:34.611709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471278087335989:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.611800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:34.611842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:34.617399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:34.617399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:34.621235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471276190613512:2048] [node 1] ICS07 socket disconnect 529 reason# EndOfStream
2025-11-27T18:18:34.621362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471276190613512:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.621516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:34.621576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471276190613512:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:34.625994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471275263009461:2048] [node 1] ICS07 socket disconnect 491 reason# EndOfStream
2025-11-27T18:18:34.618586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:34.629125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471275263009461:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.629277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:34.629353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471275263009461:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:34.629144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471275121942155:2048] [node 1] ICS01 socket: 543 reason# EndOfStream
2025-11-27T18:18:34.630080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471275121942052:2048] [node 1] ICP30 unregister session Session# [6:7577471275121942155:2048] VirtualId# [6:7577471275121942151:0]
2025-11-27T18:18:34.630119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471275121942155:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.630192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:34.630234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:34.633428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471270968042074:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4735) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:34.633541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471275263009461:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:34.633875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471270968042074:2048] [node 1] ICP30 unregister session Session# [4:7577471275263009461:2048] VirtualId# [4:7577471275263009451:0]
2025-11-27T18:18:34.634283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:34.634338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471270968042074:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.634358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.634533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471276190613416:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4735) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:34.634720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471276190613512:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:34.634872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471276190613416:2048] [node 1] ICP30 unregister session Session# [5:7577471276190613512:2048] VirtualId# [5:7577471276190613504:0]
2025-11-27T18:18:34.635266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:34.635510Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471276190613416:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.635541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.644108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471270968042074:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:34.649569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471276190613416:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:34.670856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:34.671138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471278087335988:2049] [node 2] ICS07 socket disconnect 511 reason# EndOfStream
2025-11-27T18:18:34.671182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471278087335988:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.671265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:34.671321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471278087335988:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:34.675178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471273792368595:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2462) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:34.675020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:34.673814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:34.675234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471278087335988:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:34.675257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471276190613518:2049] [node 2] ICS01 socket: 537 reason# EndOfStream
2025-11-27T18:18:34.675308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471276190613417:2049] [node 2] ICP30 unregister session Session# [5:7577471276190613518:2049] VirtualId# [5:7577471276190613510:0]
2025-11-27T18:18:34.674097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471275263009467:2049] [node 2] ICS01 socket: 509 reason# EndOfStream
2025-11-27T18:18:34.675609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471276190613518:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.674143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471270968042075:2049] [node 2] ICP30 unregister session Session# [4:7577471275263009467:2049] VirtualId# [4:7577471275263009458:0]
2025-11-27T18:18:34.675683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:34.674175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471275263009467:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.676109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:34.674256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:34.674309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:34.677698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471273792368595:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2462) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:34.677767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471278087335988:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:34.677802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471273792368595:2049] [node 2] ICP30 unregister session Session# [3:7577471278087335988:2049] VirtualId# [3:7577471278087335975:0]
2025-11-27T18:18:34.677849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:34.677881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471273792368595:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2462) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.677900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2462) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.678361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:34.678686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471275121942149:2049] [node 2] ICS07 socket disconnect 519 reason# EndOfStream
2025-11-27T18:18:34.678729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471275121942149:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.678813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:34.678875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471275121942149:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:34.680004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471275121942053:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2462) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:34.680135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471275121942149:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:34.680205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471275121942053:2049] [node 2] ICP30 unregister session Session# [6:7577471275121942149:2049] VirtualId# [6:7577471275121942139:0]
2025-11-27T18:18:34.680289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:34.680342Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471275121942053:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2462) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.680358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2462) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.687832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471273792368595:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:34.690066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471275121942053:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:34.751415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:34.751627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471275263009474:2050] [node 3] ICS01 socket: 555 reason# EndOfStream
2025-11-27T18:18:34.751668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471270968042076:2050] [node 3] ICP30 unregister session Session# [4:7577471275263009474:2050] VirtualId# [4:7577471275263009468:0]
2025-11-27T18:18:34.751723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471275263009474:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.751801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:34.751862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:34.752577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:34.752748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471276190613511:2050] [node 3] ICS07 socket disconnect 516 reason# EndOfStream
2025-11-27T18:18:34.752780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471276190613511:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.752848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:34.752901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471276190613511:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:34.753662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471276190613418:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9276) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:34.753691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:34.753820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471276190613511:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:34.753889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471276190613418:2050] [node 3] ICP30 unregister session Session# [5:7577471276190613511:2050] VirtualId# [5:7577471276190613505:0]
2025-11-27T18:18:34.753910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471275121942153:2050] [node 3] ICS07 socket disconnect 522 reason# EndOfStream
2025-11-27T18:18:34.753942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471275121942153:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.753939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:34.753982Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471276190613418:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9276) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.754021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:34.754000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9276) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.754086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471275121942153:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:34.755183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471275121942054:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9276) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:34.755298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471275121942153:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:34.755342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471275121942054:2050] [node 3] ICP30 unregister session Session# [6:7577471275121942153:2050] VirtualId# [6:7577471275121942138:0]
2025-11-27T18:18:34.755396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:34.755444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471275121942054:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9276) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.755464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9276) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:34.763614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471276190613418:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:34.765819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471275121942054:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:34.844713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:34.845043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471275121942142:2102] [node 4] ICS01 socket: 490 reason# EndOfStream
2025-11-27T18:18:34.845098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471275121942110:2102] [node 4] ICP30 unregister session Session# [6:7577471275121942142:2102] VirtualId# [6:7577471275121942133:0]
2025-11-27T18:18:34.845153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471275121942142:2102] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.845248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:34.845314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:34.899365Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:34.899663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471275121942141:2099] [node 5] ICS01 socket: 486 reason# EndOfStream
2025-11-27T18:18:34.899717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471275121942104:2099] [node 5] ICP30 unregister session Session# [6:7577471275121942141:2099] VirtualId# [6:7577471275121942132:0]
2025-11-27T18:18:34.899756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471275121942141:2099] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:34.899837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:34.899892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:35.072441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471280145728186:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:35.072564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471280145728187:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:35.090866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471280855851753:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:35.090893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471280855851754:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:35.113087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471280265744412:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:35.113141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471280265744413:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:35.138453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471279674686021:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:35.138514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471279674686022:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:35.138547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471279674686023:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:35.160185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471278586547521:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:35.160229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471278586547522:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:35.160253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471278586547523:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:35.179492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471279377076361:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:35.179529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471279377076362:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:35.179553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471279377076363:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:35.334275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471279377076424:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:35.337638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471280265744485:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:35.337637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471280855851816:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:35.337812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471278586547585:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:35.337981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471279674686084:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:35.338034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471279377076425:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:35.338465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471278586547586:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:35.338566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471280265744486:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:35.338624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471279674686085:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:35.338560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471280855851817:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:35.339124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471280145728186:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(71358235ns) {Configure:104(71232071ns) {ProcessConfigured:135(87815ns) {StartInitialHandshake:157(85307ns) {DropHandshakes:470(2901ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(611ns) PrepareNewSessionHandshake:477(1743ns) {SwitchToState:179(487ns)}}}}}
2025-11-27T18:18:35.339247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471280855851818:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:35.339374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471280265744487:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:35.339724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471280145728252:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:35.340155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471280145728253:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:35.340589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471280145728254:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.349899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471279674686099:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3940
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.350356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471280855851829:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44377
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.349757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471278586547597:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2843
2025-11-27T18:18:35.350422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471280855851830:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44377
2025-11-27T18:18:35.351834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471280855851831:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44377
2025-11-27T18:18:35.352025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471280855851832:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44377
2025-11-27T18:18:35.360643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471278586547597:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.361756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471280265744498:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59754
2025-11-27T18:18:35.361756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471279674686100:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3940
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.362018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471280265744499:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59754
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.362105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471279674686101:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3940
2025-11-27T18:18:35.362165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471280265744500:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59754
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.362466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471280265744499:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.362681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471280265744504:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59754
2025-11-27T18:18:35.362153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471280145728266:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48955
2025-11-27T18:18:35.362992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471279674686099:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.363080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471280265744500:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.363249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471280145728266:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.363280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471280265744498:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.363129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471279674686021:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:35.363488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471280145728267:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48955
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.363524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471280855851832:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.363638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471280145728268:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48955
2025-11-27T18:18:35.363633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471280855851829:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.363764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471280145728269:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48955
2025-11-27T18:18:35.364205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471279674686100:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.364340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471279674686022:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:35.363917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471280145728258:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.364255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471280855851831:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.363948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471280145728258:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.364593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471280855851830:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.364962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471280855851824:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.364981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471280855851824:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.365408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471280145728267:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.365494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471280855851836:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44377
2025-11-27T18:18:35.365532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471280145728268:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:18:35.365752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471280855851753:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-27T18:18:35.365887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471280145728270:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48955
2025-11-27T18:18:35.365897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471280265744504:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.366050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471280145728269:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.366222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471280855851824:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:35.366316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471280145728251:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.366316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471279674686104:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3940
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.366347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471280145728251:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.366721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471280855851816:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.366779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471279674686105:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3940
2025-11-27T18:18:35.367222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:35.367290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471280855851816:2107] [node 4] ICP22 created new session: [2:7577471280855851842:2107]
2025-11-27T18:18:35.367719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471280855851842:2107] [node 4] ICS09 handshake done sender: [2:7577471280855851824:2112] self: [2:7577471280855851823:0] peer: [4:7577471279674686103:0] socket: 514 qp: -1
2025-11-27T18:18:35.367883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471280145728258:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:35.367772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471280855851842:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:35.368071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:35.368500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471280145728252:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.368605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:35.369093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471279674686100:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:35.368657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471280145728252:2107] [node 4] ICP22 created new session: [1:7577471280145728275:2107]
2025-11-27T18:18:35.368726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471280145728275:2107] [node 4] ICS09 handshake done sender: [1:7577471280145728258:2113] self: [1:7577471280145728257:0] peer: [4:7577471279674686102:0] socket: 492 qp: -1
2025-11-27T18:18:35.368890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471280145728275:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:35.369076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:35.371304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471279674686022:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.371681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:35.371743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471279674686022:2049] [node 2] ICP22 created new session: [4:7577471279674686106:2049]
2025-11-27T18:18:35.371947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471279674686106:2049] [node 2] ICS09 handshake done sender: [4:7577471279674686100:2116] self: [4:7577471279674686103:0] peer: [2:7577471280855851823:0] socket: 516 qp: -1
2025-11-27T18:18:35.371899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471278586547602:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2843
2025-11-27T18:18:35.372208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471280265744412:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:35.372000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471279674686106:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:35.372365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:35.372699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471280145728251:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:35.372728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471279674686022:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:35.372724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471280265744413:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:35.373338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471279674686099:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:35.373341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471280855851822:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.373374Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471280855851822:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.373254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471280145728270:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.374261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471278586547602:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.373372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471280145728256:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.373391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471280145728256:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.374533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471280145728186:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.376271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471280855851836:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.374610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:35.374651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471280145728186:2048] [node 2] ICP22 created new session: [1:7577471280145728280:2048]
2025-11-27T18:18:35.374706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471280145728280:2048] [node 2] ICS09 handshake done sender: [1:7577471280145728251:2111] self: [1:7577471280145728250:0] peer: [2:7577471280855851839:0] socket: 487 qp: -1
2025-11-27T18:18:35.374735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471280145728280:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:35.374766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:35.376495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471278586547522:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.377062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471278586547603:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2843
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.377915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471280265744509:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59754
2025-11-27T18:18:35.378054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471280145728256:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:35.378115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471279674686101:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.383327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471280855851845:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44377
2025-11-27T18:18:35.383129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471279674686021:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.383846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471279377076439:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3337
2025-11-27T18:18:35.383239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:35.383527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471279674686021:2048] [node 1] ICP22 created new session: [4:7577471279674686108:2048]
2025-11-27T18:18:35.383579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471279674686108:2048] [node 1] ICS09 handshake done sender: [4:7577471279674686099:2115] self: [4:7577471279674686102:0] peer: [1:7577471280145728257:0] socket: 502 qp: -1
2025-11-27T18:18:35.383964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471280265744498:2125] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:35.384208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471280265744412:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.384281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:35.384322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471280265744412:2048] [node 1] ICP22 created new session: [3:7577471280265744510:2048]
2025-11-27T18:18:35.384366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471280265744510:2048] [node 1] ICS09 handshake done sender: [3:7577471280265744498:2125] self: [3:7577471280265744507:0] peer: [1:7577471280145728255:0] socket: 505 qp: -1
2025-11-27T18:18:35.384398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471280265744510:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:35.384443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:35.383626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471279674686108:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:35.383833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:35.384356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471279674686021:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:35.385610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471280145728187:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.385716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:35.385768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471280145728187:2049] [node 3] ICP22 created new session: [1:7577471280145728285:2049]
2025-11-27T18:18:35.385819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471280145728285:2049] [node 3] ICS09 handshake done sender: [1:7577471280145728256:2112] self: [1:7577471280145728255:0] peer: [3:7577471280265744507:0] socket: 491 qp: -1
2025-11-27T18:18:35.385487Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471280855851753:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471280855851829:2115] held: no
2025-11-27T18:18:35.385850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471280145728285:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:35.386478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.388112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471279674686023:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.388355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471279377076440:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3337
2025-11-27T18:18:35.385767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471280855851826:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.385792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471280855851826:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.387139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471280855851829:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:35.388427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471278586547605:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2843
2025-11-27T18:18:35.388261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471280855851822:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:35.388915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471280855851753:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.389062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471280265744489:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.389093Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471280265744489:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.389015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:35.389067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471280855851753:2048] [node 1] ICP22 created new session: [2:7577471280855851849:2048]
2025-11-27T18:18:35.389107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471280855851849:2048] [node 1] ICS09 handshake done sender: [2:7577471280855851829:2115] self: [2:7577471280855851839:0] peer: [1:7577471280145728250:0] socket: 495 qp: -1
2025-11-27T18:18:35.389161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471280855851849:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:35.389419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:35.390132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471280855851754:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.390935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:35.391027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471280855851754:2049] [node 3] ICP22 created new session: [2:7577471280855851852:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.391072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471280855851852:2049] [node 3] ICS09 handshake done sender: [2:7577471280855851822:2111] self: [2:7577471280855851821:0] peer: [3:7577471280265744508:0] socket: 513 qp: -1
2025-11-27T18:18:35.391925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471278586547603:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.392303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471280855851852:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:35.392323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471278586547606:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2843
2025-11-27T18:18:35.392338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:35.392646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471278586547521:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.393170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471278586547605:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.393428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471279674686113:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3940
2025-11-27T18:18:35.393480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471278586547585:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:35.393383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471280265744512:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59754
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.393851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471278586547608:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2843
2025-11-27T18:18:35.393652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471279377076440:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.394101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471279674686093:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.394807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471280145728260:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.394142Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471279674686093:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.394844Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471280145728260:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.395679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471279674686093:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:35.396223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471279674686084:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:35.396358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471279377076425:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-27T18:18:35.396362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471280855851826:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:35.396381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:35.396447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471279674686084:2108] [node 5] ICP22 created new session: [4:7577471279674686115:2108]
2025-11-27T18:18:35.396535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471279674686115:2108] [node 5] ICS09 handshake done sender: [4:7577471279674686093:2113] self: [4:7577471279674686092:0] peer: [5:7577471278586547609:0] socket: 537 qp: -1
2025-11-27T18:18:35.396534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471279377076441:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3337
2025-11-27T18:18:35.396600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471279674686115:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:35.396704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:35.396731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471280265744489:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:35.396898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471280855851817:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.397053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471279377076439:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.397593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471279377076361:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:35.397576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:35.397740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471280265744504:2128] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.398279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471280265744413:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22957) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471280265744504:2128] held: no
2025-11-27T18:18:35.397860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471280855851817:2108] [node 5] ICP22 created new session: [2:7577471280855851854:2108]
2025-11-27T18:18:35.398199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471278586547610:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2843
2025-11-27T18:18:35.397922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471280855851854:2108] [node 5] ICS09 handshake done sender: [2:7577471280855851826:2113] self: [2:7577471280855851825:0] peer: [5:7577471278586547604:0] socket: 506 qp: -1
2025-11-27T18:18:35.397974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471280855851854:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:35.398060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:35.398399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471280265744413:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.398491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:35.398556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471280265744413:2049] [node 2] ICP22 created new session: [3:7577471280265744515:2049]
2025-11-27T18:18:35.396772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471280145728260:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:35.397580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471280145728253:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.397835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:35.398592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471280265744515:2049] [node 2] ICS09 handshake done sender: [3:7577471280265744504:2128] self: [3:7577471280265744508:0] peer: [2:7577471280855851821:0] socket: 520 qp: -1
2025-11-27T18:18:35.397902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471280145728253:2109] [node 5] ICP22 created new session: [1:7577471280145728288:2109]
2025-11-27T18:18:35.398631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471280265744515:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:35.397956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471280145728288:2109] [node 5] ICS09 handshake done sender: [1:7577471280145728260:2114] self: [1:7577471280145728259:0] peer: [5:7577471278586547607:0] socket: 515 qp: -1
2025-11-27T18:18:35.398695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:35.398060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471280145728288:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:35.398118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:35.398904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471279674686095:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.398932Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471279674686095:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.399058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471278586547602:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:35.399987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471280265744485:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.399977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471279674686101:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:35.399904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471280145728262:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.399943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471280145728262:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.400534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:35.401239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471279674686023:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.401035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471280265744485:2107] [node 4] ICP22 created new session: [3:7577471280265744517:2107]
2025-11-27T18:18:35.401293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471280265744517:2107] [node 4] ICS09 handshake done sender: [3:7577471280265744489:2120] self: [3:7577471280265744488:0] peer: [4:7577471279674686112:0] socket: 521 qp: -1
2025-11-27T18:18:35.401341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471280265744517:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:35.401642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.401370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:35.401647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471279674686023:2050] [node 3] ICP22 created new session: [4:7577471279674686118:2050]
2025-11-27T18:18:35.402094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471279674686118:2050] [node 3] ICS09 handshake done sender: [4:7577471279674686101:2117] self: [4:7577471279674686112:0] peer: [3:7577471280265744488:0] socket: 524 qp: -1
2025-11-27T18:18:35.402148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471279674686118:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:35.402219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471279377076444:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3337
2025-11-27T18:18:35.402228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.402553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471280145728262:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:35.402584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471279377076445:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3337
2025-11-27T18:18:35.404225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471280145728254:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.404422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:35.404479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471280145728254:2110] [node 6] ICP22 created new session: [1:7577471280145728291:2110]
2025-11-27T18:18:35.405894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471280145728291:2110] [node 6] ICS09 handshake done sender: [1:7577471280145728262:2115] self: [1:7577471280145728261:0] peer: [6:7577471279377076443:0] socket: 494 qp: -1
2025-11-27T18:18:35.405946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471280145728291:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:35.406451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:35.406413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471279377076439:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:35.406615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471278586547522:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.407127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:35.407209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471278586547522:2049] [node 2] ICP22 created new session: [5:7577471278586547612:2049]
2025-11-27T18:18:35.407253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471278586547612:2049] [node 2] ICS09 handshake done sender: [5:7577471278586547602:2117] self: [5:7577471278586547604:0] peer: [2:7577471280855851825:0] socket: 532 qp: -1
2025-11-27T18:18:35.407282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471278586547612:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:35.407360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:35.409149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471279377076441:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.409639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471279674686095:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:35.410145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471278586547522:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:35.411077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471279674686085:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.411468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:35.411582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471278586547605:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:35.411583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471279377076361:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.411669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:35.411780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471279377076361:2048] [node 1] ICP22 created new session: [6:7577471279377076446:2048]
2025-11-27T18:18:35.411553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471279674686085:2109] [node 6] ICP22 created new session: [4:7577471279674686120:2109]
2025-11-27T18:18:35.411637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471279674686120:2109] [node 6] ICS09 handshake done sender: [4:7577471279674686095:2114] self: [4:7577471279674686094:0] peer: [6:7577471279377076442:0] socket: 539 qp: -1
No RDMA context found for GID: 2025-11-27T18:18:35.411852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471279377076446:2048] [node 1] ICS09 handshake done sender: [6:7577471279377076439:2115] self: [6:7577471279377076443:0] peer: [1:7577471280145728261:0] socket: 536 qp: -1
::ffff:127.1.0.0
2025-11-27T18:18:35.411899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471279377076446:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:35.411676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471279674686120:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:35.411985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:35.411808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:35.412005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471278586547611:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2843
2025-11-27T18:18:35.412273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471279377076361:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:35.412614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471278586547521:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471278586547603:2118] held: no
2025-11-27T18:18:35.413224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471278586547585:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.413313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:35.413357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471278586547585:2108] [node 4] ICP22 created new session: [5:7577471278586547614:2108]
2025-11-27T18:18:35.413400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471278586547614:2108] [node 4] ICS09 handshake done sender: [5:7577471278586547605:2119] self: [5:7577471278586547609:0] peer: [4:7577471279674686092:0] socket: 538 qp: -1
2025-11-27T18:18:35.413435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471278586547614:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:35.413466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:35.414269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471278586547603:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:35.414621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471279377076362:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:35.414714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471279377076440:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:35.415017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471278586547608:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.415087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471278586547521:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.415119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471279377076425:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.415164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:35.415199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:35.415227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471278586547521:2048] [node 1] ICP22 created new session: [5:7577471278586547616:2048]
2025-11-27T18:18:35.415251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471279377076425:2109] [node 4] ICP22 created new session: [6:7577471279377076451:2109]
2025-11-27T18:18:35.415260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471278586547616:2048] [node 1] ICS09 handshake done sender: [5:7577471278586547603:2118] self: [5:7577471278586547607:0] peer: [1:7577471280145728259:0] socket: 518 qp: -1
2025-11-27T18:18:35.415290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471279377076451:2109] [node 4] ICS09 handshake done sender: [6:7577471279377076440:2116] self: [6:7577471279377076442:0] peer: [4:7577471279674686094:0] socket: 540 qp: -1
2025-11-27T18:18:35.415238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471280855851828:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.415287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471278586547616:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:35.415317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:35.415328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471279377076451:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:35.415289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471280855851828:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.415389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.416687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471279377076448:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3337
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.417110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471280265744520:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59754
2025-11-27T18:18:35.417205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471280855851828:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:35.417064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471278586547523:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:35.417376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471280265744495:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.417420Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471280265744495:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.417737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471280855851818:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.419724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:35.419924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471280855851818:2109] [node 6] ICP22 created new session: [2:7577471280855851857:2109]
2025-11-27T18:18:35.419978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471280855851857:2109] [node 6] ICS09 handshake done sender: [2:7577471280855851828:2114] self: [2:7577471280855851827:0] peer: [6:7577471279377076449:0] socket: 546 qp: -1
2025-11-27T18:18:35.420008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471280855851857:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:35.420076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:35.419907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471279377076448:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.420367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471278586547621:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2843
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.421410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471279377076453:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3337
2025-11-27T18:18:35.421542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471279377076454:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3337
2025-11-27T18:18:35.421553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471280265744495:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:35.422453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471278586547608:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:35.423308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471278586547523:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.423455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:35.423522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471278586547523:2050] [node 3] ICP22 created new session: [5:7577471278586547622:2050]
2025-11-27T18:18:35.423578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471278586547622:2050] [node 3] ICS09 handshake done sender: [5:7577471278586547608:2121] self: [5:7577471278586547620:0] peer: [3:7577471280265744494:0] socket: 545 qp: -1
2025-11-27T18:18:35.423629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471278586547622:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:35.423640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471279377076363:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:35.423681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:35.423745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471279377076441:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:35.423786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471280265744486:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.423881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:35.424218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471280265744486:2108] [node 5] ICP22 created new session: [3:7577471280265744522:2108]
2025-11-27T18:18:35.424288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471280265744522:2108] [node 5] ICS09 handshake done sender: [3:7577471280265744495:2123] self: [3:7577471280265744494:0] peer: [5:7577471278586547620:0] socket: 543 qp: -1
2025-11-27T18:18:35.424329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471280265744522:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:35.424389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:35.424358Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471279377076362:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22957) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471279377076441:2117] held: no
2025-11-27T18:18:35.424481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471279377076362:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.424556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:35.424611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471279377076362:2049] [node 2] ICP22 created new session: [6:7577471279377076456:2049]
2025-11-27T18:18:35.424672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471279377076456:2049] [node 2] ICS09 handshake done sender: [6:7577471279377076441:2117] self: [6:7577471279377076449:0] peer: [2:7577471280855851827:0] socket: 547 qp: -1
2025-11-27T18:18:35.424712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471279377076456:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:35.424767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:35.426398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471279377076453:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.427002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471279377076424:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:35.427112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471280265744497:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.427154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471280265744497:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.427994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471280265744520:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.428052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471278586547596:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:35.428084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471278586547596:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:35.428880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471280265744497:2124] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.429211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471280265744487:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.429193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471279377076459:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3337
2025-11-27T18:18:35.429480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471278586547596:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:35.429342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:35.429397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471280265744487:2110] [node 6] ICP22 created new session: [3:7577471280265744526:2110]
2025-11-27T18:18:35.429456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471280265744526:2110] [node 6] ICS09 handshake done sender: [3:7577471280265744497:2124] self: [3:7577471280265744496:0] peer: [6:7577471279377076455:0] socket: 488 qp: -1
2025-11-27T18:18:35.429494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471280265744526:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:35.429608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:35.430904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471278586547586:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:35.430984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:35.431050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471278586547586:2110] [node 6] ICP22 created new session: [5:7577471278586547625:2110]
2025-11-27T18:18:35.431115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471278586547625:2110] [node 6] ICS09 handshake done sender: [5:7577471278586547596:2115] self: [5:7577471278586547595:0] peer: [6:7577471279377076458:0] socket: 507 qp: -1
2025-11-27T18:18:35.431170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471278586547625:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:35.431207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:35.432108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471279377076448:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:35.432490Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471279377076363:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27369) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471279377076448:2120] held: no
2025-11-27T18:18:35.432633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471279377076363:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.432692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:35.432735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471279377076363:2050] [node 3] ICP22 created new session: [6:7577471279377076461:2050]
2025-11-27T18:18:35.432789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471279377076461:2050] [node 3] ICS09 handshake done sender: [6:7577471279377076448:2120] self: [6:7577471279377076455:0] peer: [3:7577471280265744496:0] socket: 493 qp: -1
2025-11-27T18:18:35.432841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471279377076461:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:35.432876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.434828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471279377076460:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3337
2025-11-27T18:18:35.435451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471279377076424:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6923) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471279377076453:2121] held: no
2025-11-27T18:18:35.435661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471279377076453:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:35.435899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471279377076424:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:35.436006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:35.436077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471279377076424:2108] [node 5] ICP22 created new session: [6:7577471279377076463:2108]
2025-11-27T18:18:35.436123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471279377076463:2108] [node 5] ICS09 handshake done sender: [6:7577471279377076453:2121] self: [6:7577471279377076458:0] peer: [5:7577471278586547595:0] socket: 552 qp: -1
2025-11-27T18:18:35.436164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471279377076463:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:35.436200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:35.468274Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:35.468551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471280855851849:2048] [node 1] ICS01 socket: 495 reason# EndOfStream
2025-11-27T18:18:35.468598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471280855851753:2048] [node 1] ICP30 unregister session Session# [2:7577471280855851849:2048] VirtualId# [2:7577471280855851839:0]
2025-11-27T18:18:35.468639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471280855851849:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.468717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:35.468765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:35.470511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:35.470814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471280265744510:2048] [node 1] ICS01 socket: 505 reason# EndOfStream
2025-11-27T18:18:35.470866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471280265744412:2048] [node 1] ICP30 unregister session Session# [3:7577471280265744510:2048] VirtualId# [3:7577471280265744507:0]
2025-11-27T18:18:35.470905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471280265744510:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.470978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:35.471029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:35.472479Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:35.472690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471279674686108:2048] [node 1] ICS01 socket: 502 reason# EndOfStream
2025-11-27T18:18:35.472733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471279674686021:2048] [node 1] ICP30 unregister session Session# [4:7577471279674686108:2048] VirtualId# [4:7577471279674686102:0]
2025-11-27T18:18:35.472770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471279674686108:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.472841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:35.472881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:35.475428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:35.475630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471279377076446:2048] [node 1] ICS07 socket disconnect 536 reason# EndOfStream
2025-11-27T18:18:35.475662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471279377076446:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.475753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:35.475808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471279377076446:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:35.476634Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471279377076361:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15295) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:35.476693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471279377076446:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:35.476724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471279377076361:2048] [node 1] ICP30 unregister session Session# [6:7577471279377076446:2048] VirtualId# [6:7577471279377076443:0]
2025-11-27T18:18:35.476769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:35.476799Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471279377076361:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:35.476814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:35.473688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:35.473931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471278586547616:2048] [node 1] ICS01 socket: 518 reason# EndOfStream
2025-11-27T18:18:35.473967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471278586547521:2048] [node 1] ICP30 unregister session Session# [5:7577471278586547616:2048] VirtualId# [5:7577471278586547607:0]
2025-11-27T18:18:35.473995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471278586547616:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.474095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:35.474132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:35.486623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471279377076361:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:35.503762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:35.504043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471280265744515:2049] [node 2] ICS01 socket: 520 reason# EndOfStream
2025-11-27T18:18:35.504090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471280265744413:2049] [node 2] ICP30 unregister session Session# [3:7577471280265744515:2049] VirtualId# [3:7577471280265744508:0]
2025-11-27T18:18:35.504126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471280265744515:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.504216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:35.504280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:35.511803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:35.512072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471279674686106:2049] [node 2] ICS07 socket disconnect 516 reason# EndOfStream
2025-11-27T18:18:35.512116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471279674686106:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.512200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:35.512263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471279674686106:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:35.513085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471279674686022:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22957) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:35.513129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471279674686106:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:35.513155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471279674686022:2049] [node 2] ICP30 unregister session Session# [4:7577471279674686106:2049] VirtualId# [4:7577471279674686103:0]
2025-11-27T18:18:35.513197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:35.513229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471279674686022:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22957) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:35.513284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22957) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:35.513048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:35.513275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471278586547612:2049] [node 2] ICS07 socket disconnect 532 reason# EndOfStream
2025-11-27T18:18:35.513316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471278586547612:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.513388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:35.517279Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:35.513427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471278586547612:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:35.517242Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471278586547522:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22957) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:35.517328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471278586547612:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:35.517359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471278586547522:2049] [node 2] ICP30 unregister session Session# [5:7577471278586547612:2049] VirtualId# [5:7577471278586547604:0]
2025-11-27T18:18:35.517421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:35.517477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471278586547522:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22957) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:35.517526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471279377076456:2049] [node 2] ICS01 socket: 547 reason# EndOfStream
2025-11-27T18:18:35.517494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22957) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:35.517573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471279377076362:2049] [node 2] ICP30 unregister session Session# [6:7577471279377076456:2049] VirtualId# [6:7577471279377076449:0]
2025-11-27T18:18:35.517615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471279377076456:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.517720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:35.517789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:35.523375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471279674686022:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:35.527502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471278586547522:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:35.573281Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:35.573537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471279377076461:2050] [node 3] ICS01 socket: 493 reason# EndOfStream
2025-11-27T18:18:35.573590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471279377076363:2050] [node 3] ICP30 unregister session Session# [6:7577471279377076461:2050] VirtualId# [6:7577471279377076455:0]
2025-11-27T18:18:35.573625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471279377076461:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.573737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:35.573793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:35.573684Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:35.573919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471279674686118:2050] [node 3] ICS07 socket disconnect 524 reason# EndOfStream
2025-11-27T18:18:35.573947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471279674686118:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.574028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:35.574076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471279674686118:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:35.575212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471279674686023:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27369) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:35.575263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471279674686118:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:35.575320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471279674686023:2050] [node 3] ICP30 unregister session Session# [4:7577471279674686118:2050] VirtualId# [4:7577471279674686112:0]
2025-11-27T18:18:35.575371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:35.575400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471279674686023:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27369) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:35.575430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27369) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:35.574313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:35.574510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471278586547622:2050] [node 3] ICS01 socket: 545 reason# EndOfStream
2025-11-27T18:18:35.574550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471278586547523:2050] [node 3] ICP30 unregister session Session# [5:7577471278586547622:2050] VirtualId# [5:7577471278586547620:0]
2025-11-27T18:18:35.574576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471278586547622:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.574649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:35.574693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:35.584916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471279674686023:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:35.624281Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:35.624533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471278586547614:2108] [node 4] ICS01 socket: 538 reason# EndOfStream
2025-11-27T18:18:35.624577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471278586547585:2108] [node 4] ICP30 unregister session Session# [5:7577471278586547614:2108] VirtualId# [5:7577471278586547609:0]
2025-11-27T18:18:35.624612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471278586547614:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.624718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:35.624776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:35.625334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:35.625602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471279377076451:2109] [node 4] ICS07 socket disconnect 540 reason# EndOfStream
2025-11-27T18:18:35.625643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471279377076451:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.625723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:35.625770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471279377076451:2109] [node 4] ICS15 start handshake
2025-11-27T18:18:35.626743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471279377076425:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25615) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:35.626815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471279377076451:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:35.626863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471279377076425:2109] [node 4] ICP30 unregister session Session# [6:7577471279377076451:2109] VirtualId# [6:7577471279377076442:0]
2025-11-27T18:18:35.626927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:35.626963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471279377076425:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25615) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:35.626978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25615) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:35.637058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471279377076425:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:18:35.652193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:35.652455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471279377076463:2108] [node 5] ICS01 socket: 552 reason# EndOfStream
2025-11-27T18:18:35.652504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471279377076424:2108] [node 5] ICP30 unregister session Session# [6:7577471279377076463:2108] VirtualId# [6:7577471279377076458:0]
2025-11-27T18:18:35.652537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471279377076463:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:35.652624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:35.652676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:35.745929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471279671007445:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:35.745958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471279671007446:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:35.768849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471278320445483:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:35.768913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471278320445484:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:35.790514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471281696089899:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:35.790603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471281696089900:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:35.817659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471278555566038:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:35.817702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471278555566039:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:35.817895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471278555566037:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:35.841246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471281811138632:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:35.841313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471281811138633:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:35.841348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471281811138634:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:35.863650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471281686569545:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:35.863691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471281686569546:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:35.863716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471281686569547:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:35.927293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471281686569605:2104] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:35.932174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471281811138693:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-27T18:18:35.932893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471279671007508:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:36.040369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471281811138693:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.046747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471279671007445:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23589253ns) {Configure:104(23582080ns) {ProcessConfigured:135(17182100ns) {StartInitialHandshake:157(17179530ns) {DropHandshakes:470(1724ns) {DropIncomingHandshake:442(354ns) DropOutgoingHandshake:457(331ns)} GenerateSessionVirtualId:988(322ns) PrepareNewSessionHandshake:477(3442ns) {SwitchToState:179(1643ns)}}}}}
2025-11-27T18:18:36.048745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471285991057275:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:36.048938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471286106105992:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:36.061934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471285991057277:2117] [node 6] ICP01 ready to work
2025-11-27T18:18:36.065874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471285991057276:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:36.053767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471286106105993:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:36.065482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471283965974811:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.080286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471285991057284:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:313
2025-11-27T18:18:36.072200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471285981536909:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:36.081525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471282850533396:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:36.066531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471279671007508:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(132256586ns) {Configure:104(132251112ns) {ProcessConfigured:135(16220911ns) {StartInitialHandshake:157(16218235ns) {DropHandshakes:470(6973ns) {DropIncomingHandshake:442(970ns) DropOutgoingHandshake:457(517ns)} GenerateSessionVirtualId:988(1008ns) PrepareNewSessionHandshake:477(5664ns) {SwitchToState:179(2946ns)}}}}}
2025-11-27T18:18:36.087461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471278320445484:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(137001024ns) {Configure:104(136993485ns) {ProcessConfigured:135(78021ns) {StartInitialHandshake:157(76087ns) {DropHandshakes:470(2216ns) {DropIncomingHandshake:442(495ns) DropOutgoingHandshake:457(288ns)} GenerateSessionVirtualId:988(880ns) PrepareNewSessionHandshake:477(1776ns) {SwitchToState:179(580ns)}}}}}
2025-11-27T18:18:36.088540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471282850533397:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:36.090591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471278555566037:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(132776780ns) {Configure:104(132764613ns) {ProcessConfigured:135(80507ns) {StartInitialHandshake:157(77991ns) {DropHandshakes:470(2336ns) {DropIncomingHandshake:442(582ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(722ns) PrepareNewSessionHandshake:477(1548ns) {SwitchToState:179(338ns)}}}}}
2025-11-27T18:18:36.092605Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471285991057275:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27386186ns) {Configure:104(27382797ns) {ProcessConfigured:135(68987ns) {StartInitialHandshake:157(66868ns) {DropHandshakes:470(864ns) {DropIncomingHandshake:442(195ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:477(1213ns) {SwitchToState:179(202ns)}}}}}
2025-11-27T18:18:36.086553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471283965974816:2111] [node 6] ICP01 ready to work
2025-11-27T18:18:36.089703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471281811138633:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16691491ns) {Configure:104(16687557ns) {ProcessConfigured:135(14095212ns) {StartInitialHandshake:157(14091482ns) {DropHandshakes:470(6812067ns) {DropIncomingHandshake:442(1710ns) DropOutgoingHandshake:457(573ns)} GenerateSessionVirtualId:988(1520ns) PrepareNewSessionHandshake:477(2439ns) {SwitchToState:179(1296ns)}}}}}
2025-11-27T18:18:36.095420Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471281686569546:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(129846157ns) {Configure:104(129840299ns) {ProcessConfigured:135(3826031ns) {StartInitialHandshake:157(3823564ns) {DropHandshakes:470(1115ns) {DropIncomingHandshake:442(260ns) DropOutgoingHandshake:457(280ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:477(8148ns) {SwitchToState:179(1154ns)}}}}}
2025-11-27T18:18:36.097923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471278320445483:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(133287886ns) {Configure:104(133282422ns) {ProcessConfigured:135(9799746ns) {StartInitialHandshake:157(9796778ns) {DropHandshakes:470(1441ns) {DropIncomingHandshake:442(415ns) DropOutgoingHandshake:457(177ns)} GenerateSessionVirtualId:988(918ns) PrepareNewSessionHandshake:477(5022ns) {SwitchToState:179(2211ns)}}}}}
2025-11-27T18:18:36.103680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471281686569545:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16946081ns) {Configure:104(16936874ns) {ProcessConfigured:135(59687ns) {StartInitialHandshake:157(56854ns) {DropHandshakes:470(2653ns) {DropIncomingHandshake:442(744ns) DropOutgoingHandshake:457(333ns)} GenerateSessionVirtualId:988(771ns) PrepareNewSessionHandshake:477(1228ns) {SwitchToState:179(582ns)}}}}}
2025-11-27T18:18:36.104531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471282615412849:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:36.121470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471282615412850:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.123231Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471281686569547:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27596908ns) {Configure:104(27593086ns) {ProcessConfigured:135(41222ns) {StartInitialHandshake:157(39925ns) {DropHandshakes:470(759ns) {DropIncomingHandshake:442(201ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:477(614ns) {SwitchToState:179(138ns)}}}}}
2025-11-27T18:18:36.123875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471285981536916:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14900
2025-11-27T18:18:36.122044Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471278555566039:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29981275ns) {Configure:104(29976645ns) {ProcessConfigured:135(15539806ns) {StartInitialHandshake:157(15537064ns) {DropHandshakes:470(1659ns) {DropIncomingHandshake:442(534ns) DropOutgoingHandshake:457(263ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:477(5364ns) {SwitchToState:179(895ns)}}}}}
2025-11-27T18:18:36.124492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471283965974819:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49484
2025-11-27T18:18:36.124710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471281811138634:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(65319520ns) {Configure:104(65304833ns) {ProcessConfigured:135(78563ns) {StartInitialHandshake:157(77122ns) {DropHandshakes:470(1270ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(540ns) PrepareNewSessionHandshake:477(1031ns) {SwitchToState:179(310ns)}}}}}
2025-11-27T18:18:36.125473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471282615412851:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:36.127920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471285981536916:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.127964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471283965974816:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29280638ns) {Configure:104(29274672ns) {ProcessConfigured:135(2008281ns) {StartInitialHandshake:157(2006247ns) {DropHandshakes:470(2508ns) {DropIncomingHandshake:442(699ns) DropOutgoingHandshake:457(266ns)} GenerateSessionVirtualId:988(1038ns) PrepareNewSessionHandshake:477(2431ns) {SwitchToState:179(1067ns)}}}}}
2025-11-27T18:18:36.128585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471286106105993:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36627873ns) {Configure:104(36621841ns) {ProcessConfigured:135(1165539ns) {StartInitialHandshake:157(1163625ns) {DropHandshakes:470(1672ns) {DropIncomingHandshake:442(418ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(472ns) PrepareNewSessionHandshake:477(1913ns) {SwitchToState:179(689ns)}}}}}
2025-11-27T18:18:36.130991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471283965974819:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.133068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471281686569547:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.133290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471285981536909:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(26952574ns) {Configure:104(26945695ns) {ProcessConfigured:135(1411969ns) {StartInitialHandshake:157(1410285ns) {DropHandshakes:470(1547ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:477(5860ns) {SwitchToState:179(1307ns)}}}}}
2025-11-27T18:18:36.134580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471285991057283:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.134649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471285991057283:2129] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.138287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471283965974822:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49484
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.138825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471282850533406:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6490
2025-11-27T18:18:36.136102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471285991057292:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:313
2025-11-27T18:18:36.138987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471283965974823:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49484
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.136836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471285991057283:2129] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:36.139728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471283965974822:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.139845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471285981536922:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14900
2025-11-27T18:18:36.139983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471282850533406:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.141085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471285991057284:2130] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.140450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471283965974824:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49484
2025-11-27T18:18:36.142056Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471285981536916:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:36.140203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471278555566037:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.141158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471285991057277:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.142679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471281686569547:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:36.142732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
::ffff:127.1.0.0
2025-11-27T18:18:36.142420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471283965974812:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.141196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:36.142473Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471283965974812:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.142790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471281686569547:2050] [node 3] ICP22 created new session: [6:7577471285981536923:2050]
2025-11-27T18:18:36.156513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471282615412850:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20475069ns) {Configure:104(20468598ns) {ProcessConfigured:135(8553006ns) {StartInitialHandshake:157(8548966ns) {DropHandshakes:470(1249ns) {DropIncomingHandshake:442(465ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(508ns) PrepareNewSessionHandshake:477(2617ns) {SwitchToState:179(1563ns)}}}}}
2025-11-27T18:18:36.142863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471285981536923:2050] [node 3] ICS09 handshake done sender: [6:7577471285981536916:2115] self: [6:7577471285981536919:0] peer: [3:7577471285991057282:0] socket: 504 qp: -1
2025-11-27T18:18:36.141876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471285991057277:2117] [node 6] ICP22 created new session: [3:7577471285991057294:2117]
2025-11-27T18:18:36.154067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471283965974823:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.156465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471286106106008:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-27T18:18:36.142535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471285991057294:2117] [node 6] ICS09 handshake done sender: [3:7577471285991057283:2129] self: [3:7577471285991057282:0] peer: [6:7577471285981536919:0] socket: 500 qp: -1
2025-11-27T18:18:36.154471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471283965974812:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:36.143244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471285991057294:2117] [node 6] ICS10 traffic start
2025-11-27T18:18:36.143286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:36.145736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471285981536923:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:36.149415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:36.160555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471279671007507:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.151908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471281696089899:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.160601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471279671007507:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.161440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471279671007508:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.161486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.161536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471279671007508:2107] [node 4] ICP22 created new session: [1:7577471283965974829:2107]
2025-11-27T18:18:36.161592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471283965974829:2107] [node 4] ICS09 handshake done sender: [1:7577471283965974812:2113] self: [1:7577471283965974810:0] peer: [4:7577471282850533412:0] socket: 503 qp: -1
2025-11-27T18:18:36.164470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471285991057277:2117] [node 6] ICP35 event processing took too much time PendingConnection:279(17862001ns) {HandleHandshakeStatus:329(17860026ns) {DropIncomingHandshake:442(176ns) DropOutgoingHandshake:457(352ns) SwitchToState:179(229ns) ProcessPendingSessionEvents:495(2742216ns) {ForwardSessionEventToSession:612(621168ns) ForwardSessionEventToSession:612(224640ns) ForwardSessionEventToSession:612(330509ns) ForwardSessionEventToSession:612(160739ns) ForwardSessionEventToSession:612(4179ns) ForwardSessionEventToSession:612(2592ns) ForwardSessionEventToSession:612(237500ns) ForwardSessionEventToSession:612(4170ns) ForwardSessionEventToSession:612(2679ns) ForwardSessionEventToSession:612(1134813ns)}}}
2025-11-27T18:18:36.161633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471283965974829:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:36.164928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471286106106009:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-27T18:18:36.165098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471286106106008:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.161667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:36.165235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471283965974824:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.166355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471285991057292:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.166726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471279671007508:2107] [node 4] ICP18 handshake (actor [1:7577471283965974824:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.166822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471279671007507:2108] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:36.166816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471286106106009:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.166877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471282850533414:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6490
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.167023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471282615412851:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23675250ns) {Configure:104(23666176ns) {ProcessConfigured:135(386384ns) {StartInitialHandshake:157(382790ns) {DropHandshakes:470(2968ns) {DropIncomingHandshake:442(1017ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:477(1851ns) {SwitchToState:179(1014ns)}}}}}
2025-11-27T18:18:36.167245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471282850533413:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6490
2025-11-27T18:18:36.167067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471281811138632:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.169203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471279671007446:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.167138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471286106105993:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.167452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471282850533415:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6490
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.171538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:36.171250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471282850533404:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.175820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471285991057298:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:313
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.173118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471279671007446:2049] [node 3] ICP22 created new session: [1:7577471283965974833:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.176756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471286106106012:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-27T18:18:36.171288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471282850533404:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.177190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471285991057286:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.173212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471283965974815:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.172984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471282850533413:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.173239Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471283965974815:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.173239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471282850533406:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:36.175217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471283965974833:2049] [node 3] ICS09 handshake done sender: [1:7577471279671007507:2108] self: [1:7577471279671007506:0] peer: [3:7577471285991057297:0] socket: 495 qp: -1
2025-11-27T18:18:36.174439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471282850533414:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.175849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471283965974833:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:36.174602Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471278555566037:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471282850533406:2114] held: no
2025-11-27T18:18:36.177857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471282615412852:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6174
2025-11-27T18:18:36.176003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.178101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471286106106008:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:36.175394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471278555566039:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.177220Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471285991057286:2131] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.179059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471281696089899:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471285991057284:2130] held: no
2025-11-27T18:18:36.175691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471282850533404:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:36.180739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471286106105993:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.176591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471282850533396:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.180304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471285991057284:2130] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:36.179822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471282615412854:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6174
::ffff:127.1.0.0
2025-11-27T18:18:36.176618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471278555566037:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.176636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:36.191696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471283965974815:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:36.176697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471282850533396:2108] [node 5] ICP22 created new session: [4:7577471282850533420:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.191758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471281696089899:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.176719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:36.176757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471282850533420:2108] [node 5] ICS09 handshake done sender: [4:7577471282850533404:2112] self: [4:7577471282850533403:0] peer: [5:7577471286106106010:0] socket: 522 qp: -1
2025-11-27T18:18:36.176770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471278555566037:2048] [node 1] ICP22 created new session: [4:7577471282850533421:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.176796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471282850533420:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:36.176842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471282850533421:2048] [node 1] ICS09 handshake done sender: [4:7577471282850533406:2114] self: [4:7577471282850533412:0] peer: [1:7577471283965974810:0] socket: 507 qp: -1
2025-11-27T18:18:36.180468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471282615412853:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6174
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.196846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471283965974837:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49484
2025-11-27T18:18:36.191328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:36.192414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:36.191631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471286106105993:2109] [node 4] ICP22 created new session: [5:7577471286106106013:2109]
2025-11-27T18:18:36.189880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471282615412852:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.192453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471285981536926:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14900
2025-11-27T18:18:36.197858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471283965974811:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.177055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471282850533421:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:36.192488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471281696089899:2048] [node 1] ICP22 created new session: [3:7577471285991057302:2048]
2025-11-27T18:18:36.191523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471282615412862:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6174
2025-11-27T18:18:36.197642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471285981536926:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.193016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471286106106013:2109] [node 4] ICS09 handshake done sender: [5:7577471286106106008:2116] self: [5:7577471286106106010:0] peer: [4:7577471282850533403:0] socket: 523 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.199401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:36.177060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:36.191941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471282615412859:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6174
2025-11-27T18:18:36.192564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471285991057302:2048] [node 1] ICS09 handshake done sender: [3:7577471285991057284:2130] self: [3:7577471285991057297:0] peer: [1:7577471279671007506:0] socket: 496 qp: -1
2025-11-27T18:18:36.199769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471283965974811:2109] [node 5] ICP22 created new session: [1:7577471283965974838:2109]
2025-11-27T18:18:36.199374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471281686569545:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.198102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471286106106013:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:36.192628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471282615412854:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.177176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:36.192584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471285991057300:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:313
2025-11-27T18:18:36.202381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471283965974838:2109] [node 5] ICS09 handshake done sender: [1:7577471283965974815:2114] self: [1:7577471283965974814:0] peer: [5:7577471286106106011:0] socket: 533 qp: -1
2025-11-27T18:18:36.206269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471285981536928:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14900
2025-11-27T18:18:36.198164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:36.202282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471282615412853:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.207877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471285981536926:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:36.193089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471285991057286:2131] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.202427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471283965974838:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:36.206373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471282615412859:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.209868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471281686569545:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.193164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471285991057302:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:36.198674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471286106105993:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(18491773ns) {HandleHandshakeStatus:329(17684976ns) {DropIncomingHandshake:442(582ns) DropOutgoingHandshake:457(6413ns) SwitchToState:179(695ns) ProcessPendingSessionEvents:495(116751ns) {ForwardSessionEventToSession:612(55230ns) ForwardSessionEventToSession:612(3318ns) ForwardSessionEventToSession:612(25326ns) ForwardSessionEventToSession:612(2689ns) ForwardSessionEventToSession:612(2779ns) ForwardSessionEventToSession:612(4873ns) ForwardSessionEventToSession:612(3177ns) ForwardSessionEventToSession:612(2148ns) ForwardSessionEventToSession:612(7021ns) ForwardSessionEventToSession:612(2489ns)}}}
2025-11-27T18:18:36.202467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:36.210263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:36.206697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471278320445483:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.210444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471281686569545:2048] [node 1] ICP22 created new session: [6:7577471285981536929:2048]
2025-11-27T18:18:36.208503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471282615412862:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.193225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.210964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471286106106014:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-27T18:18:36.217896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471282850533424:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6490
2025-11-27T18:18:36.202900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471283965974821:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.211465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471285981536929:2048] [node 1] ICS09 handshake done sender: [6:7577471285981536926:2118] self: [6:7577471285981536927:0] peer: [1:7577471283965974820:0] socket: 542 qp: -1
2025-11-27T18:18:36.211501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471285981536929:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:36.202919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471283965974821:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.205018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471283965974821:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:36.209419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471283965974816:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.209947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:36.210007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471283965974816:2111] [node 6] ICP22 created new session: [1:7577471283965974841:2111]
2025-11-27T18:18:36.210151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471279671007513:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.210174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471279671007513:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.210274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471283965974841:2111] [node 6] ICS09 handshake done sender: [1:7577471283965974821:2116] self: [1:7577471283965974820:0] peer: [6:7577471285981536927:0] socket: 541 qp: -1
2025-11-27T18:18:36.210549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471283965974841:2111] [node 6] ICS10 traffic start
2025-11-27T18:18:36.211015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:36.214864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471281811138632:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471286106106009:2117] held: no
2025-11-27T18:18:36.194068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471285991057303:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:313
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.231055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471282850533425:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6490
2025-11-27T18:18:36.211532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:36.215237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471286106106016:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-27T18:18:36.215417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471286106106009:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:36.216207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471286106106016:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.216333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471281811138634:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.221968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471281811138632:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.201214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471285991057276:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.233943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471282850533425:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.235390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471278555566038:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.237482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471279671007513:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:36.201309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:36.235727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471282850533414:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.240659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471285981536932:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14900
2025-11-27T18:18:36.201353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471285991057276:2107] [node 4] ICP22 created new session: [3:7577471285991057306:2107]
2025-11-27T18:18:36.235710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471282615412848:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.233367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:36.243088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471283965974837:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.201389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471285991057306:2107] [node 4] ICS09 handshake done sender: [3:7577471285991057286:2131] self: [3:7577471285991057285:0] peer: [4:7577471282850533418:0] socket: 528 qp: -1
2025-11-27T18:18:36.243151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471279671007445:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.235758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471282615412848:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.201435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471285991057306:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:36.243189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:36.201468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:36.241916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471282615412856:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.241938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471282615412856:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.243236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471279671007445:2048] [node 2] ICP22 created new session: [1:7577471283965974844:2048]
2025-11-27T18:18:36.238047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471278555566039:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.218189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471285991057291:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.235721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471281811138632:2048] [node 1] ICP22 created new session: [5:7577471286106106019:2048]
2025-11-27T18:18:36.243285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471283965974844:2048] [node 2] ICS09 handshake done sender: [1:7577471279671007513:2112] self: [1:7577471279671007509:0] peer: [2:7577471282615412868:0] socket: 502 qp: -1
2025-11-27T18:18:36.244596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471282615412856:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:36.218214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471285991057291:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.238142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:36.238586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471286106106019:2048] [node 1] ICS09 handshake done sender: [5:7577471286106106009:2117] self: [5:7577471286106106011:0] peer: [1:7577471283965974814:0] socket: 534 qp: -1
2025-11-27T18:18:36.221959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471285991057303:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.245458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471285981536932:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.240292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471286106106019:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:36.244889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471282615412848:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:36.243307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471283965974844:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:36.238187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471278555566039:2050] [node 3] ICP22 created new session: [4:7577471282850533428:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.243325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:36.228874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471281696089900:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.240331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.238227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471282850533428:2050] [node 3] ICS09 handshake done sender: [4:7577471282850533414:2117] self: [4:7577471282850533418:0] peer: [3:7577471285991057285:0] socket: 531 qp: -1
2025-11-27T18:18:36.238261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471282850533428:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:36.246740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471285981536909:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.246350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471281811138632:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24019397ns) {HandleHandshakeStatus:329(24016933ns) {DropIncomingHandshake:442(1464ns) DropOutgoingHandshake:457(689ns) SwitchToState:179(467ns) ProcessPendingSessionEvents:495(909ns)}}
2025-11-27T18:18:36.238312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:36.243653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471279671007445:2048] [node 2] ICP18 handshake (actor [1:7577471283965974837:2120]) is too old
2025-11-27T18:18:36.231925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471285991057291:2132] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:36.247582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471286106106020:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-27T18:18:36.245483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471282850533430:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6490
2025-11-27T18:18:36.246458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471282850533425:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:36.238535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471285991057300:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.248445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471285981536931:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14900
2025-11-27T18:18:36.244139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471283965974816:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(23938182ns) {HandleHandshakeStatus:329(22433310ns) {DropIncomingHandshake:442(11839ns) DropOutgoingHandshake:457(486ns) SwitchToState:179(123ns) ProcessPendingSessionEvents:495(17854769ns) {ForwardSessionEventToSession:612(4304667ns) ForwardSessionEventToSession:612(482871ns) ForwardSessionEventToSession:612(3247ns) ForwardSessionEventToSession:612(3120573ns) ForwardSessionEventToSession:612(6662ns) ForwardSessionEventToSession:612(2928ns) ForwardSessionEventToSession:612(8835681ns) ForwardSessionEventToSession:612(8609ns) ForwardSessionEventToSession:612(3289ns) ForwardSessionEventToSession:612(630935ns)}}}
2025-11-27T18:18:36.249008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471281811138634:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14593) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471286106106016:2120] held: no
2025-11-27T18:18:36.238603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471285991057275:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.250353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471278320445484:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.247954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471278555566038:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.249186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471286106106016:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:36.252312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:36.238697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:36.253197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471285981536931:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.249780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471281811138634:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.248427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:36.252657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471278320445484:2049] [node 3] ICP22 created new session: [2:7577471282615412874:2049]
2025-11-27T18:18:36.254200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471281686569605:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.238744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471285991057275:2108] [node 5] ICP22 created new session: [3:7577471285991057310:2108]
2025-11-27T18:18:36.250064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471278555566038:2049] [node 2] ICP22 created new session: [4:7577471282850533431:2049]
2025-11-27T18:18:36.250770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:36.250747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471282850533431:2049] [node 2] ICS09 handshake done sender: [4:7577471282850533425:2120] self: [4:7577471282850533427:0] peer: [2:7577471282615412855:0] socket: 552 qp: -1
2025-11-27T18:18:36.238788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471285991057310:2108] [node 5] ICS09 handshake done sender: [3:7577471285991057291:2132] self: [3:7577471285991057290:0] peer: [5:7577471286106106018:0] socket: 549 qp: -1
2025-11-27T18:18:36.252846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471282615412874:2049] [node 3] ICS09 handshake done sender: [2:7577471282615412848:2114] self: [2:7577471282615412847:0] peer: [3:7577471285991057309:0] socket: 497 qp: -1
2025-11-27T18:18:36.250897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471281811138634:2050] [node 3] ICP22 created new session: [5:7577471286106106022:2050]
2025-11-27T18:18:36.251264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471282850533431:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:36.253205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471282615412874:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:36.238821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471285991057310:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:36.251984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:36.250951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471286106106022:2050] [node 3] ICS09 handshake done sender: [5:7577471286106106016:2120] self: [5:7577471286106106018:0] peer: [3:7577471285991057290:0] socket: 550 qp: -1
2025-11-27T18:18:36.254484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:36.238853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:36.255745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471282850533408:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.247156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471285991057276:2107] [node 4] ICP18 handshake (actor [3:7577471285991057300:2135]) is too old
2025-11-27T18:18:36.255769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471282850533408:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.250987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471286106106022:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:36.254490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471282615412849:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.251128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:36.257224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471282850533408:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:36.248243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471285991057312:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:313
2025-11-27T18:18:36.255457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:36.250760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471285991057303:2136] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.256575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471282615412849:2107] [node 4] ICP22 created new session: [2:7577471282615412876:2107]
2025-11-27T18:18:36.256309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471286106106006:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.257526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471282615412876:2107] [node 4] ICS09 handshake done sender: [2:7577471282615412856:2118] self: [2:7577471282615412855:0] peer: [4:7577471282850533427:0] socket: 551 qp: -1
2025-11-27T18:18:36.251045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471281696089900:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.257563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471282615412876:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:36.256337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471286106106006:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.251090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:36.252631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471281696089900:2049] [node 2] ICP22 created new session: [3:7577471285991057313:2049]
2025-11-27T18:18:36.253193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471285991057313:2049] [node 2] ICS09 handshake done sender: [3:7577471285991057303:2136] self: [3:7577471285991057309:0] peer: [2:7577471282615412847:0] socket: 547 qp: -1
2025-11-27T18:18:36.258090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471285981536935:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14900
2025-11-27T18:18:36.253232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471285991057313:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:36.258527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471286106106006:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:36.253354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:36.258183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:36.258601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471278320445483:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471282615412853:2116] held: no
2025-11-27T18:18:36.259024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471282850533397:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:36.259135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
2025-11-27T18:18:36.259132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471286106105992:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.259371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471282850533397:2109] [node 6] ICP22 created new session: [4:7577471282850533434:2109]
2025-11-27T18:18:36.259398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:36.259314Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471285981536932:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:36.259441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471286106105992:2110] [node 6] ICP22 created new session: [5:7577471286106106025:2110]
2025-11-27T18:18:36.259389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471285981536936:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14900
2025-11-27T18:18:36.259487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471286106106025:2110] [node 6] ICS09 handshake done sender: [5:7577471286106106006:2115] self: [5:7577471286106106005:0] peer: [6:7577471285981536934:0] socket: 512 qp: -1
2025-11-27T18:18:36.260112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471285981536909:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.260869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:36.260458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471282850533434:2109] [node 6] ICS09 handshake done sender: [4:7577471282850533408:2115] self: [4:7577471282850533407:0] peer: [6:7577471285981536933:0] socket: 555 qp: -1
2025-11-27T18:18:36.259690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471286106106025:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:36.260756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471282850533434:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:36.260317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.260899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:36.261118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471285981536909:2110] [node 4] ICP22 created new session: [6:7577471285981536937:2110]
2025-11-27T18:18:36.261987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471282615412873:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6174
2025-11-27T18:18:36.261549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471285981536937:2110] [node 4] ICS09 handshake done sender: [6:7577471285981536932:2121] self: [6:7577471285981536933:0] peer: [4:7577471282850533407:0] socket: 557 qp: -1
2025-11-27T18:18:36.261578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471285981536937:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:36.262022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:36.262835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471285981536909:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:36.263799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471282615412853:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:36.264281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471278320445483:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.264791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:36.264862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471286106106027:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-27T18:18:36.264923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471278320445483:2048] [node 1] ICP22 created new session: [2:7577471282615412880:2048]
2025-11-27T18:18:36.265147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471285981536931:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:36.265067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471282615412880:2048] [node 1] ICS09 handshake done sender: [2:7577471282615412853:2116] self: [2:7577471282615412868:0] peer: [1:7577471279671007509:0] socket: 510 qp: -1
2025-11-27T18:18:36.265123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471282615412880:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:36.265579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471281686569605:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23068) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471285981536931:2120] held: no
2025-11-27T18:18:36.265186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:36.265879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471281686569605:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.265918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:36.266004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471281686569605:2104] [node 5] ICP22 created new session: [6:7577471285981536939:2104]
2025-11-27T18:18:36.266082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471285981536939:2104] [node 5] ICS09 handshake done sender: [6:7577471285981536931:2120] self: [6:7577471285981536934:0] peer: [5:7577471286106106005:0] socket: 556 qp: -1
2025-11-27T18:18:36.266117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471285981536939:2104] [node 5] ICS10 traffic start
2025-11-27T18:18:36.266265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.267470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471285981536940:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14900
2025-11-27T18:18:36.267827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471286106106027:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.268004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471285981536940:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.268148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471281686569546:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.268036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471281811138633:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:36.268318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471282615412858:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.268349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471282615412858:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.268427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471282615412861:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:36.268456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471282615412861:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:36.269363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471282615412858:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.270576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471282615412850:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.270709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:36.270776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471282615412850:2108] [node 5] ICP22 created new session: [2:7577471282615412884:2108]
2025-11-27T18:18:36.270733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471286106106029:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-27T18:18:36.271115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471282615412884:2108] [node 5] ICS09 handshake done sender: [2:7577471282615412858:2119] self: [2:7577471282615412857:0] peer: [5:7577471286106106028:0] socket: 493 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:36.271033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471286106106027:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:36.271734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471285981536943:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14900
2025-11-27T18:18:36.271226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471282615412884:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:36.271549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471281811138633:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.271310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:36.271790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471282615412861:2121] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:36.272390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471282615412851:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:36.273745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471285981536940:2124] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:36.273872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:36.272455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:36.274145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471281811138633:2049] [node 2] ICP22 created new session: [5:7577471286106106030:2049]
2025-11-27T18:18:36.272512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471282615412851:2110] [node 6] ICP22 created new session: [2:7577471282615412886:2110]
2025-11-27T18:18:36.274603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471286106106030:2049] [node 2] ICS09 handshake done sender: [5:7577471286106106027:2122] self: [5:7577471286106106028:0] peer: [2:7577471282615412857:0] socket: 508 qp: -1
2025-11-27T18:18:36.276110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471281686569546:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:36.277031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:36.277084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471281686569546:2049] [node 2] ICP22 created new session: [6:7577471285981536944:2049]
2025-11-27T18:18:36.277144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471285981536944:2049] [node 2] ICS09 handshake done sender: [6:7577471285981536940:2124] self: [6:7577471285981536942:0] peer: [2:7577471282615412860:0] socket: 492 qp: -1
2025-11-27T18:18:36.277181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471285981536944:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:36.277233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:36.277483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471281686569546:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:36.272559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471282615412886:2110] [node 6] ICS09 handshake done sender: [2:7577471282615412861:2121] self: [2:7577471282615412860:0] peer: [6:7577471285981536942:0] socket: 525 qp: -1
2025-11-27T18:18:36.272588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471282615412886:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:36.272614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:36.274628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471286106106030:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:36.276092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:36.279746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471281811138633:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:36.316035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:36.316254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471282615412880:2048] [node 1] ICS01 socket: 510 reason# EndOfStream
2025-11-27T18:18:36.316296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471278320445483:2048] [node 1] ICP30 unregister session Session# [2:7577471282615412880:2048] VirtualId# [2:7577471282615412868:0]
2025-11-27T18:18:36.316330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471282615412880:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.316404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:36.316450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:36.318103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:36.318347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471285991057302:2048] [node 1] ICS01 socket: 496 reason# EndOfStream
2025-11-27T18:18:36.318381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471281696089899:2048] [node 1] ICP30 unregister session Session# [3:7577471285991057302:2048] VirtualId# [3:7577471285991057297:0]
2025-11-27T18:18:36.318410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471285991057302:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.318484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:36.318530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:36.320773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:36.321029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471286106106019:2048] [node 1] ICS01 socket: 534 reason# EndOfStream
2025-11-27T18:18:36.321063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471281811138632:2048] [node 1] ICP30 unregister session Session# [5:7577471286106106019:2048] VirtualId# [5:7577471286106106011:0]
2025-11-27T18:18:36.321092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471286106106019:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.321153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:36.321186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:36.323524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:36.323744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471285981536929:2048] [node 1] ICS01 socket: 542 reason# EndOfStream
2025-11-27T18:18:36.323778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471281686569545:2048] [node 1] ICP30 unregister session Session# [6:7577471285981536929:2048] VirtualId# [6:7577471285981536927:0]
2025-11-27T18:18:36.323804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471285981536929:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.323876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:36.323914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:36.333440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:36.333770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471282850533421:2048] [node 1] ICS01 socket: 507 reason# EndOfStream
2025-11-27T18:18:36.333824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471278555566037:2048] [node 1] ICP30 unregister session Session# [4:7577471282850533421:2048] VirtualId# [4:7577471282850533412:0]
2025-11-27T18:18:36.333878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471282850533421:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.333991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:36.334064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:36.404377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:36.404635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471285991057313:2049] [node 2] ICS01 socket: 547 reason# EndOfStream
2025-11-27T18:18:36.404701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471281696089900:2049] [node 2] ICP30 unregister session Session# [3:7577471285991057313:2049] VirtualId# [3:7577471285991057309:0]
2025-11-27T18:18:36.404738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471285991057313:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.404837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:36.404885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:36.407222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:36.407458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471282850533431:2049] [node 2] ICS01 socket: 552 reason# EndOfStream
2025-11-27T18:18:36.407501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471278555566038:2049] [node 2] ICP30 unregister session Session# [4:7577471282850533431:2049] VirtualId# [4:7577471282850533427:0]
2025-11-27T18:18:36.407532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471282850533431:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.407594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:36.407627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:36.408343Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:36.408599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471286106106030:2049] [node 2] ICS07 socket disconnect 508 reason# EndOfStream
2025-11-27T18:18:36.408632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471286106106030:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.408710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:36.408766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471286106106030:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:36.409299Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:36.409508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471285981536944:2049] [node 2] ICS01 socket: 492 reason# EndOfStream
2025-11-27T18:18:36.409567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471281686569546:2049] [node 2] ICP30 unregister session Session# [6:7577471285981536944:2049] VirtualId# [6:7577471285981536942:0]
2025-11-27T18:18:36.409610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471285981536944:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.409686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:36.409742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:36.409783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471281811138633:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7704) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:36.409862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471286106106030:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:36.409949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471281811138633:2049] [node 2] ICP30 unregister session Session# [5:7577471286106106030:2049] VirtualId# [5:7577471286106106028:0]
2025-11-27T18:18:36.409994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:36.410036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471281811138633:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7704) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:36.410050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7704) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:36.419390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471281811138633:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:36.471519Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:36.471780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471282850533428:2050] [node 3] ICS01 socket: 531 reason# EndOfStream
2025-11-27T18:18:36.471829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471278555566039:2050] [node 3] ICP30 unregister session Session# [4:7577471282850533428:2050] VirtualId# [4:7577471282850533418:0]
2025-11-27T18:18:36.471872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471282850533428:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.472014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:36.472089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:36.473550Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:36.473801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471285981536923:2050] [node 3] ICS01 socket: 504 reason# EndOfStream
2025-11-27T18:18:36.473844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471281686569547:2050] [node 3] ICP30 unregister session Session# [6:7577471285981536923:2050] VirtualId# [6:7577471285981536919:0]
2025-11-27T18:18:36.473889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471285981536923:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.473980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:36.474029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:36.475713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:36.476017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471286106106022:2050] [node 3] ICS01 socket: 550 reason# EndOfStream
2025-11-27T18:18:36.476058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471281811138634:2050] [node 3] ICP30 unregister session Session# [5:7577471286106106022:2050] VirtualId# [5:7577471286106106018:0]
2025-11-27T18:18:36.476088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471286106106022:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.476167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:36.476211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:36.501143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:36.501453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471286106106013:2109] [node 4] ICS01 socket: 523 reason# EndOfStream
2025-11-27T18:18:36.501495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471286106105993:2109] [node 4] ICP30 unregister session Session# [5:7577471286106106013:2109] VirtualId# [5:7577471286106106010:0]
2025-11-27T18:18:36.501530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471286106106013:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.501632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:36.501687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:36.503348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:36.503589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471285981536937:2110] [node 4] ICS07 socket disconnect 557 reason# EndOfStream
2025-11-27T18:18:36.503633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471285981536937:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.503709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:36.503763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471285981536937:2110] [node 4] ICS15 start handshake
2025-11-27T18:18:36.509425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471285981536909:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23065) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:36.509508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471285981536937:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:36.509737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471285981536909:2110] [node 4] ICP30 unregister session Session# [6:7577471285981536937:2110] VirtualId# [6:7577471285981536933:0]
2025-11-27T18:18:36.509806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:36.509843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471285981536909:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:36.509857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:36.519765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471285981536909:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:18:36.563276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:36.563533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471285981536939:2104] [node 5] ICS01 socket: 556 reason# EndOfStream
2025-11-27T18:18:36.563580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471281686569605:2104] [node 5] ICP30 unregister session Session# [6:7577471285981536939:2104] VirtualId# [6:7577471285981536934:0]
2025-11-27T18:18:36.563611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471285981536939:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:36.563696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:36.563754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:36.661914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471284253447006:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:36.661953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471284253447005:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:36.684077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471284151977960:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:36.684121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471284151977961:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:36.705511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471286280869334:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:36.705716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471286280869335:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:36.727887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471284678414832:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:36.727939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471284678414833:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:36.728047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471284678414831:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:36.751044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471286816141259:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:36.751045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471286816141260:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:36.751085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471286816141261:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:36.774033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471283698497548:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:36.774066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471283698497549:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:36.774182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471283698497547:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:36.862903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471284253447069:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:36.863250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471284253447070:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:36.946288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471283698497606:2104] [node 5] ICP01 ready to work
2025-11-27T18:18:36.961764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471284151978034:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:36.962292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471286816141322:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:36.968662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471286280869397:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:36.972458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471283698497609:2105] [node 4] ICP01 ready to work
2025-11-27T18:18:36.972098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471284678414895:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:36.973420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471286280869399:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:36.973052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471284678414896:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:36.978068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471284151978035:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:36.977560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471286816141323:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:36.974608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471286280869398:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:36.998824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471283698497548:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41617430ns) {Configure:104(39255397ns) {ProcessConfigured:135(76863ns) {StartInitialHandshake:157(74504ns) {DropHandshakes:470(3148ns) {DropIncomingHandshake:442(682ns) DropOutgoingHandshake:457(488ns)} GenerateSessionVirtualId:988(851ns) PrepareNewSessionHandshake:477(2162ns) {SwitchToState:179(725ns)}}}}}
2025-11-27T18:18:37.027315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471284678414831:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(138557253ns) {Configure:104(138551722ns) {ProcessConfigured:135(65841ns) {StartInitialHandshake:157(64237ns) {DropHandshakes:470(2062ns) {DropIncomingHandshake:442(530ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(624ns) PrepareNewSessionHandshake:477(1094ns) {SwitchToState:179(456ns)}}}}}
2025-11-27T18:18:36.993825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471284253447069:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(116346723ns) {Configure:104(116341866ns) {ProcessConfigured:135(8701451ns) {StartInitialHandshake:157(8699616ns) {DropHandshakes:470(2875ns) {DropIncomingHandshake:442(700ns) DropOutgoingHandshake:457(444ns)} GenerateSessionVirtualId:988(1614ns) PrepareNewSessionHandshake:477(3617ns) {SwitchToState:179(2085ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.014078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471288446945332:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:37.033723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471286816141259:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(87282272ns) {Configure:104(87275358ns) {ProcessConfigured:135(73095ns) {StartInitialHandshake:157(70909ns) {DropHandshakes:470(1964ns) {DropIncomingHandshake:442(416ns) DropOutgoingHandshake:457(468ns)} GenerateSessionVirtualId:988(691ns) PrepareNewSessionHandshake:477(1558ns) {SwitchToState:179(543ns)}}}}}
2025-11-27T18:18:37.034222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471284253447006:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(176267226ns) {Configure:104(176261789ns) {ProcessConfigured:135(19792847ns) {StartInitialHandshake:157(19791062ns) {DropHandshakes:470(2944ns) {DropIncomingHandshake:442(656ns) DropOutgoingHandshake:457(507ns)} GenerateSessionVirtualId:988(1134ns) PrepareNewSessionHandshake:477(5486ns) {SwitchToState:179(117ns)}}}}}
2025-11-27T18:18:37.018683Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471284151977961:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(148653354ns) {Configure:104(148641783ns) {ProcessConfigured:135(84482ns) {StartInitialHandshake:157(82431ns) {DropHandshakes:470(2859ns) {DropIncomingHandshake:442(644ns) DropOutgoingHandshake:457(574ns)} GenerateSessionVirtualId:988(894ns) PrepareNewSessionHandshake:477(2259ns) {SwitchToState:179(773ns)}}}}}
2025-11-27T18:18:37.041460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471288973382199:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:848
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.061722Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471286280869399:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(84581413ns) {Configure:104(84577611ns) {ProcessConfigured:135(19179838ns) {StartInitialHandshake:157(12930065ns) {DropHandshakes:470(3186ns) {DropIncomingHandshake:442(204ns) DropOutgoingHandshake:457(451ns)} GenerateSessionVirtualId:988(1426ns) PrepareNewSessionHandshake:477(5432ns) {SwitchToState:179(2763ns)}}}}}
2025-11-27T18:18:37.049685Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471283698497609:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25238884ns) {Configure:104(25232304ns) {ProcessConfigured:135(24051574ns) {StartInitialHandshake:157(24050011ns) {DropHandshakes:470(1015ns) {DropIncomingHandshake:442(219ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(630ns) PrepareNewSessionHandshake:477(4085ns) {SwitchToState:179(842ns)}}}}}
2025-11-27T18:18:37.064221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471290575836702:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27209
2025-11-27T18:18:37.046467Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471286816141260:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(74668610ns) {Configure:104(74662908ns) {ProcessConfigured:135(68283ns) {StartInitialHandshake:157(65709ns) {DropHandshakes:470(3816ns) {DropIncomingHandshake:442(796ns) DropOutgoingHandshake:457(433ns)} GenerateSessionVirtualId:988(678ns) PrepareNewSessionHandshake:477(2127ns) {SwitchToState:179(1051ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.034258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471288548414379:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:37.069827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471284151977960:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(184545959ns) {Configure:104(184541606ns) {ProcessConfigured:135(23410561ns) {StartInitialHandshake:157(23407141ns) {DropHandshakes:470(9588977ns) {DropIncomingHandshake:442(13945ns) DropOutgoingHandshake:457(39ns)} GenerateSessionVirtualId:988(1500ns) PrepareNewSessionHandshake:477(5140ns) {SwitchToState:179(2490ns)}}}}}
2025-11-27T18:18:37.071908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471283698497606:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(109906383ns) {Configure:104(109898449ns) {ProcessConfigured:135(64381ns) {StartInitialHandshake:157(61949ns) {DropHandshakes:470(2587ns) {DropIncomingHandshake:442(571ns) DropOutgoingHandshake:457(649ns)} GenerateSessionVirtualId:988(592ns) PrepareNewSessionHandshake:477(2048ns) {SwitchToState:179(767ns)}}}}}
2025-11-27T18:18:37.074260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471287993464917:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39156
2025-11-27T18:18:37.101834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471286280869335:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(126932337ns) {Configure:104(126914002ns) {ProcessConfigured:135(74533ns) {StartInitialHandshake:157(72345ns) {DropHandshakes:470(3315ns) {DropIncomingHandshake:442(1016ns) DropOutgoingHandshake:457(502ns)} GenerateSessionVirtualId:988(1195ns) PrepareNewSessionHandshake:477(2194ns) {SwitchToState:179(640ns)}}}}}
2025-11-27T18:18:37.104828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471290575836702:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.084785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471284253447070:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(166959715ns) {Configure:104(166954161ns) {ProcessConfigured:135(19078392ns) {StartInitialHandshake:157(19076206ns) {DropHandshakes:470(3289ns) {DropIncomingHandshake:442(563ns) DropOutgoingHandshake:457(399ns)} GenerateSessionVirtualId:988(757ns) PrepareNewSessionHandshake:477(4113ns) {SwitchToState:179(2148ns)}}}}}
2025-11-27T18:18:37.086965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471284151978035:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49209725ns) {Configure:104(49203122ns) {ProcessConfigured:135(2138033ns) {StartInitialHandshake:157(2135732ns) {DropHandshakes:470(3095ns) {DropIncomingHandshake:442(737ns) DropOutgoingHandshake:457(498ns)} GenerateSessionVirtualId:988(672ns) PrepareNewSessionHandshake:477(4945ns) {SwitchToState:179(2979ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.118342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471290575836708:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27209
2025-11-27T18:18:37.133770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471284678414833:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(156017322ns) {Configure:104(156005727ns) {ProcessConfigured:135(14570393ns) {StartInitialHandshake:157(14568053ns) {DropHandshakes:470(2331ns) {DropIncomingHandshake:442(440ns) DropOutgoingHandshake:457(418ns)} GenerateSessionVirtualId:988(501ns) PrepareNewSessionHandshake:477(6222ns) {SwitchToState:179(2937ns)}}}}}
2025-11-27T18:18:37.074441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471287993464918:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39156
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.134012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471288973382199:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.074498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471287993464919:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39156
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.076964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471283698497547:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(104236545ns) {Configure:104(104232066ns) {ProcessConfigured:135(6766501ns) {StartInitialHandshake:157(6763962ns) {DropHandshakes:470(2675ns) {DropIncomingHandshake:442(914ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(560ns) PrepareNewSessionHandshake:477(2899ns) {SwitchToState:179(917ns)}}}}}
2025-11-27T18:18:37.138427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471284678414832:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(135081847ns) {Configure:104(135075129ns) {ProcessConfigured:135(22863337ns) {StartInitialHandshake:157(22850264ns) {DropHandshakes:470(2900ns) {DropIncomingHandshake:442(560ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(644ns) PrepareNewSessionHandshake:477(4084ns) {SwitchToState:179(2192ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.141995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471291111108633:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13681
2025-11-27T18:18:37.147953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471284253447005:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(107672058ns) {Configure:104(107666600ns) {ProcessConfigured:135(3880994ns) {StartInitialHandshake:157(3878593ns) {DropHandshakes:470(3231ns) {DropIncomingHandshake:442(772ns) DropOutgoingHandshake:457(348ns)} GenerateSessionVirtualId:988(717ns) PrepareNewSessionHandshake:477(2456ns) {SwitchToState:179(1242ns)}}}}}
2025-11-27T18:18:37.147938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471290575836709:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27209
2025-11-27T18:18:37.121912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471287993464917:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.147953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471286280869397:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(159990762ns) {Configure:104(159985624ns) {ProcessConfigured:135(28634903ns) {StartInitialHandshake:157(28632149ns) {DropHandshakes:470(5374545ns) {DropIncomingHandshake:442(1292ns) DropOutgoingHandshake:457(3186ns)} GenerateSessionVirtualId:988(2786ns) PrepareNewSessionHandshake:477(2982ns) {SwitchToState:179(1634ns)}}}}}
2025-11-27T18:18:37.143863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471291111108634:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13681
2025-11-27T18:18:37.149830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471288973382207:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:848
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.122075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471287993464919:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.151289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471288548414385:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26682
2025-11-27T18:18:37.150475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471290575836701:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.144023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471291111108635:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13681
2025-11-27T18:18:37.152825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471288548414386:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26682
2025-11-27T18:18:37.137749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471283698497549:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.144187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471286816141329:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.150514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471290575836701:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.142014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471283698497606:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.156893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471288973382207:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.144210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471286816141329:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.158647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471288548414385:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.152896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471290575836708:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.161318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471288973382196:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.145514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471287993464926:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39156
2025-11-27T18:18:37.161358Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471288973382196:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.156144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471290575836709:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.145055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471286816141329:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:37.146025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471287993464917:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:37.150244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471291111108635:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.159972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471288446945340:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.163922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471288973382198:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.146228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471283698497606:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.159695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471288548414386:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.151992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471291111108634:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.157483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471290575836699:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.163946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471288973382198:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.152642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471286816141261:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.157515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471290575836699:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.160021Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471288446945340:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.146276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:37.159541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471286280869335:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.154253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471291111108636:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13681
2025-11-27T18:18:37.162316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471290575836701:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:37.154792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471291111108633:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:37.168083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471288548414383:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26682
2025-11-27T18:18:37.163832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471288446945340:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:37.146336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471283698497606:2104] [node 5] ICP22 created new session: [6:7577471287993464927:2104]
2025-11-27T18:18:37.164566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471288973382198:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:37.168080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471290575836716:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27209
2025-11-27T18:18:37.154861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471286816141323:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.154916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:37.169987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471290575836699:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:37.166190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471288446945334:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.146379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471287993464927:2104] [node 5] ICS09 handshake done sender: [6:7577471287993464917:2114] self: [6:7577471287993464925:0] peer: [5:7577471286816141328:0] socket: 514 qp: -1
2025-11-27T18:18:37.168702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471288548414379:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(57388179ns) {Configure:104(57380069ns) {ProcessConfigured:135(4308587ns) {StartInitialHandshake:157(4305197ns) {DropHandshakes:470(2706ns) {DropIncomingHandshake:442(416ns) DropOutgoingHandshake:457(440ns)} GenerateSessionVirtualId:988(519ns) PrepareNewSessionHandshake:477(6847ns) {SwitchToState:179(980ns)}}}}}
2025-11-27T18:18:37.165059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471288973382196:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.174410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471288548414393:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26682
2025-11-27T18:18:37.154976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471286816141323:2109] [node 6] ICP22 created new session: [5:7577471291111108640:2109]
2025-11-27T18:18:37.170511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471286280869398:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.167068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471284678414896:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.166222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471288446945334:2121] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.146417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471287993464927:2104] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.155020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471291111108640:2109] [node 6] ICS09 handshake done sender: [5:7577471286816141329:2112] self: [5:7577471286816141328:0] peer: [6:7577471287993464925:0] socket: 500 qp: -1
2025-11-27T18:18:37.171188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.168023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:37.146452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:37.167978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471288446945334:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:37.155059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471291111108640:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:37.171822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471286280869398:2109] [node 6] ICP22 created new session: [3:7577471290575836718:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.181240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471288548414383:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.168314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471284678414896:2108] [node 5] ICP22 created new session: [4:7577471288973382213:2108]
2025-11-27T18:18:37.155092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:37.168152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471284151978035:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.169867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:37.150055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471287993464918:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.169559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471288973382213:2108] [node 5] ICS09 handshake done sender: [4:7577471288973382198:2113] self: [4:7577471288973382197:0] peer: [5:7577471291111108644:0] socket: 505 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.158293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471291111108636:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.172570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471290575836718:2109] [node 6] ICS09 handshake done sender: [3:7577471290575836701:2112] self: [3:7577471290575836700:0] peer: [6:7577471287993464924:0] socket: 510 qp: -1
2025-11-27T18:18:37.181239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471288548414392:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26682
2025-11-27T18:18:37.169589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471284678414895:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.170055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471284151978035:2109] [node 5] ICP22 created new session: [2:7577471288446945349:2109]
2025-11-27T18:18:37.154551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471283698497609:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.183314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471288548414393:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.173907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471286280869399:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.170528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471287993464932:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39156
2025-11-27T18:18:37.175111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471287993464931:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39156
2025-11-27T18:18:37.181319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471283698497549:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18794) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471287993464919:2116] held: no
2025-11-27T18:18:37.186619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471287993464918:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:37.169605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471288973382213:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:37.158390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471286816141260:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.170300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:37.170853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:37.171945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471284678414895:2109] [node 6] ICP22 created new session: [4:7577471288973382215:2109]
2025-11-27T18:18:37.171992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471288973382215:2109] [node 6] ICS09 handshake done sender: [4:7577471288973382196:2112] self: [4:7577471288973382195:0] peer: [6:7577471287993464929:0] socket: 504 qp: -1
2025-11-27T18:18:37.172043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471288973382215:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:37.172097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:37.175571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471288973382209:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:848
2025-11-27T18:18:37.187678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471288973382218:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:848
2025-11-27T18:18:37.174053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471290575836718:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:37.171212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471284151977961:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.162523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471286816141322:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.197524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471288548414392:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.196541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471287993464919:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:37.174777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:37.174753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471291111108645:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13681
2025-11-27T18:18:37.196542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471288973382209:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.173335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471288446945349:2109] [node 5] ICS09 handshake done sender: [2:7577471288446945340:2124] self: [2:7577471288446945339:0] peer: [5:7577471291111108642:0] socket: 523 qp: -1
2025-11-27T18:18:37.175324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471286280869399:2108] [node 5] ICP22 created new session: [3:7577471290575836720:2108]
2025-11-27T18:18:37.201144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471283698497609:2105] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20483) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471287993464918:2115] held: no
2025-11-27T18:18:37.173374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471288446945349:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:37.200842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471284678414831:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.205155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471284253447080:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.175379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471286816141322:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20483) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471291111108633:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.203625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471283698497609:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.180240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471291111108647:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13681
2025-11-27T18:18:37.176033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:37.205196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471284253447080:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.205346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471288973382218:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.175362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:37.206630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471284678414832:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.206015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471287993464933:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39156
2025-11-27T18:18:37.178899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471290575836720:2108] [node 5] ICS09 handshake done sender: [3:7577471290575836699:2111] self: [3:7577471290575836698:0] peer: [5:7577471291111108639:0] socket: 509 qp: -1
2025-11-27T18:18:37.176089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:37.187121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471291111108646:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13681
2025-11-27T18:18:37.213301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471284253447080:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.229862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471288973382224:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:848
2025-11-27T18:18:37.189773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471291111108648:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13681
2025-11-27T18:18:37.177224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471284151977961:2049] [node 3] ICP22 created new session: [2:7577471288446945352:2049]
2025-11-27T18:18:37.178945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471290575836720:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:37.207842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471283698497549:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.179504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471288446945352:2049] [node 3] ICS09 handshake done sender: [2:7577471288446945334:2121] self: [2:7577471288446945333:0] peer: [3:7577471290575836715:0] socket: 521 qp: -1
2025-11-27T18:18:37.232650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471288548414378:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.178979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:37.194027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471291111108633:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.210241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.180168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471288446945352:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:37.232679Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471288548414378:2120] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.237353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471288973382222:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:848
2025-11-27T18:18:37.210339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.197081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471286816141261:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18794) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471291111108634:2116] held: no
2025-11-27T18:18:37.233937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471284253447069:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.189566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471290575836719:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27209
2025-11-27T18:18:37.184138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:37.239539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471288973382223:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:848
2025-11-27T18:18:37.210540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471283698497609:2105] [node 4] ICP22 created new session: [6:7577471287993464935:2105]
2025-11-27T18:18:37.196590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471286280869398:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16865353ns) {HandleHandshakeStatus:329(16861814ns) {DropIncomingHandshake:442(14188ns) DropOutgoingHandshake:457(244ns) SwitchToState:179(447ns) ProcessPendingSessionEvents:495(5529978ns) {ForwardSessionEventToSession:612(2409115ns) ForwardSessionEventToSession:612(5869ns) ForwardSessionEventToSession:612(2333296ns) ForwardSessionEventToSession:612(6771ns) ForwardSessionEventToSession:612(3171ns) ForwardSessionEventToSession:612(724417ns) ForwardSessionEventToSession:612(6566ns) ForwardSessionEventToSession:612(8007ns) ForwardSessionEventToSession:612(2882ns) ForwardSessionEventToSession:612(6078ns)}}}
2025-11-27T18:18:37.202919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471288446945354:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
2025-11-27T18:18:37.197199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471286816141322:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.236393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471288548414378:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:37.239975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471288973382222:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.210597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471287993464935:2105] [node 4] ICS09 handshake done sender: [6:7577471287993464918:2115] self: [6:7577471287993464929:0] peer: [4:7577471288973382195:0] socket: 515 qp: -1
2025-11-27T18:18:37.210673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471283698497549:2050] [node 3] ICP22 created new session: [6:7577471287993464936:2050]
2025-11-27T18:18:37.212474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471287993464936:2050] [node 3] ICS09 handshake done sender: [6:7577471287993464919:2116] self: [6:7577471287993464924:0] peer: [3:7577471290575836700:0] socket: 516 qp: -1
2025-11-27T18:18:37.212531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471287993464935:2105] [node 4] ICS10 traffic start
2025-11-27T18:18:37.213464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471287993464936:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:37.213503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:37.213658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:37.214939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471287993464933:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.197234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:37.239799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:37.243368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471284678414833:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.205518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471288446945355:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
2025-11-27T18:18:37.230979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471283698497548:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.197300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471286816141322:2108] [node 4] ICP22 created new session: [5:7577471291111108650:2108]
2025-11-27T18:18:37.205176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471286280869399:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(21012548ns) {HandleHandshakeStatus:329(21010421ns) {DropIncomingHandshake:442(11601ns) DropOutgoingHandshake:457(365ns) SwitchToState:179(236ns) ProcessPendingSessionEvents:495(9841889ns) {ForwardSessionEventToSession:612(2393799ns) ForwardSessionEventToSession:612(2528354ns) ForwardSessionEventToSession:612(1073653ns) ForwardSessionEventToSession:612(4421ns) ForwardSessionEventToSession:612(3069ns) ForwardSessionEventToSession:612(2209187ns) ForwardSessionEventToSession:612(4885ns) ForwardSessionEventToSession:612(3315ns) ForwardSessionEventToSession:612(670757ns) ForwardSessionEventToSession:612(929295ns)}}}
2025-11-27T18:18:37.244313Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471288973382209:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:37.240755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471284253447069:2107] [node 4] ICP22 created new session: [1:7577471288548414401:2107]
2025-11-27T18:18:37.197346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471291111108650:2108] [node 4] ICS09 handshake done sender: [5:7577471291111108633:2115] self: [5:7577471291111108644:0] peer: [4:7577471288973382197:0] socket: 506 qp: -1
2025-11-27T18:18:37.205608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471288446945356:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
2025-11-27T18:18:37.206097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471290575836709:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:37.241842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471288548414401:2107] [node 4] ICS09 handshake done sender: [1:7577471284253447080:2119] self: [1:7577471284253447079:0] peer: [4:7577471288973382220:0] socket: 519 qp: -1
2025-11-27T18:18:37.244890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471288973382218:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:37.205746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471288446945354:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.197383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471291111108650:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.270658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471287993464940:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39156
2025-11-27T18:18:37.271007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471287993464933:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:37.208539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471290575836716:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.206014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471284151978035:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(22319004ns) {HandleHandshakeStatus:329(22316289ns) {DropIncomingHandshake:442(915ns) DropOutgoingHandshake:457(862ns) SwitchToState:179(460ns) ProcessPendingSessionEvents:495(6995820ns) {ForwardSessionEventToSession:612(1708571ns) ForwardSessionEventToSession:612(1537506ns) ForwardSessionEventToSession:612(419131ns) ForwardSessionEventToSession:612(11675ns) ForwardSessionEventToSession:612(314560ns) ForwardSessionEventToSession:612(3253ns) ForwardSessionEventToSession:612(3247ns) ForwardSessionEventToSession:612(502375ns) ForwardSessionEventToSession:612(3065ns) ForwardSessionEventToSession:612(2470707ns)}}}
2025-11-27T18:18:37.197425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:37.253107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471284678414831:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.243033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471288548414401:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:37.271389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471283698497548:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.260566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471284678414832:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.202005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471291111108636:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:37.215319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471290575836724:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27209
2025-11-27T18:18:37.260643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:37.244559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:37.215563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471286280869335:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.215610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:37.215665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471286280869335:2049] [node 2] ICP22 created new session: [3:7577471290575836726:2049]
2025-11-27T18:18:37.215710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471290575836726:2049] [node 2] ICS09 handshake done sender: [3:7577471290575836709:2116] self: [3:7577471290575836715:0] peer: [2:7577471288446945333:0] socket: 527 qp: -1
2025-11-27T18:18:37.215743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471290575836726:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:37.215777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:37.222654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471286280869334:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.231950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471290575836724:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.239704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471290575836730:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27209
2025-11-27T18:18:37.241628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471290575836716:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:37.247081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471286280869334:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.247144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:37.247851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471286280869334:2048] [node 1] ICP22 created new session: [3:7577471290575836731:2048]
2025-11-27T18:18:37.248439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471290575836712:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.248466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471290575836712:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.251081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471290575836731:2048] [node 1] ICS09 handshake done sender: [3:7577471290575836716:2118] self: [3:7577471290575836728:0] peer: [1:7577471288548414377:0] socket: 524 qp: -1
2025-11-27T18:18:37.251136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471290575836731:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:37.251184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:37.252399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471290575836712:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:37.260494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471286280869397:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.260587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:37.261101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471286280869397:2107] [node 4] ICP22 created new session: [3:7577471290575836734:2107]
2025-11-27T18:18:37.265481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471290575836734:2107] [node 4] ICS09 handshake done sender: [3:7577471290575836712:2117] self: [3:7577471290575836710:0] peer: [4:7577471288973382225:0] socket: 560 qp: -1
2025-11-27T18:18:37.268066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471290575836734:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:37.270108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:37.210555Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471284151977961:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(26271557ns) {HandleHandshakeStatus:329(26268889ns) {DropIncomingHandshake:442(1257ns) DropOutgoingHandshake:457(632ns) SwitchToState:179(471ns) ProcessPendingSessionEvents:495(6598509ns) {ForwardSessionEventToSession:612(4961697ns) ForwardSessionEventToSession:612(5900ns) ForwardSessionEventToSession:612(373923ns) ForwardSessionEventToSession:612(3368ns) ForwardSessionEventToSession:612(345837ns) ForwardSessionEventToSession:612(7249ns) ForwardSessionEventToSession:612(2436ns) ForwardSessionEventToSession:612(867193ns) ForwardSessionEventToSession:612(4470ns) ForwardSessionEventToSession:612(2704ns)}}}
2025-11-27T18:18:37.271440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:37.206082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471291111108634:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:37.271492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471283698497548:2049] [node 2] ICP22 created new session: [6:7577471287993464941:2049]
2025-11-27T18:18:37.271536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471287993464941:2049] [node 2] ICS09 handshake done sender: [6:7577471287993464933:2121] self: [6:7577471287993464939:0] peer: [2:7577471288446945341:0] socket: 556 qp: -1
2025-11-27T18:18:37.271567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471287993464941:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:37.244613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471284253447006:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.260713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471284678414832:2049] [node 2] ICP22 created new session: [4:7577471288973382226:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.213384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471288446945336:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.213421Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471288446945336:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.214168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471286816141260:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.271602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:37.263718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471288973382226:2049] [node 2] ICS09 handshake done sender: [4:7577471288973382218:2119] self: [4:7577471288973382221:0] peer: [2:7577471288446945335:0] socket: 517 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.317508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471287993464943:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39156
2025-11-27T18:18:37.253195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:37.313593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471286280869397:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(35878109ns) {HandleHandshakeStatus:329(30366888ns) {DropIncomingHandshake:442(37519ns) DropOutgoingHandshake:457(212ns) SwitchToState:179(420ns) ProcessPendingSessionEvents:495(11267475ns) {ForwardSessionEventToSession:612(7335125ns) ForwardSessionEventToSession:612(7459ns) ForwardSessionEventToSession:612(3840ns) ForwardSessionEventToSession:612(6786ns) ForwardSessionEventToSession:612(2545ns) ForwardSessionEventToSession:612(209008ns) ForwardSessionEventToSession:612(4294ns) ForwardSessionEventToSession:612(17818ns) ForwardSessionEventToSession:612(3648436ns) ForwardSessionEventToSession:612(8359ns)}}}
2025-11-27T18:18:37.214937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471291111108645:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.265435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:37.232250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471288446945342:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.232279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471288446945342:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.240721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471288446945336:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:37.253280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471284253447006:2049] [node 3] ICP22 created new session: [1:7577471288548414403:2049]
2025-11-27T18:18:37.227257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471286816141261:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.265501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471284678414831:2048] [node 1] ICP22 created new session: [4:7577471288973382228:2048]
2025-11-27T18:18:37.253882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471288548414403:2049] [node 3] ICS09 handshake done sender: [1:7577471288548414378:2120] self: [1:7577471288548414377:0] peer: [3:7577471290575836728:0] socket: 536 qp: -1
2025-11-27T18:18:37.227311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.241935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471288446945358:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
2025-11-27T18:18:37.267600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471288973382226:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:37.261179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471288548414403:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:37.230645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:37.261254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:37.230700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471286816141261:2050] [node 3] ICP22 created new session: [5:7577471291111108653:2050]
2025-11-27T18:18:37.269898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:37.231094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471286816141260:2049] [node 2] ICP22 created new session: [5:7577471291111108652:2049]
2025-11-27T18:18:37.284797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471288973382228:2048] [node 1] ICS09 handshake done sender: [4:7577471288973382209:2118] self: [4:7577471288973382220:0] peer: [1:7577471284253447079:0] socket: 535 qp: -1
2025-11-27T18:18:37.285866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471288973382228:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:37.231839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471291111108653:2050] [node 3] ICS09 handshake done sender: [5:7577471291111108634:2116] self: [5:7577471291111108639:0] peer: [3:7577471290575836698:0] socket: 512 qp: -1
2025-11-27T18:18:37.261873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471284253447006:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(18575779ns) {HandleHandshakeStatus:329(18572198ns) {DropIncomingHandshake:442(14696ns) DropOutgoingHandshake:457(869ns) SwitchToState:179(479ns) ProcessPendingSessionEvents:495(123440ns) {ForwardSessionEventToSession:612(74659ns) ForwardSessionEventToSession:612(3634ns) ForwardSessionEventToSession:612(3077ns) ForwardSessionEventToSession:612(6618ns) ForwardSessionEventToSession:612(5663ns) ForwardSessionEventToSession:612(6472ns) ForwardSessionEventToSession:612(3655ns) ForwardSessionEventToSession:612(2591ns) ForwardSessionEventToSession:612(5804ns) ForwardSessionEventToSession:612(3041ns)}}}
2025-11-27T18:18:37.231882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471291111108652:2049] [node 2] ICS09 handshake done sender: [5:7577471291111108636:2118] self: [5:7577471291111108642:0] peer: [2:7577471288446945339:0] socket: 529 qp: -1
2025-11-27T18:18:37.246019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471288446945342:2125] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:37.262124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471288548414381:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.232145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471291111108653:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:37.262148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471288548414381:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.253200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471284151978034:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.258683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:37.265707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471288548414381:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:37.259630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471288446945358:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.232209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471291111108652:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:37.284844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471284253447069:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(33760205ns) {HandleHandshakeStatus:329(33756625ns) {DropIncomingHandshake:442(13146ns) DropOutgoingHandshake:457(515ns) SwitchToState:179(332ns) ProcessPendingSessionEvents:495(15601121ns) {ForwardSessionEventToSession:612(1839914ns) ForwardSessionEventToSession:612(6035600ns) ForwardSessionEventToSession:612(4551ns) ForwardSessionEventToSession:612(3253ns) ForwardSessionEventToSession:612(852580ns) ForwardSessionEventToSession:612(3133ns) ForwardSessionEventToSession:612(4597566ns) ForwardSessionEventToSession:612(13053ns) ForwardSessionEventToSession:612(2652ns) ForwardSessionEventToSession:612(2223946ns)}}}
2025-11-27T18:18:37.259992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471288446945332:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.233678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:37.259995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471284151978034:2107] [node 4] ICP22 created new session: [2:7577471288446945361:2107]
2025-11-27T18:18:37.286252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471284253447070:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.235194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:37.267637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:37.294477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:37.242141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471286816141261:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(23242875ns) {HandleHandshakeStatus:329(13217110ns) {DropIncomingHandshake:442(1362ns) DropOutgoingHandshake:457(416ns) SwitchToState:179(595ns) ProcessPendingSessionEvents:495(1542ns)}}
2025-11-27T18:18:37.268193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471288446945361:2107] [node 4] ICS09 handshake done sender: [2:7577471288446945336:2122] self: [2:7577471288446945335:0] peer: [4:7577471288973382221:0] socket: 555 qp: -1
2025-11-27T18:18:37.294532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471284253447070:2108] [node 5] ICP22 created new session: [1:7577471288548414408:2108]
2025-11-27T18:18:37.259917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471286816141259:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.273197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471288446945361:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:37.309560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471288548414408:2108] [node 5] ICS09 handshake done sender: [1:7577471288548414381:2121] self: [1:7577471288548414380:0] peer: [5:7577471291111108656:0] socket: 542 qp: -1
2025-11-27T18:18:37.273245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:37.275593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471288446945332:2110] [node 6] ICP22 created new session: [2:7577471288446945362:2110]
2025-11-27T18:18:37.317737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471288548414408:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:37.260851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471286816141260:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(28989707ns) {HandleHandshakeStatus:329(28985205ns) {DropIncomingHandshake:442(745ns) DropOutgoingHandshake:457(5428ns) SwitchToState:179(522ns) ProcessPendingSessionEvents:495(2262333ns) {ForwardSessionEventToSession:612(67239ns) ForwardSessionEventToSession:612(2929ns) ForwardSessionEventToSession:612(2267ns) ForwardSessionEventToSession:612(136327ns) ForwardSessionEventToSession:612(3899ns) ForwardSessionEventToSession:612(816744ns) ForwardSessionEventToSession:612(399786ns) ForwardSessionEventToSession:612(3120ns) ForwardSessionEventToSession:612(812665ns) ForwardSessionEventToSession:612(4241ns)}}}
2025-11-27T18:18:37.319144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:37.285108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471288446945362:2110] [node 6] ICS09 handshake done sender: [2:7577471288446945342:2125] self: [2:7577471288446945341:0] peer: [6:7577471287993464939:0] socket: 554 qp: -1
2025-11-27T18:18:37.313610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471291111108657:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13681
2025-11-27T18:18:37.285928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471288446945362:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:37.333365Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471291111108645:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:37.354645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471284253447070:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(68399617ns) {HandleHandshakeStatus:329(68395711ns) {DropIncomingHandshake:442(19487ns) DropOutgoingHandshake:457(557ns) SwitchToState:179(838ns) ProcessPendingSessionEvents:495(19502830ns) {ForwardSessionEventToSession:612(6680964ns) ForwardSessionEventToSession:612(8176ns) ForwardSessionEventToSession:612(8692268ns) ForwardSessionEventToSession:612(6729ns) ForwardSessionEventToSession:612(4053118ns) ForwardSessionEventToSession:612(8708ns) ForwardSessionEventToSession:612(7892ns) ForwardSessionEventToSession:612(5458ns) ForwardSessionEventToSession:612(7927ns) ForwardSessionEventToSession:612(3617ns)}}}
2025-11-27T18:18:37.354562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471287993464943:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.350051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471286816141259:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.285983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.350111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:37.354795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471283698497547:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.350158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471286816141259:2048] [node 1] ICP22 created new session: [5:7577471291111108658:2048]
2025-11-27T18:18:37.349797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471284151978034:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(56778906ns) {HandleHandshakeStatus:329(56775232ns) {DropIncomingHandshake:442(16179ns) DropOutgoingHandshake:457(499ns) SwitchToState:179(652ns) ProcessPendingSessionEvents:495(24784650ns) {ForwardSessionEventToSession:612(4242349ns) ForwardSessionEventToSession:612(2474619ns) ForwardSessionEventToSession:612(663390ns) ForwardSessionEventToSession:612(6144ns) ForwardSessionEventToSession:612(1226820ns) ForwardSessionEventToSession:612(4814ns) ForwardSessionEventToSession:612(3163ns) ForwardSessionEventToSession:612(14736397ns) ForwardSessionEventToSession:612(3281ns) ForwardSessionEventToSession:612(554511ns)}}}
2025-11-27T18:18:37.350212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471291111108658:2048] [node 1] ICS09 handshake done sender: [5:7577471291111108645:2119] self: [5:7577471291111108656:0] peer: [1:7577471288548414380:0] socket: 544 qp: -1
2025-11-27T18:18:37.353487Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471288446945332:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(54258960ns) {HandleHandshakeStatus:329(54003705ns) {DropIncomingHandshake:442(797ns) DropOutgoingHandshake:457(963ns) SwitchToState:179(691ns) ProcessPendingSessionEvents:495(23808899ns) {ForwardSessionEventToSession:612(2890089ns) ForwardSessionEventToSession:612(11764ns) ForwardSessionEventToSession:612(3413ns) ForwardSessionEventToSession:612(1311974ns) ForwardSessionEventToSession:612(4347ns) ForwardSessionEventToSession:612(3593ns) ForwardSessionEventToSession:612(5441028ns) ForwardSessionEventToSession:612(3759ns) ForwardSessionEventToSession:612(9724281ns) ForwardSessionEventToSession:612(4385293ns)}}}
2025-11-27T18:18:37.350251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471291111108658:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:37.350287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:37.355129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471288446945365:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
2025-11-27T18:18:37.350506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471286816141259:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:37.287361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:37.355533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471288548414391:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.298312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471284678414833:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18794) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471288973382222:2120] held: no
2025-11-27T18:18:37.355527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471288446945332:2110] [node 6] ICP18 handshake (actor [2:7577471288446945358:2129]) is too old
2025-11-27T18:18:37.355569Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471288548414391:2126] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.356336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471288446945365:2130] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.333700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471284678414832:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(29500251ns) {HandleHandshakeStatus:329(29496548ns) {DropIncomingHandshake:442(681ns) DropOutgoingHandshake:457(6122ns) SwitchToState:179(370ns) ProcessPendingSessionEvents:495(11173453ns) {ForwardSessionEventToSession:612(7057102ns) ForwardSessionEventToSession:612(10535ns) ForwardSessionEventToSession:612(3079ns) ForwardSessionEventToSession:612(148056ns) ForwardSessionEventToSession:612(3570ns) ForwardSessionEventToSession:612(946896ns) ForwardSessionEventToSession:612(4267ns) ForwardSessionEventToSession:612(1603324ns) ForwardSessionEventToSession:612(1049660ns) ForwardSessionEventToSession:612(10675ns)}}}
2025-11-27T18:18:37.353759Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471284678414833:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(20267784ns) {HandleHandshakeStatus:395(18844638ns) {LogHandshakeFail:472(15675ns) {UpdateErrorStateLog:545(5747ns)} DropOutgoingHandshake:457(1290ns) ProcessPendingSessionEvents:495(15107661ns) {DropSessionEvent:512(12705ns) DropSessionEvent:512(5057ns) DropSessionEvent:512(5825ns) DropSessionEvent:512(40285ns) DropSessionEvent:512(6643ns) DropSessionEvent:512(3841514ns) DropSessionEvent:512(21483ns) DropSessionEvent:512(6891ns) DropSessionEvent:512(6324ns) DropSessionEvent:512(6071ns)}}}
2025-11-27T18:18:37.356596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471284151977960:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:37.354782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471284678414831:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(81732119ns) {HandleHandshakeStatus:329(80639614ns) {DropIncomingHandshake:442(869ns) DropOutgoingHandshake:457(5214ns) SwitchToState:179(502ns) ProcessPendingSessionEvents:495(40334344ns) {ForwardSessionEventToSession:612(16657467ns) ForwardSessionEventToSession:612(3671959ns) ForwardSessionEventToSession:612(4006735ns) ForwardSessionEventToSession:612(3363ns) ForwardSessionEventToSession:612(3596196ns) ForwardSessionEventToSession:612(5487ns) ForwardSessionEventToSession:612(3982252ns) ForwardSessionEventToSession:612(7992753ns) ForwardSessionEventToSession:612(5283ns) ForwardSessionEventToSession:612(370765ns)}}}
2025-11-27T18:18:37.356966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471288548414391:2126] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:37.354877Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471284678414831:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:37.355174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471288973382227:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:848
2025-11-27T18:18:37.355550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471288973382222:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:37.355742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471284678414833:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.356451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:37.356732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471284678414833:2050] [node 3] ICP22 created new session: [4:7577471288973382231:2050]
2025-11-27T18:18:37.356980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471288973382231:2050] [node 3] ICS09 handshake done sender: [4:7577471288973382222:2120] self: [4:7577471288973382225:0] peer: [3:7577471290575836710:0] socket: 563 qp: -1
2025-11-27T18:18:37.357025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471288973382231:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:37.357086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:37.357458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471287993464945:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39156
2025-11-27T18:18:37.357585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471288548414379:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.358004Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471287993464943:2123] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:37.357710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471288548414384:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:37.357736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471288548414384:2123] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:37.357915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:37.359017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471288548414379:2109] [node 6] ICP22 created new session: [1:7577471288548414411:2109]
2025-11-27T18:18:37.361915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471284151977960:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471288446945365:2130] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:37.362003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471283698497547:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471287993464943:2123] held: no
2025-11-27T18:18:37.359312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471288548414411:2109] [node 6] ICS09 handshake done sender: [1:7577471288548414391:2126] self: [1:7577471288548414390:0] peer: [6:7577471287993464944:0] socket: 508 qp: -1
2025-11-27T18:18:37.362530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471283698497547:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.359349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471288548414411:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:37.359751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471288548414384:2123] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:37.362620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:37.359912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471284253447005:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:37.361795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:37.362552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471288446945367:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
2025-11-27T18:18:37.362279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:37.362694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471283698497547:2048] [node 1] ICP22 created new session: [6:7577471287993464946:2048]
2025-11-27T18:18:37.362553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471284253447005:2048] [node 2] ICP22 created new session: [1:7577471288548414414:2048]
2025-11-27T18:18:37.362740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471287993464946:2048] [node 1] ICS09 handshake done sender: [6:7577471287993464943:2123] self: [6:7577471287993464944:0] peer: [1:7577471288548414390:0] socket: 507 qp: -1
2025-11-27T18:18:37.362613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471288548414414:2048] [node 2] ICS09 handshake done sender: [1:7577471288548414384:2123] self: [1:7577471288548414382:0] peer: [2:7577471288446945366:0] socket: 503 qp: -1
2025-11-27T18:18:37.362842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471287993464946:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:37.362648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471288548414414:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:37.362714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:37.363057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:37.363840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471288446945365:2130] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:37.364124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471284151977960:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:37.364616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:37.364706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471284151977960:2048] [node 1] ICP22 created new session: [2:7577471288446945368:2048]
2025-11-27T18:18:37.364762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471288446945368:2048] [node 1] ICS09 handshake done sender: [2:7577471288446945365:2130] self: [2:7577471288446945366:0] peer: [1:7577471288548414382:0] socket: 518 qp: -1
2025-11-27T18:18:37.364829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471288446945368:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:37.364877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:37.403946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:37.404153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471288446945368:2048] [node 1] ICS01 socket: 518 reason# EndOfStream
2025-11-27T18:18:37.404198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471284151977960:2048] [node 1] ICP30 unregister session Session# [2:7577471288446945368:2048] VirtualId# [2:7577471288446945366:0]
2025-11-27T18:18:37.404222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471288446945368:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.404290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:37.404333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:37.405134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:37.405369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471290575836731:2048] [node 1] ICS01 socket: 524 reason# EndOfStream
2025-11-27T18:18:37.405418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471286280869334:2048] [node 1] ICP30 unregister session Session# [3:7577471290575836731:2048] VirtualId# [3:7577471290575836728:0]
2025-11-27T18:18:37.405447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471290575836731:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.405530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:37.405618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:37.407245Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:37.407480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471288973382228:2048] [node 1] ICS07 socket disconnect 535 reason# EndOfStream
2025-11-27T18:18:37.407519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471288973382228:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.407585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:37.407635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471288973382228:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:37.408464Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471284678414831:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14952) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:37.408577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471288973382228:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:37.408635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471284678414831:2048] [node 1] ICP30 unregister session Session# [4:7577471288973382228:2048] VirtualId# [4:7577471288973382220:0]
2025-11-27T18:18:37.408697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:37.408734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471284678414831:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14952) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:37.408775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14952) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:37.409211Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:37.409360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:37.409428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471287993464946:2048] [node 1] ICS01 socket: 507 reason# EndOfStream
2025-11-27T18:18:37.409482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471283698497547:2048] [node 1] ICP30 unregister session Session# [6:7577471287993464946:2048] VirtualId# [6:7577471287993464944:0]
2025-11-27T18:18:37.409519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471287993464946:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.409596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:37.409605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471291111108658:2048] [node 1] ICS01 socket: 544 reason# EndOfStream
2025-11-27T18:18:37.409644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:37.409647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471286816141259:2048] [node 1] ICP30 unregister session Session# [5:7577471291111108658:2048] VirtualId# [5:7577471291111108656:0]
2025-11-27T18:18:37.409697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471291111108658:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.409784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:37.409852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:37.418807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471284678414831:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:37.448829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:37.449093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471290575836726:2049] [node 2] ICS01 socket: 527 reason# EndOfStream
2025-11-27T18:18:37.449143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471286280869335:2049] [node 2] ICP30 unregister session Session# [3:7577471290575836726:2049] VirtualId# [3:7577471290575836715:0]
2025-11-27T18:18:37.449195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471290575836726:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.449870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:37.449960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:37.451520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:37.451790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471288973382226:2049] [node 2] ICS01 socket: 517 reason# EndOfStream
2025-11-27T18:18:37.451847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471284678414832:2049] [node 2] ICP30 unregister session Session# [4:7577471288973382226:2049] VirtualId# [4:7577471288973382221:0]
2025-11-27T18:18:37.452708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471288973382226:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.452800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:37.452866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:37.452648Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:37.453332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471291111108652:2049] [node 2] ICS07 socket disconnect 529 reason# EndOfStream
2025-11-27T18:18:37.453370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471291111108652:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.453469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:37.453526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471291111108652:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:37.454514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:37.454706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471287993464941:2049] [node 2] ICS01 socket: 556 reason# EndOfStream
2025-11-27T18:18:37.454747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471283698497548:2049] [node 2] ICP30 unregister session Session# [6:7577471287993464941:2049] VirtualId# [6:7577471287993464939:0]
2025-11-27T18:18:37.454779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471287993464941:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.454848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:37.454885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:37.455507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471286816141260:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20290) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:37.455726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471291111108652:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:37.455775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471286816141260:2049] [node 2] ICP30 unregister session Session# [5:7577471291111108652:2049] VirtualId# [5:7577471291111108642:0]
2025-11-27T18:18:37.455836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:37.455894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471286816141260:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20290) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:37.455916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20290) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:37.465996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471286816141260:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:37.532536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:37.532787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471291111108653:2050] [node 3] ICS01 socket: 512 reason# EndOfStream
2025-11-27T18:18:37.532850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471286816141261:2050] [node 3] ICP30 unregister session Session# [5:7577471291111108653:2050] VirtualId# [5:7577471291111108639:0]
2025-11-27T18:18:37.532895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471291111108653:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.533072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:37.533176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:37.537408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:37.537650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471288973382231:2050] [node 3] ICS01 socket: 563 reason# EndOfStream
2025-11-27T18:18:37.537701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471284678414833:2050] [node 3] ICP30 unregister session Session# [4:7577471288973382231:2050] VirtualId# [4:7577471288973382225:0]
2025-11-27T18:18:37.537752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471288973382231:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.537838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:37.537898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:37.538101Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:37.538306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471287993464936:2050] [node 3] ICS01 socket: 516 reason# EndOfStream
2025-11-27T18:18:37.538344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471283698497549:2050] [node 3] ICP30 unregister session Session# [6:7577471287993464936:2050] VirtualId# [6:7577471287993464924:0]
2025-11-27T18:18:37.538375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471287993464936:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.538438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:37.538476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:37.601586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:37.601860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471291111108650:2108] [node 4] ICS01 socket: 506 reason# EndOfStream
2025-11-27T18:18:37.601905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471286816141322:2108] [node 4] ICP30 unregister session Session# [5:7577471291111108650:2108] VirtualId# [5:7577471291111108644:0]
2025-11-27T18:18:37.601943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471291111108650:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.602049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:37.602110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:37.602725Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:37.602940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471287993464935:2105] [node 4] ICS01 socket: 515 reason# EndOfStream
2025-11-27T18:18:37.602999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471283698497609:2105] [node 4] ICP30 unregister session Session# [6:7577471287993464935:2105] VirtualId# [6:7577471287993464929:0]
2025-11-27T18:18:37.603049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471287993464935:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.603127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:37.603187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:37.678385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:37.678833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471287993464927:2104] [node 5] ICS01 socket: 514 reason# EndOfStream
2025-11-27T18:18:37.678931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471283698497606:2104] [node 5] ICP30 unregister session Session# [6:7577471287993464927:2104] VirtualId# [6:7577471287993464925:0]
2025-11-27T18:18:37.679001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471287993464927:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:37.679164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:37.679249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:37.796000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471288831146656:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:37.796041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471288831146657:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:37.823923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471289176739521:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:37.824778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471289176739520:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:37.849083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471288571423692:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:37.849122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471288571423693:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:37.879055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471288853416720:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:37.879108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471288853416721:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:37.879273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471288853416722:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:37.901877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471288098349110:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:37.901949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471288098349111:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:37.901985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471288098349112:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:37.925271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471289756115667:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:37.925332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471289756115669:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:37.925371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471289756115668:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:38.204848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471294051083024:2105] [node 5] ICP01 ready to work
2025-11-27T18:18:38.224452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471294051083027:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:38.263538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471294051083024:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(44101723ns) {Configure:104(44088965ns) {ProcessConfigured:135(96026ns) {StartInitialHandshake:157(92326ns) {DropHandshakes:470(2230ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(2373ns) PrepareNewSessionHandshake:477(1372ns) {SwitchToState:179(459ns)}}}}}
2025-11-27T18:18:38.274144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471293126114023:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:38.343602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471288098349110:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(66091099ns) {Configure:104(66078959ns) {ProcessConfigured:135(73691ns) {StartInitialHandshake:157(70627ns) {DropHandshakes:470(2609ns) {DropIncomingHandshake:442(603ns) DropOutgoingHandshake:457(315ns)} GenerateSessionVirtualId:988(1066ns) PrepareNewSessionHandshake:477(1575ns) {SwitchToState:179(583ns)}}}}}
2025-11-27T18:18:38.274855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471293126114024:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:38.337682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471293471706882:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:38.362191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471293471706883:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:38.302695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471288831146656:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(101021655ns) {Configure:104(101016771ns) {ProcessConfigured:135(92399ns) {StartInitialHandshake:157(88758ns) {DropHandshakes:470(2195ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(320ns)} GenerateSessionVirtualId:988(1398ns) PrepareNewSessionHandshake:477(2131ns) {SwitchToState:179(837ns)}}}}}
2025-11-27T18:18:38.368330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471293148384079:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:38.366830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471288831146657:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(158472452ns) {Configure:104(158464234ns) {ProcessConfigured:135(62187ns) {StartInitialHandshake:157(59530ns) {DropHandshakes:470(2366ns) {DropIncomingHandshake:442(610ns) DropOutgoingHandshake:457(338ns)} GenerateSessionVirtualId:988(1104ns) PrepareNewSessionHandshake:477(1353ns) {SwitchToState:179(373ns)}}}}}
2025-11-27T18:18:38.365573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471292393316473:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:38.370131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471293471706884:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:38.375008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471289176739521:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(148336206ns) {Configure:104(148332128ns) {ProcessConfigured:135(69248ns) {StartInitialHandshake:157(67205ns) {DropHandshakes:470(2352ns) {DropIncomingHandshake:442(513ns) DropOutgoingHandshake:457(663ns)} GenerateSessionVirtualId:988(925ns) PrepareNewSessionHandshake:477(2036ns) {SwitchToState:179(463ns)}}}}}
2025-11-27T18:18:38.372868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471293148384080:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:38.390474Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471289756115668:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34123463ns) {Configure:104(34115732ns) {ProcessConfigured:135(73657ns) {StartInitialHandshake:157(71123ns) {DropHandshakes:470(2645ns) {DropIncomingHandshake:442(720ns) DropOutgoingHandshake:457(297ns)} GenerateSessionVirtualId:988(732ns) PrepareNewSessionHandshake:477(1769ns) {SwitchToState:179(783ns)}}}}}
2025-11-27T18:18:38.369768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471293126114029:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:38.369797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471292866391054:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:38.369934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471292866391053:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:38.370766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471292866391055:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:38.393332Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471293126114023:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(81281670ns) {Configure:104(81274978ns) {ProcessConfigured:135(13657874ns) {StartInitialHandshake:157(13654460ns) {DropHandshakes:470(2441ns) {DropIncomingHandshake:442(872ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(1123ns) PrepareNewSessionHandshake:477(2585ns) {SwitchToState:179(556ns)}}}}}
2025-11-27T18:18:38.377325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471292393316475:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:38.402289Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471293471706883:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16551124ns) {Configure:104(16545781ns) {ProcessConfigured:135(78840ns) {StartInitialHandshake:157(75672ns) {DropHandshakes:470(5184ns) {DropIncomingHandshake:442(405ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(1264ns) PrepareNewSessionHandshake:477(1856ns) {SwitchToState:179(930ns)}}}}}
2025-11-27T18:18:38.428732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471292393316473:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(57575892ns) {Configure:104(57570149ns) {ProcessConfigured:135(67217ns) {StartInitialHandshake:157(64801ns) {DropHandshakes:470(843ns) {DropIncomingHandshake:442(214ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:477(2130ns) {SwitchToState:179(893ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.438996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471288853416720:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(112316186ns) {Configure:104(112310310ns) {ProcessConfigured:135(17434050ns) {StartInitialHandshake:157(17430121ns) {DropHandshakes:470(4685ns) {DropIncomingHandshake:442(588ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(1061ns) PrepareNewSessionHandshake:477(4099ns) {SwitchToState:179(1384ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.441680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471293148384085:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10779
2025-11-27T18:18:38.441759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471293148384086:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10779
2025-11-27T18:18:38.441919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471293148384087:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10779
No RDMA context found for GID: 2025-11-27T18:18:38.442983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471293148384086:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.430384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471293471706895:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:274
2025-11-27T18:18:38.430755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471293126114029:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(43319509ns) {Configure:104(43315731ns) {ProcessConfigured:135(89894ns) {StartInitialHandshake:157(87421ns) {DropHandshakes:470(2297ns) {DropIncomingHandshake:442(601ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(838ns) PrepareNewSessionHandshake:477(1358ns) {SwitchToState:179(458ns)}}}}}
2025-11-27T18:18:38.458045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471292393316474:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54589
2025-11-27T18:18:38.453957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471289756115667:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(107015372ns) {Configure:104(107004807ns) {ProcessConfigured:135(56041ns) {StartInitialHandshake:157(54112ns) {DropHandshakes:470(2583ns) {DropIncomingHandshake:442(591ns) DropOutgoingHandshake:457(446ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(1384ns) {SwitchToState:179(358ns)}}}}}
2025-11-27T18:18:38.433977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471293471706882:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(62833498ns) {Configure:104(62397374ns) {ProcessConfigured:135(8029519ns) {StartInitialHandshake:157(8026391ns) {DropHandshakes:470(3075ns) {DropIncomingHandshake:442(748ns) DropOutgoingHandshake:457(309ns)} GenerateSessionVirtualId:988(945ns) PrepareNewSessionHandshake:477(2531ns) {SwitchToState:179(915ns)}}}}}
2025-11-27T18:18:38.465506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471292393316474:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.464947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471292866391053:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(92621303ns) {Configure:104(92617213ns) {ProcessConfigured:135(47465ns) {StartInitialHandshake:157(44392ns) {DropHandshakes:470(1497ns) {DropIncomingHandshake:442(393ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(651ns) PrepareNewSessionHandshake:477(1272ns) {SwitchToState:179(474ns)}}}}}
2025-11-27T18:18:38.470215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471293126114041:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48635
2025-11-27T18:18:38.454514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471294051083036:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42509
2025-11-27T18:18:38.467144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471292866391056:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12352
2025-11-27T18:18:38.475986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471293148384085:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.474841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471293126114042:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48635
2025-11-27T18:18:38.467198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471292866391057:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12352
2025-11-27T18:18:38.480321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471293471706895:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.454601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471294051083037:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42509
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.487583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471292393316481:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54589
2025-11-27T18:18:38.485469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471293126114041:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.493690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471293126114039:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48635
2025-11-27T18:18:38.494106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471293126114039:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.467411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471292866391064:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12352
2025-11-27T18:18:38.487244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471288853416722:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(210618496ns) {Configure:104(210615213ns) {ProcessConfigured:135(3031482ns) {StartInitialHandshake:157(3029712ns) {DropHandshakes:470(1933ns) {DropIncomingHandshake:442(687ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(950ns) PrepareNewSessionHandshake:477(2759ns) {SwitchToState:179(914ns)}}}}}
2025-11-27T18:18:38.473361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471288571423692:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(212148575ns) {Configure:104(212135209ns) {ProcessConfigured:135(23886257ns) {StartInitialHandshake:157(23882387ns) {DropHandshakes:470(1575ns) {DropIncomingHandshake:442(562ns) DropOutgoingHandshake:457(275ns)} GenerateSessionVirtualId:988(566ns) PrepareNewSessionHandshake:477(3640ns) {SwitchToState:179(1946ns)}}}}}
2025-11-27T18:18:38.484651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471289176739520:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(248719185ns) {Configure:104(248712620ns) {ProcessConfigured:135(12469879ns) {StartInitialHandshake:157(12467639ns) {DropHandshakes:470(2804ns) {DropIncomingHandshake:442(633ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(506ns) PrepareNewSessionHandshake:477(2988ns) {SwitchToState:179(1248ns)}}}}}
2025-11-27T18:18:38.479896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471294051083027:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(235477132ns) {Configure:104(235471296ns) {ProcessConfigured:135(23808080ns) {StartInitialHandshake:157(23805533ns) {DropHandshakes:470(11675476ns) {DropIncomingHandshake:442(2185ns) DropOutgoingHandshake:457(628ns)} GenerateSessionVirtualId:988(1277ns) PrepareNewSessionHandshake:477(2413ns) {SwitchToState:179(651ns)}}}}}
2025-11-27T18:18:38.496966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471293126114031:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.497004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471293126114031:2120] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.487081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471289756115669:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(118499840ns) {Configure:104(118495736ns) {ProcessConfigured:135(5870942ns) {StartInitialHandshake:157(5867611ns) {DropHandshakes:470(2182040ns) {DropIncomingHandshake:442(410ns) DropOutgoingHandshake:457(1890ns)} GenerateSessionVirtualId:988(5985ns) PrepareNewSessionHandshake:477(3396ns) {SwitchToState:179(2110ns)}}}}}
2025-11-27T18:18:38.494377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471289176739520:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.475160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471292866391057:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.477007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471292866391056:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.499085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471294051083036:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.501087Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471288853416721:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(120705803ns) {Configure:104(120700118ns) {ProcessConfigured:135(7159092ns) {StartInitialHandshake:157(7155976ns) {DropHandshakes:470(1469ns) {DropIncomingHandshake:442(269ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(478ns) PrepareNewSessionHandshake:477(3209ns) {SwitchToState:179(1599ns)}}}}}
2025-11-27T18:18:38.501476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471293148384087:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.501545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471288853416721:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.502549Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471288098349111:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(140106077ns) {Configure:104(140099864ns) {ProcessConfigured:135(6701374ns) {StartInitialHandshake:157(6698465ns) {DropHandshakes:470(2964ns) {DropIncomingHandshake:442(731ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(1258ns) PrepareNewSessionHandshake:477(1414ns) {SwitchToState:179(959ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.500029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471289756115668:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.501660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471293126114026:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.501699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471288853416720:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.504517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471293471706892:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.485918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471288571423692:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.504554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471293471706892:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.498451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471292866391064:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.501697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471293126114026:2118] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.503082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471292393316481:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.504731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471293148384084:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.500146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471294051083037:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.502154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471293126114042:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.503459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471288098349112:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.504755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471293148384084:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.504619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471293471706886:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.499659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471288571423693:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(125397358ns) {Configure:104(125390879ns) {ProcessConfigured:135(10100756ns) {StartInitialHandshake:157(10098503ns) {DropHandshakes:470(1272ns) {DropIncomingHandshake:442(263ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(481ns) PrepareNewSessionHandshake:477(2911ns) {SwitchToState:179(1612ns)}}}}}
2025-11-27T18:18:38.511693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471293148384080:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(103469363ns) {Configure:104(103450546ns) {ProcessConfigured:135(3488224ns) {StartInitialHandshake:157(3485608ns) {DropHandshakes:470(1059353ns) {DropIncomingHandshake:442(1104ns) DropOutgoingHandshake:457(436ns)} GenerateSessionVirtualId:988(1163ns) PrepareNewSessionHandshake:477(1327ns) {SwitchToState:179(385ns)}}}}}
2025-11-27T18:18:38.513708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471293148384084:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:38.505086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471292393316475:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(74445767ns) {Configure:104(74439396ns) {ProcessConfigured:135(3080858ns) {StartInitialHandshake:157(3078671ns) {DropHandshakes:470(2477ns) {DropIncomingHandshake:442(908ns) DropOutgoingHandshake:457(328ns)} GenerateSessionVirtualId:988(875ns) PrepareNewSessionHandshake:477(1635ns) {SwitchToState:179(673ns)}}}}}
2025-11-27T18:18:38.502462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471293126114040:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48635
2025-11-27T18:18:38.502624Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471293126114031:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:38.501702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471294051083027:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.504940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471288831146657:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.501991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471288571423693:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.504640Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471293471706886:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.546715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471293148384102:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10779
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.505188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471293471706898:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:274
2025-11-27T18:18:38.504389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471292866391061:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.506913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471292393316487:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54589
2025-11-27T18:18:38.505680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471293126114026:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:38.508434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471294051083046:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42509
2025-11-27T18:18:38.547021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471292393316481:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.504419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471292866391061:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.509560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:38.507980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471293471706901:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:274
2025-11-27T18:18:38.549449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471293148384079:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.511646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471292866391072:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12352
2025-11-27T18:18:38.509705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471294051083043:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42509
2025-11-27T18:18:38.510033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471293471706888:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.511548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471293126114028:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.511825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471292866391061:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:38.539319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471294051083048:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42509
2025-11-27T18:18:38.558671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471288098349112:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16432) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471292393316481:2114] held: no
2025-11-27T18:18:38.549492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.510074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471293471706888:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.511590Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471293126114028:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.549552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471293148384079:2109] [node 6] ICP22 created new session: [4:7577471293148384103:2109]
2025-11-27T18:18:38.542068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471294051083043:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.566585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471288098349112:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.513860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471288831146657:2049] [node 3] ICP22 created new session: [1:7577471293126114048:2049]
2025-11-27T18:18:38.510475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471293471706892:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:38.577543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:38.516307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471292866391055:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(87479063ns) {Configure:104(87469300ns) {ProcessConfigured:135(1908610ns) {StartInitialHandshake:157(1907347ns) {DropHandshakes:470(938ns) {DropIncomingHandshake:442(272ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(241ns) PrepareNewSessionHandshake:477(2336ns) {SwitchToState:179(1163ns)}}}}}
2025-11-27T18:18:38.550454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471294051083036:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:38.515722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471293126114048:2049] [node 3] ICS09 handshake done sender: [1:7577471293126114031:2120] self: [1:7577471293126114030:0] peer: [3:7577471292866391068:0] socket: 507 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.549594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471293148384103:2109] [node 6] ICS09 handshake done sender: [4:7577471293148384084:2111] self: [4:7577471293148384083:0] peer: [6:7577471294051083045:0] socket: 526 qp: -1
2025-11-27T18:18:38.513284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471293471706895:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:18:38.577613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471288098349112:2050] [node 3] ICP22 created new session: [5:7577471292393316492:2050]
::ffff:127.1.0.0
2025-11-27T18:18:38.530506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471288571423692:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471292866391056:2112] held: no
2025-11-27T18:18:38.551784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471289756115669:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.515904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471293126114040:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.522128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471293471706886:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:38.549625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471293148384103:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:38.577665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471292393316492:2050] [node 3] ICS09 handshake done sender: [5:7577471292393316481:2114] self: [5:7577471292393316486:0] peer: [3:7577471292866391060:0] socket: 540 qp: -1
2025-11-27T18:18:38.541790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471292866391056:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:38.515959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471288831146656:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.546232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471292866391053:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.555000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471289756115668:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.524376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471293471706898:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.549652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:38.577703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471292393316492:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:38.516001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:38.546316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:38.550114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471293148384086:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:38.555050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:38.525383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471289176739520:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.516055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471288831146656:2048] [node 2] ICP22 created new session: [1:7577471293126114049:2048]
2025-11-27T18:18:38.577764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:38.547207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471292866391053:2108] [node 5] ICP22 created new session: [3:7577471292866391078:2108]
2025-11-27T18:18:38.525442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:38.525492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471289176739520:2048] [node 1] ICP22 created new session: [2:7577471293471706906:2048]
2025-11-27T18:18:38.525534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471293471706906:2048] [node 1] ICS09 handshake done sender: [2:7577471293471706895:2116] self: [2:7577471293471706900:0] peer: [1:7577471293126114025:0] socket: 519 qp: -1
2025-11-27T18:18:38.525566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471293471706906:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:38.525602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:38.526646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471293471706904:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:274
2025-11-27T18:18:38.527363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471293471706883:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.527445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:38.535574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471289176739521:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.537435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471293471706883:2110] [node 6] ICP22 created new session: [2:7577471293471706909:2110]
2025-11-27T18:18:38.541716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471293471706888:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:38.545044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471293471706909:2110] [node 6] ICS09 handshake done sender: [2:7577471293471706892:2115] self: [2:7577471293471706891:0] peer: [6:7577471294051083044:0] socket: 515 qp: -1
2025-11-27T18:18:38.545095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471293471706909:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:38.545248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:38.547070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471289176739521:2049] [node 3] ICP22 created new session: [2:7577471293471706910:2049]
2025-11-27T18:18:38.552733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471288853416721:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.548867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:38.552804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:38.548913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471293471706910:2049] [node 3] ICS09 handshake done sender: [2:7577471293471706886:2113] self: [2:7577471293471706885:0] peer: [3:7577471292866391071:0] socket: 509 qp: -1
2025-11-27T18:18:38.551528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471293471706884:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.553674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471288853416721:2049] [node 2] ICP22 created new session: [4:7577471293148384106:2049]
2025-11-27T18:18:38.551566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:38.558945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471293148384106:2049] [node 2] ICS09 handshake done sender: [4:7577471293148384086:2113] self: [4:7577471293148384095:0] peer: [2:7577471293471706887:0] socket: 523 qp: -1
2025-11-27T18:18:38.555095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471289756115668:2049] [node 2] ICP22 created new session: [6:7577471294051083050:2049]
2025-11-27T18:18:38.593691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471288098349112:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(26993930ns) {HandleHandshakeStatus:329(26991053ns) {DropIncomingHandshake:442(825ns) DropOutgoingHandshake:457(252ns) SwitchToState:179(636ns) ProcessPendingSessionEvents:495(909ns)}}
2025-11-27T18:18:38.547261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471292866391078:2108] [node 5] ICS09 handshake done sender: [3:7577471292866391061:2115] self: [3:7577471292866391060:0] peer: [5:7577471292393316486:0] socket: 538 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.555894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471294051083050:2049] [node 2] ICS09 handshake done sender: [6:7577471294051083036:2113] self: [6:7577471294051083044:0] peer: [2:7577471293471706891:0] socket: 530 qp: -1
2025-11-27T18:18:38.569681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471293148384106:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:38.516101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471293126114049:2048] [node 2] ICS09 handshake done sender: [1:7577471293126114026:2118] self: [1:7577471293126114025:0] peer: [2:7577471293471706900:0] socket: 512 qp: -1
2025-11-27T18:18:38.551653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471293471706910:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:38.627382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471292393316489:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54589
2025-11-27T18:18:38.547294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471292866391078:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.552204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:38.555873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471293471706884:2108] [node 4] ICP22 created new session: [2:7577471293471706912:2108]
2025-11-27T18:18:38.561467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471293471706912:2108] [node 4] ICS09 handshake done sender: [2:7577471293471706888:2114] self: [2:7577471293471706887:0] peer: [4:7577471293148384095:0] socket: 510 qp: -1
2025-11-27T18:18:38.561532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471293471706912:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:38.561579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:38.563380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471293471706904:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.564012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471293471706915:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:274
2025-11-27T18:18:38.564090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471293471706884:2108] [node 4] ICP18 handshake (actor [2:7577471293471706904:2121]) is too old
2025-11-27T18:18:38.581817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471293471706916:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:274
2025-11-27T18:18:38.613839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471293471706915:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.516134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471293126114049:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:38.569752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:38.630352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471293471706916:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.630682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471292393316490:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54589
2025-11-27T18:18:38.634196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471292393316494:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54589
2025-11-27T18:18:38.637056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471292393316489:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.645365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471292393316490:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.646544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471292393316494:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.650754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471288098349110:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.653627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471288098349111:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.654091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471292393316473:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.695162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471288098349111:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4609) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471292393316489:2118] held: no
2025-11-27T18:18:38.705891Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471292393316473:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6954) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471292393316494:2120] held: no
2025-11-27T18:18:38.713505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471292393316485:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.713546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471292393316485:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.558654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471294051083047:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42509
2025-11-27T18:18:38.547337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:38.565343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471294051083050:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:38.565415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:38.573539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471294051083048:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.585798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471289756115668:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(30580699ns) {HandleHandshakeStatus:329(30578913ns) {DropIncomingHandshake:442(375ns) DropOutgoingHandshake:457(11064ns) SwitchToState:179(143ns) ProcessPendingSessionEvents:495(135267ns) {ForwardSessionEventToSession:612(78610ns) ForwardSessionEventToSession:612(3483ns) ForwardSessionEventToSession:612(3066ns) ForwardSessionEventToSession:612(7123ns) ForwardSessionEventToSession:612(5373ns) ForwardSessionEventToSession:612(3013ns) ForwardSessionEventToSession:612(10222ns) ForwardSessionEventToSession:612(7853ns) ForwardSessionEventToSession:612(5707ns) ForwardSessionEventToSession:612(2712ns)}}}
2025-11-27T18:18:38.586216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471294051083037:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:38.587832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471294051083027:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.587895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:38.587927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471294051083027:2109] [node 4] ICP22 created new session: [6:7577471294051083052:2109]
2025-11-27T18:18:38.587964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471294051083052:2109] [node 4] ICS09 handshake done sender: [6:7577471294051083037:2114] self: [6:7577471294051083045:0] peer: [4:7577471293148384083:0] socket: 531 qp: -1
2025-11-27T18:18:38.587993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471294051083052:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:38.588027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:38.597415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471289756115667:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.637036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471294051083057:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42509
2025-11-27T18:18:38.662343Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471294051083048:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:38.670949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471294051083056:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42509
2025-11-27T18:18:38.677644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471289756115667:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471294051083048:2120] held: no
2025-11-27T18:18:38.679931Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471289756115669:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16432) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471294051083043:2117] held: no
2025-11-27T18:18:38.681059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471289756115667:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.691544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471294051083058:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42509
2025-11-27T18:18:38.693097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471294051083043:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:38.697563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:38.701354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471289756115669:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.701438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:38.701510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471289756115669:2050] [node 3] ICP22 created new session: [6:7577471294051083060:2050]
2025-11-27T18:18:38.701591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471294051083060:2050] [node 3] ICS09 handshake done sender: [6:7577471294051083043:2117] self: [6:7577471294051083049:0] peer: [3:7577471292866391062:0] socket: 542 qp: -1
2025-11-27T18:18:38.701644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471294051083060:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:38.701687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:38.704117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471294051083058:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.709864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471294051083024:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.713713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471289756115667:2048] [node 1] ICP22 created new session: [6:7577471294051083059:2048]
2025-11-27T18:18:38.721697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471294051083059:2048] [node 1] ICS09 handshake done sender: [6:7577471294051083048:2120] self: [6:7577471294051083054:0] peer: [1:7577471293126114037:0] socket: 561 qp: -1
2025-11-27T18:18:38.731839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471294051083059:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.516167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:38.547788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471288571423692:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.747348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:38.547824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:38.522292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471293126114048:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:38.524183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471293126114028:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:38.547857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471288571423692:2048] [node 1] ICP22 created new session: [3:7577471292866391080:2048]
2025-11-27T18:18:38.530312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: 2025-11-27T18:18:38.733656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471292393316501:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54589
::ffff:127.1.0.0
2025-11-27T18:18:38.547903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471292866391080:2048] [node 1] ICS09 handshake done sender: [3:7577471292866391056:2112] self: [3:7577471292866391068:0] peer: [1:7577471293126114030:0] socket: 508 qp: -1
2025-11-27T18:18:38.632660Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471293471706883:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(49240673ns) {HandleHandshakeStatus:329(49238270ns) {DropIncomingHandshake:442(2685ns) DropOutgoingHandshake:457(894ns) SwitchToState:179(453ns) ProcessPendingSessionEvents:495(20463968ns) {ForwardSessionEventToSession:612(5401309ns) ForwardSessionEventToSession:612(5343ns) ForwardSessionEventToSession:612(14228ns) ForwardSessionEventToSession:612(2350ns) ForwardSessionEventToSession:612(6029784ns) ForwardSessionEventToSession:612(7573ns) ForwardSessionEventToSession:612(4831512ns) ForwardSessionEventToSession:612(4134488ns) ForwardSessionEventToSession:612(4969ns) ForwardSessionEventToSession:612(2983ns)}}}
2025-11-27T18:18:38.579021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471288853416720:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471293148384085:2112] held: no
2025-11-27T18:18:38.547930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471292866391080:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:38.547963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:38.552128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471292866391077:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12352
2025-11-27T18:18:38.564730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471292866391063:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.564767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471292866391063:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.587603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471292866391057:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:38.596191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471292866391084:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12352
2025-11-27T18:18:38.601931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471292866391084:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.602085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471292866391063:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:38.609520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471288571423693:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.609591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:38.609655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471288571423693:2049] [node 2] ICP22 created new session: [3:7577471292866391086:2049]
2025-11-27T18:18:38.609701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471292866391086:2049] [node 2] ICS09 handshake done sender: [3:7577471292866391057:2113] self: [3:7577471292866391071:0] peer: [2:7577471293471706885:0] socket: 513 qp: -1
2025-11-27T18:18:38.609741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471292866391086:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:38.609775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:38.611279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471292866391088:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12352
2025-11-27T18:18:38.618857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471292866391054:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.621850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:38.621910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471292866391054:2110] [node 6] ICP22 created new session: [3:7577471292866391090:2110]
2025-11-27T18:18:38.632675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471292866391090:2110] [node 6] ICS09 handshake done sender: [3:7577471292866391063:2116] self: [3:7577471292866391062:0] peer: [6:7577471294051083049:0] socket: 539 qp: -1
2025-11-27T18:18:38.634382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471292866391090:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:38.638843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:38.638900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471292866391088:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.639110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471292866391075:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.639135Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471292866391075:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.642480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471292866391075:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:38.660531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471292866391055:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.660605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:38.660670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471292866391055:2107] [node 4] ICP22 created new session: [3:7577471292866391093:2107]
2025-11-27T18:18:38.660720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471292866391093:2107] [node 4] ICS09 handshake done sender: [3:7577471292866391075:2120] self: [3:7577471292866391073:0] peer: [4:7577471293148384110:0] socket: 533 qp: -1
2025-11-27T18:18:38.660762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471292866391093:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:38.660852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:38.674663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471292866391054:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(55639303ns) {HandleHandshakeStatus:329(52287018ns) {DropIncomingHandshake:442(1537ns) DropOutgoingHandshake:457(1490ns) SwitchToState:179(625ns) ProcessPendingSessionEvents:495(8562307ns) {ForwardSessionEventToSession:612(7515327ns) ForwardSessionEventToSession:612(19907ns) ForwardSessionEventToSession:612(2888ns) ForwardSessionEventToSession:612(2306ns) ForwardSessionEventToSession:612(5721ns) ForwardSessionEventToSession:612(3590ns) ForwardSessionEventToSession:612(3616ns) ForwardSessionEventToSession:612(10201ns) ForwardSessionEventToSession:612(3108ns) ForwardSessionEventToSession:612(974631ns)}}}
2025-11-27T18:18:38.674854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471292866391054:2110] [node 6] ICP18 handshake (actor [3:7577471292866391088:2123]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:38.548984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471293126114024:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.550459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:38.552959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471293126114024:2107] [node 4] ICP22 created new session: [1:7577471293126114056:2107]
2025-11-27T18:18:38.554795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471288831146656:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(35982796ns) {HandleHandshakeStatus:329(35980859ns) {DropIncomingHandshake:442(414ns) DropOutgoingHandshake:457(364ns) SwitchToState:179(362ns) ProcessPendingSessionEvents:495(7146794ns) {ForwardSessionEventToSession:612(4332839ns) ForwardSessionEventToSession:612(4427ns) ForwardSessionEventToSession:612(6519ns) ForwardSessionEventToSession:612(5259ns) ForwardSessionEventToSession:612(3321ns) ForwardSessionEventToSession:612(1342649ns) ForwardSessionEventToSession:612(2771ns) ForwardSessionEventToSession:612(1426334ns) ForwardSessionEventToSession:612(4312ns) ForwardSessionEventToSession:612(2606ns)}}}
2025-11-27T18:18:38.558636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471293126114056:2107] [node 4] ICS09 handshake done sender: [1:7577471293126114028:2119] self: [1:7577471293126114027:0] peer: [4:7577471293148384096:0] socket: 506 qp: -1
2025-11-27T18:18:38.566472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471293126114056:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:38.589761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:38.596980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471288831146657:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(47205815ns) {HandleHandshakeStatus:329(47202951ns) {DropIncomingHandshake:442(11672ns) DropOutgoingHandshake:457(346ns) SwitchToState:179(446ns) ProcessPendingSessionEvents:495(21671149ns) {ForwardSessionEventToSession:612(7340216ns) ForwardSessionEventToSession:612(4160287ns) ForwardSessionEventToSession:612(4898ns) ForwardSessionEventToSession:612(17469ns) ForwardSessionEventToSession:612(1763385ns) ForwardSessionEventToSession:612(3859ns) ForwardSessionEventToSession:612(4749130ns) ForwardSessionEventToSession:612(13130ns) ForwardSessionEventToSession:612(3822ns) ForwardSessionEventToSession:612(366706ns)}}}
2025-11-27T18:18:38.617999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471293126114038:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.618041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471293126114038:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.623837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471293126114024:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(74552643ns) {HandleHandshakeStatus:329(74549688ns) {DropIncomingHandshake:442(10682ns) DropOutgoingHandshake:457(296ns) SwitchToState:179(243ns) ProcessPendingSessionEvents:495(5339971ns) {ForwardSessionEventToSession:612(5275956ns) ForwardSessionEventToSession:612(9686ns) ForwardSessionEventToSession:612(5639ns) ForwardSessionEventToSession:612(3146ns) ForwardSessionEventToSession:612(6039ns) ForwardSessionEventToSession:612(3069ns) ForwardSessionEventToSession:612(5635ns) ForwardSessionEventToSession:612(3112ns) ForwardSessionEventToSession:612(3308ns) ForwardSessionEventToSession:612(9778ns)}}}
2025-11-27T18:18:38.629860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471293126114038:2122] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:38.634043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471293126114029:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.650236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:38.650568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471293126114029:2110] [node 6] ICP22 created new session: [1:7577471293126114059:2110]
2025-11-27T18:18:38.679322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471293126114059:2110] [node 6] ICS09 handshake done sender: [1:7577471293126114038:2122] self: [1:7577471293126114037:0] peer: [6:7577471294051083054:0] socket: 558 qp: -1
2025-11-27T18:18:38.679414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471293126114059:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:38.682442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471293126114035:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.682478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471293126114035:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.682619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:38.690758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471293126114035:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:38.697802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471293126114023:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.699137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:38.703997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471293126114023:2109] [node 5] ICP22 created new session: [1:7577471293126114062:2109]
2025-11-27T18:18:38.709789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471293126114029:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(75499254ns) {HandleHandshakeStatus:329(75496529ns) {DropIncomingHandshake:442(17700ns) DropOutgoingHandshake:457(477ns) SwitchToState:179(461ns) ProcessPendingSessionEvents:495(10402380ns) {ForwardSessionEventToSession:612(10337246ns) ForwardSessionEventToSession:612(11167ns) ForwardSessionEventToSession:612(3424ns) ForwardSessionEventToSession:612(5458ns) ForwardSessionEventToSession:612(5938ns) ForwardSessionEventToSession:612(2903ns) ForwardSessionEventToSession:612(11044ns) ForwardSessionEventToSession:612(2997ns) ForwardSessionEventToSession:612(4041ns) ForwardSessionEventToSession:612(5689ns)}}}
2025-11-27T18:18:38.717377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471293126114062:2109] [node 5] ICS09 handshake done sender: [1:7577471293126114035:2121] self: [1:7577471293126114034:0] peer: [5:7577471292393316497:0] socket: 537 qp: -1
2025-11-27T18:18:38.717448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471293126114062:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:38.717527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:38.718061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471293126114023:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(20004807ns) {HandleHandshakeStatus:329(20000357ns) {DropIncomingHandshake:442(34933ns) DropOutgoingHandshake:457(453ns) SwitchToState:179(931ns) ProcessPendingSessionEvents:495(113700ns) {ForwardSessionEventToSession:612(62842ns) ForwardSessionEventToSession:612(3905ns) ForwardSessionEventToSession:612(6212ns) ForwardSessionEventToSession:612(3224ns) ForwardSessionEventToSession:612(8373ns) ForwardSessionEventToSession:612(3767ns) ForwardSessionEventToSession:612(5149ns) ForwardSessionEventToSession:612(5862ns) ForwardSessionEventToSession:612(2788ns) ForwardSessionEventToSession:612(2872ns)}}}
2025-11-27T18:18:38.581023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471293148384108:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10779
2025-11-27T18:18:38.637803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471293471706883:2110] [node 6] ICP18 handshake (actor [2:7577471293471706898:2118]) is too old
2025-11-27T18:18:38.755685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471294051083064:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42509
2025-11-27T18:18:38.646515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471289176739521:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(75120104ns) {HandleHandshakeStatus:329(75118126ns) {DropIncomingHandshake:442(1752ns) DropOutgoingHandshake:457(1113ns) SwitchToState:179(440ns) ProcessPendingSessionEvents:495(49972535ns) {ForwardSessionEventToSession:612(26348564ns) ForwardSessionEventToSession:612(5283ns) ForwardSessionEventToSession:612(2671ns) ForwardSessionEventToSession:612(7320058ns) ForwardSessionEventToSession:612(7327616ns) ForwardSessionEventToSession:612(6240168ns) ForwardSessionEventToSession:612(3895ns) ForwardSessionEventToSession:612(2077ns) ForwardSessionEventToSession:612(2687405ns) ForwardSessionEventToSession:612(5381ns)}}}
2025-11-27T18:18:38.658951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471289176739521:2049] [node 3] ICP18 handshake (actor [2:7577471293471706915:2122]) is too old
2025-11-27T18:18:38.738688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471292393316499:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54589
2025-11-27T18:18:38.673610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471293471706896:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.673666Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471293471706896:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.745248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471288098349110:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471292393316490:2119] held: no
2025-11-27T18:18:38.681738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471293471706896:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:38.597299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471293148384101:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10779
2025-11-27T18:18:38.755224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471292393316489:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:38.768185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471289756115667:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(84325322ns) {HandleHandshakeStatus:329(84322280ns) {DropIncomingHandshake:442(2361ns) DropOutgoingHandshake:457(203ns) SwitchToState:179(949ns) ProcessPendingSessionEvents:495(1384ns)}}
2025-11-27T18:18:38.683470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471293471706882:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.621605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471293148384085:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:38.683526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:38.755900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471292393316498:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54589
2025-11-27T18:18:38.767935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471292393316490:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:38.769142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471288098349111:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.769225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:38.769296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471288098349111:2049] [node 2] ICP22 created new session: [5:7577471292393316503:2049]
2025-11-27T18:18:38.769346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471292393316503:2049] [node 2] ICS09 handshake done sender: [5:7577471292393316489:2118] self: [5:7577471292393316496:0] peer: [2:7577471293471706894:0] socket: 562 qp: -1
2025-11-27T18:18:38.769379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471292393316503:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:38.769451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:38.770055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471292393316494:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:38.636321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471293148384108:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.772247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471288098349110:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.772316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:38.636458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471288853416722:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:38.637042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471288853416720:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.772356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471288098349110:2048] [node 1] ICP22 created new session: [5:7577471292393316505:2048]
2025-11-27T18:18:38.773999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471294051083058:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:38.776724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471294051083024:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.772393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471292393316505:2048] [node 1] ICS09 handshake done sender: [5:7577471292393316490:2119] self: [5:7577471292393316497:0] peer: [1:7577471293126114034:0] socket: 536 qp: -1
2025-11-27T18:18:38.683579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471293471706882:2109] [node 5] ICP22 created new session: [2:7577471293471706919:2109]
2025-11-27T18:18:38.772428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471292393316505:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:38.782156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:38.683633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471293471706919:2109] [node 5] ICS09 handshake done sender: [2:7577471293471706896:2117] self: [2:7577471293471706894:0] peer: [5:7577471292393316496:0] socket: 560 qp: -1
2025-11-27T18:18:38.772459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:38.653599Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471288853416721:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(68992867ns) {HandleHandshakeStatus:329(68990862ns) {DropIncomingHandshake:442(1074ns) DropOutgoingHandshake:457(32568ns) SwitchToState:179(253ns) ProcessPendingSessionEvents:495(22674539ns) {ForwardSessionEventToSession:612(10379179ns) ForwardSessionEventToSession:612(11842ns) ForwardSessionEventToSession:612(2932ns) ForwardSessionEventToSession:612(3480ns) ForwardSessionEventToSession:612(9807ns) ForwardSessionEventToSession:612(2180348ns) ForwardSessionEventToSession:612(2562850ns) ForwardSessionEventToSession:612(5407ns) ForwardSessionEventToSession:612(3823444ns) ForwardSessionEventToSession:612(3675186ns)}}}
2025-11-27T18:18:38.773062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471292393316485:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:38.664542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:38.773230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471292393316473:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.665280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471288853416720:2048] [node 1] ICP22 created new session: [4:7577471293148384111:2048]
2025-11-27T18:18:38.773784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:38.673535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471293148384097:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:38.774300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471292393316475:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.673578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471293148384097:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:38.774367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:38.774396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471292393316475:2109] [node 6] ICP22 created new session: [5:7577471292393316508:2109]
2025-11-27T18:18:38.673748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471293148384111:2048] [node 1] ICS09 handshake done sender: [4:7577471293148384085:2112] self: [4:7577471293148384096:0] peer: [1:7577471293126114027:0] socket: 522 qp: -1
2025-11-27T18:18:38.782234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471294051083024:2105] [node 5] ICP22 created new session: [6:7577471294051083065:2105]
2025-11-27T18:18:38.683672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471293471706919:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:38.680079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471293148384111:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:38.683706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:38.680464Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471288853416722:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16432) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471293148384108:2120] held: no
2025-11-27T18:18:38.681071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:38.683153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471293148384097:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:38.698115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471293148384080:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:38.698183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:38.698254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471293148384080:2108] [node 5] ICP22 created new session: [4:7577471293148384115:2108]
2025-11-27T18:18:38.698307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471293148384115:2108] [node 5] ICS09 handshake done sender: [4:7577471293148384097:2117] self: [4:7577471293148384094:0] peer: [5:7577471292393316495:0] socket: 567 qp: -1
2025-11-27T18:18:38.774435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471292393316508:2109] [node 6] ICS09 handshake done sender: [5:7577471292393316485:2116] self: [5:7577471292393316484:0] peer: [6:7577471294051083062:0] socket: 535 qp: -1
2025-11-27T18:18:38.698341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471293148384115:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:38.782286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471294051083065:2105] [node 5] ICS09 handshake done sender: [6:7577471294051083058:2123] self: [6:7577471294051083062:0] peer: [5:7577471292393316484:0] socket: 554 qp: -1
2025-11-27T18:18:38.698382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:38.774457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471292393316508:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:38.782326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471294051083065:2105] [node 5] ICS10 traffic start
2025-11-27T18:18:38.705130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471293148384112:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10779
2025-11-27T18:18:38.774493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:38.726094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471293148384108:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:38.782374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:38.775727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471292393316473:2108] [node 4] ICP22 created new session: [5:7577471292393316507:2108]
2025-11-27T18:18:38.776803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471292393316507:2108] [node 4] ICS09 handshake done sender: [5:7577471292393316494:2120] self: [5:7577471292393316495:0] peer: [4:7577471293148384094:0] socket: 570 qp: -1
2025-11-27T18:18:38.731930Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471288853416720:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(76557163ns) {HandleHandshakeStatus:329(76552992ns) {DropIncomingHandshake:442(2595ns) DropOutgoingHandshake:457(327ns) SwitchToState:179(459ns) ProcessPendingSessionEvents:495(1317ns)}}
2025-11-27T18:18:38.737499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471288853416722:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:38.781689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471292393316507:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:38.737592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:38.783110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:38.737649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471288853416722:2050] [node 3] ICP22 created new session: [4:7577471293148384117:2050]
2025-11-27T18:18:38.790130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471292393316473:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(16393173ns) {HandleHandshakeStatus:329(16390029ns) {DropIncomingHandshake:442(414ns) DropOutgoingHandshake:457(184ns) SwitchToState:179(319ns) ProcessPendingSessionEvents:495(952ns)}}
2025-11-27T18:18:38.737701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471293148384117:2050] [node 3] ICS09 handshake done sender: [4:7577471293148384108:2120] self: [4:7577471293148384110:0] peer: [3:7577471292866391073:0] socket: 566 qp: -1
2025-11-27T18:18:38.737739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471293148384117:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:38.737774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:38.868400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:38.869654Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:38.869941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471293471706906:2048] [node 1] ICS01 socket: 519 reason# EndOfStream
2025-11-27T18:18:38.869993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471289176739520:2048] [node 1] ICP30 unregister session Session# [2:7577471293471706906:2048] VirtualId# [2:7577471293471706900:0]
2025-11-27T18:18:38.870030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471293471706906:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:38.870130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:38.870183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:38.868632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471292866391080:2048] [node 1] ICS01 socket: 508 reason# EndOfStream
2025-11-27T18:18:38.868683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471288571423692:2048] [node 1] ICP30 unregister session Session# [3:7577471292866391080:2048] VirtualId# [3:7577471292866391068:0]
2025-11-27T18:18:38.868720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471292866391080:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:38.868788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:38.868849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:38.885581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:38.885815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471294051083059:2048] [node 1] ICS01 socket: 561 reason# EndOfStream
2025-11-27T18:18:38.885862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471289756115667:2048] [node 1] ICP30 unregister session Session# [6:7577471294051083059:2048] VirtualId# [6:7577471294051083054:0]
2025-11-27T18:18:38.885899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471294051083059:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:38.885976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:38.886020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:38.885443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:38.885741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471293148384111:2048] [node 1] ICS01 socket: 522 reason# EndOfStream
2025-11-27T18:18:38.885789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471288853416720:2048] [node 1] ICP30 unregister session Session# [4:7577471293148384111:2048] VirtualId# [4:7577471293148384096:0]
2025-11-27T18:18:38.885828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471293148384111:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:38.885920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:38.885970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:38.886072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:38.886312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471292393316505:2048] [node 1] ICS01 socket: 536 reason# EndOfStream
2025-11-27T18:18:38.886353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471288098349110:2048] [node 1] ICP30 unregister session Session# [5:7577471292393316505:2048] VirtualId# [5:7577471292393316497:0]
2025-11-27T18:18:38.886379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471292393316505:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:38.886437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:38.886471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:38.978144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:38.978369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471292866391086:2049] [node 2] ICS01 socket: 513 reason# EndOfStream
2025-11-27T18:18:38.978414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471288571423693:2049] [node 2] ICP30 unregister session Session# [3:7577471292866391086:2049] VirtualId# [3:7577471292866391071:0]
2025-11-27T18:18:38.978457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471292866391086:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:38.978526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:38.978566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:39.000691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:39.000995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471293148384106:2049] [node 2] ICS07 socket disconnect 523 reason# EndOfStream
2025-11-27T18:18:39.001043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471293148384106:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:39.001120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:39.001193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471293148384106:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:39.009398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471288853416721:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4609) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:39.013784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:39.014019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471294051083050:2049] [node 2] ICS01 socket: 530 reason# EndOfStream
2025-11-27T18:18:39.014077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471289756115668:2049] [node 2] ICP30 unregister session Session# [6:7577471294051083050:2049] VirtualId# [6:7577471294051083044:0]
2025-11-27T18:18:39.014118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471294051083050:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:39.014229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:39.014293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:39.013340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:39.014751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471292393316503:2049] [node 2] ICS01 socket: 562 reason# EndOfStream
2025-11-27T18:18:39.025401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471293148384106:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:39.014809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471288098349111:2049] [node 2] ICP30 unregister session Session# [5:7577471292393316503:2049] VirtualId# [5:7577471292393316496:0]
2025-11-27T18:18:39.014838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471292393316503:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:39.014922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:39.014979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:39.029489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471288853416721:2049] [node 2] ICP30 unregister session Session# [4:7577471293148384106:2049] VirtualId# [4:7577471293148384095:0]
2025-11-27T18:18:39.053357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:39.053449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471288853416721:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4609) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:39.053475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4609) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:39.061927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [4:7577471288853416721:2049] [node 2] ICP35 event processing took too much time StateWork:294(51208136ns) {HandleHandshakeStatus:395(51204485ns) {LogHandshakeFail:472(9996ns) {UpdateErrorStateLog:545(2060ns)} DropOutgoingHandshake:457(2071ns) ProcessPendingSessionEvents:495(480ns) UnregisterSession:538(4082244ns) {ProcessPendingSessionEvents:495(1260ns) SwitchToInitialState:202(2638ns) {SwitchToState:179(1618ns)}} TransitToErrorState:752(91740ns) {SwitchToState:179(9156ns) ProcessPendingSessionEvents:495(603ns) DropHandshakes:470(4024ns) {DropIncomingHandshake:442(993ns) DropOutgoingHandshake:457(429ns)}}}}
2025-11-27T18:18:39.078106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471288853416721:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:39.137362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:39.137583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471293148384117:2050] [node 3] ICS01 socket: 566 reason# EndOfStream
2025-11-27T18:18:39.136231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:39.137627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471288853416722:2050] [node 3] ICP30 unregister session Session# [4:7577471293148384117:2050] VirtualId# [4:7577471293148384110:0]
2025-11-27T18:18:39.136553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471292393316492:2050] [node 3] ICS01 socket: 540 reason# EndOfStream
2025-11-27T18:18:39.137657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471293148384117:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:39.136613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471288098349112:2050] [node 3] ICP30 unregister session Session# [5:7577471292393316492:2050] VirtualId# [5:7577471292393316486:0]
2025-11-27T18:18:39.136657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471292393316492:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:39.137731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:39.137773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:39.136748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:39.136797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:39.137378Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:39.137616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471294051083060:2050] [node 3] ICS01 socket: 542 reason# EndOfStream
2025-11-27T18:18:39.137658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471289756115669:2050] [node 3] ICP30 unregister session Session# [6:7577471294051083060:2050] VirtualId# [6:7577471294051083049:0]
2025-11-27T18:18:39.137689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471294051083060:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:39.137769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:39.137802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:39.224031Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:39.224302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471292393316507:2108] [node 4] ICS01 socket: 570 reason# EndOfStream
2025-11-27T18:18:39.224348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471292393316473:2108] [node 4] ICP30 unregister session Session# [5:7577471292393316507:2108] VirtualId# [5:7577471292393316495:0]
2025-11-27T18:18:39.224389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471292393316507:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:39.224479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:39.224522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:39.225117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:39.225359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471294051083052:2109] [node 4] ICS01 socket: 531 reason# EndOfStream
2025-11-27T18:18:39.225426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471294051083027:2109] [node 4] ICP30 unregister session Session# [6:7577471294051083052:2109] VirtualId# [6:7577471294051083045:0]
2025-11-27T18:18:39.225463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471294051083052:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:39.225546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:39.225850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:39.273468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:39.273726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471294051083065:2105] [node 5] ICS07 socket disconnect 554 reason# EndOfStream
2025-11-27T18:18:39.273767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471294051083065:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:39.273878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:39.273926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471294051083065:2105] [node 5] ICS15 start handshake
2025-11-27T18:18:39.275014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471294051083024:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15829) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:39.275105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471294051083065:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:39.275145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471294051083024:2105] [node 5] ICP30 unregister session Session# [6:7577471294051083065:2105] VirtualId# [6:7577471294051083062:0]
2025-11-27T18:18:39.275233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:18:39.275297Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471294051083024:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15829) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:39.275319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15829) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:39.285699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471294051083024:2105] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:39.410536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471299751198668:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:39.410576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471299751198669:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:39.436830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471298138584433:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:39.437501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471298138584432:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:39.463583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471297063694118:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:39.464278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471297063694119:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:39.492218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471296288352833:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:39.492266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471296288352834:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:39.492295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471296288352835:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:39.519938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471296273329032:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:39.519984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471296273329033:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:39.520014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471296273329034:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:39.550826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471296204308929:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:39.550867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471296204308930:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:39.550891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471296204308931:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:39.710968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471296204308991:2106] [node 5] ICP01 ready to work
2025-11-27T18:18:39.720040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471299751198731:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:39.751499Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471297063694118:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17418742ns) {Configure:104(17413404ns) {ProcessConfigured:135(71302ns) {StartInitialHandshake:157(68243ns) {DropHandshakes:470(5327ns) {DropIncomingHandshake:442(745ns) DropOutgoingHandshake:457(343ns)} GenerateSessionVirtualId:988(1927ns) PrepareNewSessionHandshake:477(1968ns) {SwitchToState:179(781ns)}}}}}
2025-11-27T18:18:39.785657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471299751198735:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:39.751972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471297063694192:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:39.811491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471298138584501:2111] [node 6] ICP01 ready to work
2025-11-27T18:18:39.825428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471296273329105:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:39.797405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471296204308991:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18417202ns) {Configure:104(18412272ns) {ProcessConfigured:135(8851922ns) {StartInitialHandshake:157(8849816ns) {DropHandshakes:470(2939ns) {DropIncomingHandshake:442(724ns) DropOutgoingHandshake:457(427ns)} GenerateSessionVirtualId:988(614ns) PrepareNewSessionHandshake:477(3463ns) {SwitchToState:179(2047ns)}}}}}
2025-11-27T18:18:39.818547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471296288352899:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:39.818513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471298138584502:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:39.817603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471297063694194:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:39.819807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471298138584503:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:39.827135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471296204308995:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:39.817922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471297063694195:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:39.826947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471296273329104:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:39.828513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471297063694192:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(76094254ns) {Configure:104(76089929ns) {ProcessConfigured:135(4226774ns) {StartInitialHandshake:157(4224252ns) {DropHandshakes:470(3151ns) {DropIncomingHandshake:442(1028ns) DropOutgoingHandshake:457(655ns)} GenerateSessionVirtualId:988(900ns) PrepareNewSessionHandshake:477(2547ns) {SwitchToState:179(1168ns)}}}}}
2025-11-27T18:18:39.818803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471299751198736:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:39.827270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471299751198669:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(93898069ns) {Configure:104(93889550ns) {ProcessConfigured:135(62025285ns) {StartInitialHandshake:157(62021767ns) {DropHandshakes:470(3032ns) {DropIncomingHandshake:442(783ns) DropOutgoingHandshake:457(613ns)} GenerateSessionVirtualId:988(762ns) PrepareNewSessionHandshake:477(3333ns) {SwitchToState:179(1577ns)}}}}}
2025-11-27T18:18:39.830383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471296288352900:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:39.838132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471298138584433:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29496253ns) {Configure:104(29489235ns) {ProcessConfigured:135(79303ns) {StartInitialHandshake:157(77076ns) {DropHandshakes:470(2990ns) {DropIncomingHandshake:442(506ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(920ns) PrepareNewSessionHandshake:477(1486ns) {SwitchToState:179(443ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.838833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471297063694203:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40278
2025-11-27T18:18:39.832480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471299751198668:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(118882676ns) {Configure:104(118869343ns) {ProcessConfigured:135(10974466ns) {StartInitialHandshake:157(10971992ns) {DropHandshakes:470(3727ns) {DropIncomingHandshake:442(779ns) DropOutgoingHandshake:457(777ns)} GenerateSessionVirtualId:988(663ns) PrepareNewSessionHandshake:477(3026ns) {SwitchToState:179(1526ns)}}}}}
2025-11-27T18:18:39.849013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471296204308931:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(36864484ns) {Configure:104(36856749ns) {ProcessConfigured:135(75506ns) {StartInitialHandshake:157(72275ns) {DropHandshakes:470(2804ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(662ns)} GenerateSessionVirtualId:988(878ns) PrepareNewSessionHandshake:477(1522ns) {SwitchToState:179(517ns)}}}}}
2025-11-27T18:18:39.861195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471297063694119:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51576987ns) {Configure:104(51570435ns) {ProcessConfigured:135(100677ns) {StartInitialHandshake:157(98087ns) {DropHandshakes:470(2873ns) {DropIncomingHandshake:442(618ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(1343ns) PrepareNewSessionHandshake:477(1739ns) {SwitchToState:179(770ns)}}}}}
2025-11-27T18:18:39.861955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471299751198731:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(129194295ns) {Configure:104(129186494ns) {ProcessConfigured:135(10053584ns) {StartInitialHandshake:157(10049226ns) {DropHandshakes:470(2806ns) {DropIncomingHandshake:442(831ns) DropOutgoingHandshake:457(698ns)} GenerateSessionVirtualId:988(796ns) PrepareNewSessionHandshake:477(4355ns) {SwitchToState:179(2227ns)}}}}}
2025-11-27T18:18:39.865654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471296288352899:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(32138770ns) {Configure:104(32134341ns) {ProcessConfigured:135(69041ns) {StartInitialHandshake:157(66994ns) {DropHandshakes:470(1265ns) {DropIncomingHandshake:442(344ns) DropOutgoingHandshake:457(375ns)} GenerateSessionVirtualId:988(483ns) PrepareNewSessionHandshake:477(1621ns) {SwitchToState:179(737ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.870922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471298138584505:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.866057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471297063694203:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.867685Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471299751198735:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(52712987ns) {Configure:104(52705037ns) {ProcessConfigured:135(2642879ns) {StartInitialHandshake:157(2640554ns) {DropHandshakes:470(1215ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(213ns)} GenerateSessionVirtualId:988(771ns) PrepareNewSessionHandshake:477(2586ns) {SwitchToState:179(1313ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.871256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471296204308997:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53237
2025-11-27T18:18:39.870966Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471298138584505:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:39.870817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471296288352835:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52231386ns) {Configure:104(52226497ns) {ProcessConfigured:135(3180236ns) {StartInitialHandshake:157(3178024ns) {DropHandshakes:470(1848ns) {DropIncomingHandshake:442(541ns) DropOutgoingHandshake:457(296ns)} GenerateSessionVirtualId:988(553ns) PrepareNewSessionHandshake:477(1647ns) {SwitchToState:179(581ns)}}}}}
2025-11-27T18:18:39.867401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471297063694119:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.872400Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471296273329033:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(52681994ns) {Configure:104(52665901ns) {ProcessConfigured:135(3024522ns) {StartInitialHandshake:157(3023149ns) {DropHandshakes:470(1586222ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(1576ns)} GenerateSessionVirtualId:988(1699ns) PrepareNewSessionHandshake:477(1216ns) {SwitchToState:179(511ns)}}}}}
2025-11-27T18:18:39.872445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471298138584502:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(42062290ns) {Configure:104(42054684ns) {ProcessConfigured:135(2797917ns) {StartInitialHandshake:157(2796292ns) {DropHandshakes:470(4800ns) {DropIncomingHandshake:442(3730ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(766ns) PrepareNewSessionHandshake:477(1406ns) {SwitchToState:179(391ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.875026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471297063694194:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(44649674ns) {Configure:104(44642646ns) {ProcessConfigured:135(1587878ns) {StartInitialHandshake:157(1088590ns) {DropHandshakes:470(1488ns) {DropIncomingHandshake:442(284ns) DropOutgoingHandshake:457(265ns)} GenerateSessionVirtualId:988(1508ns) PrepareNewSessionHandshake:477(1444ns) {SwitchToState:179(541ns)}}}}}
2025-11-27T18:18:39.875026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471296204308929:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(58828040ns) {Configure:104(58819600ns) {ProcessConfigured:135(2612387ns) {StartInitialHandshake:157(2610128ns) {DropHandshakes:470(1121ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(283ns)} GenerateSessionVirtualId:988(248ns) PrepareNewSessionHandshake:477(1518ns) {SwitchToState:179(557ns)}}}}}
2025-11-27T18:18:39.877378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471296288352833:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(54470616ns) {Configure:104(54460217ns) {ProcessConfigured:135(885076ns) {StartInitialHandshake:157(883863ns) {DropHandshakes:470(1262ns) {DropIncomingHandshake:442(293ns) DropOutgoingHandshake:457(304ns)} GenerateSessionVirtualId:988(514ns) PrepareNewSessionHandshake:477(1687ns) {SwitchToState:179(658ns)}}}}}
2025-11-27T18:18:39.877392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471299751198740:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50797
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.873478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471296273329034:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(73367795ns) {Configure:104(73356848ns) {ProcessConfigured:135(3838523ns) {StartInitialHandshake:157(3836932ns) {DropHandshakes:470(1119ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(471ns) PrepareNewSessionHandshake:477(1703ns) {SwitchToState:179(693ns)}}}}}
2025-11-27T18:18:39.876299Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471296204308930:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(52478187ns) {Configure:104(52467042ns) {ProcessConfigured:135(3359264ns) {StartInitialHandshake:157(3357177ns) {DropHandshakes:470(1232ns) {DropIncomingHandshake:442(232ns) DropOutgoingHandshake:457(248ns)} GenerateSessionVirtualId:988(768ns) PrepareNewSessionHandshake:477(1681ns) {SwitchToState:179(711ns)}}}}}
2025-11-27T18:18:39.874388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471298138584503:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49613097ns) {Configure:104(49609627ns) {ProcessConfigured:135(3368840ns) {StartInitialHandshake:157(3366990ns) {DropHandshakes:470(537988ns) {DropIncomingHandshake:442(257ns) DropOutgoingHandshake:457(752ns)} GenerateSessionVirtualId:988(1269ns) PrepareNewSessionHandshake:477(2124ns) {SwitchToState:179(845ns)}}}}}
2025-11-27T18:18:39.879297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471296288352834:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(59839352ns) {Configure:104(59833472ns) {ProcessConfigured:135(4045648ns) {StartInitialHandshake:157(4044114ns) {DropHandshakes:470(1116ns) {DropIncomingHandshake:442(322ns) DropOutgoingHandshake:457(275ns)} GenerateSessionVirtualId:988(363ns) PrepareNewSessionHandshake:477(1530ns) {SwitchToState:179(187ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.877633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471297063694209:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40278
2025-11-27T18:18:39.878626Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471299751198736:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(35268225ns) {Configure:104(35261247ns) {ProcessConfigured:135(2292191ns) {StartInitialHandshake:157(2290737ns) {DropHandshakes:470(1997ns) {DropIncomingHandshake:442(1053ns) DropOutgoingHandshake:457(312ns)} GenerateSessionVirtualId:988(509ns) PrepareNewSessionHandshake:477(1973ns) {SwitchToState:179(747ns)}}}}}
2025-11-27T18:18:39.879313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471296204308995:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(43122045ns) {Configure:104(43116210ns) {ProcessConfigured:135(3847960ns) {StartInitialHandshake:157(3845890ns) {DropHandshakes:470(929ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(182ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:477(2052ns) {SwitchToState:179(801ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.875894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471298138584513:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41537
2025-11-27T18:18:39.876344Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471296273329032:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(51199909ns) {Configure:104(51192820ns) {ProcessConfigured:135(1552215ns) {StartInitialHandshake:157(1548400ns) {DropHandshakes:470(2619ns) {DropIncomingHandshake:442(1081ns) DropOutgoingHandshake:457(186ns)} GenerateSessionVirtualId:988(2093ns) PrepareNewSessionHandshake:477(1813ns) {SwitchToState:179(529ns)}}}}}
2025-11-27T18:18:39.879769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471296288352901:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50231
2025-11-27T18:18:39.879250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471299751198742:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50797
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.877780Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471298138584501:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(51280505ns) {Configure:104(51273514ns) {ProcessConfigured:135(2634572ns) {StartInitialHandshake:157(2632384ns) {DropHandshakes:470(1126241ns) {DropIncomingHandshake:442(1687ns) DropOutgoingHandshake:457(398ns)} GenerateSessionVirtualId:988(4065ns) PrepareNewSessionHandshake:477(1957ns) {SwitchToState:179(882ns)}}}}}
2025-11-27T18:18:39.903180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471297063694198:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.879401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471296204308997:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.903225Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471297063694198:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:39.893702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471296288352901:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.879309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471298138584505:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:39.903236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471297063694209:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.894155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471296288352913:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50231
No RDMA context found for GID: 2025-11-27T18:18:39.906078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471298138584433:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:39.913055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471297063694118:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:39.894214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471296288352914:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50231
2025-11-27T18:18:39.906127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
::ffff:127.1.0.0
2025-11-27T18:18:39.889813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471299751198740:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.881304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471296273329114:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3623
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.890383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471299751198742:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.894278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471296288352835:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:39.912026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471298138584433:2049] [node 3] ICP22 created new session: [2:7577471298138584516:2049]
2025-11-27T18:18:39.913365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471297063694201:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.902410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471296204308931:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:39.881445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471296273329117:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3623
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.906671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471296288352900:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(46081571ns) {Configure:104(46071276ns) {ProcessConfigured:135(6850768ns) {StartInitialHandshake:157(6847875ns) {DropHandshakes:470(850289ns) {DropIncomingHandshake:442(407ns) DropOutgoingHandshake:457(1852ns)} GenerateSessionVirtualId:988(1144ns) PrepareNewSessionHandshake:477(3073ns) {SwitchToState:179(1406ns)}}}}}
2025-11-27T18:18:39.913164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471298138584516:2049] [node 3] ICS09 handshake done sender: [2:7577471298138584505:2114] self: [2:7577471298138584504:0] peer: [3:7577471297063694206:0] socket: 520 qp: -1
2025-11-27T18:18:39.914485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471298138584513:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.915325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471298138584516:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:39.915908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471298138584509:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.915936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471298138584509:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:39.916336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:39.916443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471298138584432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:39.913400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471297063694201:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:39.902989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471299751198751:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50797
2025-11-27T18:18:39.907341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471296204309010:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53237
2025-11-27T18:18:39.889611Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471296273329104:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(40333707ns) {Configure:104(40327735ns) {ProcessConfigured:135(583415ns) {StartInitialHandshake:157(581734ns) {DropHandshakes:470(1031ns) {DropIncomingHandshake:442(298ns) DropOutgoingHandshake:457(125ns)} GenerateSessionVirtualId:988(489ns) PrepareNewSessionHandshake:477(1178ns) {SwitchToState:179(243ns)}}}}}
2025-11-27T18:18:39.912105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471296288352914:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.910372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471299751198751:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.916005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471297063694198:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:39.914242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471296288352913:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.913241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471296204309010:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.907810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471296273329114:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.915665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471296288352834:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:39.916863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471296288352904:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.908250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471296273329117:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.916888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471296288352904:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:39.915609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471296204309009:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53237
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.921344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471298138584509:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:39.913346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471296273329032:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:39.916450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471299751198743:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.915650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471296204308995:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:39.925247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471298138584502:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:39.922688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471296288352904:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.920677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471297063694210:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40278
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.916476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471299751198743:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:39.925506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471296273329107:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.924754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471296288352899:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:39.917773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471296204309009:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.924834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.928975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:39.924658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471297063694201:2122] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:39.925286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471296288352899:2110] [node 6] ICP22 created new session: [4:7577471296288352924:2110]
2025-11-27T18:18:39.924307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471296204308991:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:39.925535Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471296273329107:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:39.933905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471298138584522:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41537
2025-11-27T18:18:39.916837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471299751198754:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50797
2025-11-27T18:18:39.925226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471297063694211:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40278
2025-11-27T18:18:39.935008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471298138584502:2108] [node 4] ICP22 created new session: [2:7577471298138584523:2108]
2025-11-27T18:18:39.927891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471296288352924:2110] [node 6] ICS09 handshake done sender: [4:7577471296288352904:2114] self: [4:7577471296288352903:0] peer: [6:7577471296204309011:0] socket: 545 qp: -1
2025-11-27T18:18:39.916853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471299751198737:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.927533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471296204309014:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53237
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.943332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471296273329123:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3623
2025-11-27T18:18:39.925802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471297063694195:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:39.943928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471298138584524:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41537
2025-11-27T18:18:39.928771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471296288352924:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:39.925848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:39.945655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471296273329124:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3623
2025-11-27T18:18:39.916881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471299751198737:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:39.928737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471296204308995:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14276) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471296204309010:2117] held: no
2025-11-27T18:18:39.944223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471298138584523:2108] [node 4] ICS09 handshake done sender: [2:7577471298138584509:2115] self: [2:7577471298138584507:0] peer: [4:7577471296288352918:0] socket: 534 qp: -1
2025-11-27T18:18:39.919152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471299751198739:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.919182Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471299751198739:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:39.920794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471299751198743:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:39.937822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471299751198737:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:39.939769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471299751198739:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:39.941717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471299751198754:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.941782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471299751198731:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:39.945161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471299751198669:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:39.945349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471299751198668:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:39.945529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:39.945567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:39.946226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:39.946273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471299751198669:2049] [node 3] ICP22 created new session: [1:7577471299751198760:2049]
2025-11-27T18:18:39.946280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471299751198668:2048] [node 2] ICP22 created new session: [1:7577471299751198761:2048]
2025-11-27T18:18:39.947397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471299751198731:2108] [node 5] ICP22 created new session: [1:7577471299751198759:2108]
2025-11-27T18:18:39.949093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471299751198761:2048] [node 2] ICS09 handshake done sender: [1:7577471299751198739:2113] self: [1:7577471299751198738:0] peer: [2:7577471298138584518:0] socket: 528 qp: -1
2025-11-27T18:18:39.949770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471299751198760:2049] [node 3] ICS09 handshake done sender: [1:7577471299751198737:2112] self: [1:7577471299751198734:0] peer: [3:7577471297063694213:0] socket: 525 qp: -1
2025-11-27T18:18:39.950595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471299751198759:2108] [node 5] ICS09 handshake done sender: [1:7577471299751198743:2116] self: [1:7577471299751198741:0] peer: [5:7577471296273329121:0] socket: 527 qp: -1
2025-11-27T18:18:39.951582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471299751198761:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:39.957642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471299751198759:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:39.958184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471299751198760:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:39.958589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:39.959090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:39.959332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:39.930185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:39.925891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471297063694195:2107] [node 4] ICP22 created new session: [3:7577471297063694218:2107]
2025-11-27T18:18:39.945488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471296204309013:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53237
2025-11-27T18:18:39.945971Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471296273329107:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.925937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471297063694218:2107] [node 4] ICS09 handshake done sender: [3:7577471297063694198:2121] self: [3:7577471297063694197:0] peer: [4:7577471296288352916:0] socket: 512 qp: -1
2025-11-27T18:18:39.943770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471296288352920:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50231
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.959172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471296273329125:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3623
2025-11-27T18:18:39.945906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471298138584523:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:39.948029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471296204309010:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:39.925972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471297063694218:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:39.945814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471296288352921:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50231
2025-11-27T18:18:39.946473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:39.950841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471296204308997:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:39.961290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471296273329123:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.967476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471296273329117:2125] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:39.948065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471298138584513:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:39.948774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471296288352913:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:39.926005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:39.955956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471296204308995:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:39.974610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471296273329105:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:39.949445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471298138584524:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.949643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471296288352834:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:40.003450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471299751198668:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(37671416ns) {HandleHandshakeStatus:329(37669203ns) {DropIncomingHandshake:442(10726ns) DropOutgoingHandshake:457(268ns) SwitchToState:179(468ns) ProcessPendingSessionEvents:495(21605728ns) {ForwardSessionEventToSession:612(6948071ns) ForwardSessionEventToSession:612(8043946ns) ForwardSessionEventToSession:612(4269524ns) ForwardSessionEventToSession:612(9122ns) ForwardSessionEventToSession:612(689923ns) ForwardSessionEventToSession:612(2708ns) ForwardSessionEventToSession:612(3121ns) ForwardSessionEventToSession:612(743752ns) ForwardSessionEventToSession:612(2589ns) ForwardSessionEventToSession:612(874860ns)}}}
2025-11-27T18:18:39.956012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:39.926604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471297063694217:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40278
2025-11-27T18:18:39.950647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471296288352901:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:39.980462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471296273329032:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28102) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471296273329117:2125] held: no
2025-11-27T18:18:39.982049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:39.952827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471298138584433:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(24373214ns) {HandleHandshakeStatus:329(24369814ns) {DropIncomingHandshake:442(716ns) DropOutgoingHandshake:457(433ns) SwitchToState:179(337ns) ProcessPendingSessionEvents:495(9096023ns) {ForwardSessionEventToSession:612(3408432ns) ForwardSessionEventToSession:612(569330ns) ForwardSessionEventToSession:612(2995ns) ForwardSessionEventToSession:612(2526482ns) ForwardSessionEventToSession:612(4016ns) ForwardSessionEventToSession:612(2614ns) ForwardSessionEventToSession:612(1155406ns) ForwardSessionEventToSession:612(4193ns) ForwardSessionEventToSession:612(2405ns) ForwardSessionEventToSession:612(1403471ns)}}}
2025-11-27T18:18:39.953540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471298138584432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:40.007659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471299751198669:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(41824689ns) {HandleHandshakeStatus:329(41821837ns) {DropIncomingHandshake:442(12626ns) DropOutgoingHandshake:457(735ns) SwitchToState:179(517ns) ProcessPendingSessionEvents:495(10713138ns) {ForwardSessionEventToSession:612(4465576ns) ForwardSessionEventToSession:612(1290789ns) ForwardSessionEventToSession:612(5400ns) ForwardSessionEventToSession:612(3282ns) ForwardSessionEventToSession:612(840518ns) ForwardSessionEventToSession:612(3467ns) ForwardSessionEventToSession:612(3721ns) ForwardSessionEventToSession:612(1674772ns) ForwardSessionEventToSession:612(3101ns) ForwardSessionEventToSession:612(2393595ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.986628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471296273329105:2109] [node 6] ICP22 created new session: [5:7577471296273329128:2109]
2025-11-27T18:18:39.956079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471296204308995:2110] [node 4] ICP22 created new session: [6:7577471296204309018:2110]
2025-11-27T18:18:39.959290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:39.987514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471296273329033:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:39.928700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471297063694210:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.957714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:39.956129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471296204309018:2110] [node 4] ICS09 handshake done sender: [6:7577471296204309010:2117] self: [6:7577471296204309011:0] peer: [4:7577471296288352903:0] socket: 552 qp: -1
2025-11-27T18:18:40.028243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471299751198731:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(56725574ns) {HandleHandshakeStatus:329(56722290ns) {DropIncomingHandshake:442(14025ns) DropOutgoingHandshake:457(730ns) SwitchToState:179(485ns) ProcessPendingSessionEvents:495(17377396ns) {ForwardSessionEventToSession:612(5283015ns) ForwardSessionEventToSession:612(7117ns) ForwardSessionEventToSession:612(418161ns) ForwardSessionEventToSession:612(7555ns) ForwardSessionEventToSession:612(3779ns) ForwardSessionEventToSession:612(9782265ns) ForwardSessionEventToSession:612(17921ns) ForwardSessionEventToSession:612(4259ns) ForwardSessionEventToSession:612(1821220ns) ForwardSessionEventToSession:612(3914ns)}}}
2025-11-27T18:18:39.960503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471296288352835:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:39.990466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471296273329128:2109] [node 6] ICS09 handshake done sender: [5:7577471296273329107:2119] self: [5:7577471296273329106:0] peer: [6:7577471296204309012:0] socket: 539 qp: -1
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.960425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471298138584433:2049] [node 3] ICP18 handshake (actor [2:7577471298138584524:2120]) is too old
2025-11-27T18:18:39.930470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471297063694192:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:39.990521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471296273329128:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:39.960585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:39.956166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471296204309018:2110] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.930508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:40.036490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471304046166064:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50797
2025-11-27T18:18:39.960446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471298138584432:2048] [node 1] ICP22 created new session: [2:7577471298138584526:2048]
2025-11-27T18:18:39.956203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:39.964240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471296288352834:2049] [node 2] ICP22 created new session: [4:7577471296288352926:2049]
2025-11-27T18:18:39.996604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471296273329032:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:39.934663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471297063694214:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40278
2025-11-27T18:18:39.961365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471298138584526:2048] [node 1] ICS09 handshake done sender: [2:7577471298138584513:2118] self: [2:7577471298138584518:0] peer: [1:7577471299751198738:0] socket: 529 qp: -1
2025-11-27T18:18:40.041429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471299751198745:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.975417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471296204308931:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:39.996912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:39.968016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471296288352926:2049] [node 2] ICS09 handshake done sender: [4:7577471296288352913:2119] self: [4:7577471296288352918:0] peer: [2:7577471298138584507:0] socket: 537 qp: -1
2025-11-27T18:18:40.041479Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471299751198745:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.975495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:39.982105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471296204308931:2050] [node 3] ICP22 created new session: [6:7577471296204309020:2050]
2025-11-27T18:18:39.986327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471296204309020:2050] [node 3] ICS09 handshake done sender: [6:7577471296204308997:2111] self: [6:7577471296204309008:0] peer: [3:7577471297063694200:0] socket: 522 qp: -1
2025-11-27T18:18:39.986699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471296204309015:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53237
2025-11-27T18:18:39.986829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471296204309020:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:39.986885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:39.999005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471296204309015:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.001603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471296204308991:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9998) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471296204309009:2116] held: no
2025-11-27T18:18:40.004441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471296204309016:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53237
2025-11-27T18:18:40.013761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471296204308930:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:40.021620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471296204309022:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53237
2025-11-27T18:18:40.025693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471296204309009:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:40.028226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471296204308991:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:40.033616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471296204309022:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.034642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:40.034972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471296204308931:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(59917628ns) {HandleHandshakeStatus:329(59913981ns) {DropIncomingHandshake:442(2716ns) DropOutgoingHandshake:457(15211ns) SwitchToState:179(624ns) ProcessPendingSessionEvents:495(19421206ns) {ForwardSessionEventToSession:612(4553742ns) ForwardSessionEventToSession:612(5477ns) ForwardSessionEventToSession:612(10698ns) ForwardSessionEventToSession:612(954154ns) ForwardSessionEventToSession:612(4832323ns) ForwardSessionEventToSession:612(5315ns) ForwardSessionEventToSession:612(4064ns) ForwardSessionEventToSession:612(7650631ns) ForwardSessionEventToSession:612(4704ns) ForwardSessionEventToSession:612(7600ns)}}}
2025-11-27T18:18:40.035109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471296204308991:2106] [node 5] ICP22 created new session: [6:7577471300499276321:2106]
2025-11-27T18:18:40.035354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471296204308929:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:40.035778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471300499276321:2106] [node 5] ICS09 handshake done sender: [6:7577471296204309009:2116] self: [6:7577471296204309012:0] peer: [5:7577471296273329106:0] socket: 544 qp: -1
2025-11-27T18:18:40.035834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471300499276321:2106] [node 5] ICS10 traffic start
2025-11-27T18:18:40.041404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:39.964347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471298138584526:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:39.998969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:39.968023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471296288352835:2050] [node 3] ICP22 created new session: [4:7577471296288352927:2050]
2025-11-27T18:18:40.047101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471299751198750:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.980453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471296288352926:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:39.934872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471297063694212:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40278
2025-11-27T18:18:40.047139Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471299751198750:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:39.937662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471297063694192:2110] [node 6] ICP22 created new session: [3:7577471297063694221:2110]
2025-11-27T18:18:39.975422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:39.985935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471298138584527:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41537
2025-11-27T18:18:39.980515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:40.054378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471296204308991:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(19030812ns) {HandleHandshakeStatus:329(19027929ns) {DropIncomingHandshake:442(1002ns) DropOutgoingHandshake:457(351ns) SwitchToState:179(813ns) ProcessPendingSessionEvents:495(860ns)}}
2025-11-27T18:18:40.001804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471296273329032:2048] [node 1] ICP22 created new session: [5:7577471296273329131:2048]
2025-11-27T18:18:39.990126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471298138584502:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(39609796ns) {HandleHandshakeStatus:329(39607224ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(256ns) SwitchToState:179(213ns) ProcessPendingSessionEvents:495(15981497ns) {ForwardSessionEventToSession:612(1262007ns) ForwardSessionEventToSession:612(574395ns) ForwardSessionEventToSession:612(7105892ns) ForwardSessionEventToSession:612(16213ns) ForwardSessionEventToSession:612(1271724ns) ForwardSessionEventToSession:612(14984ns) ForwardSessionEventToSession:612(2750ns) ForwardSessionEventToSession:612(1877301ns) ForwardSessionEventToSession:612(3523ns) ForwardSessionEventToSession:612(3822335ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:40.047390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471304046166064:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.054168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471299751198745:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:40.069641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471299751198750:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:40.074454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471299751198735:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:39.941833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471297063694203:2123] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:39.982009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471296288352927:2050] [node 3] ICS09 handshake done sender: [4:7577471296288352901:2113] self: [4:7577471296288352916:0] peer: [3:7577471297063694197:0] socket: 514 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:40.007718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471296273329125:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.001828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471298138584527:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.944299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471297063694221:2110] [node 6] ICS09 handshake done sender: [3:7577471297063694201:2122] self: [3:7577471297063694200:0] peer: [6:7577471296204309008:0] socket: 513 qp: -1
2025-11-27T18:18:40.088606Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471296204308930:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471296204309015:2120] held: no
2025-11-27T18:18:40.085922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471299751198736:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:40.008366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471296273329131:2048] [node 1] ICS09 handshake done sender: [5:7577471296273329117:2125] self: [5:7577471296273329121:0] peer: [1:7577471299751198741:0] socket: 535 qp: -1
2025-11-27T18:18:40.008201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471298138584510:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:39.982060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471296288352927:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:40.087120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:39.945726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471297063694221:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:39.982937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:40.018629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471296273329131:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:40.008242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471298138584510:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:40.088841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471299751198735:2110] [node 6] ICP22 created new session: [1:7577471304046166068:2110]
2025-11-27T18:18:39.945765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:39.984629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471296288352899:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(27676913ns) {HandleHandshakeStatus:329(27674809ns) {DropIncomingHandshake:442(20537ns) DropOutgoingHandshake:457(560ns) SwitchToState:179(330ns) ProcessPendingSessionEvents:495(5675359ns) {ForwardSessionEventToSession:612(2581502ns) ForwardSessionEventToSession:612(6238ns) ForwardSessionEventToSession:612(1268677ns) ForwardSessionEventToSession:612(6385ns) ForwardSessionEventToSession:612(574638ns) ForwardSessionEventToSession:612(4484ns) ForwardSessionEventToSession:612(3319ns) ForwardSessionEventToSession:612(732937ns) ForwardSessionEventToSession:612(4228ns) ForwardSessionEventToSession:612(2413ns)}}}
2025-11-27T18:18:40.021611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471296273329127:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3623
2025-11-27T18:18:40.018429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471298138584510:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:40.096959Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471296204308929:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28102) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471296204309022:2122] held: no
2025-11-27T18:18:39.947160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471297063694214:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.088975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:40.021798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:40.101376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471300499276325:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53237
2025-11-27T18:18:40.093544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471299751198736:2107] [node 4] ICP22 created new session: [1:7577471304046166069:2107]
2025-11-27T18:18:40.033314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471296288352911:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.019221Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471298138584432:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(44960757ns) {HandleHandshakeStatus:329(44957442ns) {DropIncomingHandshake:442(1441ns) DropOutgoingHandshake:457(13682ns) SwitchToState:179(392ns) ProcessPendingSessionEvents:495(13870965ns) {ForwardSessionEventToSession:612(2656536ns) ForwardSessionEventToSession:612(4542ns) ForwardSessionEventToSession:612(3595ns) ForwardSessionEventToSession:612(365097ns) ForwardSessionEventToSession:612(3482ns) ForwardSessionEventToSession:612(3646ns) ForwardSessionEventToSession:612(8947334ns) ForwardSessionEventToSession:612(8333ns) ForwardSessionEventToSession:612(1849145ns) ForwardSessionEventToSession:612(5613ns)}}}
2025-11-27T18:18:39.947252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471297063694212:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.022053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471296273329032:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25367262ns) {HandleHandshakeStatus:329(25364203ns) {DropIncomingHandshake:442(1078ns) DropOutgoingHandshake:457(433ns) SwitchToState:179(250ns) ProcessPendingSessionEvents:495(1170ns)}}
2025-11-27T18:18:40.027145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471298138584503:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:40.093600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471304046166069:2107] [node 4] ICS09 handshake done sender: [1:7577471299751198750:2118] self: [1:7577471299751198747:0] peer: [4:7577471300583320228:0] socket: 515 qp: -1
2025-11-27T18:18:40.101448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471300499276324:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53237
2025-11-27T18:18:40.033361Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471296288352911:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:40.028244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:40.025705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471296273329104:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:39.948014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471297063694119:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:40.093637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471304046166069:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:40.033581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471296288352930:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50231
2025-11-27T18:18:40.030705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471298138584512:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.027145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471296273329127:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.028389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471296273329034:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:40.035079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471296273329105:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(52469256ns) {HandleHandshakeStatus:329(52466427ns) {DropIncomingHandshake:442(16189ns) DropOutgoingHandshake:457(1123ns) SwitchToState:179(647ns) ProcessPendingSessionEvents:495(8337012ns) {ForwardSessionEventToSession:612(6830048ns) ForwardSessionEventToSession:612(553813ns) ForwardSessionEventToSession:612(420576ns) ForwardSessionEventToSession:612(5020ns) ForwardSessionEventToSession:612(3289ns) ForwardSessionEventToSession:612(19561ns) ForwardSessionEventToSession:612(3405ns) ForwardSessionEventToSession:612(5456ns) ForwardSessionEventToSession:612(3062ns) ForwardSessionEventToSession:612(473217ns)}}}
2025-11-27T18:18:40.045024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471300568296432:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3623
2025-11-27T18:18:40.053381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471296273329123:2126] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:40.061848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471296273329033:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:40.066172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:40.074127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471296273329033:2049] [node 2] ICP22 created new session: [5:7577471300568296435:2049]
2025-11-27T18:18:40.082003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471300568296436:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3623
2025-11-27T18:18:40.084016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471300568296435:2049] [node 2] ICS09 handshake done sender: [5:7577471296273329123:2126] self: [5:7577471296273329129:0] peer: [2:7577471298138584508:0] socket: 557 qp: -1
2025-11-27T18:18:40.084064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471300568296435:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:40.086130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471296273329127:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:40.089037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:40.091896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471296273329034:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:40.091985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:40.093461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471296273329034:2050] [node 3] ICP22 created new session: [5:7577471300568296438:2050]
2025-11-27T18:18:40.096954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471300568296438:2050] [node 3] ICS09 handshake done sender: [5:7577471296273329127:2129] self: [5:7577471300568296433:0] peer: [3:7577471297063694207:0] socket: 510 qp: -1
2025-11-27T18:18:40.097942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471300568296438:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:40.097981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:40.099892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471300568296434:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3623
2025-11-27T18:18:40.104412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471296273329125:2128] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:40.104772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471296273329033:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(37500154ns) {HandleHandshakeStatus:329(37497284ns) {DropIncomingHandshake:442(1566ns) DropOutgoingHandshake:457(24409ns) SwitchToState:179(837ns) ProcessPendingSessionEvents:495(1560378ns) {ForwardSessionEventToSession:612(323104ns) ForwardSessionEventToSession:612(4576ns) ForwardSessionEventToSession:612(9845ns) ForwardSessionEventToSession:612(2357ns) ForwardSessionEventToSession:612(3200ns) ForwardSessionEventToSession:612(468480ns) ForwardSessionEventToSession:612(3261ns) ForwardSessionEventToSession:612(728149ns) ForwardSessionEventToSession:612(3745ns) ForwardSessionEventToSession:612(2865ns)}}}
2025-11-27T18:18:40.109697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471296273329104:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:40.102345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471296204309015:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:39.949174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:40.119048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:40.103280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471296204309022:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:40.121970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471296273329104:2108] [node 4] ICP22 created new session: [5:7577471300568296440:2108]
2025-11-27T18:18:40.093921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471304046166068:2110] [node 6] ICS09 handshake done sender: [1:7577471299751198745:2117] self: [1:7577471299751198744:0] peer: [6:7577471300499276322:0] socket: 571 qp: -1
2025-11-27T18:18:40.123821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471300568296440:2108] [node 4] ICS09 handshake done sender: [5:7577471296273329125:2128] self: [5:7577471300568296430:0] peer: [4:7577471296288352910:0] socket: 569 qp: -1
2025-11-27T18:18:40.030743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471298138584512:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:40.103459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471296204308930:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:40.094218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471304046166068:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:40.033815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471296288352835:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(42971513ns) {HandleHandshakeStatus:329(42968449ns) {DropIncomingHandshake:442(1121ns) DropOutgoingHandshake:457(15145ns) SwitchToState:179(646ns) ProcessPendingSessionEvents:495(13195806ns) {ForwardSessionEventToSession:612(7889830ns) ForwardSessionEventToSession:612(540031ns) ForwardSessionEventToSession:612(314087ns) ForwardSessionEventToSession:612(1313091ns) ForwardSessionEventToSession:612(4405ns) ForwardSessionEventToSession:612(3653ns) ForwardSessionEventToSession:612(2108228ns) ForwardSessionEventToSession:612(4007ns) ForwardSessionEventToSession:612(2976ns) ForwardSessionEventToSession:612(996649ns)}}}
2025-11-27T18:18:40.130557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471300568296440:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:40.135386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:40.103640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:40.033662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471298138584503:2109] [node 5] ICP22 created new session: [2:7577471302433551828:2109]
2025-11-27T18:18:40.109433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471296204308930:2049] [node 2] ICP22 created new session: [6:7577471300499276326:2049]
2025-11-27T18:18:39.950462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471297063694119:2049] [node 2] ICP22 created new session: [3:7577471297063694224:2049]
2025-11-27T18:18:40.035380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471302433551828:2109] [node 5] ICS09 handshake done sender: [2:7577471298138584510:2116] self: [2:7577471298138584508:0] peer: [5:7577471296273329129:0] socket: 555 qp: -1
2025-11-27T18:18:39.958504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471297063694209:2126] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:40.136709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471296273329034:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(31262093ns) {HandleHandshakeStatus:329(31259389ns) {DropIncomingHandshake:442(1461ns) DropOutgoingHandshake:457(7142ns) SwitchToState:179(1144ns) ProcessPendingSessionEvents:495(20613909ns) {ForwardSessionEventToSession:612(1863846ns) ForwardSessionEventToSession:612(1732812ns) ForwardSessionEventToSession:612(9537ns) ForwardSessionEventToSession:612(3031ns) ForwardSessionEventToSession:612(3428554ns) ForwardSessionEventToSession:612(3385ns) ForwardSessionEventToSession:612(1694644ns) ForwardSessionEventToSession:612(4711ns) ForwardSessionEventToSession:612(3795ns) ForwardSessionEventToSession:612(11843410ns)}}}
2025-11-27T18:18:40.036107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471296288352834:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(76034496ns) {HandleHandshakeStatus:329(76031429ns) {DropIncomingHandshake:442(535ns) DropOutgoingHandshake:457(4601ns) SwitchToState:179(294ns) ProcessPendingSessionEvents:495(25276049ns) {ForwardSessionEventToSession:612(4121034ns) ForwardSessionEventToSession:612(10689ns) ForwardSessionEventToSession:612(6770223ns) ForwardSessionEventToSession:612(3803ns) ForwardSessionEventToSession:612(7341234ns) ForwardSessionEventToSession:612(2843075ns) ForwardSessionEventToSession:612(3548ns) ForwardSessionEventToSession:612(4150127ns) ForwardSessionEventToSession:612(4167ns) ForwardSessionEventToSession:612(5288ns)}}}
2025-11-27T18:18:40.094792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:40.036145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471302433551828:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:40.096931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:40.036159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471296288352930:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.036519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471302433551827:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41537
2025-11-27T18:18:40.036449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471296288352833:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:40.036611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:40.118223Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471299751198735:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(29776373ns) {HandleHandshakeStatus:329(29773692ns) {DropIncomingHandshake:442(13292ns) DropOutgoingHandshake:457(260ns) SwitchToState:179(371ns) ProcessPendingSessionEvents:495(4002772ns) {ForwardSessionEventToSession:612(1785797ns) ForwardSessionEventToSession:612(313130ns) ForwardSessionEventToSession:612(4551ns) ForwardSessionEventToSession:612(573758ns) ForwardSessionEventToSession:612(373852ns) ForwardSessionEventToSession:612(3773ns) ForwardSessionEventToSession:612(3601ns) ForwardSessionEventToSession:612(914908ns) ForwardSessionEventToSession:612(3736ns) ForwardSessionEventToSession:612(10376ns)}}}
2025-11-27T18:18:40.052470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471296288352911:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:40.109505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471300499276326:2049] [node 2] ICS09 handshake done sender: [6:7577471296204309015:2120] self: [6:7577471300499276320:0] peer: [2:7577471298138584511:0] socket: 536 qp: -1
2025-11-27T18:18:40.046210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471302433551827:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.109609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471296204308929:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:40.061321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471300583320227:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50231
2025-11-27T18:18:40.049114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471298138584512:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:40.113294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471300499276326:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:40.065771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471296288352900:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:40.061814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471298138584501:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:40.121352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:40.061885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:40.121598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471296204308930:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(18053261ns) {HandleHandshakeStatus:329(18050798ns) {DropIncomingHandshake:442(1032ns) DropOutgoingHandshake:457(190ns) SwitchToState:179(1567ns) ProcessPendingSessionEvents:495(749ns)}}
2025-11-27T18:18:40.082229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:40.084015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471296288352900:2108] [node 5] ICP22 created new session: [4:7577471300583320231:2108]
2025-11-27T18:18:40.121957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:40.070011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471298138584501:2111] [node 6] ICP22 created new session: [2:7577471302433551831:2111]
2025-11-27T18:18:40.087171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471300583320231:2108] [node 5] ICS09 handshake done sender: [4:7577471296288352911:2118] self: [4:7577471296288352910:0] peer: [5:7577471300568296430:0] socket: 566 qp: -1
2025-11-27T18:18:39.959032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471297063694224:2049] [node 2] ICS09 handshake done sender: [3:7577471297063694203:2123] self: [3:7577471297063694206:0] peer: [2:7577471298138584504:0] socket: 521 qp: -1
2025-11-27T18:18:40.122764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471296204308929:2048] [node 1] ICP22 created new session: [6:7577471300499276328:2048]
2025-11-27T18:18:40.078642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471302433551831:2111] [node 6] ICS09 handshake done sender: [2:7577471298138584512:2117] self: [2:7577471298138584511:0] peer: [6:7577471300499276320:0] socket: 572 qp: -1
2025-11-27T18:18:40.130559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471300499276328:2048] [node 1] ICS09 handshake done sender: [6:7577471296204309022:2122] self: [6:7577471300499276322:0] peer: [1:7577471299751198744:0] socket: 540 qp: -1
2025-11-27T18:18:40.088860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471300583320227:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.084072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471302433551831:2111] [node 6] ICS10 traffic start
2025-11-27T18:18:40.088922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471300583320231:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:40.134562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471300499276328:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:40.130707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471299751198736:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(19682334ns) {HandleHandshakeStatus:329(19679712ns) {DropIncomingHandshake:442(46851ns) DropOutgoingHandshake:457(584ns) SwitchToState:179(676ns) ProcessPendingSessionEvents:495(3543551ns) {ForwardSessionEventToSession:612(1249227ns) ForwardSessionEventToSession:612(6348ns) ForwardSessionEventToSession:612(270624ns) ForwardSessionEventToSession:612(2796ns) ForwardSessionEventToSession:612(2788ns) ForwardSessionEventToSession:612(760683ns) ForwardSessionEventToSession:612(3471ns) ForwardSessionEventToSession:612(485342ns) ForwardSessionEventToSession:612(745544ns) ForwardSessionEventToSession:612(3186ns)}}}
2025-11-27T18:18:40.084145Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471298138584503:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(30586029ns) {HandleHandshakeStatus:329(30583554ns) {DropIncomingHandshake:442(2508ns) DropOutgoingHandshake:457(2807ns) SwitchToState:179(907ns) ProcessPendingSessionEvents:495(12688646ns) {ForwardSessionEventToSession:612(2186219ns) ForwardSessionEventToSession:612(3922ns) ForwardSessionEventToSession:612(5042681ns) ForwardSessionEventToSession:612(15044ns) ForwardSessionEventToSession:612(3369ns) ForwardSessionEventToSession:612(2037001ns) ForwardSessionEventToSession:612(3372ns) ForwardSessionEventToSession:612(3363921ns) ForwardSessionEventToSession:612(5319ns) ForwardSessionEventToSession:612(3394ns)}}}
2025-11-27T18:18:39.961216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471297063694224:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:40.135115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:40.093483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471300583320230:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50231
2025-11-27T18:18:40.086214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:40.093902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471298138584503:2109] [node 5] ICP18 handshake (actor [2:7577471302433551827:2122]) is too old
2025-11-27T18:18:39.968048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:40.137709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471296204308929:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(26473961ns) {HandleHandshakeStatus:329(26471752ns) {DropIncomingHandshake:442(730ns) DropOutgoingHandshake:457(227ns) SwitchToState:179(639ns) ProcessPendingSessionEvents:495(1119ns)}}
2025-11-27T18:18:40.093724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:39.980564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471297063694118:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:40.119009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471298138584501:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(39644622ns) {HandleHandshakeStatus:329(39640609ns) {DropIncomingHandshake:442(18620ns) DropOutgoingHandshake:457(564ns) SwitchToState:179(586ns) ProcessPendingSessionEvents:495(7772674ns) {ForwardSessionEventToSession:612(600136ns) ForwardSessionEventToSession:612(537904ns) ForwardSessionEventToSession:612(17485ns) ForwardSessionEventToSession:612(3471ns) ForwardSessionEventToSession:612(848731ns) ForwardSessionEventToSession:612(3739ns) ForwardSessionEventToSession:612(2381656ns) ForwardSessionEventToSession:612(6966ns) ForwardSessionEventToSession:612(2684ns) ForwardSessionEventToSession:612(1245455ns)}}}
2025-11-27T18:18:40.095486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471296288352833:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28102) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471296288352930:2123] held: no
2025-11-27T18:18:40.100219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471296288352930:2123] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:39.980621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:40.102517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471296288352833:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:40.138745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471296273329104:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(28792335ns) {HandleHandshakeStatus:329(28790086ns) {DropIncomingHandshake:442(1783ns) DropOutgoingHandshake:457(13813ns) SwitchToState:179(1156ns) ProcessPendingSessionEvents:495(1908540ns) {ForwardSessionEventToSession:612(662313ns) ForwardSessionEventToSession:612(339221ns) ForwardSessionEventToSession:612(20157ns) ForwardSessionEventToSession:612(2917ns) ForwardSessionEventToSession:612(8041ns) ForwardSessionEventToSession:612(249556ns) ForwardSessionEventToSession:612(3784ns) ForwardSessionEventToSession:612(400037ns) ForwardSessionEventToSession:612(3411ns) ForwardSessionEventToSession:612(197870ns)}}}
2025-11-27T18:18:39.981166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471297063694195:2107] [node 4] ICP18 handshake (actor [3:7577471297063694214:2130]) is too old
2025-11-27T18:18:40.104880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:39.982967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471297063694118:2048] [node 1] ICP22 created new session: [3:7577471297063694226:2048]
2025-11-27T18:18:40.105405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471296288352833:2048] [node 1] ICP22 created new session: [4:7577471300583320233:2048]
2025-11-27T18:18:39.986395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471297063694226:2048] [node 1] ICS09 handshake done sender: [3:7577471297063694209:2126] self: [3:7577471297063694213:0] peer: [1:7577471299751198734:0] socket: 526 qp: -1
2025-11-27T18:18:40.109516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471300583320233:2048] [node 1] ICS09 handshake done sender: [4:7577471296288352930:2123] self: [4:7577471300583320228:0] peer: [1:7577471299751198747:0] socket: 530 qp: -1
2025-11-27T18:18:39.987543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471297063694226:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:40.117531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471300583320233:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:39.987584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:40.122045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471296288352900:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(36792157ns) {HandleHandshakeStatus:329(36789907ns) {DropIncomingHandshake:442(2520ns) DropOutgoingHandshake:457(2085ns) SwitchToState:179(764ns) ProcessPendingSessionEvents:495(7034094ns) {ForwardSessionEventToSession:612(2342592ns) ForwardSessionEventToSession:612(700813ns) ForwardSessionEventToSession:612(1653715ns) ForwardSessionEventToSession:612(4045ns) ForwardSessionEventToSession:612(2882ns) ForwardSessionEventToSession:612(1744168ns) ForwardSessionEventToSession:612(5459ns) ForwardSessionEventToSession:612(328971ns) ForwardSessionEventToSession:612(4117ns) ForwardSessionEventToSession:612(227296ns)}}}
2025-11-27T18:18:39.993837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471297063694119:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(45529511ns) {HandleHandshakeStatus:329(45527011ns) {DropIncomingHandshake:442(872ns) DropOutgoingHandshake:457(13104ns) SwitchToState:179(665ns) ProcessPendingSessionEvents:495(5985502ns) {ForwardSessionEventToSession:612(5898192ns) ForwardSessionEventToSession:612(8070ns) ForwardSessionEventToSession:612(8583ns) ForwardSessionEventToSession:612(3414ns) ForwardSessionEventToSession:612(3339ns) ForwardSessionEventToSession:612(6012ns) ForwardSessionEventToSession:612(3960ns) ForwardSessionEventToSession:612(7992ns) ForwardSessionEventToSession:612(4041ns) ForwardSessionEventToSession:612(3321ns)}}}
2025-11-27T18:18:40.122256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:39.997885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471297063694192:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(39071967ns) {HandleHandshakeStatus:329(39069302ns) {DropIncomingHandshake:442(1045ns) DropOutgoingHandshake:457(394ns) SwitchToState:179(209ns) ProcessPendingSessionEvents:495(18209114ns) {ForwardSessionEventToSession:612(9272573ns) ForwardSessionEventToSession:612(6867ns) ForwardSessionEventToSession:612(1242223ns) ForwardSessionEventToSession:612(1440611ns) ForwardSessionEventToSession:612(5116ns) ForwardSessionEventToSession:612(2988ns) ForwardSessionEventToSession:612(5054514ns) ForwardSessionEventToSession:612(4403ns) ForwardSessionEventToSession:612(1141417ns) ForwardSessionEventToSession:612(7034ns)}}}
2025-11-27T18:18:39.999392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471297063694192:2110] [node 6] ICP18 handshake (actor [3:7577471297063694210:2127]) is too old
2025-11-27T18:18:40.027634Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471297063694118:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(53392461ns) {HandleHandshakeStatus:329(53390271ns) {DropIncomingHandshake:442(2980ns) DropOutgoingHandshake:457(5676ns) SwitchToState:179(699ns) ProcessPendingSessionEvents:495(19924606ns) {ForwardSessionEventToSession:612(6177530ns) ForwardSessionEventToSession:612(9249ns) ForwardSessionEventToSession:612(2708ns) ForwardSessionEventToSession:612(2993ns) ForwardSessionEventToSession:612(207869ns) ForwardSessionEventToSession:612(5540208ns) ForwardSessionEventToSession:612(7118ns) ForwardSessionEventToSession:612(4848471ns) ForwardSessionEventToSession:612(2349ns) ForwardSessionEventToSession:612(3107890ns)}}}
2025-11-27T18:18:40.122416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471296288352833:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19970606ns) {HandleHandshakeStatus:329(19968371ns) {DropIncomingHandshake:442(1072ns) DropOutgoingHandshake:457(162ns) SwitchToState:179(641ns) ProcessPendingSessionEvents:495(1143ns)}}
2025-11-27T18:18:40.027726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471297063694118:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:40.123241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471296288352900:2108] [node 5] ICP18 handshake (actor [4:7577471300583320227:2124]) is too old
2025-11-27T18:18:40.036540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471297063694208:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:40.036567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471297063694208:2125] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:40.073552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471297063694208:2125] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:40.073898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471297063694194:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:40.073979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:40.078676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471297063694194:2109] [node 5] ICP22 created new session: [3:7577471301358661526:2109]
2025-11-27T18:18:40.086059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471301358661526:2109] [node 5] ICS09 handshake done sender: [3:7577471297063694208:2125] self: [3:7577471297063694207:0] peer: [5:7577471300568296433:0] socket: 541 qp: -1
2025-11-27T18:18:40.092098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471301358661526:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:40.092158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:40.109553Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471297063694194:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(28147323ns) {HandleHandshakeStatus:329(28143917ns) {DropIncomingHandshake:442(26549ns) DropOutgoingHandshake:457(620ns) SwitchToState:179(702ns) ProcessPendingSessionEvents:495(7958270ns) {ForwardSessionEventToSession:612(1525716ns) ForwardSessionEventToSession:612(2228173ns) ForwardSessionEventToSession:612(9998ns) ForwardSessionEventToSession:612(2761ns) ForwardSessionEventToSession:612(588460ns) ForwardSessionEventToSession:612(3788ns) ForwardSessionEventToSession:612(2914ns) ForwardSessionEventToSession:612(1979408ns) ForwardSessionEventToSession:612(2956ns) ForwardSessionEventToSession:612(1595246ns)}}}
2025-11-27T18:18:40.212814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:40.213100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471298138584526:2048] [node 1] ICS07 socket disconnect 529 reason# EndOfStream
2025-11-27T18:18:40.213147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471298138584526:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.213244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:40.213318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471298138584526:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:40.214591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:40.214769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471297063694226:2048] [node 1] ICS01 socket: 526 reason# EndOfStream
2025-11-27T18:18:40.214804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471297063694118:2048] [node 1] ICP30 unregister session Session# [3:7577471297063694226:2048] VirtualId# [3:7577471297063694213:0]
2025-11-27T18:18:40.214826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471297063694226:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.214910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:40.214949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:40.215844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471298138584432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28102) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:40.215896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471298138584526:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:40.217497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471298138584432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28102) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:40.217698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471298138584526:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:40.217752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471298138584432:2048] [node 1] ICP30 unregister session Session# [2:7577471298138584526:2048] VirtualId# [2:7577471298138584518:0]
2025-11-27T18:18:40.217805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:40.217855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471298138584432:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28102) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.217871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28102) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.219665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:40.219902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471296273329131:2048] [node 1] ICS01 socket: 535 reason# EndOfStream
2025-11-27T18:18:40.219945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471296273329032:2048] [node 1] ICP30 unregister session Session# [5:7577471296273329131:2048] VirtualId# [5:7577471296273329121:0]
2025-11-27T18:18:40.219976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471296273329131:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.220041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:40.220107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:40.221565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:40.221763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471300499276328:2048] [node 1] ICS01 socket: 540 reason# EndOfStream
2025-11-27T18:18:40.221805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471296204308929:2048] [node 1] ICP30 unregister session Session# [6:7577471300499276328:2048] VirtualId# [6:7577471300499276322:0]
2025-11-27T18:18:40.221836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471300499276328:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.221901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:40.221944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:40.221417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:40.221630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471300583320233:2048] [node 1] ICS01 socket: 530 reason# EndOfStream
2025-11-27T18:18:40.221660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471296288352833:2048] [node 1] ICP30 unregister session Session# [4:7577471300583320233:2048] VirtualId# [4:7577471300583320228:0]
2025-11-27T18:18:40.221696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471300583320233:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.221762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:40.221820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:40.228398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471298138584432:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:40.290700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:40.290981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471297063694224:2049] [node 2] ICS01 socket: 521 reason# EndOfStream
2025-11-27T18:18:40.291037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471297063694119:2049] [node 2] ICP30 unregister session Session# [3:7577471297063694224:2049] VirtualId# [3:7577471297063694206:0]
2025-11-27T18:18:40.291074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471297063694224:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.291158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:40.291210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:40.293984Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:40.294215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471296288352926:2049] [node 2] ICS07 socket disconnect 537 reason# EndOfStream
2025-11-27T18:18:40.294260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471296288352926:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.294341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:40.294439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471296288352926:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:40.294958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:40.295128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471300568296435:2049] [node 2] ICS07 socket disconnect 557 reason# EndOfStream
2025-11-27T18:18:40.295174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471300568296435:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.295242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:40.295280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471296288352834:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16802) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:40.295281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471300568296435:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:40.295598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471296288352926:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:40.295660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471296288352834:2049] [node 2] ICP30 unregister session Session# [4:7577471296288352926:2049] VirtualId# [4:7577471296288352918:0]
2025-11-27T18:18:40.295721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:40.295773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471296288352834:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16802) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.295795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16802) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.296327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471296273329033:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16802) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:40.296352Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:40.296385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471300568296435:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:40.296437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471296273329033:2049] [node 2] ICP30 unregister session Session# [5:7577471300568296435:2049] VirtualId# [5:7577471296273329129:0]
2025-11-27T18:18:40.296501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:40.296538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471296273329033:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16802) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.296573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16802) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.296540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471300499276326:2049] [node 2] ICS01 socket: 536 reason# EndOfStream
2025-11-27T18:18:40.296596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471296204308930:2049] [node 2] ICP30 unregister session Session# [6:7577471300499276326:2049] VirtualId# [6:7577471300499276320:0]
2025-11-27T18:18:40.296627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471300499276326:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.296709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:40.296743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:40.305656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471296288352834:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:40.306479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471296273329033:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:40.404762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:40.405058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471300568296438:2050] [node 3] ICS07 socket disconnect 510 reason# EndOfStream
2025-11-27T18:18:40.405097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471300568296438:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.405201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:40.405279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471300568296438:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:40.404847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:40.405085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471296288352927:2050] [node 3] ICS07 socket disconnect 514 reason# EndOfStream
2025-11-27T18:18:40.405112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471296288352927:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.405215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:40.405300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471296288352927:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:40.406364Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471296288352835:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22173) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:40.406450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471296288352927:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:40.406482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471296288352835:2050] [node 3] ICP30 unregister session Session# [4:7577471296288352927:2050] VirtualId# [4:7577471296288352916:0]
2025-11-27T18:18:40.406508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471296273329034:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22173) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:40.406295Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:40.410790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:40.414135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471296204309020:2050] [node 3] ICS07 socket disconnect 522 reason# EndOfStream
2025-11-27T18:18:40.414127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471300568296438:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:40.413613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471296288352835:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.414187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471296204309020:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.413640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.424819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471296273329034:2050] [node 3] ICP30 unregister session Session# [5:7577471300568296438:2050] VirtualId# [5:7577471300568296433:0]
2025-11-27T18:18:40.420819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:40.420920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471296204309020:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:40.424891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:40.429759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471296288352835:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:40.429424Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471296273329034:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.429462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.429855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [5:7577471296273329034:2050] [node 3] ICP35 event processing took too much time StateWork:294(23066834ns) {HandleHandshakeStatus:395(23052338ns) {LogHandshakeFail:472(9486ns) {UpdateErrorStateLog:545(1949ns)} DropOutgoingHandshake:457(717ns) ProcessPendingSessionEvents:495(426ns) UnregisterSession:538(4056477ns) {ProcessPendingSessionEvents:495(1672ns) SwitchToInitialState:202(3436ns) {SwitchToState:179(2171ns)}} TransitToErrorState:752(108572ns) {SwitchToState:179(6708ns) ProcessPendingSessionEvents:495(530ns) DropHandshakes:470(3059ns) {DropIncomingHandshake:442(722ns) DropOutgoingHandshake:457(425ns)}}}}
2025-11-27T18:18:40.430413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471296204308931:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22173) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:40.430539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471296204309020:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:40.430590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471296204308931:2050] [node 3] ICP30 unregister session Session# [6:7577471296204309020:2050] VirtualId# [6:7577471296204309008:0]
2025-11-27T18:18:40.430636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:40.430677Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471296204308931:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.430703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.439642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471296273329034:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:40.440641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471296204308931:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:40.506242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:40.506492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471300568296440:2108] [node 4] ICS07 socket disconnect 569 reason# EndOfStream
2025-11-27T18:18:40.506536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471300568296440:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.506613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:40.506674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471300568296440:2108] [node 4] ICS15 start handshake
2025-11-27T18:18:40.507702Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:40.507946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471296204309018:2110] [node 4] ICS01 socket: 552 reason# EndOfStream
2025-11-27T18:18:40.507987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471296204308995:2110] [node 4] ICP30 unregister session Session# [6:7577471296204309018:2110] VirtualId# [6:7577471296204309011:0]
2025-11-27T18:18:40.508018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471296204309018:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.508084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:40.508128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:40.509248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471296273329104:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14276) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:40.509487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471300568296440:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:40.509548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471296273329104:2108] [node 4] ICP30 unregister session Session# [5:7577471300568296440:2108] VirtualId# [5:7577471300568296430:0]
2025-11-27T18:18:40.509613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:40.509664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471296273329104:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14276) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.509693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14276) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:40.556886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:40.557124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471300499276321:2106] [node 5] ICS01 socket: 544 reason# EndOfStream
2025-11-27T18:18:40.557179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471296204308991:2106] [node 5] ICP30 unregister session Session# [6:7577471300499276321:2106] VirtualId# [6:7577471296204309012:0]
2025-11-27T18:18:40.557224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471300499276321:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:40.557369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:40.557438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:40.698930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471302851619151:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:40.698979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471302851619152:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:40.719937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471302761446610:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:40.720021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471302761446611:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:40.741612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471303126859482:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:40.741691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471303126859483:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:40.763935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471300627227507:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:40.764003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471300627227508:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:40.764042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471300627227509:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:40.789502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471301017261012:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:40.789561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471301017261013:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:40.789606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471301017261014:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:40.802946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471300946852777:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:40.803016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471300946852778:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:40.803050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471300946852779:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:40.872699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471300946852839:2105] [node 5] ICP01 ready to work
2025-11-27T18:18:40.894540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471300946852839:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(21512429ns) {Configure:104(21506897ns) {ProcessConfigured:135(65325ns) {StartInitialHandshake:157(61677ns) {DropHandshakes:470(3430ns) {DropIncomingHandshake:442(683ns) DropOutgoingHandshake:457(460ns)} GenerateSessionVirtualId:988(789ns) PrepareNewSessionHandshake:477(2000ns) {SwitchToState:179(732ns)}}}}}
2025-11-27T18:18:40.959417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471302851619214:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:40.959830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471302851619215:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:40.962719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471300946852844:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:40.962865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471301017261075:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:40.962508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471300627227570:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:40.963176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471303126859545:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:40.963173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471302851619216:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:40.963360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471302761446682:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:40.963901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471301017261076:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:40.965363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471300627227571:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:40.965531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471302761446683:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:40.965313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471303126859546:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:40.965712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471303126859547:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:40.971433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471302761446686:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:40.993243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471302851619214:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17184129ns) {Configure:104(17179897ns) {ProcessConfigured:135(532184ns) {StartInitialHandshake:157(529184ns) {DropHandshakes:470(2685ns) {DropIncomingHandshake:442(693ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(1206ns) PrepareNewSessionHandshake:477(5666ns) {SwitchToState:179(1358ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:40.994554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471303126859558:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11555
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.007471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471307421826858:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11555
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.008251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471304922194881:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16395
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.009581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471307421826858:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.009724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471303126859558:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.009783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471307146586523:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21015
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.009968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471307146586524:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21015
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.010042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471301017261081:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52813
2025-11-27T18:18:41.010116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471307146586525:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21015
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.013779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471307146586523:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.013701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471307056413989:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51480
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.018248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471307421826859:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11555
2025-11-27T18:18:41.018350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471307421826860:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11555
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.015248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471307146586524:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.014101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471307056413990:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51480
2025-11-27T18:18:41.018955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471304922194882:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16395
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.019526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471307056413989:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.019451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471307146586529:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21015
2025-11-27T18:18:41.019643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471304922194883:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16395
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.020609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471307056413996:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51480
2025-11-27T18:18:41.021152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471307056413997:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51480
2025-11-27T18:18:41.022213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471301017261081:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.022700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471304922194881:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.023080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471300627227509:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.023772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471307056413990:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.023762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471303126859553:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.023799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471303126859553:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.024355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471307421826860:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.024680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471307421826863:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11555
2025-11-27T18:18:41.025557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471303126859553:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.025695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471304922194885:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16395
2025-11-27T18:18:41.026140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471303126859545:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.026413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:41.026696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471303126859545:2107] [node 4] ICP22 created new session: [3:7577471307421826866:2107]
2025-11-27T18:18:41.026925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471307421826866:2107] [node 4] ICS09 handshake done sender: [3:7577471303126859553:2112] self: [3:7577471303126859552:0] peer: [4:7577471304922194884:0] socket: 528 qp: -1
2025-11-27T18:18:41.026967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471307421826866:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:41.026932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471304922194881:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:41.027020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:41.027040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471307146586525:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.027209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471307146586529:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.027473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471307146586530:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21015
2025-11-27T18:18:41.027280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471300627227509:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.027341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:41.027385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471300627227509:2050] [node 3] ICP22 created new session: [4:7577471304922194886:2050]
2025-11-27T18:18:41.027416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471304922194886:2050] [node 3] ICS09 handshake done sender: [4:7577471304922194881:2115] self: [4:7577471304922194884:0] peer: [3:7577471303126859552:0] socket: 530 qp: -1
2025-11-27T18:18:41.027449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471304922194886:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:41.027486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:41.030267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471307421826859:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.030389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471307056413997:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.030700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471307146586530:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.030831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471307056413996:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.030180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471304922194882:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.030871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471307421826863:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.030524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471300627227507:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.030905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471302851619222:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.030934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471302851619222:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.031124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471302761446610:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.031160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471303126859482:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.032691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471303126859483:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.032808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471302851619218:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.032844Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471302851619218:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.033501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471304922194883:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.033620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471300627227508:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.033240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471302761446688:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.033297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471302761446688:2120] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.035152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471302851619220:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.035190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471302851619220:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.035265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305312228388:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52813
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.036483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305312228389:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52813
2025-11-27T18:18:41.036658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471305312228388:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.037120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471304922194891:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16395
2025-11-27T18:18:41.035522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471302761446690:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.035546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471302761446690:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.036779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471302761446688:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.037825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471307421826871:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11555
2025-11-27T18:18:41.038927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471302761446690:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:41.038587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471305312228389:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.039354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471301017261075:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.039748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471302851619222:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:41.039882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471302761446611:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.040045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471301017261014:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.039990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:41.040043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471302761446611:2049] [node 3] ICP22 created new session: [2:7577471307056414004:2049]
2025-11-27T18:18:41.040087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471307056414004:2049] [node 3] ICS09 handshake done sender: [2:7577471302761446688:2120] self: [2:7577471302761446687:0] peer: [3:7577471307421826870:0] socket: 549 qp: -1
2025-11-27T18:18:41.040127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471307056414004:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:41.041375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471300627227579:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.040162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:41.041418Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471300627227579:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.041579Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471302851619218:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:41.041547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471303126859555:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.041652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471302761446682:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.041964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:41.041584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471303126859555:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.042180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471304922194883:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:41.042204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471302761446682:2107] [node 4] ICP22 created new session: [2:7577471307056414006:2107]
2025-11-27T18:18:41.042249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471307056414006:2107] [node 4] ICS09 handshake done sender: [2:7577471302761446690:2121] self: [2:7577471302761446689:0] peer: [4:7577471304922194890:0] socket: 553 qp: -1
2025-11-27T18:18:41.042685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471307056414006:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:41.042967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:41.043086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471307421826863:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:41.043183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471300627227508:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.043195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471302851619220:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:41.043240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:41.043329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471300627227508:2049] [node 2] ICP22 created new session: [4:7577471304922194894:2049]
2025-11-27T18:18:41.043387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471304922194894:2049] [node 2] ICS09 handshake done sender: [4:7577471304922194883:2117] self: [4:7577471304922194890:0] peer: [2:7577471302761446689:0] socket: 554 qp: -1
2025-11-27T18:18:41.043444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471304922194894:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:41.043492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:41.043672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471302851619214:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.043754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:41.043794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471302851619214:2107] [node 4] ICP22 created new session: [1:7577471307146586539:2107]
2025-11-27T18:18:41.043715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471303126859483:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.043847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471307146586539:2107] [node 4] ICS09 handshake done sender: [1:7577471302851619222:2112] self: [1:7577471302851619221:0] peer: [4:7577471304922194889:0] socket: 540 qp: -1
2025-11-27T18:18:41.043765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:41.043808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471303126859483:2049] [node 2] ICP22 created new session: [3:7577471307421826874:2049]
2025-11-27T18:18:41.043912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471307146586539:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:41.043946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:41.043849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471307421826874:2049] [node 2] ICS09 handshake done sender: [3:7577471307421826863:2119] self: [3:7577471307421826870:0] peer: [2:7577471302761446687:0] socket: 557 qp: -1
2025-11-27T18:18:41.043879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471307421826874:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:41.043937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:41.045292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471302851619151:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.045813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:41.045863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471302851619151:2048] [node 2] ICP22 created new session: [1:7577471307146586541:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.046011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471307146586541:2048] [node 2] ICS09 handshake done sender: [1:7577471302851619218:2110] self: [1:7577471302851619217:0] peer: [2:7577471307056414001:0] socket: 537 qp: -1
2025-11-27T18:18:41.046076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471307146586541:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.046523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471304922194892:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16395
2025-11-27T18:18:41.046110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:41.046942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305312228392:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52813
2025-11-27T18:18:41.046781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471302851619152:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.047162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:41.047247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471302851619152:2049] [node 3] ICP22 created new session: [1:7577471307146586543:2049]
2025-11-27T18:18:41.047332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471307146586543:2049] [node 3] ICS09 handshake done sender: [1:7577471302851619220:2111] self: [1:7577471302851619219:0] peer: [3:7577471307421826869:0] socket: 539 qp: -1
2025-11-27T18:18:41.047374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471307146586543:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:41.047409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:41.047507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471304922194882:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.047720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471307056414008:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51480
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.050767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305312228393:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52813
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.051541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471300627227579:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.051989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305312228394:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52813
2025-11-27T18:18:41.052005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471307056413996:2126] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:41.052042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471305241820151:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.052409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471300627227507:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.052635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:41.052701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471300627227507:2048] [node 1] ICP22 created new session: [4:7577471304922194897:2048]
2025-11-27T18:18:41.052737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471307421826872:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11555
2025-11-27T18:18:41.052906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471304922194897:2048] [node 1] ICS09 handshake done sender: [4:7577471304922194882:2116] self: [4:7577471304922194889:0] peer: [1:7577471302851619221:0] socket: 544 qp: -1
2025-11-27T18:18:41.053141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471302761446610:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.053076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471304922194897:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:41.053158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471305312228388:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:41.053213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.055086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471305241820152:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39759
2025-11-27T18:18:41.053600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471301017261075:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.053650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:41.053694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471301017261075:2108] [node 4] ICP22 created new session: [5:7577471305312228395:2108]
2025-11-27T18:18:41.053744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471305312228395:2108] [node 4] ICS09 handshake done sender: [5:7577471305312228388:2116] self: [5:7577471305312228390:0] peer: [4:7577471300627227578:0] socket: 560 qp: -1
2025-11-27T18:18:41.053775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471305312228395:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:41.053823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:41.053141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:41.053295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471302761446610:2048] [node 1] ICP22 created new session: [2:7577471307056414011:2048]
2025-11-27T18:18:41.053462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471307056414011:2048] [node 1] ICS09 handshake done sender: [2:7577471307056413996:2126] self: [2:7577471307056414001:0] peer: [1:7577471302851619217:0] socket: 550 qp: -1
2025-11-27T18:18:41.053492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471307056414011:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:41.056589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.058296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305312228398:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52813
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.058298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471305241820152:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.060082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471305241820153:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39759
2025-11-27T18:18:41.060653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471300946852839:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.060965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471300627227570:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.060996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471303126859555:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:41.061052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:41.061013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471305312228383:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.061297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471300627227570:2108] [node 5] ICP22 created new session: [4:7577471304922194899:2108]
2025-11-27T18:18:41.061578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471304922194899:2108] [node 5] ICS09 handshake done sender: [4:7577471300627227579:2113] self: [4:7577471300627227578:0] peer: [5:7577471305312228390:0] socket: 520 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.061047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471305312228383:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.062265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471304922194899:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:41.062456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471305241820155:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39759
2025-11-27T18:18:41.062491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:41.063751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471300627227507:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:41.063701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471302761446610:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:41.063760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471305241820153:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.064172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471307146586522:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.064210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471305312228398:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.063912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471300946852777:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.064472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471305312228389:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:41.064203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471307146586522:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.064109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471305241820151:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.064204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471300946852778:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.064537Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471307056413992:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.064573Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471307421826859:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:41.065833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471307146586522:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.064570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471307056413992:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.066472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471305241820158:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39759
2025-11-27T18:18:41.067188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471307056413992:2125] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:41.066367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471305312228394:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.067921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471305241820159:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39759
2025-11-27T18:18:41.066416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471301017261014:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.068574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471305241820153:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:41.068832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471305241820160:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39759
2025-11-27T18:18:41.069057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:41.069304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471303126859482:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5970) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471307421826859:2117] held: no
2025-11-27T18:18:41.069120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471301017261014:2050] [node 3] ICP22 created new session: [5:7577471305312228400:2050]
2025-11-27T18:18:41.069463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471303126859482:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.069404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471302851619216:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.069494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:41.069496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:41.070424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471303126859482:2048] [node 1] ICP22 created new session: [3:7577471307421826876:2048]
2025-11-27T18:18:41.069545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471302851619216:2109] [node 6] ICP22 created new session: [1:7577471307146586548:2109]
2025-11-27T18:18:41.071098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471307421826876:2048] [node 1] ICS09 handshake done sender: [3:7577471307421826859:2117] self: [3:7577471307421826869:0] peer: [1:7577471302851619219:0] socket: 547 qp: -1
2025-11-27T18:18:41.071292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471302761446686:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.069602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471307146586548:2109] [node 6] ICS09 handshake done sender: [1:7577471307146586522:2114] self: [1:7577471307146586521:0] peer: [6:7577471305241820156:0] socket: 552 qp: -1
2025-11-27T18:18:41.071713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471307421826876:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:41.069633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471307146586548:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:41.071762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:41.071333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:41.073616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471300946852777:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.071392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471302761446686:2109] [node 6] ICP22 created new session: [2:7577471307056414014:2109]
2025-11-27T18:18:41.069667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:41.072183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471303126859546:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.071436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471307056414014:2109] [node 6] ICS09 handshake done sender: [2:7577471307056413992:2125] self: [2:7577471307056413991:0] peer: [6:7577471305241820157:0] socket: 573 qp: -1
2025-11-27T18:18:41.073692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:41.072225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:41.069188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471305312228400:2050] [node 3] ICS09 handshake done sender: [5:7577471305312228389:2117] self: [5:7577471305312228391:0] peer: [3:7577471303126859554:0] socket: 561 qp: -1
2025-11-27T18:18:41.072268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471303126859546:2108] [node 5] ICP22 created new session: [3:7577471307421826878:2108]
2025-11-27T18:18:41.073525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471307056414014:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:41.073737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471300946852777:2048] [node 1] ICP22 created new session: [6:7577471305241820161:2048]
2025-11-27T18:18:41.069228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471305312228400:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:41.072360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471307421826878:2108] [node 5] ICS09 handshake done sender: [3:7577471303126859555:2113] self: [3:7577471303126859554:0] peer: [5:7577471305312228391:0] socket: 523 qp: -1
2025-11-27T18:18:41.073592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:41.069305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:41.072396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471307421826878:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:41.072426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:41.074177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471305312228383:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:41.074243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471305241820161:2048] [node 1] ICS09 handshake done sender: [6:7577471305241820153:2117] self: [6:7577471305241820156:0] peer: [1:7577471307146586521:0] socket: 577 qp: -1
2025-11-27T18:18:41.074426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471305241820161:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:41.074467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:41.077540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471301017261012:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.077605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471301017261076:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.077723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:41.077789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471301017261076:2109] [node 6] ICP22 created new session: [5:7577471305312228403:2109]
2025-11-27T18:18:41.077839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471305312228403:2109] [node 6] ICS09 handshake done sender: [5:7577471305312228383:2115] self: [5:7577471305312228382:0] peer: [6:7577471305241820154:0] socket: 527 qp: -1
2025-11-27T18:18:41.077885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471305312228403:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:41.077926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:41.078002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471305241820151:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:41.078236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471300946852777:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:41.078441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471305241820155:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.078621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471300946852778:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.078663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:41.078703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471300946852778:2049] [node 2] ICP22 created new session: [6:7577471305241820164:2049]
2025-11-27T18:18:41.078747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471305241820164:2049] [node 2] ICS09 handshake done sender: [6:7577471305241820151:2115] self: [6:7577471305241820157:0] peer: [2:7577471307056413991:0] socket: 576 qp: -1
2025-11-27T18:18:41.078782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471305241820164:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:41.078815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:41.078790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471302851619224:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.078835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471302851619224:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.079525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471301017261013:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.079734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471302851619224:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:41.079990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471307056413988:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.080028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471307056413988:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.080214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471302851619215:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.080428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471305241820152:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:41.080407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:41.080820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305312228406:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52813
2025-11-27T18:18:41.080587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471302851619215:2108] [node 5] ICP22 created new session: [1:7577471307146586551:2108]
2025-11-27T18:18:41.081831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471305312228398:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:41.081953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471307056413988:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:41.080840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471307146586551:2108] [node 5] ICS09 handshake done sender: [1:7577471302851619224:2113] self: [1:7577471302851619223:0] peer: [5:7577471305312228402:0] socket: 536 qp: -1
2025-11-27T18:18:41.081321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471307146586551:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:41.081360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:41.083373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471301017261012:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.083419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:41.083497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471301017261012:2048] [node 1] ICP22 created new session: [5:7577471305312228408:2048]
2025-11-27T18:18:41.083541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471305312228408:2048] [node 1] ICS09 handshake done sender: [5:7577471305312228398:2121] self: [5:7577471305312228402:0] peer: [1:7577471302851619223:0] socket: 578 qp: -1
2025-11-27T18:18:41.083584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471305312228408:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:41.083659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:41.083343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471302761446683:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.083418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:41.083465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471302761446683:2108] [node 5] ICP22 created new session: [2:7577471307056414017:2108]
2025-11-27T18:18:41.083524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471307056414017:2108] [node 5] ICS09 handshake done sender: [2:7577471307056413988:2122] self: [2:7577471307056413987:0] peer: [5:7577471305312228405:0] socket: 531 qp: -1
2025-11-27T18:18:41.083574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471307056414017:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:41.083611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:41.084129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471300946852839:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.084175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:41.084235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471300946852839:2105] [node 5] ICP22 created new session: [6:7577471305241820167:2105]
2025-11-27T18:18:41.084276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471305241820167:2105] [node 5] ICS09 handshake done sender: [6:7577471305241820152:2116] self: [6:7577471305241820154:0] peer: [5:7577471305312228382:0] socket: 534 qp: -1
2025-11-27T18:18:41.084302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471305241820167:2105] [node 5] ICS10 traffic start
2025-11-27T18:18:41.084326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: 2025-11-27T18:18:41.084581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471300946852839:2105] [node 5] ICP27 obsolete handshake fail ignored
::ffff:127.1.0.0
2025-11-27T18:18:41.085051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471305241820163:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39759
2025-11-27T18:18:41.085311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471300946852844:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.085815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471305241820163:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.085877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471300627227581:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.085925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471300627227581:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.086877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471300946852779:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.087426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471303126859557:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.087459Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471303126859557:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.087645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471304922194901:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16395
2025-11-27T18:18:41.087721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305312228407:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52813
2025-11-27T18:18:41.087784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471300627227581:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:41.088227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471301017261013:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6345) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471305312228394:2120] held: no
2025-11-27T18:18:41.088218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471300627227571:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.088287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:41.088333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471300627227571:2109] [node 6] ICP22 created new session: [4:7577471304922194903:2109]
2025-11-27T18:18:41.088407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471303126859557:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:41.088487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471304922194903:2109] [node 6] ICS09 handshake done sender: [4:7577471300627227581:2114] self: [4:7577471300627227580:0] peer: [6:7577471305241820169:0] socket: 518 qp: -1
2025-11-27T18:18:41.088836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471305312228394:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:41.088517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471304922194903:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:41.088838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:41.089289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471301017261013:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.089356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.089399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471301017261013:2049] [node 2] ICP22 created new session: [5:7577471305312228410:2049]
2025-11-27T18:18:41.089458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471305312228410:2049] [node 2] ICS09 handshake done sender: [5:7577471305312228394:2120] self: [5:7577471305312228405:0] peer: [2:7577471307056413987:0] socket: 575 qp: -1
2025-11-27T18:18:41.089488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471305312228410:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:41.089489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471305241820172:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39759
2025-11-27T18:18:41.089523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.090123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471305241820173:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39759
2025-11-27T18:18:41.090740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471305241820155:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:41.091040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471305241820163:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:41.091369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471300946852844:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.091292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471303126859547:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.091428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:41.091366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:41.091465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471300946852844:2110] [node 4] ICP22 created new session: [6:7577471305241820174:2110]
2025-11-27T18:18:41.091421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471303126859547:2109] [node 6] ICP22 created new session: [3:7577471307421826882:2109]
2025-11-27T18:18:41.091463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471307421826882:2109] [node 6] ICS09 handshake done sender: [3:7577471303126859557:2114] self: [3:7577471303126859556:0] peer: [6:7577471305241820171:0] socket: 535 qp: -1
2025-11-27T18:18:41.091510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471305241820174:2110] [node 4] ICS09 handshake done sender: [6:7577471305241820155:2118] self: [6:7577471305241820169:0] peer: [4:7577471300627227580:0] socket: 526 qp: -1
2025-11-27T18:18:41.091540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471305241820174:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:41.091492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471307421826882:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:41.091524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:41.091598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:41.092041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471304922194901:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.092434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471300946852779:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.092486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:41.092453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471300627227571:2109] [node 6] ICP18 handshake (actor [4:7577471304922194901:2121]) is too old
2025-11-27T18:18:41.092535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471300946852779:2050] [node 3] ICP22 created new session: [6:7577471305241820176:2050]
2025-11-27T18:18:41.092582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471305241820176:2050] [node 3] ICS09 handshake done sender: [6:7577471305241820163:2122] self: [6:7577471305241820171:0] peer: [3:7577471303126859556:0] socket: 524 qp: -1
2025-11-27T18:18:41.092636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471305241820176:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:41.092668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:41.121003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:41.121244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471307056414011:2048] [node 1] ICS01 socket: 550 reason# EndOfStream
2025-11-27T18:18:41.121438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471302761446610:2048] [node 1] ICP30 unregister session Session# [2:7577471307056414011:2048] VirtualId# [2:7577471307056414001:0]
2025-11-27T18:18:41.121486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471307056414011:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.121571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:41.121612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:41.122313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:41.122474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471307421826876:2048] [node 1] ICS01 socket: 547 reason# EndOfStream
2025-11-27T18:18:41.122498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471303126859482:2048] [node 1] ICP30 unregister session Session# [3:7577471307421826876:2048] VirtualId# [3:7577471307421826869:0]
2025-11-27T18:18:41.122517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471307421826876:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.122578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:41.122617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:41.125032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:41.125203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471304922194897:2048] [node 1] ICS07 socket disconnect 544 reason# EndOfStream
2025-11-27T18:18:41.125230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471304922194897:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.125324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:41.125372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471304922194897:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:41.126262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:41.126285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471300627227507:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5970) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:41.126352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471304922194897:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:41.126406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471300627227507:2048] [node 1] ICP30 unregister session Session# [4:7577471304922194897:2048] VirtualId# [4:7577471304922194889:0]
2025-11-27T18:18:41.126463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:41.126480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471305312228408:2048] [node 1] ICS07 socket disconnect 578 reason# EndOfStream
2025-11-27T18:18:41.126497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471300627227507:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5970) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.126512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471305312228408:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.126512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5970) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.126577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:41.126617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471305312228408:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:41.127473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471301017261012:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5970) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:41.127520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471305312228408:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:41.127549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471301017261012:2048] [node 1] ICP30 unregister session Session# [5:7577471305312228408:2048] VirtualId# [5:7577471305312228402:0]
2025-11-27T18:18:41.127584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:41.127636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471301017261012:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5970) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.127663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5970) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.127601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:41.127888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471305241820161:2048] [node 1] ICS07 socket disconnect 577 reason# EndOfStream
2025-11-27T18:18:41.127925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471305241820161:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.128008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:41.128058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471305241820161:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:41.128748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471300946852777:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5970) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:41.128818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471305241820161:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:41.128862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471300946852777:2048] [node 1] ICP30 unregister session Session# [6:7577471305241820161:2048] VirtualId# [6:7577471305241820156:0]
2025-11-27T18:18:41.128899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:41.128930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471300946852777:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5970) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.128948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5970) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.135956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471300627227507:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:41.142161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471300946852777:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:41.141372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471301017261012:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:41.152722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:41.152973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471307421826874:2049] [node 2] ICS01 socket: 557 reason# EndOfStream
2025-11-27T18:18:41.153013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471303126859483:2049] [node 2] ICP30 unregister session Session# [3:7577471307421826874:2049] VirtualId# [3:7577471307421826870:0]
2025-11-27T18:18:41.153053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471307421826874:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.153144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:41.153223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:41.155223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:41.155383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471304922194894:2049] [node 2] ICS01 socket: 554 reason# EndOfStream
2025-11-27T18:18:41.155439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471300627227508:2049] [node 2] ICP30 unregister session Session# [4:7577471304922194894:2049] VirtualId# [4:7577471304922194890:0]
2025-11-27T18:18:41.155470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471304922194894:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.155526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:41.155565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:41.156189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:41.156401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471305312228410:2049] [node 2] ICS01 socket: 575 reason# EndOfStream
2025-11-27T18:18:41.156458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471301017261013:2049] [node 2] ICP30 unregister session Session# [5:7577471305312228410:2049] VirtualId# [5:7577471305312228405:0]
2025-11-27T18:18:41.156500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471305312228410:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.156580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:41.156618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:41.156944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:41.157141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471305241820164:2049] [node 2] ICS07 socket disconnect 576 reason# EndOfStream
2025-11-27T18:18:41.157181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471305241820164:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.157251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:41.157317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471305241820164:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:41.158120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471300946852778:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6345) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:41.158177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471305241820164:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:41.158207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471300946852778:2049] [node 2] ICP30 unregister session Session# [6:7577471305241820164:2049] VirtualId# [6:7577471305241820157:0]
2025-11-27T18:18:41.158261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:41.158291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471300946852778:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6345) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.158303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6345) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.169177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471300946852778:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:41.216270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:41.216533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471305312228400:2050] [node 3] ICS07 socket disconnect 561 reason# EndOfStream
2025-11-27T18:18:41.216573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471305312228400:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.216674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:41.216727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471305312228400:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:41.217189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:41.217514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471305241820176:2050] [node 3] ICS01 socket: 524 reason# EndOfStream
2025-11-27T18:18:41.217564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471300946852779:2050] [node 3] ICP30 unregister session Session# [6:7577471305241820176:2050] VirtualId# [6:7577471305241820171:0]
2025-11-27T18:18:41.217608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471305241820176:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.217629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471301017261014:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9005) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:41.217675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:41.217682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471305312228400:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:41.217721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471301017261014:2050] [node 3] ICP30 unregister session Session# [5:7577471305312228400:2050] VirtualId# [5:7577471305312228391:0]
2025-11-27T18:18:41.217768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:41.217727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:41.217817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471301017261014:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9005) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.217835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9005) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.217482Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:41.217685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471304922194886:2050] [node 3] ICS01 socket: 530 reason# EndOfStream
2025-11-27T18:18:41.217729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471300627227509:2050] [node 3] ICP30 unregister session Session# [4:7577471304922194886:2050] VirtualId# [4:7577471304922194884:0]
2025-11-27T18:18:41.217771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471304922194886:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.217847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:41.217899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:41.228095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471301017261014:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:41.261018Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:41.261250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471305312228395:2108] [node 4] ICS01 socket: 560 reason# EndOfStream
2025-11-27T18:18:41.261306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471301017261075:2108] [node 4] ICP30 unregister session Session# [5:7577471305312228395:2108] VirtualId# [5:7577471305312228390:0]
2025-11-27T18:18:41.261334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471305312228395:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.261432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:41.261508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:41.261904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:41.262136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471305241820174:2110] [node 4] ICS07 socket disconnect 526 reason# EndOfStream
2025-11-27T18:18:41.262170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471305241820174:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.262244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:41.262289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471305241820174:2110] [node 4] ICS15 start handshake
2025-11-27T18:18:41.263125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471300946852844:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2880) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:41.263182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471305241820174:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:41.263218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471300946852844:2110] [node 4] ICP30 unregister session Session# [6:7577471305241820174:2110] VirtualId# [6:7577471305241820169:0]
2025-11-27T18:18:41.263265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:41.263295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471300946852844:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2880) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.263315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2880) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.273493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471300946852844:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:18:41.290460Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:41.290722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471305241820167:2105] [node 5] ICS01 socket: 534 reason# EndOfStream
2025-11-27T18:18:41.290772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471300946852839:2105] [node 5] ICP30 unregister session Session# [6:7577471305241820167:2105] VirtualId# [6:7577471305241820154:0]
2025-11-27T18:18:41.290810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471305241820167:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.290882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:41.290937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:41.381953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471305925939112:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:41.381954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471305925939113:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:41.398510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471306471613969:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:41.398512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471306471613970:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:41.419029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471308365462509:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:41.419823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471308365462510:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:41.441242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471305486011297:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:41.441381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471305486011299:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:41.441992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471305486011298:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:41.463030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471305948015766:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:41.463079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471305948015767:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:41.463109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471305948015768:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:41.486303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471306308524822:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:41.486365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471306308524824:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:41.486388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471306308524823:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:41.562147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471306308524882:2106] [node 5] ICP01 ready to work
2025-11-27T18:18:41.611599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471308365462590:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:41.641321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471305948015829:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:41.645593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471308365462591:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:41.641688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471305948015830:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:41.646205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471308365462592:2115] [node 6] ICP01 ready to work
2025-11-27T18:18:41.650115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471306471613969:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(96997531ns) {Configure:104(88010960ns) {ProcessConfigured:135(95772ns) {StartInitialHandshake:157(92912ns) {DropHandshakes:470(1255ns) {DropIncomingHandshake:442(229ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(184ns) PrepareNewSessionHandshake:477(6503ns) {SwitchToState:179(411ns)}}}}}
2025-11-27T18:18:41.650553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471306471614035:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:41.651848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471306308524822:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(64462645ns) {Configure:104(64452395ns) {ProcessConfigured:135(404066ns) {StartInitialHandshake:157(401052ns) {DropHandshakes:470(1374ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(1118ns) PrepareNewSessionHandshake:477(3002ns) {SwitchToState:179(1602ns)}}}}}
2025-11-27T18:18:41.652320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471306308524891:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:41.653419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471306308524882:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(88758669ns) {Configure:104(88750624ns) {ProcessConfigured:135(51266ns) {StartInitialHandshake:157(48190ns) {DropHandshakes:470(1364ns) {DropIncomingHandshake:442(433ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(463ns) PrepareNewSessionHandshake:477(915ns) {SwitchToState:179(416ns)}}}}}
2025-11-27T18:18:41.653782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471305925939113:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(99515374ns) {Configure:104(99512226ns) {ProcessConfigured:135(324352ns) {StartInitialHandshake:157(321487ns) {DropHandshakes:470(3410ns) {DropIncomingHandshake:442(789ns) DropOutgoingHandshake:457(548ns)} GenerateSessionVirtualId:988(919ns) PrepareNewSessionHandshake:477(2182ns) {SwitchToState:179(767ns)}}}}}
2025-11-27T18:18:41.658167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471305925939112:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(102153253ns) {Configure:104(102147308ns) {ProcessConfigured:135(2052785ns) {StartInitialHandshake:157(2049810ns) {DropHandshakes:470(2879ns) {DropIncomingHandshake:442(713ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(1497ns) PrepareNewSessionHandshake:477(1543ns) {SwitchToState:179(799ns)}}}}}
2025-11-27T18:18:41.653964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471306471614036:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:41.661965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471305925939179:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:41.661782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471308365462509:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(83620707ns) {Configure:104(83609731ns) {ProcessConfigured:135(596146ns) {StartInitialHandshake:157(593718ns) {DropHandshakes:470(2288ns) {DropIncomingHandshake:442(932ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(1359ns) PrepareNewSessionHandshake:477(3436ns) {SwitchToState:179(1909ns)}}}}}
2025-11-27T18:18:41.662387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471306471614039:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:41.662364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471305486011297:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(62740136ns) {Configure:104(62727723ns) {ProcessConfigured:135(7715660ns) {StartInitialHandshake:157(7704433ns) {DropHandshakes:470(1280ns) {DropIncomingHandshake:442(361ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(617ns) PrepareNewSessionHandshake:477(4666ns) {SwitchToState:179(971ns)}}}}}
2025-11-27T18:18:41.663121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471305925939180:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:41.662445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471306471613970:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(109189660ns) {Configure:104(109186898ns) {ProcessConfigured:135(696973ns) {StartInitialHandshake:157(694825ns) {DropHandshakes:470(279237ns) {DropIncomingHandshake:442(350ns) DropOutgoingHandshake:457(670ns)} GenerateSessionVirtualId:988(629ns) PrepareNewSessionHandshake:477(1791ns) {SwitchToState:179(1111ns)}}}}}
2025-11-27T18:18:41.662516Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471305948015766:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16690810ns) {Configure:104(16684134ns) {ProcessConfigured:135(734264ns) {StartInitialHandshake:157(732168ns) {DropHandshakes:470(1858ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(297ns)} GenerateSessionVirtualId:988(533ns) PrepareNewSessionHandshake:477(1697ns) {SwitchToState:179(945ns)}}}}}
2025-11-27T18:18:41.662778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471305486011298:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(81376135ns) {Configure:104(81367607ns) {ProcessConfigured:135(3465130ns) {StartInitialHandshake:157(3462893ns) {DropHandshakes:470(1129ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:477(4118ns) {SwitchToState:179(2155ns)}}}}}
2025-11-27T18:18:41.664682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471305925939181:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:41.664684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471308365462590:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16542959ns) {Configure:104(16537040ns) {ProcessConfigured:135(602341ns) {StartInitialHandshake:157(600780ns) {DropHandshakes:470(76065ns) {DropIncomingHandshake:442(170ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(703ns) PrepareNewSessionHandshake:477(1874ns) {SwitchToState:179(990ns)}}}}}
2025-11-27T18:18:41.663443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471305486011382:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:41.665578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471305486011383:2115] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.690396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305948015839:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35875
2025-11-27T18:18:41.703867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471305925939180:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(34404409ns) {Configure:104(34400234ns) {ProcessConfigured:135(485382ns) {StartInitialHandshake:157(483525ns) {DropHandshakes:470(1270ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(172ns)} GenerateSessionVirtualId:988(479ns) PrepareNewSessionHandshake:477(934ns) {SwitchToState:179(349ns)}}}}}
2025-11-27T18:18:41.704612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471305486011299:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34123522ns) {Configure:104(34116474ns) {ProcessConfigured:135(6262999ns) {StartInitialHandshake:157(6259821ns) {DropHandshakes:470(2863ns) {DropIncomingHandshake:442(563ns) DropOutgoingHandshake:457(483ns)} GenerateSessionVirtualId:988(961ns) PrepareNewSessionHandshake:477(3822ns) {SwitchToState:179(2383ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.703816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471306471614039:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(26762601ns) {Configure:104(26757803ns) {ProcessConfigured:135(40848ns) {StartInitialHandshake:157(39429ns) {DropHandshakes:470(1872ns) {DropIncomingHandshake:442(427ns) DropOutgoingHandshake:457(310ns)} GenerateSessionVirtualId:988(1068ns) PrepareNewSessionHandshake:477(731ns) {SwitchToState:179(395ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.711414Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471305925939179:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(44714371ns) {Configure:104(41986952ns) {ProcessConfigured:135(52312ns) {StartInitialHandshake:157(49484ns) {DropHandshakes:470(1400ns) {DropIncomingHandshake:442(384ns) DropOutgoingHandshake:457(283ns)} GenerateSessionVirtualId:988(855ns) PrepareNewSessionHandshake:477(1417ns) {SwitchToState:179(455ns)}}}}}
2025-11-27T18:18:41.704216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471306471614035:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(35977075ns) {Configure:104(35970988ns) {ProcessConfigured:135(1989600ns) {StartInitialHandshake:157(1986017ns) {DropHandshakes:470(2421ns) {DropIncomingHandshake:442(806ns) DropOutgoingHandshake:457(295ns)} GenerateSessionVirtualId:988(887ns) PrepareNewSessionHandshake:477(2793ns) {SwitchToState:179(1474ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.720088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471308365462592:2115] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36069652ns) {Configure:104(36065619ns) {ProcessConfigured:135(2782689ns) {StartInitialHandshake:157(2780861ns) {DropHandshakes:470(1760ns) {DropIncomingHandshake:442(355ns) DropOutgoingHandshake:457(412ns)} GenerateSessionVirtualId:988(688ns) PrepareNewSessionHandshake:477(2803ns) {SwitchToState:179(2003ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.723533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471305486011383:2115] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18692925ns) {Configure:104(18689246ns) {ProcessConfigured:135(58126ns) {StartInitialHandshake:157(56042ns) {DropHandshakes:470(1283ns) {DropIncomingHandshake:442(239ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(837ns) PrepareNewSessionHandshake:477(2387ns) {SwitchToState:179(879ns)}}}}}
2025-11-27T18:18:41.724569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471305486011388:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1384
2025-11-27T18:18:41.724658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471305486011389:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1384
2025-11-27T18:18:41.712993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471305925939188:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5435
2025-11-27T18:18:41.709711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471306471614046:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24913
2025-11-27T18:18:41.721620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471305948015839:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.720956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471308365462603:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12816
2025-11-27T18:18:41.739408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471305948015829:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(50723584ns) {Configure:104(50717685ns) {ProcessConfigured:135(5320136ns) {StartInitialHandshake:157(5316845ns) {DropHandshakes:470(2665ns) {DropIncomingHandshake:442(549ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(636ns) PrepareNewSessionHandshake:477(2758ns) {SwitchToState:179(1157ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.743904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471308365462608:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12816
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.713125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471305925939189:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5435
2025-11-27T18:18:41.735431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471306471614046:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.746355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471308365462609:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12816
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.713180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471305925939190:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5435
2025-11-27T18:18:41.739964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471306471614050:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24913
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.751662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471308365462608:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.751766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471308365462609:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.751890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471308365462509:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.716453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471305925939188:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.744665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471305925939181:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(40576045ns) {Configure:104(40572368ns) {ProcessConfigured:135(1663142ns) {StartInitialHandshake:157(1660714ns) {DropHandshakes:470(2397ns) {DropIncomingHandshake:442(807ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(453ns) PrepareNewSessionHandshake:477(5645ns) {SwitchToState:179(1170ns)}}}}}
2025-11-27T18:18:41.744835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471305925939190:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.745598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471305925939193:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5435
2025-11-27T18:18:41.749798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471305925939193:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.752489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471305925939176:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.749654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305948015845:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35875
2025-11-27T18:18:41.750146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471305486011382:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(63844379ns) {Configure:104(63838442ns) {ProcessConfigured:135(3782682ns) {StartInitialHandshake:157(3779560ns) {DropHandshakes:470(2980ns) {DropIncomingHandshake:442(721ns) DropOutgoingHandshake:457(696ns)} GenerateSessionVirtualId:988(561ns) PrepareNewSessionHandshake:477(3828ns) {SwitchToState:179(2051ns)}}}}}
2025-11-27T18:18:41.742778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471306471614051:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24913
2025-11-27T18:18:41.752510Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471305925939176:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.753690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471305925939176:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:41.760474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471308365462607:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12816
2025-11-27T18:18:41.758662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305948015846:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35875
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.758924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471305948015845:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.759045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471305948015767:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.760908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471305925939192:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5435
2025-11-27T18:18:41.748313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471306471614050:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.763196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471308365462612:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12816
2025-11-27T18:18:41.764109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471308365462607:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.764245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471308365462510:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.749163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471306471614052:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24913
2025-11-27T18:18:41.765190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471305925939178:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.749820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471306471613969:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.765744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471305486011389:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.766228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471305948015846:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.769387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471305486011299:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.771512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471308365462603:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.765217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471305925939178:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.758736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471306471614052:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.770030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471305925939192:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.771261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471305486011388:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.760459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471306471614051:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.770088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471305925939113:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.770145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:41.762268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471306471614041:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.770181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471305925939113:2049] [node 3] ICP22 created new session: [1:7577471305925939200:2049]
2025-11-27T18:18:41.762293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471306471614041:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.770229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471305925939200:2049] [node 3] ICS09 handshake done sender: [1:7577471305925939176:2110] self: [1:7577471305925939175:0] peer: [3:7577471308365462610:0] socket: 540 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.766199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471306471614038:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.769456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471305948015766:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.779136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305948015850:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35875
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.780526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471305486011397:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1384
2025-11-27T18:18:41.766229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471306471614038:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.771256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471305925939200:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:41.773332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471308365462596:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.772571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:41.773570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471305925939183:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.773597Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471305925939183:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.775431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471305925939189:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.775589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471305925939113:2049] [node 3] ICP18 handshake (actor [1:7577471305925939192:2117]) is too old
2025-11-27T18:18:41.780966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471305925939178:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:41.793389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471305925939183:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:41.780942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471305948015845:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:41.769485Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471306471614041:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:41.773376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471308365462596:2129] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.793678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471305948015767:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.771473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471306471614038:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:41.793748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:41.776621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471308365462608:2135] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.799933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471305486011398:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1384
2025-11-27T18:18:41.793794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471305948015767:2049] [node 2] ICP22 created new session: [5:7577471305948015851:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.801171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471305486011397:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.781612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471306471614036:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.800912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471305925939112:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.800977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:41.802398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471305925939112:2048] [node 2] ICP22 created new session: [1:7577471305925939206:2048]
2025-11-27T18:18:41.802454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471305925939206:2048] [node 2] ICS09 handshake done sender: [1:7577471305925939178:2111] self: [1:7577471305925939177:0] peer: [2:7577471306471614054:0] socket: 539 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.802658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471305925939206:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:41.793841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471305948015851:2049] [node 2] ICS09 handshake done sender: [5:7577471305948015845:2116] self: [5:7577471305948015848:0] peer: [2:7577471306471614040:0] socket: 549 qp: -1
2025-11-27T18:18:41.793872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471305948015851:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:41.793903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:41.802393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471305486011389:2133] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:41.802866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471305486011297:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.777080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471308365462509:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.802920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471306308524901:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20807
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.806520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471306308524902:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20807
2025-11-27T18:18:41.803127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305948015853:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35875
2025-11-27T18:18:41.781668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:41.807226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471305486011299:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.807278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:41.777133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.807316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471305486011299:2050] [node 3] ICP22 created new session: [4:7577471305486011400:2050]
2025-11-27T18:18:41.803123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:41.807698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471305948015844:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.807729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471305948015844:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.815895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471305948015846:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:41.807367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471305486011400:2050] [node 3] ICS09 handshake done sender: [4:7577471305486011389:2133] self: [4:7577471305486011395:0] peer: [3:7577471308365462595:0] socket: 552 qp: -1
2025-11-27T18:18:41.807412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471305486011400:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:41.807447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:41.781711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471306471614036:2108] [node 5] ICP22 created new session: [2:7577471306471614059:2108]
2025-11-27T18:18:41.806941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471306308524901:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.777178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471308365462509:2048] [node 1] ICP22 created new session: [3:7577471308365462616:2048]
2025-11-27T18:18:41.809708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471305486011402:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1384
2025-11-27T18:18:41.803430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471305925939180:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.807095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471306308524882:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.781753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471306471614059:2108] [node 5] ICS09 handshake done sender: [2:7577471306471614041:2113] self: [2:7577471306471614040:0] peer: [5:7577471305948015848:0] socket: 533 qp: -1
2025-11-27T18:18:41.816210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471305948015766:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.777223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471308365462616:2048] [node 1] ICS09 handshake done sender: [3:7577471308365462608:2135] self: [3:7577471308365462610:0] peer: [1:7577471305925939175:0] socket: 548 qp: -1
2025-11-27T18:18:41.812850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471305486011402:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.807384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471306308524902:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.803480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:41.781785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471306471614059:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:41.777276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471308365462616:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:41.812960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471305486011298:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.807496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471306308524823:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.777310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:41.781822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:41.825458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:41.818659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471305486011387:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.818699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471305486011387:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.825249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471305486011387:2131] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:41.838706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471305486011383:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.810312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471306308524905:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20807
2025-11-27T18:18:41.807082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471305925939180:2108] [node 5] ICP22 created new session: [1:7577471305925939208:2108]
2025-11-27T18:18:41.778013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471308365462614:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12816
2025-11-27T18:18:41.844814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:41.827905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471305948015844:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:41.808176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471305925939208:2108] [node 5] ICS09 handshake done sender: [1:7577471305925939183:2112] self: [1:7577471305925939182:0] peer: [5:7577471305948015849:0] socket: 542 qp: -1
2025-11-27T18:18:41.782680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471306471613970:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.812524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471306308524906:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20807
2025-11-27T18:18:41.795189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471308365462596:2129] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:41.782729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:41.829127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471305948015766:2048] [node 1] ICP22 created new session: [5:7577471305948015856:2048]
2025-11-27T18:18:41.814015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471305925939208:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:41.814352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471306308524905:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.846888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471305486011383:2115] [node 6] ICP22 created new session: [4:7577471305486011406:2115]
2025-11-27T18:18:41.802611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471308365462590:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.814060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:41.792005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471306471613970:2049] [node 3] ICP22 created new session: [2:7577471306471614062:2049]
2025-11-27T18:18:41.829182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471305948015856:2048] [node 1] ICS09 handshake done sender: [5:7577471305948015846:2117] self: [5:7577471305948015849:0] peer: [1:7577471305925939182:0] socket: 561 qp: -1
2025-11-27T18:18:41.850502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471305486011406:2115] [node 6] ICS09 handshake done sender: [4:7577471305486011387:2131] self: [4:7577471305486011386:0] peer: [6:7577471306308524907:0] socket: 555 qp: -1
2025-11-27T18:18:41.816673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471306308524891:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.802666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:41.827990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471305925939185:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.792060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471306471614062:2049] [node 3] ICS09 handshake done sender: [2:7577471306471614038:2112] self: [2:7577471306471614037:0] peer: [3:7577471308365462613:0] socket: 531 qp: -1
2025-11-27T18:18:41.828016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471305925939185:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.792102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471306471614062:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:41.835326Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471305925939180:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(32487638ns) {HandleHandshakeStatus:329(32484723ns) {DropIncomingHandshake:442(12102ns) DropOutgoingHandshake:457(225ns) SwitchToState:179(146ns) ProcessPendingSessionEvents:495(9810983ns) {ForwardSessionEventToSession:612(5168254ns) ForwardSessionEventToSession:612(4593ns) ForwardSessionEventToSession:612(2553ns) ForwardSessionEventToSession:612(4586945ns) ForwardSessionEventToSession:612(16771ns) ForwardSessionEventToSession:612(2767ns) ForwardSessionEventToSession:612(5254ns) ForwardSessionEventToSession:612(2728ns) ForwardSessionEventToSession:612(8901ns) ForwardSessionEventToSession:612(2748ns)}}}
2025-11-27T18:18:41.835999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471305925939185:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:41.829218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471305948015856:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:41.803359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471308365462590:2107] [node 4] ICP22 created new session: [3:7577471308365462619:2107]
2025-11-27T18:18:41.855844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471306308524823:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20833) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471306308524902:2117] held: no
2025-11-27T18:18:41.845291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471305925939179:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.845374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:41.845420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471305925939179:2107] [node 4] ICP22 created new session: [1:7577471305925939212:2107]
2025-11-27T18:18:41.845475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471305925939212:2107] [node 4] ICS09 handshake done sender: [1:7577471305925939185:2113] self: [1:7577471305925939184:0] peer: [4:7577471305486011399:0] socket: 570 qp: -1
2025-11-27T18:18:41.845506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471305925939212:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:41.845548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:41.792151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.851081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471305486011406:2115] [node 6] ICS10 traffic start
2025-11-27T18:18:41.806969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471308365462619:2107] [node 4] ICS09 handshake done sender: [3:7577471308365462596:2129] self: [3:7577471308365462595:0] peer: [4:7577471305486011395:0] socket: 546 qp: -1
2025-11-27T18:18:41.860562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471306308524902:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:41.829434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:41.851210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:41.864193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471305486011297:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15125) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471305486011397:2135] held: no
2025-11-27T18:18:41.807987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471308365462619:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.808117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471306471614061:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24913
2025-11-27T18:18:41.864362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471305486011407:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1384
2025-11-27T18:18:41.811244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:41.850923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471305948015829:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.811393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471306471614045:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.850990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.851232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471305948015829:2109] [node 6] ICP22 created new session: [5:7577471305948015858:2109]
2025-11-27T18:18:41.814243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471308365462607:2134] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:41.811417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471306471614045:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.865421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471306308524908:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20807
2025-11-27T18:18:41.852920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471305948015858:2109] [node 6] ICS09 handshake done sender: [5:7577471305948015844:2115] self: [5:7577471305948015843:0] peer: [6:7577471306308524903:0] socket: 557 qp: -1
2025-11-27T18:18:41.814875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471308365462510:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20833) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471308365462607:2134] held: no
2025-11-27T18:18:41.865722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471306308524823:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.853366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471305948015858:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:41.866241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:41.853401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:41.866288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471306308524823:2049] [node 2] ICP22 created new session: [6:7577471306308524912:2049]
2025-11-27T18:18:41.861405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471305948015766:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(35194988ns) {HandleHandshakeStatus:329(35193108ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(3403ns) SwitchToState:179(377ns) ProcessPendingSessionEvents:495(12576346ns) {ForwardSessionEventToSession:612(11780472ns) ForwardSessionEventToSession:612(5881ns) ForwardSessionEventToSession:612(3084ns) ForwardSessionEventToSession:612(308993ns) ForwardSessionEventToSession:612(3087ns) ForwardSessionEventToSession:612(217172ns) ForwardSessionEventToSession:612(79126ns) ForwardSessionEventToSession:612(2882ns) ForwardSessionEventToSession:612(157189ns) ForwardSessionEventToSession:612(2862ns)}}}
2025-11-27T18:18:41.814966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471308365462510:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.814662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471306471614045:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:41.815039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:41.866349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471306308524912:2049] [node 2] ICS09 handshake done sender: [6:7577471306308524902:2117] self: [6:7577471306308524904:0] peer: [2:7577471306471614044:0] socket: 578 qp: -1
2025-11-27T18:18:41.816689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471306471614043:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.816709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471306471614043:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.817432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471306471614039:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.866223Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471305486011383:2115] [node 6] ICP35 event processing took too much time PendingConnection:279(28610944ns) {HandleHandshakeStatus:329(27103563ns) {DropIncomingHandshake:442(1614ns) DropOutgoingHandshake:457(1140ns) SwitchToState:179(611ns) ProcessPendingSessionEvents:495(11251857ns) {ForwardSessionEventToSession:612(4121327ns) ForwardSessionEventToSession:612(4815ns) ForwardSessionEventToSession:612(2542ns) ForwardSessionEventToSession:612(1406222ns) ForwardSessionEventToSession:612(3504ns) ForwardSessionEventToSession:612(3124301ns) ForwardSessionEventToSession:612(4507ns) ForwardSessionEventToSession:612(3462ns) ForwardSessionEventToSession:612(2555044ns) ForwardSessionEventToSession:612(7505ns)}}}
2025-11-27T18:18:41.815080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471308365462510:2049] [node 2] ICP22 created new session: [3:7577471308365462621:2049]
2025-11-27T18:18:41.866392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471306308524912:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:41.817509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:41.815127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471308365462621:2049] [node 2] ICS09 handshake done sender: [3:7577471308365462607:2134] self: [3:7577471308365462613:0] peer: [2:7577471306471614037:0] socket: 544 qp: -1
2025-11-27T18:18:41.815157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471308365462621:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:41.866418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:41.815203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:41.817550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471306471614039:2109] [node 6] ICP22 created new session: [2:7577471306471614067:2109]
2025-11-27T18:18:41.817588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471306471614067:2109] [node 6] ICS09 handshake done sender: [2:7577471306471614045:2115] self: [2:7577471306471614044:0] peer: [6:7577471306308524904:0] socket: 545 qp: -1
2025-11-27T18:18:41.817620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471306471614067:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:41.866232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471305486011408:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1384
2025-11-27T18:18:41.852240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471308365462590:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(42251932ns) {HandleHandshakeStatus:329(42250028ns) {DropIncomingHandshake:442(12721ns) DropOutgoingHandshake:457(276ns) SwitchToState:179(413ns) ProcessPendingSessionEvents:495(28260369ns) {ForwardSessionEventToSession:612(8738635ns) ForwardSessionEventToSession:612(3659116ns) ForwardSessionEventToSession:612(7304ns) ForwardSessionEventToSession:612(2752ns) ForwardSessionEventToSession:612(396972ns) ForwardSessionEventToSession:612(5572369ns) ForwardSessionEventToSession:612(5109ns) ForwardSessionEventToSession:612(1362400ns) ForwardSessionEventToSession:612(4913978ns) ForwardSessionEventToSession:612(3579382ns)}}}
2025-11-27T18:18:41.817648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.817958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471306471614050:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:41.833123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471306471613969:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.869069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305948015860:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35875
2025-11-27T18:18:41.833200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:41.833251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471306471613969:2048] [node 1] ICP22 created new session: [2:7577471306471614070:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.833319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471306471614070:2048] [node 1] ICS09 handshake done sender: [2:7577471306471614050:2117] self: [2:7577471306471614054:0] peer: [1:7577471305925939177:0] socket: 541 qp: -1
2025-11-27T18:18:41.833363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471306471614070:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:41.833398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.841891Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471306471613969:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:41.869526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471305948015860:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.869394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471305486011402:2137] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.869543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305948015861:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35875
2025-11-27T18:18:41.869646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471305486011298:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20833) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471305486011402:2137] held: no
2025-11-27T18:18:41.869864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471306308524909:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20807
2025-11-27T18:18:41.845195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471306471614043:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:41.869840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471305486011298:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.869782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471305948015768:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.869882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:41.851538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471306471614035:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.870049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471306308524910:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20807
2025-11-27T18:18:41.870135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471305486011298:2049] [node 2] ICP22 created new session: [4:7577471305486011411:2049]
2025-11-27T18:18:41.855443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:41.870218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471305486011411:2049] [node 2] ICS09 handshake done sender: [4:7577471305486011402:2137] self: [4:7577471305486011404:0] peer: [2:7577471306471614042:0] socket: 579 qp: -1
2025-11-27T18:18:41.856015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471306471614035:2107] [node 4] ICP22 created new session: [2:7577471306471614072:2107]
2025-11-27T18:18:41.870225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471305486011397:2135] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:41.856356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471306471614072:2107] [node 4] ICS09 handshake done sender: [2:7577471306471614043:2114] self: [2:7577471306471614042:0] peer: [4:7577471305486011404:0] socket: 547 qp: -1
2025-11-27T18:18:41.856403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471306471614072:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:41.856445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:41.870547Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471306308524882:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9100) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471306308524901:2116] held: no
2025-11-27T18:18:41.870253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471305486011411:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:41.871193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471306308524901:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:41.870753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471308365462600:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.871813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471306308524910:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.870285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:41.872058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471306308524905:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.873078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471305486011297:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.873781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:41.874956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471305486011297:2048] [node 1] ICP22 created new session: [4:7577471305486011413:2048]
2025-11-27T18:18:41.875018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471305486011413:2048] [node 1] ICS09 handshake done sender: [4:7577471305486011397:2135] self: [4:7577471305486011399:0] peer: [1:7577471305925939184:0] socket: 571 qp: -1
2025-11-27T18:18:41.875050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471305486011413:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:41.870794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471308365462600:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.875157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:41.875423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471308365462600:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:41.875490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471308365462625:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12816
2025-11-27T18:18:41.875232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471306308524882:2106] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.875281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:41.875337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471306308524882:2106] [node 5] ICP22 created new session: [6:7577471306308524915:2106]
2025-11-27T18:18:41.877701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471308365462591:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.876812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305948015863:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35875
2025-11-27T18:18:41.877962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:41.875381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471306308524915:2106] [node 5] ICS09 handshake done sender: [6:7577471306308524901:2116] self: [6:7577471306308524903:0] peer: [5:7577471305948015843:0] socket: 577 qp: -1
2025-11-27T18:18:41.879231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471308365462591:2108] [node 5] ICP22 created new session: [3:7577471308365462627:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.882075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471305486011415:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1384
2025-11-27T18:18:41.878220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471305948015860:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:41.875414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471306308524915:2106] [node 5] ICS10 traffic start
2025-11-27T18:18:41.875450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:41.875820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471306308524911:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20807
2025-11-27T18:18:41.875972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471306308524891:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.877131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:41.877162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471306308524891:2109] [node 4] ICP22 created new session: [6:7577471306308524918:2109]
2025-11-27T18:18:41.877192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471306308524918:2109] [node 4] ICS09 handshake done sender: [6:7577471306308524905:2118] self: [6:7577471306308524907:0] peer: [4:7577471305486011386:0] socket: 580 qp: -1
2025-11-27T18:18:41.877227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471306308524918:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:41.877246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:41.884012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471306308524911:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.879580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471308365462627:2108] [node 5] ICS09 handshake done sender: [3:7577471308365462600:2131] self: [3:7577471308365462599:0] peer: [5:7577471305948015862:0] socket: 585 qp: -1
2025-11-27T18:18:41.878541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471305948015768:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.879561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:41.879881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471305948015768:2050] [node 3] ICP22 created new session: [5:7577471305948015864:2050]
2025-11-27T18:18:41.879926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471305948015864:2050] [node 3] ICS09 handshake done sender: [5:7577471305948015860:2120] self: [5:7577471305948015862:0] peer: [3:7577471308365462599:0] socket: 526 qp: -1
2025-11-27T18:18:41.879954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471305948015864:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:41.879991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:41.880658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471308365462627:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:41.880722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:41.885661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471306308524822:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.885985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471306308524824:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.886068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471305925939195:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.886107Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471305925939195:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.887390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471305925939195:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:41.886697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471308365462602:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.886728Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471308365462602:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.887550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471305925939181:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.887630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:41.887683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471305925939181:2109] [node 6] ICP22 created new session: [1:7577471305925939215:2109]
2025-11-27T18:18:41.887726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471305925939215:2109] [node 6] ICS09 handshake done sender: [1:7577471305925939195:2119] self: [1:7577471305925939194:0] peer: [6:7577471306308524920:0] socket: 530 qp: -1
2025-11-27T18:18:41.887766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471305925939215:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:41.887800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:41.889044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471305948015861:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.895794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471306308524822:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15125) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471306308524910:2122] held: no
2025-11-27T18:18:41.889227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471305948015830:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:41.894961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471308365462625:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.896957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471305486011394:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:41.895264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471308365462602:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:41.897515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471306308524922:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20807
2025-11-27T18:18:41.897007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471305486011394:2134] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:41.895589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471308365462592:2115] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.898299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471306308524923:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20807
2025-11-27T18:18:41.895627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:41.896032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471308365462592:2115] [node 6] ICP22 created new session: [3:7577471308365462630:2115]
2025-11-27T18:18:41.896081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471308365462630:2115] [node 6] ICS09 handshake done sender: [3:7577471308365462602:2132] self: [3:7577471308365462601:0] peer: [6:7577471306308524921:0] socket: 586 qp: -1
2025-11-27T18:18:41.896117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471308365462630:2115] [node 6] ICS10 traffic start
2025-11-27T18:18:41.896187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:41.896485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471308365462592:2115] [node 6] ICP18 handshake (actor [3:7577471308365462625:2139]) is too old
2025-11-27T18:18:41.899375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471305486011394:2134] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:41.903368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471305948015830:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26629) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471305948015861:2121] held: no
2025-11-27T18:18:41.900914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471305486011382:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:41.900979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:41.901019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471305486011382:2108] [node 5] ICP22 created new session: [4:7577471305486011417:2108]
2025-11-27T18:18:41.901068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471305486011417:2108] [node 5] ICS09 handshake done sender: [4:7577471305486011394:2134] self: [4:7577471305486011393:0] peer: [5:7577471305948015866:0] socket: 559 qp: -1
2025-11-27T18:18:41.901102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471305486011417:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:41.901132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:41.905756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471306308524824:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4146) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471306308524911:2123] held: no
2025-11-27T18:18:41.905915Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471306308524910:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:41.906366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471306308524911:2123] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:41.906711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471306308524822:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.906768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:41.906808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471306308524822:2048] [node 1] ICP22 created new session: [6:7577471306308524924:2048]
2025-11-27T18:18:41.906853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471306308524924:2048] [node 1] ICS09 handshake done sender: [6:7577471306308524910:2122] self: [6:7577471306308524920:0] peer: [1:7577471305925939194:0] socket: 524 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:41.907097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471306308524924:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:41.907144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:41.907595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471306308524824:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.907630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:41.907669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471306308524824:2050] [node 3] ICP22 created new session: [6:7577471306308524926:2050]
2025-11-27T18:18:41.907749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471306308524926:2050] [node 3] ICS09 handshake done sender: [6:7577471306308524911:2123] self: [6:7577471306308524921:0] peer: [3:7577471308365462601:0] socket: 554 qp: -1
2025-11-27T18:18:41.907781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471306308524926:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:41.907815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:41.907135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471305948015867:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35875
2025-11-27T18:18:41.909305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471305948015861:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:41.909933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471305948015830:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:41.909975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:41.910020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471305948015830:2108] [node 4] ICP22 created new session: [5:7577471305948015868:2108]
2025-11-27T18:18:41.910065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471305948015868:2108] [node 4] ICS09 handshake done sender: [5:7577471305948015861:2121] self: [5:7577471305948015866:0] peer: [4:7577471305486011393:0] socket: 587 qp: -1
2025-11-27T18:18:41.910098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471305948015868:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:41.910128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:41.968598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:41.968857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471306471614070:2048] [node 1] ICS01 socket: 541 reason# EndOfStream
2025-11-27T18:18:41.968901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471306471613969:2048] [node 1] ICP30 unregister session Session# [2:7577471306471614070:2048] VirtualId# [2:7577471306471614054:0]
2025-11-27T18:18:41.968933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471306471614070:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.969010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:41.969056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:41.973637Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:41.974048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471308365462616:2048] [node 1] ICS01 socket: 548 reason# EndOfStream
2025-11-27T18:18:41.974086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471308365462509:2048] [node 1] ICP30 unregister session Session# [3:7577471308365462616:2048] VirtualId# [3:7577471308365462610:0]
2025-11-27T18:18:41.974117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471308365462616:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.974188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:41.974230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:41.973931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:41.974077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471305486011413:2048] [node 1] ICS01 socket: 571 reason# EndOfStream
2025-11-27T18:18:41.974104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471305486011297:2048] [node 1] ICP30 unregister session Session# [4:7577471305486011413:2048] VirtualId# [4:7577471305486011399:0]
2025-11-27T18:18:41.974125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471305486011413:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.974173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:41.974200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:41.975399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:41.975623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471305948015856:2048] [node 1] ICS07 socket disconnect 561 reason# EndOfStream
2025-11-27T18:18:41.975650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471305948015856:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.975712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:41.975756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471305948015856:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:41.976857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471305948015766:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15125) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:41.976906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471305948015856:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:41.976939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471305948015766:2048] [node 1] ICP30 unregister session Session# [5:7577471305948015856:2048] VirtualId# [5:7577471305948015849:0]
2025-11-27T18:18:41.976980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:41.977009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471305948015766:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15125) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.977023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15125) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:41.980347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:41.981142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471306308524924:2048] [node 1] ICS01 socket: 524 reason# EndOfStream
2025-11-27T18:18:41.981200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471306308524822:2048] [node 1] ICP30 unregister session Session# [6:7577471306308524924:2048] VirtualId# [6:7577471306308524920:0]
2025-11-27T18:18:41.981229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471306308524924:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:41.981470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:41.981530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:41.985853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471305948015766:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:42.039467Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:42.039682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471308365462621:2049] [node 2] ICS01 socket: 544 reason# EndOfStream
2025-11-27T18:18:42.039734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471308365462510:2049] [node 2] ICP30 unregister session Session# [3:7577471308365462621:2049] VirtualId# [3:7577471308365462613:0]
2025-11-27T18:18:42.039765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471308365462621:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:42.039842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:42.039904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:42.047304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:42.047579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471305486011411:2049] [node 2] ICS01 socket: 579 reason# EndOfStream
2025-11-27T18:18:42.047633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471305486011298:2049] [node 2] ICP30 unregister session Session# [4:7577471305486011411:2049] VirtualId# [4:7577471305486011404:0]
2025-11-27T18:18:42.047666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471305486011411:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:42.047740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:42.047784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:42.048439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:42.048633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471305948015851:2049] [node 2] ICS01 socket: 549 reason# EndOfStream
2025-11-27T18:18:42.048668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471305948015767:2049] [node 2] ICP30 unregister session Session# [5:7577471305948015851:2049] VirtualId# [5:7577471305948015848:0]
2025-11-27T18:18:42.048700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471305948015851:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:42.048764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:42.048815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:42.050138Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:42.050342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471306308524912:2049] [node 2] ICS01 socket: 578 reason# EndOfStream
2025-11-27T18:18:42.050382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471306308524823:2049] [node 2] ICP30 unregister session Session# [6:7577471306308524912:2049] VirtualId# [6:7577471306308524904:0]
2025-11-27T18:18:42.050407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471306308524912:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:42.050479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:42.050524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:42.097338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:42.097564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471306308524926:2050] [node 3] ICS01 socket: 554 reason# EndOfStream
2025-11-27T18:18:42.097601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471306308524824:2050] [node 3] ICP30 unregister session Session# [6:7577471306308524926:2050] VirtualId# [6:7577471306308524921:0]
2025-11-27T18:18:42.097630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471306308524926:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:42.097705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:42.097743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:42.098163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:42.098363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471305486011400:2050] [node 3] ICS01 socket: 552 reason# EndOfStream
2025-11-27T18:18:42.098500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471305486011299:2050] [node 3] ICP30 unregister session Session# [4:7577471305486011400:2050] VirtualId# [4:7577471305486011395:0]
2025-11-27T18:18:42.098531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471305486011400:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:42.098598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:42.098637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:42.096324Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:42.096555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471305948015864:2050] [node 3] ICS01 socket: 526 reason# EndOfStream
2025-11-27T18:18:42.096595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471305948015768:2050] [node 3] ICP30 unregister session Session# [5:7577471305948015864:2050] VirtualId# [5:7577471305948015862:0]
2025-11-27T18:18:42.096628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471305948015864:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:42.096704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:42.096750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:42.154514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:42.154770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471305948015868:2108] [node 4] ICS01 socket: 587 reason# EndOfStream
2025-11-27T18:18:42.154812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471305948015830:2108] [node 4] ICP30 unregister session Session# [5:7577471305948015868:2108] VirtualId# [5:7577471305948015866:0]
2025-11-27T18:18:42.154857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471305948015868:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:42.154931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:42.154982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:42.161432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:42.161691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471306308524918:2109] [node 4] ICS07 socket disconnect 580 reason# EndOfStream
2025-11-27T18:18:42.161721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471306308524918:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:42.161802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:42.161842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471306308524918:2109] [node 4] ICS15 start handshake
2025-11-27T18:18:42.162820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471306308524891:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26629) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:42.162881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471306308524918:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:42.162919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471306308524891:2109] [node 4] ICP30 unregister session Session# [6:7577471306308524918:2109] VirtualId# [6:7577471306308524907:0]
2025-11-27T18:18:42.162960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:42.162992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471306308524891:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:42.163011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:42.173368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471306308524891:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:18:42.209943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:42.210184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471306308524915:2106] [node 5] ICS01 socket: 577 reason# EndOfStream
2025-11-27T18:18:42.210237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471306308524882:2106] [node 5] ICP30 unregister session Session# [6:7577471306308524915:2106] VirtualId# [6:7577471306308524903:0]
2025-11-27T18:18:42.210272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471306308524915:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:42.210358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:42.210404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:42.334770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471311031424669:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:42.335012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471311031424668:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:42.379514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471311246486963:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:42.379575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471311246486964:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:42.392367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471310540050985:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:42.392415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471310540050986:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:42.415858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471309143410300:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:42.415908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471309143410301:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:42.415938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471309143410302:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:42.442547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471310552530603:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:42.442576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471310552530604:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:42.443037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471310552530602:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:42.464501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471310349190256:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:42.464528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471310349190257:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:42.464565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471310349190258:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:42.597690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471310349190318:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:42.624012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471311031424733:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:42.663263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471311031424732:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:42.728227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471311031424735:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:42.728217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471309143410365:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:42.719947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471311246487027:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:42.729749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471311246487028:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:42.703052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471310540051050:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:42.731100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471310540051052:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:42.731141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471311246487029:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:42.719985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471310349190323:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:42.728461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471310552530666:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:42.731580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471310540051051:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:42.732365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471310552530667:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:42.732597Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471311031424668:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(109539004ns) {Configure:104(109534330ns) {ProcessConfigured:135(47721848ns) {StartInitialHandshake:157(47718399ns) {DropHandshakes:470(2222ns) {DropIncomingHandshake:442(505ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(991ns) PrepareNewSessionHandshake:477(5258ns) {SwitchToState:179(2495ns)}}}}}
2025-11-27T18:18:42.743170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471310552530602:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22840592ns) {Configure:104(22834226ns) {ProcessConfigured:135(83909ns) {StartInitialHandshake:157(80402ns) {DropHandshakes:470(3307ns) {DropIncomingHandshake:442(860ns) DropOutgoingHandshake:457(585ns)} GenerateSessionVirtualId:988(847ns) PrepareNewSessionHandshake:477(4317ns) {SwitchToState:179(676ns)}}}}}
2025-11-27T18:18:42.737620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471309143410366:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.748975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471310552530663:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64065
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.753373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471311031424733:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23331871ns) {Configure:104(23324927ns) {ProcessConfigured:135(44152ns) {StartInitialHandshake:157(42037ns) {DropHandshakes:470(1762ns) {DropIncomingHandshake:442(406ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(625ns) {SwitchToState:179(188ns)}}}}}
2025-11-27T18:18:42.755626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471311031424741:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12654
2025-11-27T18:18:42.758317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471309143410301:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26894762ns) {Configure:104(26884837ns) {ProcessConfigured:135(73036ns) {StartInitialHandshake:157(69578ns) {DropHandshakes:470(2483ns) {DropIncomingHandshake:442(543ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(1245ns) PrepareNewSessionHandshake:477(1751ns) {SwitchToState:179(545ns)}}}}}
2025-11-27T18:18:42.763960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471309143410300:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53149417ns) {Configure:104(53143497ns) {ProcessConfigured:135(70772ns) {StartInitialHandshake:157(65595ns) {DropHandshakes:470(1837ns) {DropIncomingHandshake:442(674ns) DropOutgoingHandshake:457(213ns)} GenerateSessionVirtualId:988(540ns) PrepareNewSessionHandshake:477(1270ns) {SwitchToState:179(422ns)}}}}}
2025-11-27T18:18:42.757035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471310349190258:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(117614396ns) {Configure:104(117606715ns) {ProcessConfigured:135(565309ns) {StartInitialHandshake:157(561930ns) {DropHandshakes:470(1929ns) {DropIncomingHandshake:442(553ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(674ns) PrepareNewSessionHandshake:477(3900ns) {SwitchToState:179(2042ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.775142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471310349190256:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35989300ns) {Configure:104(35983850ns) {ProcessConfigured:135(3934466ns) {StartInitialHandshake:157(3931804ns) {DropHandshakes:470(1896ns) {DropIncomingHandshake:442(433ns) DropOutgoingHandshake:457(438ns)} GenerateSessionVirtualId:988(616ns) PrepareNewSessionHandshake:477(2471ns) {SwitchToState:179(835ns)}}}}}
2025-11-27T18:18:42.792950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471311031424669:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(82032183ns) {Configure:104(73034513ns) {ProcessConfigured:135(6721568ns) {StartInitialHandshake:157(6720127ns) {DropHandshakes:470(1715ns) {DropIncomingHandshake:442(551ns) DropOutgoingHandshake:457(346ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(1652ns) {SwitchToState:179(1008ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.785832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471310540051050:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(53682571ns) {Configure:104(53677696ns) {ProcessConfigured:135(81556ns) {StartInitialHandshake:157(79224ns) {DropHandshakes:470(3426ns) {DropIncomingHandshake:442(740ns) DropOutgoingHandshake:457(444ns)} GenerateSessionVirtualId:988(1104ns) PrepareNewSessionHandshake:477(2248ns) {SwitchToState:179(864ns)}}}}}
2025-11-27T18:18:42.791465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471310552530663:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.775364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471310349190323:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25493745ns) {Configure:104(25481574ns) {ProcessConfigured:135(448184ns) {StartInitialHandshake:157(445229ns) {DropHandshakes:470(1975ns) {DropIncomingHandshake:442(509ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(462ns) PrepareNewSessionHandshake:477(1624ns) {SwitchToState:179(535ns)}}}}}
2025-11-27T18:18:42.796226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471309143410378:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52253
2025-11-27T18:18:42.805760Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471311031424732:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(67440202ns) {Configure:104(67433996ns) {ProcessConfigured:135(53297ns) {StartInitialHandshake:157(50783ns) {DropHandshakes:470(2244ns) {DropIncomingHandshake:442(437ns) DropOutgoingHandshake:457(380ns)} GenerateSessionVirtualId:988(1070ns) PrepareNewSessionHandshake:477(1328ns) {SwitchToState:179(570ns)}}}}}
2025-11-27T18:18:42.791799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471310552530678:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64065
2025-11-27T18:18:42.793598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471310540050986:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(61324245ns) {Configure:104(61320782ns) {ProcessConfigured:135(69353ns) {StartInitialHandshake:157(66478ns) {DropHandshakes:470(1971ns) {DropIncomingHandshake:442(392ns) DropOutgoingHandshake:457(457ns)} GenerateSessionVirtualId:988(1389ns) PrepareNewSessionHandshake:477(1786ns) {SwitchToState:179(1308ns)}}}}}
2025-11-27T18:18:42.796909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471310552530666:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(48439852ns) {Configure:104(48432793ns) {ProcessConfigured:135(10956859ns) {StartInitialHandshake:157(10953693ns) {DropHandshakes:470(4339013ns) {DropIncomingHandshake:442(561ns) DropOutgoingHandshake:457(2727ns)} GenerateSessionVirtualId:988(2305ns) PrepareNewSessionHandshake:477(3456ns) {SwitchToState:179(2204ns)}}}}}
2025-11-27T18:18:42.795861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471310540051059:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21533
2025-11-27T18:18:42.811131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471311246487028:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(60832284ns) {Configure:104(60827679ns) {ProcessConfigured:135(15743667ns) {StartInitialHandshake:157(15740908ns) {DropHandshakes:470(4985167ns) {DropIncomingHandshake:442(2340ns) DropOutgoingHandshake:457(966ns)} GenerateSessionVirtualId:988(2908ns) PrepareNewSessionHandshake:477(3530ns) {SwitchToState:179(1791ns)}}}}}
2025-11-27T18:18:42.812728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471311246486964:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(71378211ns) {Configure:104(71371067ns) {ProcessConfigured:135(9233385ns) {StartInitialHandshake:157(9230887ns) {DropHandshakes:470(2760ns) {DropIncomingHandshake:442(723ns) DropOutgoingHandshake:457(609ns)} GenerateSessionVirtualId:988(761ns) PrepareNewSessionHandshake:477(2727ns) {SwitchToState:179(1365ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.807662Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471310540050985:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(76230665ns) {Configure:104(76223208ns) {ProcessConfigured:135(8428994ns) {StartInitialHandshake:157(8424329ns) {DropHandshakes:470(1000ns) {DropIncomingHandshake:442(423ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(291ns) PrepareNewSessionHandshake:477(3884ns) {SwitchToState:179(2535ns)}}}}}
2025-11-27T18:18:42.822077Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471310552530603:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(79777069ns) {Configure:104(79772268ns) {ProcessConfigured:135(1996871ns) {StartInitialHandshake:157(1995273ns) {DropHandshakes:470(2414ns) {DropIncomingHandshake:442(340ns) DropOutgoingHandshake:457(542ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(1882ns) {SwitchToState:179(810ns)}}}}}
2025-11-27T18:18:42.817234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471309143410302:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(72387664ns) {Configure:104(72380061ns) {ProcessConfigured:135(13699612ns) {StartInitialHandshake:157(13698093ns) {DropHandshakes:470(1280ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(505ns) PrepareNewSessionHandshake:477(2924ns) {SwitchToState:179(1260ns)}}}}}
2025-11-27T18:18:42.818313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471311246487029:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(76905002ns) {Configure:104(76899560ns) {ProcessConfigured:135(10010592ns) {StartInitialHandshake:157(10007396ns) {DropHandshakes:470(2055ns) {DropIncomingHandshake:442(446ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(900ns) PrepareNewSessionHandshake:477(5072ns) {SwitchToState:179(2885ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.811508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471310540051059:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.827178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471311246487036:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43635
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.829800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471311246487042:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43635
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.831770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471310540051068:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21533
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.824933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471310552530678:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.822683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471311031424741:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.821478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471309143410381:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52253
2025-11-27T18:18:42.831943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471309143410378:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.836861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471309143410300:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.829951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471311246487043:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43635
2025-11-27T18:18:42.824165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471311031424745:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12654
2025-11-27T18:18:42.836549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471311246487042:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.840551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471310540051052:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24789653ns) {Configure:104(24782848ns) {ProcessConfigured:135(1532202ns) {StartInitialHandshake:157(1529485ns) {DropHandshakes:470(2654ns) {DropIncomingHandshake:442(606ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(779ns) PrepareNewSessionHandshake:477(2302ns) {SwitchToState:179(869ns)}}}}}
2025-11-27T18:18:42.840258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471311031424745:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.840242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471310349190330:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59210
2025-11-27T18:18:42.836354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471310552530682:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64065
2025-11-27T18:18:42.840231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471310552530666:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.845535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471309143410381:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.838373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471311246487036:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.845355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471311031424738:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.850437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471310349190257:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(75486298ns) {Configure:104(75479632ns) {ProcessConfigured:135(8797307ns) {StartInitialHandshake:157(8795132ns) {DropHandshakes:470(1748ns) {DropIncomingHandshake:442(727ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(369ns) PrepareNewSessionHandshake:477(2019ns) {SwitchToState:179(1186ns)}}}}}
2025-11-27T18:18:42.850449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471310540051058:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.848446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471309143410370:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.845398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471311031424738:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.847741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471310552530682:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.851544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471311246487043:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.852001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471311031424738:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:42.853174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471310349190330:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.850489Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471310540051058:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.851970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471310540051054:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.848471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471309143410302:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.848501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471310552530604:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.852026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471310540051054:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.848497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471309143410370:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.852563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471311246486963:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.855948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471310349190318:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.855367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471310552530685:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64065
2025-11-27T18:18:42.852871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471310540051069:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21533
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.856738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471311246487034:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.851597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471309143410370:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.855180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471310540051068:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.856387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471311031424736:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.856450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471310552530685:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.856777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471311246487034:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.856178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471309143410365:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.856388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471311031424733:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.858314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471310349190332:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59210
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.858298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471310552530673:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.855493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471310540050986:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.858417Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471311246487027:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(39243114ns) {Configure:104(39236656ns) {ProcessConfigured:135(789336ns) {StartInitialHandshake:157(786580ns) {DropHandshakes:470(2360ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(292ns)} GenerateSessionVirtualId:988(606ns) PrepareNewSessionHandshake:477(1589ns) {SwitchToState:179(687ns)}}}}}
2025-11-27T18:18:42.863448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471310349190336:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59210
2025-11-27T18:18:42.856463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:42.856217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471310540051054:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:42.858328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471310552530673:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.856430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471311031424736:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.863866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471311246487034:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:42.856523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471309143410365:2108] [node 5] ICP22 created new session: [4:7577471309143410386:2108]
2025-11-27T18:18:42.861087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471310552530687:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64065
2025-11-27T18:18:42.861283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471310552530603:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.864192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471310552530678:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:42.865233Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471310552530673:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:42.856448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.865007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471311246487031:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.865094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471310540051058:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:42.856581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471309143410386:2108] [node 5] ICS09 handshake done sender: [4:7577471309143410370:2113] self: [4:7577471309143410369:0] peer: [5:7577471310552530684:0] socket: 539 qp: -1
2025-11-27T18:18:42.868868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471310540051073:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21533
2025-11-27T18:18:42.871159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471310349190332:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.856775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471309143410385:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52253
2025-11-27T18:18:42.865038Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471311246487031:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.858321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471311031424733:2107] [node 4] ICP22 created new session: [1:7577471311031424754:2107]
2025-11-27T18:18:42.872364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471310552530688:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64065
2025-11-27T18:18:42.872681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471310349190338:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59210
2025-11-27T18:18:42.874309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471311246487031:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:42.872064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471310540051050:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.872493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471310552530666:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.858884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471311031424754:2107] [node 4] ICS09 handshake done sender: [1:7577471311031424738:2112] self: [1:7577471311031424737:0] peer: [4:7577471309143410382:0] socket: 537 qp: -1
2025-11-27T18:18:42.856809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471309143410386:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:42.872130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:42.872919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471310349190256:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.874973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471311246486964:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.861320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471311031424754:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:42.872993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:42.857357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:42.863487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:42.875257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:42.872198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471310540051069:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.873820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471310349190330:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:42.873788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471310552530666:2109] [node 4] ICP22 created new session: [5:7577471310552530691:2109]
2025-11-27T18:18:42.864239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471309143410300:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471309143410378:2116] held: no
2025-11-27T18:18:42.875498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471310349190336:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.875848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471311246486964:2049] [node 3] ICP22 created new session: [2:7577471311246487052:2049]
2025-11-27T18:18:42.872389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471310540051051:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.867865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471309143410378:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:42.872350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471311031424736:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:42.874018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471310552530691:2109] [node 4] ICS09 handshake done sender: [5:7577471310552530678:2114] self: [5:7577471310552530684:0] peer: [4:7577471309143410369:0] socket: 544 qp: -1
2025-11-27T18:18:42.875918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471310349190318:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.878526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471311246487052:2049] [node 3] ICS09 handshake done sender: [2:7577471311246487034:2112] self: [2:7577471311246487032:0] peer: [3:7577471310540051071:0] socket: 559 qp: -1
2025-11-27T18:18:42.872497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471310540051050:2109] [node 5] ICP22 created new session: [3:7577471310540051074:2109]
2025-11-27T18:18:42.874443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471310552530691:2109] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.872419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471309143410300:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.873035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471311031424753:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12654
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.873101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:42.879782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:42.878569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471311246487052:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:42.872942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:42.874746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:42.882023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471310349190318:2107] [node 5] ICP22 created new session: [6:7577471310349190341:2107]
2025-11-27T18:18:42.878605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:42.873802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471309143410300:2048] [node 1] ICP22 created new session: [4:7577471309143410390:2048]
2025-11-27T18:18:42.874890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471311031424668:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.873601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471310540051051:2107] [node 4] ICP22 created new session: [3:7577471310540051075:2107]
2025-11-27T18:18:42.874933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:42.874705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471309143410390:2048] [node 1] ICS09 handshake done sender: [4:7577471309143410378:2116] self: [4:7577471309143410382:0] peer: [1:7577471311031424737:0] socket: 546 qp: -1
2025-11-27T18:18:42.875170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471310552530667:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.883377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471310349190341:2107] [node 5] ICS09 handshake done sender: [6:7577471310349190330:2114] self: [6:7577471310349190337:0] peer: [5:7577471310552530672:0] socket: 542 qp: -1
2025-11-27T18:18:42.873856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471310540051075:2107] [node 4] ICS09 handshake done sender: [3:7577471310540051058:2114] self: [3:7577471310540051057:0] peer: [4:7577471309143410383:0] socket: 552 qp: -1
2025-11-27T18:18:42.879587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471311246487028:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.875444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471311031424740:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.875226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:42.884463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471310349190341:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:42.879628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:42.874797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471309143410388:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52253
2025-11-27T18:18:42.874682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471310540051074:2109] [node 5] ICS09 handshake done sender: [3:7577471310540051054:2112] self: [3:7577471310540051053:0] peer: [5:7577471310552530686:0] socket: 548 qp: -1
2025-11-27T18:18:42.875841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471310552530667:2110] [node 6] ICP22 created new session: [5:7577471310552530695:2110]
2025-11-27T18:18:42.875481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471311031424740:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.874942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471309143410390:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:42.884690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:42.875049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471310540051075:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:42.879682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471311246487028:2109] [node 5] ICP22 created new session: [2:7577471311246487054:2109]
2025-11-27T18:18:42.876300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471310552530695:2110] [node 6] ICS09 handshake done sender: [5:7577471310552530673:2113] self: [5:7577471310552530672:0] peer: [6:7577471310349190337:0] socket: 533 qp: -1
2025-11-27T18:18:42.881807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471311246487054:2109] [node 5] ICS09 handshake done sender: [2:7577471311246487031:2111] self: [2:7577471311246487030:0] peer: [5:7577471310552530689:0] socket: 558 qp: -1
2025-11-27T18:18:42.875160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:42.882740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471311246487054:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:42.878186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471310552530695:2110] [node 6] ICS10 traffic start
No RDMA context found for GID: No RDMA context found for GID: 2025-11-27T18:18:42.883306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:18:42.882032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471309143410381:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:42.878693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:42.893463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471309143410302:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.875106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:42.875508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471311031424668:2048] [node 2] ICP22 created new session: [1:7577471311031424758:2048]
2025-11-27T18:18:42.893294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471310349190257:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.894492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471310552530682:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:42.893915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:42.893970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471311246487051:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43635
2025-11-27T18:18:42.878127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471311031424758:2048] [node 2] ICS09 handshake done sender: [1:7577471311031424736:2111] self: [1:7577471311031424734:0] peer: [2:7577471311246487044:0] socket: 536 qp: -1
2025-11-27T18:18:42.875159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471310540051074:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:42.894965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471310349190340:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59210
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.878435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471311031424758:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:42.875207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:42.896531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471310552530667:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(18056321ns) {HandleHandshakeStatus:329(18053819ns) {DropIncomingHandshake:442(10915ns) DropOutgoingHandshake:457(143ns) SwitchToState:179(287ns) ProcessPendingSessionEvents:495(11305856ns) {ForwardSessionEventToSession:612(1427250ns) ForwardSessionEventToSession:612(1220367ns) ForwardSessionEventToSession:612(878161ns) ForwardSessionEventToSession:612(2321ns) ForwardSessionEventToSession:612(365158ns) ForwardSessionEventToSession:612(21101ns) ForwardSessionEventToSession:612(3551ns) ForwardSessionEventToSession:612(3222587ns) ForwardSessionEventToSession:612(3218ns) ForwardSessionEventToSession:612(873944ns)}}}
2025-11-27T18:18:42.882902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471310540051068:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:42.878976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471311031424733:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17602181ns) {HandleHandshakeStatus:329(17599131ns) {DropIncomingHandshake:442(13359ns) DropOutgoingHandshake:457(403ns) SwitchToState:179(329ns) ProcessPendingSessionEvents:495(4225480ns) {ForwardSessionEventToSession:612(1619444ns) ForwardSessionEventToSession:612(659148ns) ForwardSessionEventToSession:612(3618ns) ForwardSessionEventToSession:612(3395ns) ForwardSessionEventToSession:612(433100ns) ForwardSessionEventToSession:612(3491ns) ForwardSessionEventToSession:612(995398ns) ForwardSessionEventToSession:612(320121ns) ForwardSessionEventToSession:612(3091ns) ForwardSessionEventToSession:612(170454ns)}}}
2025-11-27T18:18:42.879228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:42.897457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471310552530604:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7508) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471310552530682:2116] held: no
2025-11-27T18:18:42.879815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471311031424753:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.896468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471311246486964:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(18757621ns) {HandleHandshakeStatus:329(18560059ns) {DropIncomingHandshake:442(12163ns) DropOutgoingHandshake:457(196ns) SwitchToState:179(451ns) ProcessPendingSessionEvents:495(11417797ns) {ForwardSessionEventToSession:612(2725802ns) ForwardSessionEventToSession:612(8700ns) ForwardSessionEventToSession:612(3369ns) ForwardSessionEventToSession:612(838786ns) ForwardSessionEventToSession:612(3534ns) ForwardSessionEventToSession:612(3255ns) ForwardSessionEventToSession:612(5743879ns) ForwardSessionEventToSession:612(2962ns) ForwardSessionEventToSession:612(2054996ns) ForwardSessionEventToSession:612(12682ns)}}}
2025-11-27T18:18:42.895473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471310540050986:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29610) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471310540051068:2118] held: no
2025-11-27T18:18:42.894581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471309143410302:2050] [node 3] ICP22 created new session: [4:7577471309143410393:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.897122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471310540050986:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.909914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471310349190318:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(20338343ns) {HandleHandshakeStatus:329(20332992ns) {DropIncomingHandshake:442(366ns) DropOutgoingHandshake:457(7555ns) SwitchToState:179(2567ns) ProcessPendingSessionEvents:495(2656559ns) {ForwardSessionEventToSession:612(1057663ns) ForwardSessionEventToSession:612(392623ns) ForwardSessionEventToSession:612(361462ns) ForwardSessionEventToSession:612(5256ns) ForwardSessionEventToSession:612(2942ns) ForwardSessionEventToSession:612(15192ns) ForwardSessionEventToSession:612(4219ns) ForwardSessionEventToSession:612(2848ns) ForwardSessionEventToSession:612(409964ns) ForwardSessionEventToSession:612(387609ns)}}}
2025-11-27T18:18:42.898414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471310552530692:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64065
2025-11-27T18:18:42.894813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471311031424740:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:42.894874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471309143410393:2050] [node 3] ICS09 handshake done sender: [4:7577471309143410381:2118] self: [4:7577471309143410383:0] peer: [3:7577471310540051057:0] socket: 554 qp: -1
2025-11-27T18:18:42.898315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471311246487035:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.902081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471310552530604:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.897159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:42.896842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471311031424668:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(19350193ns) {HandleHandshakeStatus:329(19347981ns) {DropIncomingHandshake:442(240ns) DropOutgoingHandshake:457(243ns) SwitchToState:179(505ns) ProcessPendingSessionEvents:495(11377009ns) {ForwardSessionEventToSession:612(2038871ns) ForwardSessionEventToSession:612(4079ns) ForwardSessionEventToSession:612(1081918ns) ForwardSessionEventToSession:612(3152ns) ForwardSessionEventToSession:612(6331712ns) ForwardSessionEventToSession:612(910175ns) ForwardSessionEventToSession:612(6611ns) ForwardSessionEventToSession:612(665887ns) ForwardSessionEventToSession:612(3635ns) ForwardSessionEventToSession:612(2867ns)}}}
2025-11-27T18:18:42.898359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471311246487035:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.911958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471310349190344:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59210
2025-11-27T18:18:42.895091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471309143410393:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:42.901654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471310540050986:2049] [node 2] ICP22 created new session: [3:7577471310540051083:2049]
2025-11-27T18:18:42.897305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471311031424735:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.909459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:42.895396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.911179Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471311246487028:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(17505817ns) {HandleHandshakeStatus:329(17503328ns) {DropIncomingHandshake:442(313ns) DropOutgoingHandshake:457(291ns) SwitchToState:179(328ns) ProcessPendingSessionEvents:495(3084407ns) {ForwardSessionEventToSession:612(1413433ns) ForwardSessionEventToSession:612(6644ns) ForwardSessionEventToSession:612(373688ns) ForwardSessionEventToSession:612(3297ns) ForwardSessionEventToSession:612(968764ns) ForwardSessionEventToSession:612(8010ns) ForwardSessionEventToSession:612(2856ns) ForwardSessionEventToSession:612(279434ns) ForwardSessionEventToSession:612(2941ns) ForwardSessionEventToSession:612(2556ns)}}}
No RDMA context found for GID: 2025-11-27T18:18:42.913328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471310349190332:2115] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.910205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471310552530604:2050] [node 3] ICP22 created new session: [5:7577471310552530697:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.911219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471311246487036:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:42.913346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471310349190340:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.915879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471310349190346:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59210
2025-11-27T18:18:42.913011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471311246486963:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.911951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471309143410302:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16612926ns) {HandleHandshakeStatus:329(16610085ns) {DropIncomingHandshake:442(2029ns) DropOutgoingHandshake:457(6348ns) SwitchToState:179(571ns) ProcessPendingSessionEvents:495(13211661ns) {ForwardSessionEventToSession:612(571208ns) ForwardSessionEventToSession:612(4727ns) ForwardSessionEventToSession:612(952564ns) ForwardSessionEventToSession:612(3854ns) ForwardSessionEventToSession:612(4708782ns) ForwardSessionEventToSession:612(4297ns) ForwardSessionEventToSession:612(2630ns) ForwardSessionEventToSession:612(6941093ns) ForwardSessionEventToSession:612(4371ns) ForwardSessionEventToSession:612(2845ns)}}}
2025-11-27T18:18:42.897349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:42.903121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471310540051083:2049] [node 2] ICS09 handshake done sender: [3:7577471310540051068:2118] self: [3:7577471310540051071:0] peer: [2:7577471311246487032:0] socket: 564 qp: -1
2025-11-27T18:18:42.916828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471310349190256:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471310349190332:2115] held: no
2025-11-27T18:18:42.919629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471309143410395:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52253
2025-11-27T18:18:42.917080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471310349190256:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.903295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471311031424735:2109] [node 6] ICP22 created new session: [1:7577471311031424764:2109]
2025-11-27T18:18:42.914514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:42.910643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471310552530697:2050] [node 3] ICS09 handshake done sender: [5:7577471310552530682:2116] self: [5:7577471310552530686:0] peer: [3:7577471310540051053:0] socket: 549 qp: -1
2025-11-27T18:18:42.910341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471310540051083:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:42.910178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471311031424764:2109] [node 6] ICS09 handshake done sender: [1:7577471311031424740:2113] self: [1:7577471311031424739:0] peer: [6:7577471310349190339:0] socket: 543 qp: -1
2025-11-27T18:18:42.919657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471309143410396:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52253
2025-11-27T18:18:42.915132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471311246486963:2048] [node 1] ICP22 created new session: [2:7577471311246487059:2048]
2025-11-27T18:18:42.911010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471310552530697:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:42.917463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:42.910665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:42.916116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471311246487059:2048] [node 1] ICS09 handshake done sender: [2:7577471311246487036:2114] self: [2:7577471311246487044:0] peer: [1:7577471311031424734:0] socket: 555 qp: -1
2025-11-27T18:18:42.910221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471311031424764:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:42.919109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471310349190256:2048] [node 1] ICP22 created new session: [6:7577471310349190347:2048]
2025-11-27T18:18:42.910279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:42.911283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:42.911852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471310552530694:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64065
2025-11-27T18:18:42.916165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471311246487059:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:42.912956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471310540051079:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21533
2025-11-27T18:18:42.916867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471310552530685:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:42.919688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471310349190347:2048] [node 1] ICS09 handshake done sender: [6:7577471310349190332:2115] self: [6:7577471310349190339:0] peer: [1:7577471311031424739:0] socket: 557 qp: -1
2025-11-27T18:18:42.920736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471310552530694:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.920924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471310349190347:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:42.920765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471310552530603:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.920966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:42.917038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:42.919253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471311246487035:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:42.920991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:42.916609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471311031424763:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12654
2025-11-27T18:18:42.915171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471310540051051:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:42.926363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471309143410395:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.921465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471310552530603:2049] [node 2] ICP22 created new session: [5:7577471310552530700:2049]
2025-11-27T18:18:42.919599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471310540051081:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21533
2025-11-27T18:18:42.923387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471310349190323:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.919541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471311031424766:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12654
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.922005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471310552530700:2049] [node 2] ICS09 handshake done sender: [5:7577471310552530685:2118] self: [5:7577471310552530689:0] peer: [2:7577471311246487030:0] socket: 561 qp: -1
2025-11-27T18:18:42.929012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471311246487029:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.921361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471311031424766:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.928997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471310349190346:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.921407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471310540051081:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.922160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471310552530700:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:42.926252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471311031424749:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.929295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:42.922189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471310552530602:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.929494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471311246487029:2110] [node 6] ICP22 created new session: [2:7577471311246487062:2110]
2025-11-27T18:18:42.930435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471310349190348:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59210
2025-11-27T18:18:42.922202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:42.926266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471311031424747:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.926291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471311031424749:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.926310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471311031424747:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.922168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471310540050985:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.930828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471310349190258:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.930808Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471311031424668:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:42.930568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471311246487062:2110] [node 6] ICS09 handshake done sender: [2:7577471311246487035:2113] self: [2:7577471311246487033:0] peer: [6:7577471310349190343:0] socket: 560 qp: -1
2025-11-27T18:18:42.929405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471309143410372:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.930959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471310349190336:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:42.930744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471311246487062:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:42.929448Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471309143410372:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.932224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471309143410372:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:42.935727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471310540051067:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.930806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:42.935037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471311031424749:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:42.932240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471309143410396:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.933497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471311246487061:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43635
2025-11-27T18:18:42.935510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471311031424747:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:42.935775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471310540051067:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.937095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471311031424732:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.937204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471311031424669:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:42.937237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:18:42.937284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
::ffff:127.1.0.0
2025-11-27T18:18:42.937336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471311031424669:2049] [node 3] ICP22 created new session: [1:7577471311031424771:2049]
2025-11-27T18:18:42.937570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471311031424732:2108] [node 5] ICP22 created new session: [1:7577471311031424770:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.934932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471309143410301:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:42.935578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471309143410366:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.936570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471310349190257:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.936359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:42.940141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471311246487047:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:42.937233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:42.940185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471311246487047:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:42.937639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471311031424770:2108] [node 5] ICS09 handshake done sender: [1:7577471311031424749:2117] self: [1:7577471311031424748:0] peer: [5:7577471310552530701:0] socket: 582 qp: -1
2025-11-27T18:18:42.936433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471309143410366:2110] [node 6] ICP22 created new session: [4:7577471309143410400:2110]
2025-11-27T18:18:42.939023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471310540051086:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21533
2025-11-27T18:18:42.938704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471311031424770:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:42.939034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471310552530703:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64065
2025-11-27T18:18:42.937583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471310349190257:2049] [node 2] ICP22 created new session: [6:7577471310349190354:2049]
2025-11-27T18:18:42.940082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471310540051067:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:42.940827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471311246487063:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43635
2025-11-27T18:18:42.940113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471310552530602:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471310552530694:2122] held: no
2025-11-27T18:18:42.936495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471309143410400:2110] [node 6] ICS09 handshake done sender: [4:7577471309143410372:2114] self: [4:7577471309143410371:0] peer: [6:7577471310349190350:0] socket: 583 qp: -1
2025-11-27T18:18:42.938903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471311031424771:2049] [node 3] ICS09 handshake done sender: [1:7577471311031424747:2116] self: [1:7577471311031424746:0] peer: [3:7577471310540051085:0] socket: 550 qp: -1
2025-11-27T18:18:42.938899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471310349190354:2049] [node 2] ICS09 handshake done sender: [6:7577471310349190336:2117] self: [6:7577471310349190343:0] peer: [2:7577471311246487033:0] socket: 565 qp: -1
2025-11-27T18:18:42.942058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471310552530694:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:42.936560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471309143410400:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:42.942900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471311246487029:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:42.936605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:42.938912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:42.940973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471310540051081:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:42.938938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471310349190354:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:42.943655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471310552530602:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.943695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471311246487047:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:42.939032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:42.943699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:42.943204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471310540050985:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.938952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471311031424771:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:42.947078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471310552530602:2048] [node 1] ICP22 created new session: [5:7577471310552530704:2048]
2025-11-27T18:18:42.939298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:42.946362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471310540051052:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.947022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: No RDMA context found for GID: 2025-11-27T18:18:42.947340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471310540050985:2048] [node 1] ICP22 created new session: [3:7577471310540051088:2048]
::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:18:42.948052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471311246487027:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:42.948050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471310552530704:2048] [node 1] ICS09 handshake done sender: [5:7577471310552530694:2122] self: [5:7577471310552530701:0] peer: [1:7577471311031424748:0] socket: 586 qp: -1
2025-11-27T18:18:42.947696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471310540051088:2048] [node 1] ICS09 handshake done sender: [3:7577471310540051081:2122] self: [3:7577471310540051085:0] peer: [1:7577471311031424746:0] socket: 563 qp: -1
2025-11-27T18:18:42.947927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471310349190323:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7628) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471310349190340:2119] held: no
2025-11-27T18:18:42.948289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471310552530704:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:42.948459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471309143410402:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52253
2025-11-27T18:18:42.947739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:42.948429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471310349190353:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59210
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:42.947806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471310540051052:2110] [node 6] ICP22 created new session: [3:7577471310540051089:2110]
2025-11-27T18:18:42.948930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:42.948516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:42.949531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471310349190340:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:42.949684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471311246487027:2107] [node 4] ICP22 created new session: [2:7577471311246487066:2107]
2025-11-27T18:18:42.950341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471310349190356:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59210
2025-11-27T18:18:42.950453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471309143410396:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:42.947811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471310540051088:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:42.953687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471311246487066:2107] [node 4] ICS09 handshake done sender: [2:7577471311246487047:2118] self: [2:7577471311246487046:0] peer: [4:7577471309143410399:0] socket: 545 qp: -1
2025-11-27T18:18:42.962610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471310349190323:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.955918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471309143410301:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.962674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:42.962734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471310349190323:2110] [node 4] ICP22 created new session: [6:7577471310349190357:2110]
2025-11-27T18:18:42.962808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471310349190357:2110] [node 4] ICS09 handshake done sender: [6:7577471310349190340:2119] self: [6:7577471310349190350:0] peer: [4:7577471309143410371:0] socket: 585 qp: -1
2025-11-27T18:18:42.962851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471310349190357:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:42.962894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:42.971164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471310349190258:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7508) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471310349190346:2121] held: no
2025-11-27T18:18:42.971364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471310349190346:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:42.973414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471310349190258:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:42.973459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:42.973521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471310349190258:2050] [node 3] ICP22 created new session: [6:7577471310349190359:2050]
2025-11-27T18:18:42.973565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471310349190359:2050] [node 3] ICS09 handshake done sender: [6:7577471310349190346:2121] self: [6:7577471310349190352:0] peer: [3:7577471310540051066:0] socket: 593 qp: -1
2025-11-27T18:18:42.973597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471310349190359:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:42.973640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:42.956245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471311246487066:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:42.956304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:42.947872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471310540051089:2110] [node 6] ICS09 handshake done sender: [3:7577471310540051067:2117] self: [3:7577471310540051066:0] peer: [6:7577471310349190352:0] socket: 592 qp: -1
2025-11-27T18:18:42.947883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:42.947910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471310540051089:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:42.947941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:42.956127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:42.962070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471309143410301:2049] [node 2] ICP22 created new session: [4:7577471309143410403:2049]
2025-11-27T18:18:42.971584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471309143410403:2049] [node 2] ICS09 handshake done sender: [4:7577471309143410396:2122] self: [4:7577471309143410399:0] peer: [2:7577471311246487046:0] socket: 596 qp: -1
2025-11-27T18:18:42.971645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471309143410403:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:42.971704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:42.972671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471309143410301:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16560289ns) {HandleHandshakeStatus:329(16557154ns) {DropIncomingHandshake:442(898ns) DropOutgoingHandshake:457(13633ns) SwitchToState:179(591ns) ProcessPendingSessionEvents:495(206031ns) {ForwardSessionEventToSession:612(147748ns) ForwardSessionEventToSession:612(3328ns) ForwardSessionEventToSession:612(6199ns) ForwardSessionEventToSession:612(3177ns) ForwardSessionEventToSession:612(5596ns) ForwardSessionEventToSession:612(8788ns) ForwardSessionEventToSession:612(3407ns) ForwardSessionEventToSession:612(14660ns) ForwardSessionEventToSession:612(2569ns) ForwardSessionEventToSession:612(2274ns)}}}
2025-11-27T18:18:42.974854Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471311246487027:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(24293592ns) {HandleHandshakeStatus:329(24291790ns) {DropIncomingHandshake:442(258248ns) DropOutgoingHandshake:457(209ns) SwitchToState:179(372ns) ProcessPendingSessionEvents:495(3468498ns) {ForwardSessionEventToSession:612(3060691ns) ForwardSessionEventToSession:612(222270ns) ForwardSessionEventToSession:612(3395ns) ForwardSessionEventToSession:612(6247ns) ForwardSessionEventToSession:612(3883ns) ForwardSessionEventToSession:612(6081ns) ForwardSessionEventToSession:612(5168ns) ForwardSessionEventToSession:612(2516ns) ForwardSessionEventToSession:612(3396ns) ForwardSessionEventToSession:612(142638ns)}}}
2025-11-27T18:18:43.019005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:43.019277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471310540051088:2048] [node 1] ICS07 socket disconnect 563 reason# EndOfStream
2025-11-27T18:18:43.019306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471310540051088:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.019389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:43.019444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471310540051088:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:43.021341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471310540050985:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28209) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:43.021470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471310540051088:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:43.021512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471310540050985:2048] [node 1] ICP30 unregister session Session# [3:7577471310540051088:2048] VirtualId# [3:7577471310540051085:0]
2025-11-27T18:18:43.021576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:43.021620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471310540050985:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28209) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.021648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28209) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.021707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:43.021983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471311246487059:2048] [node 1] ICS07 socket disconnect 555 reason# EndOfStream
2025-11-27T18:18:43.022045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471311246487059:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.022133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:43.022188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471311246487059:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:43.022233Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:43.022479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471309143410390:2048] [node 1] ICS01 socket: 546 reason# EndOfStream
2025-11-27T18:18:43.022524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471309143410300:2048] [node 1] ICP30 unregister session Session# [4:7577471309143410390:2048] VirtualId# [4:7577471309143410382:0]
2025-11-27T18:18:43.022559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471309143410390:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.022628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:43.022667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:43.023121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471311246486963:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28209) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:43.023210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471311246487059:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:43.023262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471311246486963:2048] [node 1] ICP30 unregister session Session# [2:7577471311246487059:2048] VirtualId# [2:7577471311246487044:0]
2025-11-27T18:18:43.023306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:43.023356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471311246486963:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28209) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.023377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28209) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.023326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:43.023526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471310552530704:2048] [node 1] ICS01 socket: 586 reason# EndOfStream
2025-11-27T18:18:43.023578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471310552530602:2048] [node 1] ICP30 unregister session Session# [5:7577471310552530704:2048] VirtualId# [5:7577471310552530701:0]
2025-11-27T18:18:43.023608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471310552530704:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.023701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:43.023761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:43.024839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:43.025041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471310349190347:2048] [node 1] ICS01 socket: 557 reason# EndOfStream
2025-11-27T18:18:43.025089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471310349190256:2048] [node 1] ICP30 unregister session Session# [6:7577471310349190347:2048] VirtualId# [6:7577471310349190339:0]
2025-11-27T18:18:43.025136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471310349190347:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.025228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:43.025304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:43.031465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471310540050985:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:43.068376Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:43.068596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471310540051083:2049] [node 2] ICS01 socket: 564 reason# EndOfStream
2025-11-27T18:18:43.068654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471310540050986:2049] [node 2] ICP30 unregister session Session# [3:7577471310540051083:2049] VirtualId# [3:7577471310540051071:0]
2025-11-27T18:18:43.068714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471310540051083:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.068819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:43.068923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:43.071272Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:43.071487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471310552530700:2049] [node 2] ICS07 socket disconnect 561 reason# EndOfStream
2025-11-27T18:18:43.071521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471310552530700:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.071596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:43.071639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471310552530700:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:43.072609Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471310552530603:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29610) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:43.072682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471310552530700:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:43.071928Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:43.072721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471310552530603:2049] [node 2] ICP30 unregister session Session# [5:7577471310552530700:2049] VirtualId# [5:7577471310552530689:0]
2025-11-27T18:18:43.072141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471309143410403:2049] [node 2] ICS01 socket: 596 reason# EndOfStream
2025-11-27T18:18:43.072186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471309143410301:2049] [node 2] ICP30 unregister session Session# [4:7577471309143410403:2049] VirtualId# [4:7577471309143410399:0]
2025-11-27T18:18:43.072771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:43.072218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471309143410403:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.072829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471310552530603:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29610) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.072857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29610) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.072767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:43.072296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:43.073011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471310349190354:2049] [node 2] ICS07 socket disconnect 565 reason# EndOfStream
2025-11-27T18:18:43.072344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:43.073035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471310349190354:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.073129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:43.073178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471310349190354:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:43.073993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471310349190257:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29610) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:43.075401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471310349190354:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:43.075441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471310349190257:2049] [node 2] ICP30 unregister session Session# [6:7577471310349190354:2049] VirtualId# [6:7577471310349190343:0]
2025-11-27T18:18:43.075531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:43.075564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471310349190257:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29610) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.075580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29610) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.082573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471310552530603:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:43.085671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471310349190257:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:43.158946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:43.159206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471309143410393:2050] [node 3] ICS07 socket disconnect 554 reason# EndOfStream
2025-11-27T18:18:43.159251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471309143410393:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.159345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:43.159423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471309143410393:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:43.160305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471309143410302:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7508) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:43.160380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471309143410393:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:43.160417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:43.160424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471309143410302:2050] [node 3] ICP30 unregister session Session# [4:7577471309143410393:2050] VirtualId# [4:7577471309143410383:0]
2025-11-27T18:18:43.160498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:43.160605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471309143410302:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7508) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.160621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7508) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.160687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471310552530697:2050] [node 3] ICS01 socket: 549 reason# EndOfStream
2025-11-27T18:18:43.160740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471310552530604:2050] [node 3] ICP30 unregister session Session# [5:7577471310552530697:2050] VirtualId# [5:7577471310552530686:0]
2025-11-27T18:18:43.160789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471310552530697:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.160888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:43.160971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:43.161108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:43.161321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471310349190359:2050] [node 3] ICS01 socket: 593 reason# EndOfStream
2025-11-27T18:18:43.161371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471310349190258:2050] [node 3] ICP30 unregister session Session# [6:7577471310349190359:2050] VirtualId# [6:7577471310349190352:0]
2025-11-27T18:18:43.161402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471310349190359:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.161477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:43.161551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:43.218618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:43.218850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471310552530691:2109] [node 4] ICS07 socket disconnect 544 reason# EndOfStream
2025-11-27T18:18:43.218879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471310552530691:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.218980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:43.219035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471310552530691:2109] [node 4] ICS15 start handshake
2025-11-27T18:18:43.220089Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471310552530666:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7628) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:43.220137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471310552530691:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:43.220174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471310552530666:2109] [node 4] ICP30 unregister session Session# [5:7577471310552530691:2109] VirtualId# [5:7577471310552530684:0]
2025-11-27T18:18:43.220232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:43.220260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471310552530666:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.220274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:43.221472Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:43.221672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471310349190357:2110] [node 4] ICS01 socket: 585 reason# EndOfStream
2025-11-27T18:18:43.221698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471310349190323:2110] [node 4] ICP30 unregister session Session# [6:7577471310349190357:2110] VirtualId# [6:7577471310349190350:0]
2025-11-27T18:18:43.221720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471310349190357:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.221779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:43.221808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:43.230225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471310552530666:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:18:43.290292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:43.290536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471310349190341:2107] [node 5] ICS01 socket: 542 reason# EndOfStream
2025-11-27T18:18:43.290575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471310349190318:2107] [node 5] ICP30 unregister session Session# [6:7577471310349190341:2107] VirtualId# [6:7577471310349190337:0]
2025-11-27T18:18:43.290607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471310349190341:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:43.290697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:43.290747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:43.402517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471313651843673:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:43.402579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471313651843674:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:43.425724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471316673730420:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:43.425767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471316673730421:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:43.468578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471316505212297:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:43.468616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471316505212298:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:43.497159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471316068602860:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:43.497201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471316068602862:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:43.497210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471316068602861:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:43.527996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471314487703333:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:43.528037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471314487703334:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:43.528082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471314487703335:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:43.552671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471314262366240:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:43.552709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471314262366241:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:43.552734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471314262366242:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:43.650562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471314262366301:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:43.724193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471313651843737:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:43.744872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471316673730485:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:43.737445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471316505212361:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:43.749859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471316068602941:2118] [node 6] ICP01 ready to work
2025-11-27T18:18:43.749889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471316673730487:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.745423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471314262366308:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:43.743502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471316505212362:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:43.749646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471316673730486:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:43.734351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471313651843736:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:43.747139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471316505212363:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:43.752546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471314487703397:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:43.749646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471313651843739:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:43.754940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471314487703333:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16797846ns) {Configure:104(16791308ns) {ProcessConfigured:135(60110ns) {StartInitialHandshake:157(57398ns) {DropHandshakes:470(1999ns) {DropIncomingHandshake:442(454ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(1616ns) PrepareNewSessionHandshake:477(1076ns) {SwitchToState:179(273ns)}}}}}
2025-11-27T18:18:43.770422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471314262366241:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(79205980ns) {Configure:104(79199191ns) {ProcessConfigured:135(79874ns) {StartInitialHandshake:157(74777ns) {DropHandshakes:470(2781ns) {DropIncomingHandshake:442(557ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(1233ns) PrepareNewSessionHandshake:477(1672ns) {SwitchToState:179(568ns)}}}}}
2025-11-27T18:18:43.762853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471316068602942:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.766551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471313651843674:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(89024198ns) {Configure:104(89011981ns) {ProcessConfigured:135(11966308ns) {StartInitialHandshake:157(11963444ns) {DropHandshakes:470(2985ns) {DropIncomingHandshake:442(598ns) DropOutgoingHandshake:457(572ns)} GenerateSessionVirtualId:988(1207ns) PrepareNewSessionHandshake:477(3769ns) {SwitchToState:179(1902ns)}}}}}
2025-11-27T18:18:43.756354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471314487703394:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2880
2025-11-27T18:18:43.778718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471314262366242:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(87248164ns) {Configure:104(87243063ns) {ProcessConfigured:135(56771ns) {StartInitialHandshake:157(53836ns) {DropHandshakes:470(2015ns) {DropIncomingHandshake:442(598ns) DropOutgoingHandshake:457(294ns)} GenerateSessionVirtualId:988(723ns) PrepareNewSessionHandshake:477(1316ns) {SwitchToState:179(362ns)}}}}}
2025-11-27T18:18:43.779084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471314262366313:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22870
2025-11-27T18:18:43.757604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471314487703398:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:43.787731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471313651843673:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(105138446ns) {Configure:104(105129737ns) {ProcessConfigured:135(16458329ns) {StartInitialHandshake:157(16455677ns) {DropHandshakes:470(3316ns) {DropIncomingHandshake:442(771ns) DropOutgoingHandshake:457(479ns)} GenerateSessionVirtualId:988(746ns) PrepareNewSessionHandshake:477(3482ns) {SwitchToState:179(756ns)}}}}}
2025-11-27T18:18:43.807571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471314487703394:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.807368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471316673730485:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(53974982ns) {Configure:104(53970319ns) {ProcessConfigured:135(1255508ns) {StartInitialHandshake:157(1252276ns) {DropHandshakes:470(1249ns) {DropIncomingHandshake:442(391ns) DropOutgoingHandshake:457(172ns)} GenerateSessionVirtualId:988(803ns) PrepareNewSessionHandshake:477(2687ns) {SwitchToState:179(1179ns)}}}}}
2025-11-27T18:18:43.807229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471313651843736:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(58748671ns) {Configure:104(58740186ns) {ProcessConfigured:135(1632200ns) {StartInitialHandshake:157(1629973ns) {DropHandshakes:470(1470ns) {DropIncomingHandshake:442(313ns) DropOutgoingHandshake:457(308ns)} GenerateSessionVirtualId:988(670ns) PrepareNewSessionHandshake:477(2554ns) {SwitchToState:179(1203ns)}}}}}
2025-11-27T18:18:43.807741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471314262366313:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.816775Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471316068602860:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(115121051ns) {Configure:104(106106578ns) {ProcessConfigured:135(2765745ns) {StartInitialHandshake:157(2763270ns) {DropHandshakes:470(1018ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(460ns) PrepareNewSessionHandshake:477(2840ns) {SwitchToState:179(1297ns)}}}}}
2025-11-27T18:18:43.817606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471316068602862:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51983945ns) {Configure:104(51977509ns) {ProcessConfigured:135(70476ns) {StartInitialHandshake:157(67285ns) {DropHandshakes:470(2320ns) {DropIncomingHandshake:442(462ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:477(962ns) {SwitchToState:179(422ns)}}}}}
2025-11-27T18:18:43.816485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471316505212363:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(49666216ns) {Configure:104(49657730ns) {ProcessConfigured:135(2134834ns) {StartInitialHandshake:157(264326ns) {DropHandshakes:470(1689ns) {DropIncomingHandshake:442(1061ns) DropOutgoingHandshake:457(120ns)} GenerateSessionVirtualId:988(1189ns) PrepareNewSessionHandshake:477(1486ns) {SwitchToState:179(829ns)}}}}}
2025-11-27T18:18:43.813847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471314487703335:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47065097ns) {Configure:104(47056359ns) {ProcessConfigured:135(6897221ns) {StartInitialHandshake:157(6893483ns) {DropHandshakes:470(1663ns) {DropIncomingHandshake:442(443ns) DropOutgoingHandshake:457(295ns)} GenerateSessionVirtualId:988(632ns) PrepareNewSessionHandshake:477(3729ns) {SwitchToState:179(1868ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.822556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471313651843737:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32304925ns) {Configure:104(32296629ns) {ProcessConfigured:135(44211ns) {StartInitialHandshake:157(41819ns) {DropHandshakes:470(1944ns) {DropIncomingHandshake:442(530ns) DropOutgoingHandshake:457(296ns)} GenerateSessionVirtualId:988(481ns) PrepareNewSessionHandshake:477(1464ns) {SwitchToState:179(474ns)}}}}}
2025-11-27T18:18:43.816358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471316673730487:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(49560573ns) {Configure:104(49550898ns) {ProcessConfigured:135(2427156ns) {StartInitialHandshake:157(2424680ns) {DropHandshakes:470(391396ns) {DropIncomingHandshake:442(1734ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(1646ns) PrepareNewSessionHandshake:477(2421ns) {SwitchToState:179(1152ns)}}}}}
2025-11-27T18:18:43.819017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471314487703398:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(50566101ns) {Configure:104(50561670ns) {ProcessConfigured:135(3473998ns) {StartInitialHandshake:157(1622920ns) {DropHandshakes:470(2253ns) {DropIncomingHandshake:442(237ns) DropOutgoingHandshake:457(172ns)} GenerateSessionVirtualId:988(1196ns) PrepareNewSessionHandshake:477(2057ns) {SwitchToState:179(1209ns)}}}}}
2025-11-27T18:18:43.818330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471316068602950:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34086
2025-11-27T18:18:43.819027Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471316673730421:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(75922190ns) {Configure:104(75912525ns) {ProcessConfigured:135(1265666ns) {StartInitialHandshake:157(1263736ns) {DropHandshakes:470(1316ns) {DropIncomingHandshake:442(340ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(594ns) PrepareNewSessionHandshake:477(1631ns) {SwitchToState:179(738ns)}}}}}
2025-11-27T18:18:43.823002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471313651843743:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9087
2025-11-27T18:18:43.823487Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471314487703334:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(50232671ns) {Configure:104(50214198ns) {ProcessConfigured:135(4490893ns) {StartInitialHandshake:157(4488836ns) {DropHandshakes:470(1278ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:477(2431ns) {SwitchToState:179(1218ns)}}}}}
2025-11-27T18:18:43.823558Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471316068602861:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(122281369ns) {Configure:104(122275870ns) {ProcessConfigured:135(1416545ns) {StartInitialHandshake:157(1414288ns) {DropHandshakes:470(558869ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(1129ns)} GenerateSessionVirtualId:988(1419ns) PrepareNewSessionHandshake:477(2306ns) {SwitchToState:179(1352ns)}}}}}
2025-11-27T18:18:43.811672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471314262366240:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(67832299ns) {Configure:104(67821178ns) {ProcessConfigured:135(2148158ns) {StartInitialHandshake:157(2145562ns) {DropHandshakes:470(2185ns) {DropIncomingHandshake:442(511ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(873ns) PrepareNewSessionHandshake:477(1656ns) {SwitchToState:179(792ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.816898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471316505212297:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(75211813ns) {Configure:104(75205400ns) {ProcessConfigured:135(2561782ns) {StartInitialHandshake:157(2559086ns) {DropHandshakes:470(2203ns) {DropIncomingHandshake:442(598ns) DropOutgoingHandshake:457(342ns)} GenerateSessionVirtualId:988(701ns) PrepareNewSessionHandshake:477(1758ns) {SwitchToState:179(516ns)}}}}}
2025-11-27T18:18:43.825615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471316068602950:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.826132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471316068602955:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34086
2025-11-27T18:18:43.824364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471314487703402:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.823210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471313651843748:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9087
2025-11-27T18:18:43.811816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471314262366301:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:43.819275Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471316505212361:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49030744ns) {Configure:104(49021600ns) {ProcessConfigured:135(8827759ns) {StartInitialHandshake:157(8824345ns) {DropHandshakes:470(2694ns) {DropIncomingHandshake:442(577ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(1171ns) PrepareNewSessionHandshake:477(3102ns) {SwitchToState:179(1449ns)}}}}}
2025-11-27T18:18:43.842166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471316673730497:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17010
2025-11-27T18:18:43.842663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471313651843743:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.857069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471316068602941:2118] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(52890828ns) {Configure:104(52882539ns) {ProcessConfigured:135(7644691ns) {StartInitialHandshake:157(7641321ns) {DropHandshakes:470(2379ns) {DropIncomingHandshake:442(925ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(600ns) PrepareNewSessionHandshake:477(1718ns) {SwitchToState:179(666ns)}}}}}
2025-11-27T18:18:43.819677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471316505212362:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(48799098ns) {Configure:104(48791817ns) {ProcessConfigured:135(4149614ns) {StartInitialHandshake:157(4147207ns) {DropHandshakes:470(1487ns) {DropIncomingHandshake:442(398ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(790ns) PrepareNewSessionHandshake:477(2307ns) {SwitchToState:179(295ns)}}}}}
2025-11-27T18:18:43.846538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471316673730497:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.824405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471314487703402:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.859352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471313651843752:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9087
2025-11-27T18:18:43.868417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471314262366301:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16395) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471314262366313:2114] held: no
2025-11-27T18:18:43.878004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471316673730502:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17010
2025-11-27T18:18:43.839488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471314487703413:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2880
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.889330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471316505212375:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9265
2025-11-27T18:18:43.878072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471316673730503:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17010
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.881517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471314262366322:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22870
2025-11-27T18:18:43.891700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471313651843748:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.894122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471316068602955:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.850547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471314487703402:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:43.899513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471316505212375:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.897908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471316673730496:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17010
2025-11-27T18:18:43.901164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471316068602860:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:43.901999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471316505212376:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9265
2025-11-27T18:18:43.850889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471314487703397:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.902748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471313651843754:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9087
2025-11-27T18:18:43.902539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471316673730502:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.910773Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471314262366301:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(21410332ns) {HandleHandshakeStatus:395(11900096ns) {LogHandshakeFail:472(22349ns) {UpdateErrorStateLog:545(5307ns)} DropOutgoingHandshake:457(1717ns) ProcessPendingSessionEvents:495(9826972ns) {DropSessionEvent:512(7992ns) DropSessionEvent:512(16789ns) DropSessionEvent:512(3876ns) DropSessionEvent:512(3984ns) DropSessionEvent:512(3700ns) DropSessionEvent:512(3234ns) DropSessionEvent:512(4805ns) DropSessionEvent:512(4383ns) DropSessionEvent:512(3399ns) DropSessionEvent:512(45079ns)}}}
2025-11-27T18:18:43.853797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:43.902906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471313651843752:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.905890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471316673730486:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(107505213ns) {Configure:104(106355676ns) {ProcessConfigured:135(29889775ns) {StartInitialHandshake:157(29885480ns) {DropHandshakes:470(8563185ns) {DropIncomingHandshake:442(2671ns) DropOutgoingHandshake:457(409ns)} GenerateSessionVirtualId:988(2039ns) PrepareNewSessionHandshake:477(2709ns) {SwitchToState:179(1186ns)}}}}}
2025-11-27T18:18:43.925915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471316068602957:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34086
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.925762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471316505212376:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.863862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471314487703397:2110] [node 6] ICP22 created new session: [5:7577471314487703415:2110]
2025-11-27T18:18:43.907326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471316673730496:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.916511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471314262366321:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22870
2025-11-27T18:18:43.903735Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471313651843739:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(107347799ns) {Configure:104(107339627ns) {ProcessConfigured:135(8679658ns) {StartInitialHandshake:157(8678093ns) {DropHandshakes:470(3059ns) {DropIncomingHandshake:442(479ns) DropOutgoingHandshake:457(291ns)} GenerateSessionVirtualId:988(1932ns) PrepareNewSessionHandshake:477(1910ns) {SwitchToState:179(1082ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.926129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471316068602958:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34086
2025-11-27T18:18:43.925389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471316673730420:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:43.921561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471313651843745:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.863928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471314487703415:2110] [node 6] ICS09 handshake done sender: [5:7577471314487703402:2112] self: [5:7577471314487703401:0] peer: [6:7577471314262366320:0] socket: 537 qp: -1
2025-11-27T18:18:43.929452Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471316673730420:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19377202ns) {IncomingHandshake:261(19374170ns) {DropIncomingHandshake:442(625ns) PrepareNewSessionHandshake:477(5125508ns) {SwitchToState:179(3553ns)} GenerateSessionVirtualId:988(1199ns) IssueIncomingHandshakeReply:193(85542ns) {SwitchToState:179(461ns)}}}
2025-11-27T18:18:43.921615Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471313651843745:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:43.863977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471314487703415:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:43.927619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471313651843742:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.864044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:43.927657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471313651843742:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:43.916523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471314262366313:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:43.930065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471316505212379:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9265
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.926156Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471314487703397:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(50991397ns) {HandleHandshakeStatus:329(50988391ns) {DropIncomingHandshake:442(15496ns) DropOutgoingHandshake:457(476ns) SwitchToState:179(1455ns) ProcessPendingSessionEvents:495(20334524ns) {ForwardSessionEventToSession:612(13877209ns) ForwardSessionEventToSession:612(8456ns) ForwardSessionEventToSession:612(3201ns) ForwardSessionEventToSession:612(645773ns) ForwardSessionEventToSession:612(4169ns) ForwardSessionEventToSession:612(3110ns) ForwardSessionEventToSession:612(437263ns) ForwardSessionEventToSession:612(3093ns) ForwardSessionEventToSession:612(1020231ns) ForwardSessionEventToSession:612(3652ns)}}}
2025-11-27T18:18:43.930245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471313651843745:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:43.927802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471316068602957:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.933636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471316673730494:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.926057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471314262366301:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:43.933676Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471316673730494:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:43.930114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471314487703413:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.931764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471316505212368:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.929678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471316068602861:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:43.927469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:43.931148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471314487703335:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:43.931786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471316505212368:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:43.936131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471313651843742:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:43.931153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471316068602942:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(107919846ns) {Configure:104(107912157ns) {ProcessConfigured:135(5959874ns) {StartInitialHandshake:157(5956087ns) {DropHandshakes:470(2284ns) {DropIncomingHandshake:442(541ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(1663ns) PrepareNewSessionHandshake:477(2372ns) {SwitchToState:179(877ns)}}}}}
2025-11-27T18:18:43.934531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471316673730503:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.941694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471313651843736:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:43.929623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471314262366321:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.933739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471316068602958:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.941752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:43.941810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471313651843736:2107] [node 4] ICP22 created new session: [1:7577471313651843761:2107]
2025-11-27T18:18:43.942240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471313651843673:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:43.935814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471316068602963:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34086
::ffff:127.1.0.0
2025-11-27T18:18:43.933852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471316505212381:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9265
2025-11-27T18:18:43.934888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471316673730492:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.942289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.956559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471314487703420:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2880
2025-11-27T18:18:43.929748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471314262366301:2107] [node 5] ICP22 created new session: [6:7577471314262366324:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.934921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471316505212381:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.941723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471316505212368:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:43.943771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471316505212361:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:43.943816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:43.949432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471316505212379:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.958709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471316505212361:2109] [node 5] ICP22 created new session: [3:7577471316505212386:2109]
2025-11-27T18:18:43.943716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471316068602955:2133] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.934929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471316673730492:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:43.942612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471313651843761:2107] [node 4] ICS09 handshake done sender: [1:7577471313651843745:2113] self: [1:7577471313651843744:0] peer: [4:7577471316068602959:0] socket: 551 qp: -1
2025-11-27T18:18:43.931019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471314262366324:2107] [node 5] ICS09 handshake done sender: [6:7577471314262366313:2114] self: [6:7577471314262366320:0] peer: [5:7577471314487703401:0] socket: 539 qp: -1
2025-11-27T18:18:43.958663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471314487703420:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.967268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471316505212385:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9265
2025-11-27T18:18:43.938541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471316673730494:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:43.942817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471313651843761:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:43.968176Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471316068602860:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32547) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471316068602955:2133] held: no
2025-11-27T18:18:43.931057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471314262366324:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:43.965618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471314487703419:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2880
2025-11-27T18:18:43.938968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471316673730487:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:43.967291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471316505212386:2109] [node 5] ICS09 handshake done sender: [3:7577471316505212368:2112] self: [3:7577471316505212366:0] peer: [5:7577471314487703418:0] socket: 558 qp: -1
2025-11-27T18:18:43.974598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471316068602860:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:43.967297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471314487703333:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:43.969833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471316505212386:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.931131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471314262366241:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:43.942885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471313651843754:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.939030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.976536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:43.971919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471316505212297:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:43.931884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.943031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.975843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471314487703413:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:43.974531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:43.939083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471316673730487:2107] [node 4] ICP22 created new session: [2:7577471316673730511:2107]
2025-11-27T18:18:43.979653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471316505212385:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.980433Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471314487703335:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12580) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471314487703413:2116] held: no
2025-11-27T18:18:43.979569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471316068602861:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29250) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471316068602957:2134] held: no
2025-11-27T18:18:43.943028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471313651843673:2048] [node 2] ICP22 created new session: [1:7577471313651843762:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.967912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471314262366328:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22870
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.979590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471316068602860:2048] [node 1] ICP22 created new session: [4:7577471316068602966:2048]
2025-11-27T18:18:43.939124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471316673730511:2107] [node 4] ICS09 handshake done sender: [2:7577471316673730494:2114] self: [2:7577471316673730491:0] peer: [4:7577471316068602962:0] socket: 562 qp: -1
2025-11-27T18:18:43.981702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471316505212298:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:43.997934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471314487703335:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:43.975009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471314262366241:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29250) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471314262366321:2117] held: no
2025-11-27T18:18:43.943646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471313651843762:2048] [node 2] ICS09 handshake done sender: [1:7577471313651843742:2111] self: [1:7577471313651843740:0] peer: [2:7577471316673730506:0] socket: 535 qp: -1
2025-11-27T18:18:43.981415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471316068602966:2048] [node 1] ICS09 handshake done sender: [4:7577471316068602955:2133] self: [4:7577471316068602959:0] peer: [1:7577471313651843744:0] socket: 553 qp: -1
2025-11-27T18:18:43.981978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471316068602966:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:43.983264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471316068602965:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34086
2025-11-27T18:18:43.997970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:43.985121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471316068602957:2134] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:43.992026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471316505212391:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9265
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.997739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471314262366329:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22870
2025-11-27T18:18:43.943699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471313651843762:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:43.939166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471316673730511:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:43.992235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471316505212385:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:43.988762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:43.998009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471314487703335:2050] [node 3] ICP22 created new session: [5:7577471314487703424:2050]
2025-11-27T18:18:43.999980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471314262366328:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.939198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:43.943735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:43.999108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471316068602861:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.992476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471316505212298:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:44.002744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471314262366327:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22870
2025-11-27T18:18:43.998037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471314487703424:2050] [node 3] ICS09 handshake done sender: [5:7577471314487703413:2116] self: [5:7577471314487703418:0] peer: [3:7577471316505212366:0] socket: 559 qp: -1
2025-11-27T18:18:43.958149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471313651843736:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16254171ns) {HandleHandshakeStatus:329(16251234ns) {DropIncomingHandshake:442(13940ns) DropOutgoingHandshake:457(349ns) SwitchToState:179(313ns) ProcessPendingSessionEvents:495(14125540ns) {ForwardSessionEventToSession:612(14065916ns) ForwardSessionEventToSession:612(5829ns) ForwardSessionEventToSession:612(8843ns) ForwardSessionEventToSession:612(2845ns) ForwardSessionEventToSession:612(5536ns) ForwardSessionEventToSession:612(2189ns) ForwardSessionEventToSession:612(6545ns) ForwardSessionEventToSession:612(5109ns) ForwardSessionEventToSession:612(9284ns) ForwardSessionEventToSession:612(3013ns)}}}
2025-11-27T18:18:43.974355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471313651843747:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.974395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471313651843747:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:43.992549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:43.979534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471313651843741:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.999028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471316505212298:2049] [node 2] ICP22 created new session: [3:7577471316505212392:2049]
2025-11-27T18:18:43.979574Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471313651843741:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:44.002558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471316505212393:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9265
2025-11-27T18:18:43.980408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471313651843747:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:43.999008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471313651843737:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:44.007029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471316505212392:2049] [node 2] ICS09 handshake done sender: [3:7577471316505212385:2120] self: [3:7577471316505212390:0] peer: [2:7577471316673730493:0] socket: 585 qp: -1
2025-11-27T18:18:43.999090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:44.007062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471316505212392:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:44.000624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471313651843737:2108] [node 5] ICP22 created new session: [1:7577471313651843770:2108]
2025-11-27T18:18:44.007100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:44.002693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471313651843770:2108] [node 5] ICS09 handshake done sender: [1:7577471313651843747:2114] self: [1:7577471313651843746:0] peer: [5:7577471314487703421:0] socket: 550 qp: -1
2025-11-27T18:18:44.009712Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471316505212379:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:44.011835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471316505212297:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:44.002825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471313651843673:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(38084841ns) {HandleHandshakeStatus:329(38082523ns) {DropIncomingHandshake:442(9709ns) DropOutgoingHandshake:457(261ns) SwitchToState:179(315ns) ProcessPendingSessionEvents:495(12984220ns) {ForwardSessionEventToSession:612(4623102ns) ForwardSessionEventToSession:612(2992039ns) ForwardSessionEventToSession:612(4122ns) ForwardSessionEventToSession:612(3955ns) ForwardSessionEventToSession:612(745467ns) ForwardSessionEventToSession:612(2536ns) ForwardSessionEventToSession:612(3820609ns) ForwardSessionEventToSession:612(8126ns) ForwardSessionEventToSession:612(2418ns) ForwardSessionEventToSession:612(762974ns)}}}
2025-11-27T18:18:44.014334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:44.002948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471313651843741:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:44.016905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471316505212361:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(57713172ns) {HandleHandshakeStatus:329(57710545ns) {DropIncomingHandshake:442(9963ns) DropOutgoingHandshake:457(218ns) SwitchToState:179(298ns) ProcessPendingSessionEvents:495(21103250ns) {ForwardSessionEventToSession:612(2332082ns) ForwardSessionEventToSession:612(6161045ns) ForwardSessionEventToSession:612(3774ns) ForwardSessionEventToSession:612(2699ns) ForwardSessionEventToSession:612(429482ns) ForwardSessionEventToSession:612(4158ns) ForwardSessionEventToSession:612(4203888ns) ForwardSessionEventToSession:612(6682ns) ForwardSessionEventToSession:612(7062737ns) ForwardSessionEventToSession:612(858449ns)}}}
2025-11-27T18:18:44.007356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471313651843770:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:44.017028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471316505212297:2048] [node 1] ICP22 created new session: [3:7577471320800179692:2048]
2025-11-27T18:18:44.024989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471320800179692:2048] [node 1] ICS09 handshake done sender: [3:7577471316505212379:2118] self: [3:7577471316505212388:0] peer: [1:7577471313651843738:0] socket: 570 qp: -1
2025-11-27T18:18:44.008235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:44.025424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471320800179692:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:44.020632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471313651843674:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:44.025828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:44.020692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:44.028266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471316505212370:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:44.020734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471313651843674:2049] [node 3] ICP22 created new session: [1:7577471317946811069:2049]
2025-11-27T18:18:44.028285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471316505212370:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:44.020788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471317946811069:2049] [node 3] ICS09 handshake done sender: [1:7577471313651843741:2110] self: [1:7577471313651843738:0] peer: [3:7577471316505212388:0] socket: 566 qp: -1
2025-11-27T18:18:44.020820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471317946811069:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:44.030573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471316505212298:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(34263508ns) {HandleHandshakeStatus:329(34260894ns) {DropIncomingHandshake:442(540ns) DropOutgoingHandshake:457(10614ns) SwitchToState:179(365ns) ProcessPendingSessionEvents:495(6722581ns) {ForwardSessionEventToSession:612(5285002ns) ForwardSessionEventToSession:612(7017ns) ForwardSessionEventToSession:612(2926ns) ForwardSessionEventToSession:612(438570ns) ForwardSessionEventToSession:612(16247ns) ForwardSessionEventToSession:612(621782ns) ForwardSessionEventToSession:612(4071ns) ForwardSessionEventToSession:612(2716ns) ForwardSessionEventToSession:612(321956ns) ForwardSessionEventToSession:612(6625ns)}}}
2025-11-27T18:18:43.943342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471316673730492:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:44.000594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:43.998085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471314487703424:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:43.998116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:43.998510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471314487703422:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2880
2025-11-27T18:18:44.004990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471314487703333:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32547) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471314487703420:2118] held: no
2025-11-27T18:18:44.005208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471314487703426:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2880
2025-11-27T18:18:44.007187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471314487703422:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:44.008121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471314487703334:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:44.014714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471314487703423:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2880
2025-11-27T18:18:44.025236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471314487703426:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:44.025776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471314487703420:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:44.026713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471314487703398:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:44.028066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471314487703333:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:44.029207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:44.020855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:44.003796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471314262366240:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:43.958755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471316673730508:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17010
2025-11-27T18:18:44.030782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471316505212297:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18744775ns) {HandleHandshakeStatus:329(18742435ns) {DropIncomingHandshake:442(716ns) DropOutgoingHandshake:457(5321ns) SwitchToState:179(321ns) ProcessPendingSessionEvents:495(2418754ns) {ForwardSessionEventToSession:612(2357448ns) ForwardSessionEventToSession:612(10199ns) ForwardSessionEventToSession:612(2510ns) ForwardSessionEventToSession:612(3068ns) ForwardSessionEventToSession:612(15387ns) ForwardSessionEventToSession:612(2318ns) ForwardSessionEventToSession:612(6147ns) ForwardSessionEventToSession:612(2506ns) ForwardSessionEventToSession:612(5584ns) ForwardSessionEventToSession:612(4332ns)}}}
2025-11-27T18:18:44.014764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471314262366321:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:43.968508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471316673730485:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:44.034128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471316505212370:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:44.016309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471314262366329:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.968568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:44.033994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471314487703333:2048] [node 1] ICP22 created new session: [5:7577471318782670726:2048]
2025-11-27T18:18:44.000635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471316068602861:2049] [node 2] ICP22 created new session: [4:7577471316068602971:2049]
2025-11-27T18:18:44.025118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471313651843756:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:44.046342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471318782670726:2048] [node 1] ICS09 handshake done sender: [5:7577471314487703420:2118] self: [5:7577471314487703421:0] peer: [1:7577471313651843746:0] socket: 565 qp: -1
2025-11-27T18:18:44.016417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471314262366308:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:43.968624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471316673730485:2110] [node 6] ICP22 created new session: [2:7577471316673730515:2110]
2025-11-27T18:18:44.002664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471316068602971:2049] [node 2] ICS09 handshake done sender: [4:7577471316068602957:2134] self: [4:7577471316068602962:0] peer: [2:7577471316673730491:0] socket: 563 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:44.025154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471313651843756:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:44.024838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471314262366241:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:43.968681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471316673730515:2110] [node 6] ICS09 handshake done sender: [2:7577471316673730492:2113] self: [2:7577471316673730490:0] peer: [6:7577471314262366325:0] socket: 561 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:44.046467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471318782670727:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2880
2025-11-27T18:18:44.047833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471316505212363:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:44.029151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471317946811068:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9087
2025-11-27T18:18:44.003554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471316068602971:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:43.968716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471316673730515:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:44.025689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471318557333628:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22870
2025-11-27T18:18:44.047411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471318782670726:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:44.049593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:44.025965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:44.040207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471313651843756:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:44.053083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:44.008328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471316068602860:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27075919ns) {HandleHandshakeStatus:329(27074521ns) {DropIncomingHandshake:442(800ns) DropOutgoingHandshake:457(214ns) SwitchToState:179(418ns) ProcessPendingSessionEvents:495(823ns)}}
2025-11-27T18:18:43.968753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:44.052909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471316505212373:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:44.026497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471314262366241:2049] [node 2] ICP22 created new session: [6:7577471318557333630:2049]
2025-11-27T18:18:44.068674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471314487703333:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(39222349ns) {HandleHandshakeStatus:329(38922063ns) {DropIncomingHandshake:442(1162ns) DropOutgoingHandshake:457(259ns) SwitchToState:179(283ns) ProcessPendingSessionEvents:495(1108ns)}}
2025-11-27T18:18:44.009486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:43.967529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471316673730420:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32547) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471316673730502:2118] held: no
2025-11-27T18:18:44.026968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471318557333630:2049] [node 2] ICS09 handshake done sender: [6:7577471314262366321:2117] self: [6:7577471314262366325:0] peer: [2:7577471316673730490:0] socket: 564 qp: -1
2025-11-27T18:18:44.052951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471316505212373:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:44.043737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471313651843737:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(34180170ns) {HandleHandshakeStatus:329(34176780ns) {DropIncomingHandshake:442(19372ns) DropOutgoingHandshake:457(378ns) SwitchToState:179(386ns) ProcessPendingSessionEvents:495(12462208ns) {ForwardSessionEventToSession:612(9700191ns) ForwardSessionEventToSession:612(623756ns) ForwardSessionEventToSession:612(2707ns) ForwardSessionEventToSession:612(2578ns) ForwardSessionEventToSession:612(891130ns) ForwardSessionEventToSession:612(3129ns) ForwardSessionEventToSession:612(653336ns) ForwardSessionEventToSession:612(3690ns) ForwardSessionEventToSession:612(2857ns) ForwardSessionEventToSession:612(564818ns)}}}
2025-11-27T18:18:44.023648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471316068602861:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(30543056ns) {HandleHandshakeStatus:329(24439591ns) {DropIncomingHandshake:442(3578ns) DropOutgoingHandshake:457(190ns) SwitchToState:179(328ns) ProcessPendingSessionEvents:495(859ns)}}
2025-11-27T18:18:44.068737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471314487703422:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:44.046863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471317946811068:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:44.027057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471318557333628:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:44.026545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471316068602954:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:43.976173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471316673730502:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:44.053401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471316505212363:2110] [node 6] ICP22 created new session: [3:7577471320800179695:2110]
2025-11-27T18:18:44.027251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471318557333630:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:43.980422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471316673730420:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:44.068176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471313651843739:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:44.069490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471314487703334:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:44.070557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:44.026592Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471316068602954:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:44.071059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471314487703334:2049] [node 2] ICP22 created new session: [5:7577471318782670730:2049]
2025-11-27T18:18:44.073283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471318782670730:2049] [node 2] ICS09 handshake done sender: [5:7577471314487703422:2119] self: [5:7577471318782670724:0] peer: [2:7577471316673730504:0] socket: 548 qp: -1
2025-11-27T18:18:44.073680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471318782670728:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2880
2025-11-27T18:18:44.073742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471318782670730:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:44.073801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:44.066246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471320800179695:2110] [node 6] ICS09 handshake done sender: [3:7577471316505212370:2113] self: [3:7577471316505212369:0] peer: [6:7577471318557333631:0] socket: 536 qp: -1
2025-11-27T18:18:44.068967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:43.981471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:44.028095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:44.027305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471316068602970:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34086
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:44.068446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471316505212373:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:44.070421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471313651843739:2109] [node 6] ICP22 created new session: [1:7577471317946811073:2109]
2025-11-27T18:18:44.028209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471314262366242:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:44.030986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471316068602961:2136] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:44.049332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471318557333635:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22870
2025-11-27T18:18:43.982056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471316673730420:2048] [node 1] ICP22 created new session: [2:7577471316673730518:2048]
2025-11-27T18:18:44.068924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471320800179695:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:44.031009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471316068602961:2136] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:44.073159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471317946811073:2109] [node 6] ICS09 handshake done sender: [1:7577471313651843756:2118] self: [1:7577471313651843755:0] peer: [6:7577471318557333627:0] socket: 591 qp: -1
2025-11-27T18:18:44.073197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471317946811073:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:44.073232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:44.087412Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471313651843739:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(19081989ns) {HandleHandshakeStatus:329(19078571ns) {DropIncomingHandshake:442(15984ns) DropOutgoingHandshake:457(331ns) SwitchToState:179(809ns) ProcessPendingSessionEvents:495(9652030ns) {ForwardSessionEventToSession:612(9597610ns) ForwardSessionEventToSession:612(8381ns) ForwardSessionEventToSession:612(5058ns) ForwardSessionEventToSession:612(9037ns) ForwardSessionEventToSession:612(3223ns) ForwardSessionEventToSession:612(2524ns) ForwardSessionEventToSession:612(8267ns) ForwardSessionEventToSession:612(1822ns) ForwardSessionEventToSession:612(2354ns) ForwardSessionEventToSession:612(5550ns)}}}
2025-11-27T18:18:44.061919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471318557333633:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22870
2025-11-27T18:18:43.990572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471316673730495:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:44.109551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471314487703398:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471314487703426:2121] held: no
2025-11-27T18:18:44.109697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471314487703426:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:44.110280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471314487703398:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:44.110315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:44.037058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471316068602970:2139] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:44.068970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:43.990606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471316673730495:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:44.071199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471316505212362:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:44.065462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471318557333634:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22870
2025-11-27T18:18:44.110365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471314487703398:2109] [node 4] ICP22 created new session: [5:7577471318782670732:2109]
2025-11-27T18:18:44.037345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471316068602954:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:44.071559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471314262366242:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12580) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471318557333628:2122] held: no
2025-11-27T18:18:43.991457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471316673730495:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:44.037872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471316068602862:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:44.044016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471316068602961:2136] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:44.047691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471316068602941:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:44.049530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:44.053213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471316068602941:2118] [node 6] ICP22 created new session: [4:7577471320363570272:2118]
2025-11-27T18:18:44.062001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471316068602942:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:44.062009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471320363570272:2118] [node 6] ICS09 handshake done sender: [4:7577471316068602954:2132] self: [4:7577471316068602947:0] peer: [6:7577471318557333629:0] socket: 593 qp: -1
2025-11-27T18:18:44.062064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:44.062110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471316068602942:2108] [node 5] ICP22 created new session: [4:7577471320363570273:2108]
2025-11-27T18:18:44.065579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471320363570272:2118] [node 6] ICS10 traffic start
2025-11-27T18:18:44.066356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:44.068369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471320363570273:2108] [node 5] ICS09 handshake done sender: [4:7577471316068602961:2136] self: [4:7577471316068602960:0] peer: [5:7577471318782670725:0] socket: 594 qp: -1
2025-11-27T18:18:44.069392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471320363570273:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:44.070573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:44.086572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471316068602942:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(28120304ns) {HandleHandshakeStatus:329(28117777ns) {DropIncomingHandshake:442(18016ns) DropOutgoingHandshake:457(327ns) SwitchToState:179(446ns) ProcessPendingSessionEvents:495(12820436ns) {ForwardSessionEventToSession:612(4191848ns) ForwardSessionEventToSession:612(8577795ns) ForwardSessionEventToSession:612(4849ns) ForwardSessionEventToSession:612(6195ns) ForwardSessionEventToSession:612(5260ns) ForwardSessionEventToSession:612(3163ns) ForwardSessionEventToSession:612(2672ns) ForwardSessionEventToSession:612(5403ns) ForwardSessionEventToSession:612(2042ns) ForwardSessionEventToSession:612(9143ns)}}}
2025-11-27T18:18:44.086668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471316068602862:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12580) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471316068602970:2139] held: no
2025-11-27T18:18:44.106729Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471316068602941:2118] [node 6] ICP35 event processing took too much time PendingConnection:279(25558412ns) {HandleHandshakeStatus:329(25555173ns) {DropIncomingHandshake:442(13072ns) DropOutgoingHandshake:457(775ns) SwitchToState:179(409ns) ProcessPendingSessionEvents:495(4231257ns) {ForwardSessionEventToSession:612(239337ns) ForwardSessionEventToSession:612(242796ns) ForwardSessionEventToSession:612(4427ns) ForwardSessionEventToSession:612(975270ns) ForwardSessionEventToSession:612(617972ns) ForwardSessionEventToSession:612(4016ns) ForwardSessionEventToSession:612(2757ns) ForwardSessionEventToSession:612(564986ns) ForwardSessionEventToSession:612(2406ns) ForwardSessionEventToSession:612(1563348ns)}}}
2025-11-27T18:18:44.110028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471320363570276:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34086
2025-11-27T18:18:44.127840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471316068602970:2139] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:44.130923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471316068602862:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:44.132743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:44.137741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471316068602862:2050] [node 3] ICP22 created new session: [4:7577471320363570277:2050]
2025-11-27T18:18:44.139343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471320363570277:2050] [node 3] ICS09 handshake done sender: [4:7577471316068602970:2139] self: [4:7577471320363570271:0] peer: [3:7577471316505212372:0] socket: 560 qp: -1
2025-11-27T18:18:44.139389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471320363570277:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:44.139440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:44.071565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471314262366328:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:44.073504Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471314262366308:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471314262366329:2121] held: no
2025-11-27T18:18:44.110404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471318782670732:2109] [node 4] ICS09 handshake done sender: [5:7577471314487703426:2121] self: [5:7577471318782670725:0] peer: [4:7577471316068602960:0] socket: 598 qp: -1
2025-11-27T18:18:44.077620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471318557333628:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:44.102018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471314262366329:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:44.104437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471314262366240:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:44.110436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471318782670732:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:44.104626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471314262366242:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:44.104695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:44.110469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:44.104750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471314262366242:2050] [node 3] ICP22 created new session: [6:7577471318557333636:2050]
2025-11-27T18:18:44.131007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471314487703334:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(39287030ns) {HandleHandshakeStatus:329(39284544ns) {DropIncomingHandshake:442(2292ns) DropOutgoingHandshake:457(8452ns) SwitchToState:179(414ns) ProcessPendingSessionEvents:495(7028510ns) {ForwardSessionEventToSession:612(4155090ns) ForwardSessionEventToSession:612(333174ns) ForwardSessionEventToSession:612(224956ns) ForwardSessionEventToSession:612(408870ns) ForwardSessionEventToSession:612(5285ns) ForwardSessionEventToSession:612(506004ns) ForwardSessionEventToSession:612(2642ns) ForwardSessionEventToSession:612(1366251ns) ForwardSessionEventToSession:612(3728ns) ForwardSessionEventToSession:612(3218ns)}}}
2025-11-27T18:18:44.104810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471318557333636:2050] [node 3] ICS09 handshake done sender: [6:7577471318557333628:2122] self: [6:7577471318557333631:0] peer: [3:7577471316505212369:0] socket: 571 qp: -1
2025-11-27T18:18:44.104848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471318557333636:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:44.073196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:44.104887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:44.073620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471316505212362:2107] [node 4] ICP22 created new session: [3:7577471320800179698:2107]
2025-11-27T18:18:44.106956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471314262366308:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:44.107401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:44.077578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471320800179698:2107] [node 4] ICS09 handshake done sender: [3:7577471316505212373:2115] self: [3:7577471316505212372:0] peer: [4:7577471320363570271:0] socket: 544 qp: -1
2025-11-27T18:18:44.081594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471320800179698:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:44.107635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:44.101657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:44.107687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471314262366308:2110] [node 4] ICP22 created new session: [6:7577471318557333639:2110]
2025-11-27T18:18:44.108818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471316505212363:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(29777657ns) {HandleHandshakeStatus:329(29774698ns) {DropIncomingHandshake:442(12261ns) DropOutgoingHandshake:457(259ns) SwitchToState:179(366ns) ProcessPendingSessionEvents:495(7151956ns) {ForwardSessionEventToSession:612(2739961ns) ForwardSessionEventToSession:612(110669ns) ForwardSessionEventToSession:612(225314ns) ForwardSessionEventToSession:612(5624ns) ForwardSessionEventToSession:612(2788ns) ForwardSessionEventToSession:612(3873415ns) ForwardSessionEventToSession:612(5110ns) ForwardSessionEventToSession:612(3054ns) ForwardSessionEventToSession:612(6120ns) ForwardSessionEventToSession:612(9061ns)}}}
2025-11-27T18:18:44.107734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471318557333639:2110] [node 4] ICS09 handshake done sender: [6:7577471314262366329:2121] self: [6:7577471318557333629:0] peer: [4:7577471316068602947:0] socket: 595 qp: -1
2025-11-27T18:18:44.126341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471316505212362:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(50229741ns) {HandleHandshakeStatus:329(50226938ns) {DropIncomingHandshake:442(20995ns) DropOutgoingHandshake:457(246ns) SwitchToState:179(1697ns) ProcessPendingSessionEvents:495(14649432ns) {ForwardSessionEventToSession:612(2935264ns) ForwardSessionEventToSession:612(4919ns) ForwardSessionEventToSession:612(497355ns) ForwardSessionEventToSession:612(10277ns) ForwardSessionEventToSession:612(4018ns) ForwardSessionEventToSession:612(2964509ns) ForwardSessionEventToSession:612(10375ns) ForwardSessionEventToSession:612(2727ns) ForwardSessionEventToSession:612(8197631ns) ForwardSessionEventToSession:612(3837ns)}}}
2025-11-27T18:18:44.107770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471318557333639:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:44.107814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:43.991724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471316673730421:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:44.108751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471314262366240:2048] [node 1] ICP22 created new session: [6:7577471318557333638:2048]
2025-11-27T18:18:43.991805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:44.109431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471318557333638:2048] [node 1] ICS09 handshake done sender: [6:7577471314262366328:2120] self: [6:7577471318557333627:0] peer: [1:7577471313651843755:0] socket: 592 qp: -1
2025-11-27T18:18:43.992845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471316673730518:2048] [node 1] ICS09 handshake done sender: [2:7577471316673730502:2118] self: [2:7577471316673730506:0] peer: [1:7577471313651843740:0] socket: 569 qp: -1
2025-11-27T18:18:44.111949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471318557333638:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:44.121216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:43.999049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471316673730518:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:43.992763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471316673730421:2049] [node 3] ICP22 created new session: [2:7577471316673730520:2049]
2025-11-27T18:18:43.999495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471316673730520:2049] [node 3] ICS09 handshake done sender: [2:7577471316673730495:2115] self: [2:7577471316673730493:0] peer: [3:7577471316505212390:0] socket: 552 qp: -1
2025-11-27T18:18:43.999541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471316673730520:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:43.999579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:44.002095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:44.014503Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471316673730420:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(31183947ns) {HandleHandshakeStatus:329(31181744ns) {DropIncomingHandshake:442(585ns) DropOutgoingHandshake:457(239ns) SwitchToState:179(471ns) ProcessPendingSessionEvents:495(935ns)}}
2025-11-27T18:18:44.023559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471316673730505:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:44.023601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471316673730505:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:44.030546Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471316673730421:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(33470667ns) {HandleHandshakeStatus:329(33467914ns) {DropIncomingHandshake:442(9533ns) DropOutgoingHandshake:457(265ns) SwitchToState:179(408ns) ProcessPendingSessionEvents:495(12943568ns) {ForwardSessionEventToSession:612(1214675ns) ForwardSessionEventToSession:612(3318899ns) ForwardSessionEventToSession:612(11895ns) ForwardSessionEventToSession:612(13643ns) ForwardSessionEventToSession:612(3543274ns) ForwardSessionEventToSession:612(4142ns) ForwardSessionEventToSession:612(4040270ns) ForwardSessionEventToSession:612(358493ns) ForwardSessionEventToSession:612(4414ns) ForwardSessionEventToSession:612(416115ns)}}}
2025-11-27T18:18:44.043826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471316673730505:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:44.058003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471320968697819:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17010
2025-11-27T18:18:44.061395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471316673730486:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:44.062218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:44.066003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471316673730486:2108] [node 5] ICP22 created new session: [2:7577471320968697821:2108]
2025-11-27T18:18:44.066074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471320968697821:2108] [node 5] ICS09 handshake done sender: [2:7577471316673730505:2120] self: [2:7577471316673730504:0] peer: [5:7577471318782670724:0] socket: 547 qp: -1
2025-11-27T18:18:44.067788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471320968697821:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:44.067847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:44.068916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471320968697819:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:44.107047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471316673730486:2108] [node 5] ICP18 handshake (actor [2:7577471320968697819:2122]) is too old
2025-11-27T18:18:44.142952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471314262366240:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(35706445ns) {HandleHandshakeStatus:329(35702172ns) {DropIncomingHandshake:442(493ns) DropOutgoingHandshake:457(7060ns) SwitchToState:179(462ns) ProcessPendingSessionEvents:495(12321392ns) {ForwardSessionEventToSession:612(4902833ns) ForwardSessionEventToSession:612(4443ns) ForwardSessionEventToSession:612(1398493ns) ForwardSessionEventToSession:612(3148ns) ForwardSessionEventToSession:612(3587644ns) ForwardSessionEventToSession:612(588422ns) ForwardSessionEventToSession:612(9042ns) ForwardSessionEventToSession:612(836729ns) ForwardSessionEventToSession:612(3616ns) ForwardSessionEventToSession:612(2687ns)}}}
2025-11-27T18:18:44.143026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471314262366240:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:44.201380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:44.201656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471316673730518:2048] [node 1] ICS01 socket: 569 reason# EndOfStream
2025-11-27T18:18:44.201710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471316673730420:2048] [node 1] ICP30 unregister session Session# [2:7577471316673730518:2048] VirtualId# [2:7577471316673730506:0]
2025-11-27T18:18:44.201748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471316673730518:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.201834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:44.201911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:44.204069Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:44.204298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471320800179692:2048] [node 1] ICS07 socket disconnect 570 reason# EndOfStream
2025-11-27T18:18:44.204332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471320800179692:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.204407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:44.204454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471320800179692:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:44.206782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471316505212297:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32547) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:44.206848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471320800179692:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:44.207231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:44.207606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471316068602966:2048] [node 1] ICS01 socket: 553 reason# EndOfStream
2025-11-27T18:18:44.207664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471316068602860:2048] [node 1] ICP30 unregister session Session# [4:7577471316068602966:2048] VirtualId# [4:7577471316068602959:0]
2025-11-27T18:18:44.207698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471316068602966:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.207820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:44.207904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:44.207871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471316505212297:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32547) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:44.207942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471320800179692:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:44.207983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471316505212297:2048] [node 1] ICP30 unregister session Session# [3:7577471320800179692:2048] VirtualId# [3:7577471316505212388:0]
2025-11-27T18:18:44.208088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:44.208127Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471316505212297:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:44.208149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:44.208765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:44.208969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471318782670726:2048] [node 1] ICS01 socket: 565 reason# EndOfStream
2025-11-27T18:18:44.209005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471314487703333:2048] [node 1] ICP30 unregister session Session# [5:7577471318782670726:2048] VirtualId# [5:7577471314487703421:0]
2025-11-27T18:18:44.209024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471318782670726:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.209080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:44.209128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:44.209794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:44.209990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471318557333638:2048] [node 1] ICS07 socket disconnect 592 reason# EndOfStream
2025-11-27T18:18:44.210043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471318557333638:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.210100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:44.210136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471318557333638:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:44.210817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471314262366240:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32547) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:44.210937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471318557333638:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:44.210971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471314262366240:2048] [node 1] ICP30 unregister session Session# [6:7577471318557333638:2048] VirtualId# [6:7577471318557333627:0]
2025-11-27T18:18:44.211034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:44.211070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471314262366240:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:44.211083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:44.218160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471316505212297:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:44.221185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471314262366240:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:44.288158Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:44.288378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471316505212392:2049] [node 2] ICS07 socket disconnect 585 reason# EndOfStream
2025-11-27T18:18:44.288412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471316505212392:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.288484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:44.288531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471316505212392:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:44.291650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471316505212298:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29250) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:44.291716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471316505212392:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:44.292164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:44.292393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471316068602971:2049] [node 2] ICS01 socket: 563 reason# EndOfStream
2025-11-27T18:18:44.292452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471316068602861:2049] [node 2] ICP30 unregister session Session# [4:7577471316068602971:2049] VirtualId# [4:7577471316068602962:0]
2025-11-27T18:18:44.292490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471316068602971:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.292569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:44.292628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:44.292819Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471316505212298:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29250) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:44.293357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471316505212392:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:44.293420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471316505212298:2049] [node 2] ICP30 unregister session Session# [3:7577471316505212392:2049] VirtualId# [3:7577471316505212390:0]
2025-11-27T18:18:44.293470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:44.293503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471316505212298:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:44.293518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:44.293853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:44.294086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471318782670730:2049] [node 2] ICS07 socket disconnect 548 reason# EndOfStream
2025-11-27T18:18:44.294113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471318782670730:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.294178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:44.294215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471318782670730:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:44.295886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:44.295856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471314487703334:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29250) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:44.298085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471318557333630:2049] [node 2] ICS01 socket: 564 reason# EndOfStream
2025-11-27T18:18:44.298079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471318782670730:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:44.298151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471314487703334:2049] [node 2] ICP30 unregister session Session# [5:7577471318782670730:2049] VirtualId# [5:7577471318782670724:0]
2025-11-27T18:18:44.298144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471314262366241:2049] [node 2] ICP30 unregister session Session# [6:7577471318557333630:2049] VirtualId# [6:7577471314262366325:0]
2025-11-27T18:18:44.298259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:44.298179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471318557333630:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.298301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471314487703334:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:44.298255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:44.298315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:44.298294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:44.303151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471316505212298:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:44.308273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471314487703334:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:44.358644Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:44.358915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471320363570277:2050] [node 3] ICS01 socket: 560 reason# EndOfStream
2025-11-27T18:18:44.358974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471316068602862:2050] [node 3] ICP30 unregister session Session# [4:7577471320363570277:2050] VirtualId# [4:7577471320363570271:0]
2025-11-27T18:18:44.359011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471320363570277:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.359089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:44.359136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:44.359730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:44.359944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471314487703424:2050] [node 3] ICS01 socket: 559 reason# EndOfStream
2025-11-27T18:18:44.359984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471314487703335:2050] [node 3] ICP30 unregister session Session# [5:7577471314487703424:2050] VirtualId# [5:7577471314487703418:0]
2025-11-27T18:18:44.360016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471314487703424:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.360082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:44.360137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:44.360690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:44.360904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471318557333636:2050] [node 3] ICS01 socket: 571 reason# EndOfStream
2025-11-27T18:18:44.360946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471314262366242:2050] [node 3] ICP30 unregister session Session# [6:7577471318557333636:2050] VirtualId# [6:7577471318557333631:0]
2025-11-27T18:18:44.360973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471318557333636:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.361047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:44.361092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:44.460203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:44.460413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471318557333639:2110] [node 4] ICS01 socket: 595 reason# EndOfStream
2025-11-27T18:18:44.460461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471314262366308:2110] [node 4] ICP30 unregister session Session# [6:7577471318557333639:2110] VirtualId# [6:7577471318557333629:0]
2025-11-27T18:18:44.460490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471318557333639:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.460597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:44.460653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:44.464227Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:44.464524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471318782670732:2109] [node 4] ICS01 socket: 598 reason# EndOfStream
2025-11-27T18:18:44.464576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471314487703398:2109] [node 4] ICP30 unregister session Session# [5:7577471318782670732:2109] VirtualId# [5:7577471318782670725:0]
2025-11-27T18:18:44.464613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471318782670732:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.464704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:44.464760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:44.506109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:44.506336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471314262366324:2107] [node 5] ICS01 socket: 539 reason# EndOfStream
2025-11-27T18:18:44.506381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471314262366301:2107] [node 5] ICP30 unregister session Session# [6:7577471314262366324:2107] VirtualId# [6:7577471314262366320:0]
2025-11-27T18:18:44.506416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471314262366324:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:44.506510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:44.506575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:44.628962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471317666212597:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:44.628986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471317666212598:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:44.651129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471318847815892:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:44.651172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471318847815893:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:44.675532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471321025023291:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:44.675580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471321025023292:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:44.703470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471321248021068:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:44.703524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471321248021069:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:44.703560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471321248021070:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:44.728739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471318466340760:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:44.728797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471318466340761:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:44.728830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471318466340762:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:44.756517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471317613509945:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:44.756583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471317613509947:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:44.756644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471317613509946:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:44.848320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471317613510007:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:44.865204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471317666212660:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:44.865195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471318847815958:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:44.867815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471318847815960:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:44.870570Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471317666212597:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20960962ns) {Configure:104(20957112ns) {ProcessConfigured:135(94804ns) {StartInitialHandshake:157(91389ns) {DropHandshakes:470(4371ns) {DropIncomingHandshake:442(1205ns) DropOutgoingHandshake:457(657ns)} GenerateSessionVirtualId:988(835ns) PrepareNewSessionHandshake:477(2056ns) {SwitchToState:179(600ns)}}}}}
2025-11-27T18:18:44.875555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471317666212662:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:44.900037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471317666212669:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:44.923388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471321025023354:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:44.940506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471321248021132:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:44.939685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471321025023355:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:44.971191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471317613509947:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21600084ns) {Configure:104(21591175ns) {ProcessConfigured:135(86647ns) {StartInitialHandshake:157(83366ns) {DropHandshakes:470(2211ns) {DropIncomingHandshake:442(613ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(1116ns) PrepareNewSessionHandshake:477(1597ns) {SwitchToState:179(589ns)}}}}}
2025-11-27T18:18:44.960289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471318466340826:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:44.981881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471317613510014:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:44.954111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471318847815961:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35441
2025-11-27T18:18:44.959014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471321025023356:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:44.986875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471321025023291:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(84020704ns) {Configure:104(81515715ns) {ProcessConfigured:135(53637ns) {StartInitialHandshake:157(51951ns) {DropHandshakes:470(1821ns) {DropIncomingHandshake:442(479ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:477(1030ns) {SwitchToState:179(373ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:44.973374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471321248021134:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:44.958968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471318847815966:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:44.984160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471318466340827:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.000802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471321248021133:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-11-27T18:18:44.977692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471318847815893:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(94313591ns) {Configure:104(94307515ns) {ProcessConfigured:135(36541495ns) {StartInitialHandshake:157(36537862ns) {DropHandshakes:470(2844ns) {DropIncomingHandshake:442(1039ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(1151ns) PrepareNewSessionHandshake:477(3066ns) {SwitchToState:179(1693ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.001016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471317666212669:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(81296224ns) {Configure:104(81293119ns) {ProcessConfigured:135(22157619ns) {StartInitialHandshake:157(22155266ns) {DropHandshakes:470(3079ns) {DropIncomingHandshake:442(765ns) DropOutgoingHandshake:457(341ns)} GenerateSessionVirtualId:988(973ns) PrepareNewSessionHandshake:477(2354ns) {SwitchToState:179(723ns)}}}}}
2025-11-27T18:18:44.992057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471318847815961:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.005537Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471317666212598:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(129791908ns) {Configure:104(129564785ns) {ProcessConfigured:135(16078328ns) {StartInitialHandshake:157(16064332ns) {DropHandshakes:470(2202ns) {DropIncomingHandshake:442(516ns) DropOutgoingHandshake:457(371ns)} GenerateSessionVirtualId:988(1402ns) PrepareNewSessionHandshake:477(3239ns) {SwitchToState:179(1247ns)}}}}}
2025-11-27T18:18:45.007424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471321248021132:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(49400044ns) {Configure:104(49395028ns) {ProcessConfigured:135(1022581ns) {StartInitialHandshake:157(1020443ns) {DropHandshakes:470(3273ns) {DropIncomingHandshake:442(693ns) DropOutgoingHandshake:457(358ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:477(3183ns) {SwitchToState:179(1047ns)}}}}}
2025-11-27T18:18:45.006029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471321961179971:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60716
2025-11-27T18:18:44.997508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471318847815892:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:45.013631Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471321025023354:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(59290540ns) {Configure:104(59276382ns) {ProcessConfigured:135(41996ns) {StartInitialHandshake:157(41009ns) {DropHandshakes:470(990ns) {DropIncomingHandshake:442(331ns) DropOutgoingHandshake:457(167ns)} GenerateSessionVirtualId:988(509ns) PrepareNewSessionHandshake:477(719ns) {SwitchToState:179(337ns)}}}}}
2025-11-27T18:18:45.006749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471317666212664:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.006741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471318847815960:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(122205914ns) {Configure:104(122196451ns) {ProcessConfigured:135(7162972ns) {StartInitialHandshake:157(7159852ns) {DropHandshakes:470(2581ns) {DropIncomingHandshake:442(567ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(435ns) PrepareNewSessionHandshake:477(10942ns) {SwitchToState:179(979ns)}}}}}
2025-11-27T18:18:45.015356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471321025023356:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(37489969ns) {Configure:104(37482932ns) {ProcessConfigured:135(42674ns) {StartInitialHandshake:157(41106ns) {DropHandshakes:470(1205ns) {DropIncomingHandshake:442(367ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(427ns) PrepareNewSessionHandshake:477(622ns) {SwitchToState:179(216ns)}}}}}
2025-11-27T18:18:45.015716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471321248021068:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(51983302ns) {Configure:104(46688630ns) {ProcessConfigured:135(4171715ns) {StartInitialHandshake:157(4169346ns) {DropHandshakes:470(1667ns) {DropIncomingHandshake:442(450ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(414ns) PrepareNewSessionHandshake:477(2602ns) {SwitchToState:179(1133ns)}}}}}
2025-11-27T18:18:45.006794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471317666212664:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.018804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471321248021133:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.020420Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471321248021069:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(83849023ns) {Configure:104(83844589ns) {ProcessConfigured:135(2422080ns) {StartInitialHandshake:157(2419810ns) {DropHandshakes:470(1006194ns) {DropIncomingHandshake:442(504ns) DropOutgoingHandshake:457(1016ns)} GenerateSessionVirtualId:988(791ns) PrepareNewSessionHandshake:477(1534ns) {SwitchToState:179(673ns)}}}}}
2025-11-27T18:18:45.007878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471317666212662:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(125634364ns) {Configure:104(125627954ns) {ProcessConfigured:135(2620118ns) {StartInitialHandshake:157(2616153ns) {DropHandshakes:470(2271ns) {DropIncomingHandshake:442(642ns) DropOutgoingHandshake:457(370ns)} GenerateSessionVirtualId:988(1607ns) PrepareNewSessionHandshake:477(3397ns) {SwitchToState:179(895ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.021608Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471321248021070:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(67911599ns) {Configure:104(67901204ns) {ProcessConfigured:135(2874903ns) {StartInitialHandshake:157(2873269ns) {DropHandshakes:470(1176ns) {DropIncomingHandshake:442(277ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(364ns) PrepareNewSessionHandshake:477(1114ns) {SwitchToState:179(484ns)}}}}}
2025-11-27T18:18:45.021825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471325319990657:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46858
2025-11-27T18:18:45.020035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471317613509945:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(93267808ns) {Configure:104(93259315ns) {ProcessConfigured:135(1951776ns) {StartInitialHandshake:157(1950217ns) {DropHandshakes:470(1232ns) {DropIncomingHandshake:442(272ns) DropOutgoingHandshake:457(192ns)} GenerateSessionVirtualId:988(600ns) PrepareNewSessionHandshake:477(2551ns) {SwitchToState:179(1514ns)}}}}}
2025-11-27T18:18:45.007983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471317666212661:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60716
2025-11-27T18:18:45.024601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471323142783268:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35441
2025-11-27T18:18:45.020473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471317613509946:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(60580349ns) {Configure:104(60576429ns) {ProcessConfigured:135(3427043ns) {StartInitialHandshake:157(2468191ns) {DropHandshakes:470(2069ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(182ns)} GenerateSessionVirtualId:988(1408ns) PrepareNewSessionHandshake:477(2573ns) {SwitchToState:179(1624ns)}}}}}
2025-11-27T18:18:45.024830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471321025023292:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(88149069ns) {Configure:104(88109082ns) {ProcessConfigured:135(3416082ns) {StartInitialHandshake:157(3413491ns) {DropHandshakes:470(1372ns) {DropIncomingHandshake:442(403ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(425ns) PrepareNewSessionHandshake:477(2367ns) {SwitchToState:179(955ns)}}}}}
2025-11-27T18:18:45.025104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471325319990656:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46858
2025-11-27T18:18:45.025775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471318847815892:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11501) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471318847815961:2112] held: no
2025-11-27T18:18:45.026304Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471321025023355:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(56660437ns) {Configure:104(56653280ns) {ProcessConfigured:135(1824826ns) {StartInitialHandshake:157(1822029ns) {DropHandshakes:470(1689ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(665ns) PrepareNewSessionHandshake:477(4204ns) {SwitchToState:179(1904ns)}}}}}
2025-11-27T18:18:45.026645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471325319990657:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.012269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471317666212661:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.032681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471318466340762:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27344313ns) {Configure:104(27338309ns) {ProcessConfigured:135(1038300ns) {StartInitialHandshake:157(859206ns) {DropHandshakes:470(1944ns) {DropIncomingHandshake:442(1088ns) DropOutgoingHandshake:457(177ns)} GenerateSessionVirtualId:988(1206ns) PrepareNewSessionHandshake:477(1501ns) {SwitchToState:179(700ns)}}}}}
2025-11-27T18:18:45.033108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471321248021069:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:45.016674Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471317666212664:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:45.034207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471325319990656:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.021768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471317666212597:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.033243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471318847815966:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48446386ns) {Configure:104(48440524ns) {ProcessConfigured:135(3791037ns) {StartInitialHandshake:157(3788603ns) {DropHandshakes:470(1584ns) {DropIncomingHandshake:442(443ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(712ns) PrepareNewSessionHandshake:477(2485ns) {SwitchToState:179(788ns)}}}}}
2025-11-27T18:18:45.034427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471321908477319:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19721
2025-11-27T18:18:45.034412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471325542988435:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-11-27T18:18:45.024692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 2025-11-27T18:18:45.025370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471317666212597:2048] [node 2] ICP22 created new session: [1:7577471321961179976:2048]
::ffff:127.1.0.02025-11-27T18:18:45.035721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471325542988434:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-11-27T18:18:45.033891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471318847815961:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:45.026500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471321961179976:2048] [node 2] ICS09 handshake done sender: [1:7577471317666212664:2111] self: [1:7577471317666212663:0] peer: [2:7577471318847815970:0] socket: 546 qp: -1
2025-11-27T18:18:45.036953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471321248021134:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24603618ns) {Configure:104(24598258ns) {ProcessConfigured:135(965948ns) {StartInitialHandshake:157(963972ns) {DropHandshakes:470(1627ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(305ns)} GenerateSessionVirtualId:988(429ns) PrepareNewSessionHandshake:477(1331ns) {SwitchToState:179(127ns)}}}}}
2025-11-27T18:18:45.033180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471318466340761:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(67243543ns) {Configure:104(67236698ns) {ProcessConfigured:135(1091087ns) {StartInitialHandshake:157(1089387ns) {DropHandshakes:470(1429ns) {DropIncomingHandshake:442(427ns) DropOutgoingHandshake:457(186ns)} GenerateSessionVirtualId:988(333ns) PrepareNewSessionHandshake:477(2513ns) {SwitchToState:179(1482ns)}}}}}
2025-11-27T18:18:45.035517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471318847815892:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.035767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471321025023292:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.035579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:45.039626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471325542988435:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.039873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471321908477319:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.036454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471318847815892:2048] [node 1] ICP22 created new session: [2:7577471323142783272:2048]
2025-11-27T18:18:45.034123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471318466340827:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23085093ns) {Configure:104(23076565ns) {ProcessConfigured:135(885233ns) {StartInitialHandshake:157(882918ns) {DropHandshakes:470(1029ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(338ns) PrepareNewSessionHandshake:477(1625ns) {SwitchToState:179(891ns)}}}}}
2025-11-27T18:18:45.026543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471321961179976:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.040099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471317613509945:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:45.034160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471318466340826:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(37537508ns) {Configure:104(37524980ns) {ProcessConfigured:135(4619994ns) {StartInitialHandshake:157(4240686ns) {DropHandshakes:470(1540ns) {DropIncomingHandshake:442(401ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(913ns) PrepareNewSessionHandshake:477(1755ns) {SwitchToState:179(204ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.036982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471323142783272:2048] [node 1] ICS09 handshake done sender: [2:7577471318847815961:2112] self: [2:7577471318847815970:0] peer: [1:7577471317666212663:0] socket: 548 qp: -1
2025-11-27T18:18:45.035310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471318466340760:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46457613ns) {Configure:104(46451642ns) {ProcessConfigured:135(4373665ns) {StartInitialHandshake:157(4371522ns) {DropHandshakes:470(1244ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(386ns) PrepareNewSessionHandshake:477(2159ns) {SwitchToState:179(897ns)}}}}}
2025-11-27T18:18:45.045832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471321248021068:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.030700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:45.037034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471323142783272:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.044919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471318466340825:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:104
2025-11-27T18:18:45.050077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471325319990670:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46858
2025-11-27T18:18:45.048114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471325542988434:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.049565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471321908477320:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19721
2025-11-27T18:18:45.045916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471321961179972:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.037054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471318847815967:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.044968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471318466340821:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:104
2025-11-27T18:18:45.048165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471325542988442:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.052347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471321908477320:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.045919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471321961179971:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.045945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471321961179972:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.048065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471317666212672:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.047817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471318466340825:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.051969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471325542988449:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-11-27T18:18:45.037077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:45.054797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471317613509947:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:45.037086Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471318847815967:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.048094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471317666212672:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.058618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471325319990661:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.049630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471318466340760:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.052581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471325542988442:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.068978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471321908477322:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19721
2025-11-27T18:18:45.058658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471325319990661:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.054303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471321248021070:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:45.039934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471318847815963:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.050123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471317666212672:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:45.050235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471318466340821:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.039986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471318847815963:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.071809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471322761308134:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:104
2025-11-27T18:18:45.053016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471317666212666:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.061368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471325319990659:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.074435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471321908477322:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.056785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471325542988451:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-11-27T18:18:45.061396Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471325319990659:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.045105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471318847815967:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:45.074647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471321908477326:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19721
2025-11-27T18:18:45.074607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471322761308134:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.063366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471325319990656:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:45.053046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471317666212666:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.057472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471325542988453:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.069827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471321025023292:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.076783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471321908477324:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19721
2025-11-27T18:18:45.045455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471323142783274:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35441
2025-11-27T18:18:45.053427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471321961179972:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.075732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471318466340761:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.084268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471322761308143:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:104
2025-11-27T18:18:45.077696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471317613510014:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:45.081013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471321908477324:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.053864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471317666212669:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.081084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471321908477320:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:45.064433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471321248021133:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:45.070000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471325319990672:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46858
2025-11-27T18:18:45.046251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471318847815963:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:45.084757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471322761308134:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:45.053897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:45.070131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471325319990659:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:45.068821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471325542988435:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:45.081523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471317613509947:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2743) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471321908477320:2116] held: no
2025-11-27T18:18:45.047296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471318847815893:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.053958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471317666212669:2107] [node 4] ICP22 created new session: [1:7577471321961179983:2107]
2025-11-27T18:18:45.084876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471318466340761:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.070622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471325319990661:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:45.070692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471321248021069:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.084930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:45.054001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471321961179983:2107] [node 4] ICS09 handshake done sender: [1:7577471317666212672:2113] self: [1:7577471317666212670:0] peer: [4:7577471325542988448:0] socket: 560 qp: -1
2025-11-27T18:18:45.082170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471317613509946:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:45.047337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:45.072217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471321248021068:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.070649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:45.083049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471321908477325:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19721
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.072243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:45.072283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:45.073808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471321248021069:2049] [node 2] ICP22 created new session: [4:7577471325542988455:2049]
2025-11-27T18:18:45.074312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471321248021068:2048] [node 1] ICP22 created new session: [4:7577471325542988456:2048]
2025-11-27T18:18:45.076747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471325542988456:2048] [node 1] ICS09 handshake done sender: [4:7577471325542988435:2114] self: [4:7577471325542988448:0] peer: [1:7577471317666212670:0] socket: 562 qp: -1
2025-11-27T18:18:45.076841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471325542988453:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.077461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471325542988455:2049] [node 2] ICS09 handshake done sender: [4:7577471321248021133:2111] self: [4:7577471325542988443:0] peer: [2:7577471318847815962:0] socket: 552 qp: -1
2025-11-27T18:18:45.079910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471325542988455:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:45.079948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:45.080028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471325542988456:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:45.080064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:45.082479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471325542988454:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-11-27T18:18:45.085710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471325542988432:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.085735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471325542988432:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.086357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471321248021069:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16239423ns) {HandleHandshakeStatus:329(15486476ns) {DropIncomingHandshake:442(469ns) DropOutgoingHandshake:457(8174ns) SwitchToState:179(309ns) ProcessPendingSessionEvents:495(98294ns) {ForwardSessionEventToSession:612(57085ns) ForwardSessionEventToSession:612(3230ns) ForwardSessionEventToSession:612(5330ns) ForwardSessionEventToSession:612(2741ns) ForwardSessionEventToSession:612(2887ns) ForwardSessionEventToSession:612(4994ns) ForwardSessionEventToSession:612(2255ns) ForwardSessionEventToSession:612(4887ns) ForwardSessionEventToSession:612(5606ns) ForwardSessionEventToSession:612(2033ns)}}}
2025-11-27T18:18:45.087055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471325542988442:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:45.087269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471321248021070:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.087325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:45.087372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471321248021070:2050] [node 3] ICP22 created new session: [4:7577471325542988462:2050]
2025-11-27T18:18:45.091435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471325542988462:2050] [node 3] ICS09 handshake done sender: [4:7577471325542988442:2118] self: [4:7577471325542988452:0] peer: [3:7577471325319990658:0] socket: 543 qp: -1
2025-11-27T18:18:45.091461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471325542988462:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:45.092408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:45.095704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471325542988432:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:45.096044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471321248021132:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.096099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:45.096139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471321248021132:2110] [node 6] ICP22 created new session: [4:7577471325542988465:2110]
2025-11-27T18:18:45.105136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471325542988465:2110] [node 6] ICS09 handshake done sender: [4:7577471325542988432:2112] self: [4:7577471325542988431:0] peer: [6:7577471321908477327:0] socket: 571 qp: -1
2025-11-27T18:18:45.106163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471325542988465:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:45.106211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:45.047374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471318847815893:2049] [node 3] ICP22 created new session: [2:7577471323142783277:2049]
2025-11-27T18:18:45.054034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471321961179983:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:45.084971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471318466340761:2049] [node 2] ICP22 created new session: [5:7577471322761308144:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.087483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471317613509947:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.072143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471321025023292:2049] [node 2] ICP22 created new session: [3:7577471325319990675:2049]
2025-11-27T18:18:45.074330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471325319990675:2049] [node 2] ICS09 handshake done sender: [3:7577471325319990656:2111] self: [3:7577471325319990669:0] peer: [2:7577471318847815965:0] socket: 559 qp: -1
2025-11-27T18:18:45.085011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471322761308144:2049] [node 2] ICS09 handshake done sender: [5:7577471322761308134:2118] self: [5:7577471322761308142:0] peer: [2:7577471318847815969:0] socket: 579 qp: -1
2025-11-27T18:18:45.054063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:45.047416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471323142783277:2049] [node 3] ICS09 handshake done sender: [2:7577471318847815967:2114] self: [2:7577471318847815965:0] peer: [3:7577471325319990669:0] socket: 555 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.074366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471325319990675:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:45.091380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:45.085036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471322761308144:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.047445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471323142783277:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.085075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:45.092263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471317613509947:2050] [node 3] ICP22 created new session: [6:7577471321908477329:2050]
2025-11-27T18:18:45.074826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471321025023356:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.054336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471317666212597:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(24203549ns) {HandleHandshakeStatus:329(24200225ns) {DropIncomingHandshake:442(15834ns) DropOutgoingHandshake:457(353ns) SwitchToState:179(363ns) ProcessPendingSessionEvents:495(11390121ns) {ForwardSessionEventToSession:612(2051139ns) ForwardSessionEventToSession:612(6530ns) ForwardSessionEventToSession:612(562294ns) ForwardSessionEventToSession:612(3369ns) ForwardSessionEventToSession:612(3984279ns) ForwardSessionEventToSession:612(3809752ns) ForwardSessionEventToSession:612(5495ns) ForwardSessionEventToSession:612(484362ns) ForwardSessionEventToSession:612(451414ns) ForwardSessionEventToSession:612(5691ns)}}}
2025-11-27T18:18:45.047474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:45.085545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471322761308140:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:104
2025-11-27T18:18:45.146406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471321248021070:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(38693924ns) {HandleHandshakeStatus:329(38691654ns) {DropIncomingHandshake:442(353ns) DropOutgoingHandshake:457(3151ns) SwitchToState:179(157ns) ProcessPendingSessionEvents:495(23239052ns) {ForwardSessionEventToSession:612(4294564ns) ForwardSessionEventToSession:612(4503ns) ForwardSessionEventToSession:612(2816ns) ForwardSessionEventToSession:612(8521984ns) ForwardSessionEventToSession:612(3115ns) ForwardSessionEventToSession:612(2036969ns) ForwardSessionEventToSession:612(4328ns) ForwardSessionEventToSession:612(5052084ns) ForwardSessionEventToSession:612(3281090ns) ForwardSessionEventToSession:612(4331ns)}}}
2025-11-27T18:18:45.094586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471321908477329:2050] [node 3] ICS09 handshake done sender: [6:7577471321908477320:2116] self: [6:7577471321908477323:0] peer: [3:7577471325319990660:0] socket: 569 qp: -1
2025-11-27T18:18:45.054793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471321961179979:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60716
2025-11-27T18:18:45.050680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471318847815958:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.052911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:45.075376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:45.093249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471318466340825:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:45.061282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471317666212662:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.075812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:45.096243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471321908477319:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:45.053360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471318847815958:2107] [node 4] ICP22 created new session: [2:7577471323142783280:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.061334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:45.075860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471321025023356:2109] [node 6] ICP22 created new session: [3:7577471325319990678:2109]
2025-11-27T18:18:45.103508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471318466340760:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.100645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471321908477329:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.057631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471323142783280:2107] [node 4] ICS09 handshake done sender: [2:7577471318847815963:2113] self: [2:7577471318847815962:0] peer: [4:7577471325542988443:0] socket: 549 qp: -1
2025-11-27T18:18:45.064107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471317666212662:2109] [node 6] ICP22 created new session: [1:7577471321961179986:2109]
2025-11-27T18:18:45.105124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471322761308141:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:104
2025-11-27T18:18:45.077346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471321025023354:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.106099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:45.077419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471325319990671:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46858
2025-11-27T18:18:45.106144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471318466340760:2048] [node 1] ICP22 created new session: [5:7577471322761308147:2048]
2025-11-27T18:18:45.078521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471325319990678:2109] [node 6] ICS09 handshake done sender: [3:7577471325319990661:2114] self: [3:7577471325319990660:0] peer: [6:7577471321908477323:0] socket: 568 qp: -1
2025-11-27T18:18:45.122846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471322761308147:2048] [node 1] ICS09 handshake done sender: [5:7577471318466340825:2112] self: [5:7577471322761308137:0] peer: [1:7577471317666212665:0] socket: 551 qp: -1
2025-11-27T18:18:45.080888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471325319990678:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:45.080902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:45.125643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471322761308141:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.080937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:45.082064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471321025023354:2107] [node 4] ICP22 created new session: [3:7577471325319990679:2107]
2025-11-27T18:18:45.125750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471322761308147:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:45.100702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471317613509945:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.085821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471325319990671:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.085922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471325319990679:2107] [node 4] ICS09 handshake done sender: [3:7577471325319990659:2113] self: [3:7577471325319990658:0] peer: [4:7577471325542988452:0] socket: 567 qp: -1
2025-11-27T18:18:45.128944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:45.091603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471325319990679:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:45.100687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:45.091648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:45.138346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471318466340762:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:45.103495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:45.091954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471325319990672:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.141121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471322761308146:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:104
2025-11-27T18:18:45.103548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471317613509945:2048] [node 1] ICP22 created new session: [6:7577471321908477333:2048]
2025-11-27T18:18:45.101005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471321025023291:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:45.148232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471322761308151:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:104
2025-11-27T18:18:45.114670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471321025023292:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(22523965ns) {HandleHandshakeStatus:329(22521170ns) {DropIncomingHandshake:442(953ns) DropOutgoingHandshake:457(12112ns) SwitchToState:179(704ns) ProcessPendingSessionEvents:495(11988512ns) {ForwardSessionEventToSession:612(1776542ns) ForwardSessionEventToSession:612(2777ns) ForwardSessionEventToSession:612(830287ns) ForwardSessionEventToSession:612(2771ns) ForwardSessionEventToSession:612(2680453ns) ForwardSessionEventToSession:612(3922ns) ForwardSessionEventToSession:612(1860226ns) ForwardSessionEventToSession:612(4810735ns) ForwardSessionEventToSession:612(2988ns) ForwardSessionEventToSession:612(1934ns)}}}
2025-11-27T18:18:45.113463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471321908477333:2048] [node 1] ICS09 handshake done sender: [6:7577471321908477319:2115] self: [6:7577471321908477321:0] peer: [1:7577471321961179970:0] socket: 566 qp: -1
2025-11-27T18:18:45.157863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471321248021132:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(50525420ns) {HandleHandshakeStatus:329(50523122ns) {DropIncomingHandshake:442(12476ns) DropOutgoingHandshake:457(267ns) SwitchToState:179(331ns) ProcessPendingSessionEvents:495(11302181ns) {ForwardSessionEventToSession:612(1725196ns) ForwardSessionEventToSession:612(1355161ns) ForwardSessionEventToSession:612(2434791ns) ForwardSessionEventToSession:612(6501ns) ForwardSessionEventToSession:612(1910041ns) ForwardSessionEventToSession:612(2540ns) ForwardSessionEventToSession:612(3211137ns) ForwardSessionEventToSession:612(4542ns) ForwardSessionEventToSession:612(2242ns) ForwardSessionEventToSession:612(618599ns)}}}
2025-11-27T18:18:45.158203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471325542988444:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.058476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471323142783280:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:45.071135Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471317666212666:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:45.150405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471322761308146:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.128660Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471321025023356:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(26701813ns) {HandleHandshakeStatus:329(26372090ns) {DropIncomingHandshake:442(11249ns) DropOutgoingHandshake:457(195ns) SwitchToState:179(212ns) ProcessPendingSessionEvents:495(15398840ns) {ForwardSessionEventToSession:612(6274363ns) ForwardSessionEventToSession:612(6031ns) ForwardSessionEventToSession:612(237715ns) ForwardSessionEventToSession:612(17893ns) ForwardSessionEventToSession:612(3732ns) ForwardSessionEventToSession:612(3633278ns) ForwardSessionEventToSession:612(3693ns) ForwardSessionEventToSession:612(2776ns) ForwardSessionEventToSession:612(2857448ns) ForwardSessionEventToSession:612(3542ns)}}}
2025-11-27T18:18:45.058710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:45.064460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471323142783274:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.080184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471323142783267:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.080234Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471323142783267:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.082228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471323142783282:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35441
2025-11-27T18:18:45.084177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471323142783267:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:45.084364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471318847815960:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.091267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:45.115979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471321908477333:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:45.133103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471325319990685:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46858
2025-11-27T18:18:45.135852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471325319990682:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46858
2025-11-27T18:18:45.138605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471325319990672:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:45.143069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471321025023291:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.143123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:45.143162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471321025023291:2048] [node 1] ICP22 created new session: [3:7577471325319990686:2048]
2025-11-27T18:18:45.143208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471325319990686:2048] [node 1] ICS09 handshake done sender: [3:7577471325319990672:2119] self: [3:7577471325319990684:0] peer: [1:7577471317666212671:0] socket: 597 qp: -1
2025-11-27T18:18:45.143243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471325319990686:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:45.143270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:45.144534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471321025023354:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(57832174ns) {HandleHandshakeStatus:329(57829740ns) {DropIncomingHandshake:442(948ns) DropOutgoingHandshake:457(565ns) SwitchToState:179(311ns) ProcessPendingSessionEvents:495(30694974ns) {ForwardSessionEventToSession:612(3377421ns) ForwardSessionEventToSession:612(691246ns) ForwardSessionEventToSession:612(4379ns) ForwardSessionEventToSession:612(2526ns) ForwardSessionEventToSession:612(6587469ns) ForwardSessionEventToSession:612(9157ns) ForwardSessionEventToSession:612(3749ns) ForwardSessionEventToSession:612(7863662ns) ForwardSessionEventToSession:612(4439269ns) ForwardSessionEventToSession:612(7676464ns)}}}
2025-11-27T18:18:45.146984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471325319990665:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.147013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471325319990665:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.149240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471325319990665:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:45.155159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471321025023355:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.155226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:45.157895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471321025023355:2108] [node 5] ICP22 created new session: [3:7577471325319990689:2108]
2025-11-27T18:18:45.160846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471325319990689:2108] [node 5] ICS09 handshake done sender: [3:7577471325319990665:2116] self: [3:7577471325319990664:0] peer: [5:7577471322761308150:0] socket: 604 qp: -1
2025-11-27T18:18:45.160907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471325319990689:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:45.161374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:45.071604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471317666212660:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.071664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:45.071727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471317666212660:2108] [node 5] ICP22 created new session: [1:7577471321961179988:2108]
2025-11-27T18:18:45.071765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471321961179986:2109] [node 6] ICS09 handshake done sender: [1:7577471321961179972:2116] self: [1:7577471321961179970:0] peer: [6:7577471321908477321:0] socket: 545 qp: -1
2025-11-27T18:18:45.071796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471321961179986:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:45.072184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471321961179979:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.072665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:45.074042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471321961179988:2108] [node 5] ICS09 handshake done sender: [1:7577471317666212666:2112] self: [1:7577471317666212665:0] peer: [5:7577471322761308137:0] socket: 547 qp: -1
2025-11-27T18:18:45.074487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471321961179988:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:45.075843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:45.083096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471317666212669:2107] [node 4] ICP18 handshake (actor [1:7577471321961179979:2117]) is too old
2025-11-27T18:18:45.085668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471317666212662:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16649702ns) {HandleHandshakeStatus:329(16646694ns) {DropIncomingHandshake:442(1541ns) DropOutgoingHandshake:457(999ns) SwitchToState:179(683ns) ProcessPendingSessionEvents:495(4713704ns) {ForwardSessionEventToSession:612(1347707ns) ForwardSessionEventToSession:612(3132ns) ForwardSessionEventToSession:612(176342ns) ForwardSessionEventToSession:612(1901ns) ForwardSessionEventToSession:612(1025016ns) ForwardSessionEventToSession:612(2675ns) ForwardSessionEventToSession:612(1936ns) ForwardSessionEventToSession:612(309544ns) ForwardSessionEventToSession:612(1826930ns) ForwardSessionEventToSession:612(5492ns)}}}
2025-11-27T18:18:45.106116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471317666212673:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.106148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471317666212673:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.113538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471317666212660:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(19854444ns) {HandleHandshakeStatus:329(19852076ns) {DropIncomingHandshake:442(243ns) DropOutgoingHandshake:457(279ns) SwitchToState:179(447ns) ProcessPendingSessionEvents:495(11530960ns) {ForwardSessionEventToSession:612(1252968ns) ForwardSessionEventToSession:612(313696ns) ForwardSessionEventToSession:612(3188ns) ForwardSessionEventToSession:612(2324ns) ForwardSessionEventToSession:612(698737ns) ForwardSessionEventToSession:612(2468ns) ForwardSessionEventToSession:612(1161440ns) ForwardSessionEventToSession:612(2361490ns) ForwardSessionEventToSession:612(3292ns) ForwardSessionEventToSession:612(5715048ns)}}}
2025-11-27T18:18:45.135407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471317666212673:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:45.138947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471317666212598:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.140998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:45.141057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471317666212598:2049] [node 3] ICP22 created new session: [1:7577471321961179992:2049]
2025-11-27T18:18:45.146515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471321961179992:2049] [node 3] ICS09 handshake done sender: [1:7577471317666212673:2114] self: [1:7577471317666212671:0] peer: [3:7577471325319990684:0] socket: 596 qp: -1
2025-11-27T18:18:45.148458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471321961179992:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:45.149349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:45.154092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471318466340826:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:45.091320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471318847815960:2108] [node 5] ICP22 created new session: [2:7577471323142783284:2108]
2025-11-27T18:18:45.158228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471325542988444:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.160914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471325542988444:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:45.116024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:45.154167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471322761308141:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:45.161321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471318466340762:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.161785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471322761308153:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:104
2025-11-27T18:18:45.162840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471322761308146:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:45.163756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471318466340826:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.163825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:45.163879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471318466340826:2109] [node 4] ICP22 created new session: [5:7577471322761308154:2109]
2025-11-27T18:18:45.163931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471322761308154:2109] [node 4] ICS09 handshake done sender: [5:7577471322761308146:2122] self: [5:7577471322761308152:0] peer: [4:7577471325542988441:0] socket: 557 qp: -1
2025-11-27T18:18:45.163969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471322761308154:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:45.164020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:45.164482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471318466340760:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(47609655ns) {HandleHandshakeStatus:329(46895341ns) {DropIncomingHandshake:442(3483ns) DropOutgoingHandshake:457(9729ns) SwitchToState:179(11960ns) ProcessPendingSessionEvents:495(12174495ns) {ForwardSessionEventToSession:612(2954321ns) ForwardSessionEventToSession:612(4715222ns) ForwardSessionEventToSession:612(667464ns) ForwardSessionEventToSession:612(3792ns) ForwardSessionEventToSession:612(2901ns) ForwardSessionEventToSession:612(1951842ns) ForwardSessionEventToSession:612(2788ns) ForwardSessionEventToSession:612(1097497ns) ForwardSessionEventToSession:612(3611ns) ForwardSessionEventToSession:612(756616ns)}}}
2025-11-27T18:18:45.173712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:45.173766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471318466340762:2050] [node 3] ICP22 created new session: [5:7577471322761308156:2050]
2025-11-27T18:18:45.173815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471322761308156:2050] [node 3] ICS09 handshake done sender: [5:7577471322761308141:2120] self: [5:7577471322761308150:0] peer: [3:7577471325319990664:0] socket: 605 qp: -1
2025-11-27T18:18:45.173846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471322761308156:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:45.173926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:45.181896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471322761308131:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.181933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471322761308131:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.162268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471321248021134:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.087511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471318847815958:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(24274258ns) {HandleHandshakeStatus:329(24270981ns) {DropIncomingHandshake:442(263ns) DropOutgoingHandshake:457(486ns) SwitchToState:179(378ns) ProcessPendingSessionEvents:495(10773737ns) {ForwardSessionEventToSession:612(7810739ns) ForwardSessionEventToSession:612(5953ns) ForwardSessionEventToSession:612(91587ns) ForwardSessionEventToSession:612(6806ns) ForwardSessionEventToSession:612(3314ns) ForwardSessionEventToSession:612(870324ns) ForwardSessionEventToSession:612(1459183ns) ForwardSessionEventToSession:612(3223ns) ForwardSessionEventToSession:612(499137ns) ForwardSessionEventToSession:612(2882ns)}}}
2025-11-27T18:18:45.189679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471322761308131:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:45.162328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:45.129091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471317613509947:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(30018090ns) {HandleHandshakeStatus:329(30015792ns) {DropIncomingHandshake:442(4305ns) DropOutgoingHandshake:457(201ns) SwitchToState:179(121ns) ProcessPendingSessionEvents:495(1438ns)}}
2025-11-27T18:18:45.190095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471318466340827:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.190386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471318466340762:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(25872044ns) {HandleHandshakeStatus:329(25745834ns) {DropIncomingHandshake:442(1952ns) DropOutgoingHandshake:457(16181ns) SwitchToState:179(608ns) ProcessPendingSessionEvents:495(12277396ns) {ForwardSessionEventToSession:612(4822407ns) ForwardSessionEventToSession:612(4655ns) ForwardSessionEventToSession:612(467344ns) ForwardSessionEventToSession:612(4015ns) ForwardSessionEventToSession:612(2730987ns) ForwardSessionEventToSession:612(3031234ns) ForwardSessionEventToSession:612(2336ns) ForwardSessionEventToSession:612(1191518ns) ForwardSessionEventToSession:612(5013ns) ForwardSessionEventToSession:612(2479ns)}}}
2025-11-27T18:18:45.191976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:45.092133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471323142783270:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.164242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471321248021134:2108] [node 5] ICP22 created new session: [4:7577471325542988468:2108]
2025-11-27T18:18:45.193621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471318466340827:2111] [node 6] ICP22 created new session: [5:7577471322761308159:2111]
2025-11-27T18:18:45.092154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471323142783270:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:45.092181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471323142783282:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.093185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471323142783284:2108] [node 5] ICS09 handshake done sender: [2:7577471323142783267:2115] self: [2:7577471318847815969:0] peer: [5:7577471322761308142:0] socket: 570 qp: -1
2025-11-27T18:18:45.093240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471323142783284:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:45.094116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471323142783270:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:45.094336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471318847815966:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:45.094364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:45.094390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471318847815966:2110] [node 6] ICP22 created new session: [2:7577471323142783286:2110]
2025-11-27T18:18:45.094420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471323142783286:2110] [node 6] ICS09 handshake done sender: [2:7577471323142783270:2117] self: [2:7577471323142783269:0] peer: [6:7577471321908477328:0] socket: 588 qp: -1
2025-11-27T18:18:45.094438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471323142783286:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:45.094456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:45.100649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:45.128851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471323142783289:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35441
2025-11-27T18:18:45.135941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471318847815958:2107] [node 4] ICP18 handshake (actor [2:7577471323142783274:2118]) is too old
2025-11-27T18:18:45.136131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471318847815966:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(21638960ns) {HandleHandshakeStatus:329(21636385ns) {DropIncomingHandshake:442(219ns) DropOutgoingHandshake:457(568ns) SwitchToState:179(152ns) ProcessPendingSessionEvents:495(21412597ns) {ForwardSessionEventToSession:612(137343ns) ForwardSessionEventToSession:612(1609663ns) ForwardSessionEventToSession:612(2307048ns) ForwardSessionEventToSession:612(21454ns) ForwardSessionEventToSession:612(927652ns) ForwardSessionEventToSession:612(2939ns) ForwardSessionEventToSession:612(7528962ns) ForwardSessionEventToSession:612(4300ns) ForwardSessionEventToSession:612(3264ns) ForwardSessionEventToSession:612(2295153ns)}}}
2025-11-27T18:18:45.145948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471318847815960:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(45297916ns) {HandleHandshakeStatus:329(45295276ns) {DropIncomingHandshake:442(711ns) DropOutgoingHandshake:457(344ns) SwitchToState:179(386ns) ProcessPendingSessionEvents:495(24434826ns) {ForwardSessionEventToSession:612(9253116ns) ForwardSessionEventToSession:612(5463ns) ForwardSessionEventToSession:612(8468730ns) ForwardSessionEventToSession:612(6317ns) ForwardSessionEventToSession:612(3052ns) ForwardSessionEventToSession:612(5617685ns) ForwardSessionEventToSession:612(4207ns) ForwardSessionEventToSession:612(3153ns) ForwardSessionEventToSession:612(1045613ns) ForwardSessionEventToSession:612(3492ns)}}}
2025-11-27T18:18:45.146859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471323142783289:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.148575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471318847815960:2108] [node 5] ICP18 handshake (actor [2:7577471323142783282:2119]) is too old
2025-11-27T18:18:45.149398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471318847815966:2110] [node 6] ICP18 handshake (actor [2:7577471323142783289:2120]) is too old
2025-11-27T18:18:45.133342Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471317613510014:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8407) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471321908477322:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:45.148540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471321908477332:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19721
2025-11-27T18:18:45.149460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471321908477334:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19721
2025-11-27T18:18:45.159236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471317613509946:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29066) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471321908477324:2118] held: no
2025-11-27T18:18:45.159431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471321908477324:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:45.160589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471317613509945:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(52668508ns) {HandleHandshakeStatus:329(52666762ns) {DropIncomingHandshake:442(1227ns) DropOutgoingHandshake:457(4695ns) SwitchToState:179(282ns) ProcessPendingSessionEvents:495(14172455ns) {ForwardSessionEventToSession:612(9043046ns) ForwardSessionEventToSession:612(9971ns) ForwardSessionEventToSession:612(3311ns) ForwardSessionEventToSession:612(798310ns) ForwardSessionEventToSession:612(2194ns) ForwardSessionEventToSession:612(1738773ns) ForwardSessionEventToSession:612(3859ns) ForwardSessionEventToSession:612(2609ns) ForwardSessionEventToSession:612(2544857ns) ForwardSessionEventToSession:612(8136ns)}}}
2025-11-27T18:18:45.161308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471321908477322:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:45.162590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471321908477336:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19721
2025-11-27T18:18:45.169724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471317613509946:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.173321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471321908477336:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:45.174052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:45.174355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471317613510014:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.174424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:45.174488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471317613510014:2110] [node 4] ICP22 created new session: [6:7577471321908477338:2110]
2025-11-27T18:18:45.174541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471321908477338:2110] [node 4] ICS09 handshake done sender: [6:7577471321908477322:2117] self: [6:7577471321908477327:0] peer: [4:7577471325542988431:0] socket: 572 qp: -1
2025-11-27T18:18:45.174568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471321908477338:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:45.174592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:45.175082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471317613509946:2049] [node 2] ICP22 created new session: [6:7577471321908477337:2049]
2025-11-27T18:18:45.175140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471317613510007:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:45.179431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471321908477337:2049] [node 2] ICS09 handshake done sender: [6:7577471321908477324:2118] self: [6:7577471321908477328:0] peer: [2:7577471323142783269:0] socket: 590 qp: -1
2025-11-27T18:18:45.186296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471321908477337:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:45.187277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:45.189650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471317613509946:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(25162514ns) {HandleHandshakeStatus:329(19847593ns) {DropIncomingHandshake:442(928ns) DropOutgoingHandshake:457(267ns) SwitchToState:179(785ns) ProcessPendingSessionEvents:495(1030ns)}}
2025-11-27T18:18:45.199722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471317613510007:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26624) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471321908477336:2123] held: no
2025-11-27T18:18:45.200007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471321908477342:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19721
2025-11-27T18:18:45.170894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471325542988468:2108] [node 5] ICS09 handshake done sender: [4:7577471325542988444:2119] self: [4:7577471325542988441:0] peer: [5:7577471322761308152:0] socket: 606 qp: -1
2025-11-27T18:18:45.174316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471325542988468:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:45.174400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:45.193695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471321248021134:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(26892669ns) {HandleHandshakeStatus:329(26890473ns) {DropIncomingHandshake:442(14188ns) DropOutgoingHandshake:457(300ns) SwitchToState:179(560ns) ProcessPendingSessionEvents:495(13247104ns) {ForwardSessionEventToSession:612(6984405ns) ForwardSessionEventToSession:612(9976ns) ForwardSessionEventToSession:612(2986ns) ForwardSessionEventToSession:612(2370ns) ForwardSessionEventToSession:612(4342751ns) ForwardSessionEventToSession:612(4473ns) ForwardSessionEventToSession:612(3098ns) ForwardSessionEventToSession:612(1028822ns) ForwardSessionEventToSession:612(3036ns) ForwardSessionEventToSession:612(209200ns)}}}
2025-11-27T18:18:45.201287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471322761308159:2111] [node 6] ICS09 handshake done sender: [5:7577471322761308131:2115] self: [5:7577471322761308126:0] peer: [6:7577471321908477340:0] socket: 558 qp: -1
2025-11-27T18:18:45.201352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471322761308159:2111] [node 6] ICS10 traffic start
2025-11-27T18:18:45.201402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:45.202411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471321908477336:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:45.202656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471317613510007:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:45.202711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:45.202755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471317613510007:2107] [node 5] ICP22 created new session: [6:7577471321908477343:2107]
2025-11-27T18:18:45.202795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471321908477343:2107] [node 5] ICS09 handshake done sender: [6:7577471321908477336:2123] self: [6:7577471321908477340:0] peer: [5:7577471322761308126:0] socket: 553 qp: -1
2025-11-27T18:18:45.202826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471321908477343:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:45.202860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:45.275906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:45.276177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471323142783272:2048] [node 1] ICS01 socket: 548 reason# EndOfStream
2025-11-27T18:18:45.276243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471318847815892:2048] [node 1] ICP30 unregister session Session# [2:7577471323142783272:2048] VirtualId# [2:7577471318847815970:0]
2025-11-27T18:18:45.276284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471323142783272:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.276362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:45.276414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:45.276215Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:45.276401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471325319990686:2048] [node 1] ICS01 socket: 597 reason# EndOfStream
2025-11-27T18:18:45.276445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471321025023291:2048] [node 1] ICP30 unregister session Session# [3:7577471325319990686:2048] VirtualId# [3:7577471325319990684:0]
2025-11-27T18:18:45.276471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471325319990686:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.276565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:45.276610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:45.280127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:45.280363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471322761308147:2048] [node 1] ICS07 socket disconnect 551 reason# EndOfStream
2025-11-27T18:18:45.280410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471322761308147:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.280498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:45.280549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471322761308147:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:45.281234Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:45.281473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471321908477333:2048] [node 1] ICS07 socket disconnect 566 reason# EndOfStream
2025-11-27T18:18:45.281507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471321908477333:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.281452Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:45.281655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471325542988456:2048] [node 1] ICS07 socket disconnect 562 reason# EndOfStream
2025-11-27T18:18:45.281584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:45.281678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471325542988456:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.281771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:45.281633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471321908477333:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:45.283019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471318466340760:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11501) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:45.283229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471322761308147:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:45.282458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471325542988456:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:45.283407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471318466340760:2048] [node 1] ICP30 unregister session Session# [5:7577471322761308147:2048] VirtualId# [5:7577471322761308137:0]
2025-11-27T18:18:45.283460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:45.283497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471318466340760:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11501) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.283525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11501) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.283922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471317613509945:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11501) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:45.284008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471321908477333:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:45.284131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471317613509945:2048] [node 1] ICP30 unregister session Session# [6:7577471321908477333:2048] VirtualId# [6:7577471321908477321:0]
2025-11-27T18:18:45.284208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:45.284238Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471317613509945:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11501) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.284258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11501) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.285610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471321248021068:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11501) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:45.285896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471325542988456:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:45.285941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471321248021068:2048] [node 1] ICP30 unregister session Session# [4:7577471325542988456:2048] VirtualId# [4:7577471325542988448:0]
2025-11-27T18:18:45.285994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:45.286025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471321248021068:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11501) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.286043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11501) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.293430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471318466340760:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:45.294385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471317613509945:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:45.296113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471321248021068:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:45.382119Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:45.382345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471325319990675:2049] [node 2] ICS07 socket disconnect 559 reason# EndOfStream
2025-11-27T18:18:45.382377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471325319990675:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.382445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:45.382504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471325319990675:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:45.383598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471321025023292:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29066) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:45.383834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471325319990675:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:45.383863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471321025023292:2049] [node 2] ICP30 unregister session Session# [3:7577471325319990675:2049] VirtualId# [3:7577471325319990669:0]
2025-11-27T18:18:45.383912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:45.383944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471321025023292:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29066) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.384724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:45.383957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29066) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.384956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471325542988455:2049] [node 2] ICS01 socket: 552 reason# EndOfStream
2025-11-27T18:18:45.385004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471321248021069:2049] [node 2] ICP30 unregister session Session# [4:7577471325542988455:2049] VirtualId# [4:7577471325542988443:0]
2025-11-27T18:18:45.385039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471325542988455:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.385105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:45.385147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:45.385866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:45.386058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471322761308144:2049] [node 2] ICS01 socket: 579 reason# EndOfStream
2025-11-27T18:18:45.386098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471318466340761:2049] [node 2] ICP30 unregister session Session# [5:7577471322761308144:2049] VirtualId# [5:7577471322761308142:0]
2025-11-27T18:18:45.386121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471322761308144:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.386200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:45.386242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:45.392741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:45.393058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471321908477337:2049] [node 2] ICS01 socket: 590 reason# EndOfStream
2025-11-27T18:18:45.397686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471321025023292:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:45.397724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471317613509946:2049] [node 2] ICP30 unregister session Session# [6:7577471321908477337:2049] VirtualId# [6:7577471321908477328:0]
2025-11-27T18:18:45.397765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471321908477337:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.397860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:45.397904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:45.478348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:45.478574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471325542988462:2050] [node 3] ICS07 socket disconnect 543 reason# EndOfStream
2025-11-27T18:18:45.478603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471325542988462:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.478677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:45.478723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471325542988462:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:45.479549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471321248021070:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2743) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:45.479608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471325542988462:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:45.479702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471321248021070:2050] [node 3] ICP30 unregister session Session# [4:7577471325542988462:2050] VirtualId# [4:7577471325542988452:0]
2025-11-27T18:18:45.479765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:45.479807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471321248021070:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2743) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.479831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2743) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.479794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:45.479965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471322761308156:2050] [node 3] ICS07 socket disconnect 605 reason# EndOfStream
2025-11-27T18:18:45.479990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471322761308156:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.480051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:45.480084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471322761308156:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:45.489896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471321248021070:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:45.481019Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:45.481208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471321908477329:2050] [node 3] ICS01 socket: 569 reason# EndOfStream
2025-11-27T18:18:45.481247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471317613509947:2050] [node 3] ICP30 unregister session Session# [6:7577471321908477329:2050] VirtualId# [6:7577471321908477323:0]
2025-11-27T18:18:45.481293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471321908477329:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.481357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:45.481423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:45.487284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471318466340762:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2743) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:45.497410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471322761308156:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:45.497473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471318466340762:2050] [node 3] ICP30 unregister session Session# [5:7577471322761308156:2050] VirtualId# [5:7577471322761308150:0]
2025-11-27T18:18:45.497532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:45.497567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471318466340762:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2743) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.497584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2743) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:45.513444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471318466340762:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:45.548461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:45.548691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471321908477338:2110] [node 4] ICS01 socket: 572 reason# EndOfStream
2025-11-27T18:18:45.548739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471317613510014:2110] [node 4] ICP30 unregister session Session# [6:7577471321908477338:2110] VirtualId# [6:7577471321908477327:0]
2025-11-27T18:18:45.548796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471321908477338:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.548890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:45.548945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:45.549393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:45.549643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471322761308154:2109] [node 4] ICS01 socket: 557 reason# EndOfStream
2025-11-27T18:18:45.549745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471318466340826:2109] [node 4] ICP30 unregister session Session# [5:7577471322761308154:2109] VirtualId# [5:7577471322761308152:0]
2025-11-27T18:18:45.553385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471322761308154:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.553503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:45.553576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:45.621388Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:45.621633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471321908477343:2107] [node 5] ICS01 socket: 553 reason# EndOfStream
2025-11-27T18:18:45.621724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471317613510007:2107] [node 5] ICP30 unregister session Session# [6:7577471321908477343:2107] VirtualId# [6:7577471321908477340:0]
2025-11-27T18:18:45.621783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471321908477343:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:45.621904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:45.621992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:45.723567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471322598652480:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:45.723625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471322598652481:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:45.744855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471323182784839:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:45.744917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471323182784840:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:45.764386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471322422875178:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:45.764427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471322422875179:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:45.783241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471324324681383:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:45.783288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471324324681384:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:45.783368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471324324681385:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:45.802440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471323336140280:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:45.802505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471323336140281:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:45.802547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471323336140282:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:45.824449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471325139186838:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:45.824484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471325139186839:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:45.824509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471325139186840:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:45.987997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471325139186898:2105] [node 5] ICP01 ready to work
2025-11-27T18:18:46.029032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471322598652480:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19238272ns) {Configure:104(19233421ns) {ProcessConfigured:135(10284501ns) {StartInitialHandshake:157(10272769ns) {DropHandshakes:470(3269ns) {DropIncomingHandshake:442(786ns) DropOutgoingHandshake:457(407ns)} GenerateSessionVirtualId:988(999ns) PrepareNewSessionHandshake:477(8336ns) {SwitchToState:179(3926ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.050319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471327631107637:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1616
2025-11-27T18:18:46.049162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471325139186898:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(54729446ns) {Configure:104(54724720ns) {ProcessConfigured:135(18980888ns) {StartInitialHandshake:157(18976202ns) {DropHandshakes:470(4137ns) {DropIncomingHandshake:442(702ns) DropOutgoingHandshake:457(518ns)} GenerateSessionVirtualId:988(1152ns) PrepareNewSessionHandshake:477(12778ns) {SwitchToState:179(10614ns)}}}}}
2025-11-27T18:18:46.056027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471327477752199:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:46.065220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471327477752200:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:46.068148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471327477752201:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:46.068414Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471322598652481:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(38966800ns) {Configure:104(38959638ns) {ProcessConfigured:135(130025ns) {StartInitialHandshake:157(88948ns) {DropHandshakes:470(1769ns) {DropIncomingHandshake:442(567ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(959ns) PrepareNewSessionHandshake:477(2391ns) {SwitchToState:179(808ns)}}}}}
2025-11-27T18:18:46.068892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471326893619843:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:46.068283Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471322422875178:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(61581474ns) {Configure:104(49848432ns) {ProcessConfigured:135(299816ns) {StartInitialHandshake:157(157240ns) {DropHandshakes:470(4317ns) {DropIncomingHandshake:442(1263ns) DropOutgoingHandshake:457(376ns)} GenerateSessionVirtualId:988(1444ns) PrepareNewSessionHandshake:477(1745ns) {SwitchToState:179(1087ns)}}}}}
2025-11-27T18:18:46.077417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471326893619844:2108] [node 6] ICP01 ready to work
2025-11-27T18:18:46.108458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471326893619845:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:46.129906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471329434154207:2113] [node 4] ICP01 ready to work
2025-11-27T18:18:46.068911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471326717842539:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:46.117495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471324324681383:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28035322ns) {Configure:104(28025462ns) {ProcessConfigured:135(48006ns) {StartInitialHandshake:157(46490ns) {DropHandshakes:470(1382ns) {DropIncomingHandshake:442(413ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(1210ns) PrepareNewSessionHandshake:477(1454ns) {SwitchToState:179(666ns)}}}}}
2025-11-27T18:18:46.069349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471326717842540:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:46.131196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471326717842541:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:46.182387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471323182784839:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(114001388ns) {Configure:104(113994584ns) {ProcessConfigured:135(62826ns) {StartInitialHandshake:157(60111ns) {DropHandshakes:470(1791ns) {DropIncomingHandshake:442(367ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(738ns) PrepareNewSessionHandshake:477(1122ns) {SwitchToState:179(406ns)}}}}}
2025-11-27T18:18:46.168107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471328619648745:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:46.184823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471328619648746:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.192205Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471329434154207:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(61990464ns) {Configure:104(61986277ns) {ProcessConfigured:135(35247ns) {StartInitialHandshake:157(32552ns) {DropHandshakes:470(555ns) {DropIncomingHandshake:442(89ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:477(472ns) {SwitchToState:179(95ns)}}}}}
2025-11-27T18:18:46.192741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471329434154212:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57329
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.202960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471326717842553:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25195
2025-11-27T18:18:46.204071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471329434154212:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.203636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471326717842553:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.204221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471325139186840:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.204486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471326717842545:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.204511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471326717842545:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.206809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471326717842545:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:46.208682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471326717842541:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.214039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:46.219144Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471326893619845:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(21220358ns) {Configure:104(21212459ns) {ProcessConfigured:135(669247ns) {StartInitialHandshake:157(666391ns) {DropHandshakes:470(3274ns) {DropIncomingHandshake:442(954ns) DropOutgoingHandshake:457(613ns)} GenerateSessionVirtualId:988(656ns) PrepareNewSessionHandshake:477(2852ns) {SwitchToState:179(1011ns)}}}}}
2025-11-27T18:18:46.218307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471326717842541:2109] [node 6] ICP22 created new session: [3:7577471326717842556:2109]
2025-11-27T18:18:46.207495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471329434154214:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57329
2025-11-27T18:18:46.208901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471329434154212:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:46.218838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471326717842556:2109] [node 6] ICS09 handshake done sender: [3:7577471326717842545:2112] self: [3:7577471326717842544:0] peer: [6:7577471329434154213:0] socket: 551 qp: -1
2025-11-27T18:18:46.209157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471325139186840:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.209212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:46.225754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471326717842556:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:46.209278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471325139186840:2050] [node 3] ICP22 created new session: [6:7577471329434154215:2050]
2025-11-27T18:18:46.226162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:46.217934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471329434154215:2050] [node 3] ICS09 handshake done sender: [6:7577471329434154212:2115] self: [6:7577471329434154213:0] peer: [3:7577471326717842544:0] socket: 556 qp: -1
2025-11-27T18:18:46.233832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471326717842541:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(22955164ns) {HandleHandshakeStatus:329(22951126ns) {DropIncomingHandshake:442(17082ns) DropOutgoingHandshake:457(751ns) SwitchToState:179(5967ns) ProcessPendingSessionEvents:495(637494ns) {ForwardSessionEventToSession:612(137635ns) ForwardSessionEventToSession:612(5040ns) ForwardSessionEventToSession:612(3423ns) ForwardSessionEventToSession:612(6685ns) ForwardSessionEventToSession:612(3078ns) ForwardSessionEventToSession:612(3225ns) ForwardSessionEventToSession:612(292410ns) ForwardSessionEventToSession:612(6685ns) ForwardSessionEventToSession:612(160571ns) ForwardSessionEventToSession:612(3562ns)}}}
2025-11-27T18:18:46.217994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471329434154215:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:46.232632Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471328619648746:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23012342ns) {Configure:104(23005036ns) {ProcessConfigured:135(61312ns) {StartInitialHandshake:157(59367ns) {DropHandshakes:470(10724ns) {DropIncomingHandshake:442(592ns) DropOutgoingHandshake:457(418ns)} GenerateSessionVirtualId:988(567ns) PrepareNewSessionHandshake:477(1577ns) {SwitchToState:179(667ns)}}}}}
2025-11-27T18:18:46.231803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471323336140280:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(161181501ns) {Configure:104(161177056ns) {ProcessConfigured:135(141212ns) {StartInitialHandshake:157(137403ns) {DropHandshakes:470(3088ns) {DropIncomingHandshake:442(749ns) DropOutgoingHandshake:457(554ns)} GenerateSessionVirtualId:988(970ns) PrepareNewSessionHandshake:477(2649ns) {SwitchToState:179(879ns)}}}}}
2025-11-27T18:18:46.218042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.236233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471327631107642:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:46.236125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471328619648751:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50443
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.236736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471327631107643:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:46.236092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471326893619843:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16420969ns) {Configure:104(16416227ns) {ProcessConfigured:135(373520ns) {StartInitialHandshake:157(370565ns) {DropHandshakes:470(1860ns) {DropIncomingHandshake:442(541ns) DropOutgoingHandshake:457(458ns)} GenerateSessionVirtualId:988(476ns) PrepareNewSessionHandshake:477(3369ns) {SwitchToState:179(1611ns)}}}}}
2025-11-27T18:18:46.242596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471328619648756:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50443
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.253049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471326717842558:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25195
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.253292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471326893619850:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23605
2025-11-27T18:18:46.254358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471326893619851:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23605
2025-11-27T18:18:46.254518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471326893619852:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23605
2025-11-27T18:18:46.254907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471326893619857:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23605
2025-11-27T18:18:46.256145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471328619648751:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.256340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471329434154218:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57329
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.257245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471327477752215:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40225
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.257461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471327477752216:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40225
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.257987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471326893619852:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.257523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471327477752217:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40225
2025-11-27T18:18:46.257584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471327477752218:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40225
2025-11-27T18:18:46.257729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471327477752215:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.258272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471326717842560:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25195
2025-11-27T18:18:46.258769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471327477752205:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.259019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471326893619850:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.259271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471326893619851:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.258397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471326717842558:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.258800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471327477752205:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.258516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471322422875179:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.261646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471327477752217:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.262029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471327477752205:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.262352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471328619648756:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.262560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471326717842562:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25195
2025-11-27T18:18:46.262714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471323182784840:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.264343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471327631107637:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.263051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471324324681385:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.270726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471329434154218:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.270306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471326893619857:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.265216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471327631107652:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1616
2025-11-27T18:18:46.273313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:46.270864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471325139186838:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.272205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471326893619862:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23605
2025-11-27T18:18:46.272343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471326893619847:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.273388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471323182784840:2049] [node 3] ICP22 created new session: [2:7577471327477752222:2049]
2025-11-27T18:18:46.274262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471324324681384:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.273881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471327477752222:2049] [node 3] ICS09 handshake done sender: [2:7577471327477752205:2112] self: [2:7577471327477752204:0] peer: [3:7577471326717842561:0] socket: 561 qp: -1
2025-11-27T18:18:46.273988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471329434154220:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57329
2025-11-27T18:18:46.272368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471326893619847:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.274045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471327477752222:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:46.274401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:46.284644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471326717842543:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.273143Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471326893619847:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:46.284238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471329434154218:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:46.286526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471326893619844:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.286636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:46.286691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471326893619844:2108] [node 6] ICP22 created new session: [1:7577471326893619865:2108]
2025-11-27T18:18:46.284537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471325139186838:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.284709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471326717842543:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.286744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471326893619865:2108] [node 6] ICS09 handshake done sender: [1:7577471326893619847:2112] self: [1:7577471326893619846:0] peer: [6:7577471329434154219:0] socket: 578 qp: -1
2025-11-27T18:18:46.286174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471327631107652:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.285712Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471326717842558:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:46.286789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471326893619865:2108] [node 6] ICS10 traffic start
2025-11-27T18:18:46.288844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471326717842560:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.285993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:46.286831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:46.287273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471325139186838:2048] [node 1] ICP22 created new session: [6:7577471329434154221:2048]
2025-11-27T18:18:46.288891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471322422875179:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.291299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:46.287329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471329434154221:2048] [node 1] ICS09 handshake done sender: [6:7577471329434154218:2117] self: [6:7577471329434154219:0] peer: [1:7577471326893619846:0] socket: 579 qp: -1
2025-11-27T18:18:46.291524Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471323182784840:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(26156260ns) {HandleHandshakeStatus:329(26153861ns) {DropIncomingHandshake:442(22108ns) DropOutgoingHandshake:457(623ns) SwitchToState:179(261ns) ProcessPendingSessionEvents:495(7100017ns) {ForwardSessionEventToSession:612(4151957ns) ForwardSessionEventToSession:612(7291ns) ForwardSessionEventToSession:612(1335350ns) ForwardSessionEventToSession:612(4780ns) ForwardSessionEventToSession:612(3189ns) ForwardSessionEventToSession:612(1453369ns) ForwardSessionEventToSession:612(4907ns) ForwardSessionEventToSession:612(2967ns) ForwardSessionEventToSession:612(114632ns) ForwardSessionEventToSession:612(4045ns)}}}
2025-11-27T18:18:46.287368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471329434154221:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:46.291513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471322422875179:2049] [node 2] ICP22 created new session: [3:7577471326717842565:2049]
2025-11-27T18:18:46.287429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:46.291637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471326717842565:2049] [node 2] ICS09 handshake done sender: [3:7577471326717842558:2116] self: [3:7577471326717842561:0] peer: [2:7577471327477752204:0] socket: 558 qp: -1
2025-11-27T18:18:46.291801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471326717842565:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:46.291855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:46.294144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471327477752216:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.294312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471327477752207:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.294261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471326893619862:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.294339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471327477752207:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.293527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471327631107642:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.302035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471326717842563:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25195
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.309579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471327631107658:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1616
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.312890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471327631107659:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1616
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.316571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471329434154224:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57329
2025-11-27T18:18:46.307877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471328619648750:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.311125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471327477752218:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.317791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471326717842563:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.317757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471327631107658:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.317967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471326717842543:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:46.311507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471327477752224:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40225
2025-11-27T18:18:46.317870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471323336140280:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.307939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471328619648750:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.321984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471322422875178:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.322148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471326717842540:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.322208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:46.322275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471326717842540:2108] [node 4] ICP22 created new session: [3:7577471326717842569:2108]
2025-11-27T18:18:46.322323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471326717842569:2108] [node 4] ICS09 handshake done sender: [3:7577471326717842543:2111] self: [3:7577471326717842542:0] peer: [4:7577471328619648760:0] socket: 550 qp: -1
2025-11-27T18:18:46.322360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471326717842569:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:46.322416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:46.311749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471323182784839:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.320347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471322598652544:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.330401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471329434154224:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.330552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471325139186839:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.328720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471326717842572:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25195
2025-11-27T18:18:46.320400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471322598652544:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.313070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471327477752207:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:46.313481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471327477752224:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.313527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471327477752199:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.313581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:46.313636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471327477752199:2107] [node 4] ICP22 created new session: [2:7577471327477752229:2107]
2025-11-27T18:18:46.313679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471327477752229:2107] [node 4] ICS09 handshake done sender: [2:7577471327477752207:2113] self: [2:7577471327477752206:0] peer: [4:7577471328619648761:0] socket: 562 qp: -1
2025-11-27T18:18:46.313712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471327477752229:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:46.313757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:46.330098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471327477752233:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40225
2025-11-27T18:18:46.326098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471326893619849:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.326130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471326893619849:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.326716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471326893619842:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.326737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471326893619842:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.329035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471322598652544:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.332107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471327477752211:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.332139Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471327477752211:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.332803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471327477752218:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.332206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471327631107659:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.332747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471327631107661:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1616
2025-11-27T18:18:46.332906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471327631107662:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1616
2025-11-27T18:18:46.337775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471323336140281:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.341541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471327631107661:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.341713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471323336140282:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.347737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471327631107658:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:46.348049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471323336140280:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.348140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:46.348193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471323336140280:2048] [node 1] ICP22 created new session: [5:7577471327631107665:2048]
2025-11-27T18:18:46.341414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471323182784839:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.348244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471327631107665:2048] [node 1] ICS09 handshake done sender: [5:7577471327631107658:2117] self: [5:7577471327631107660:0] peer: [1:7577471326893619848:0] socket: 595 qp: -1
2025-11-27T18:18:46.348280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471327631107665:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:46.348317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:46.350982Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471324324681384:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471328619648756:2117] held: no
2025-11-27T18:18:46.351476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471328619648762:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50443
2025-11-27T18:18:46.351632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471328619648763:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50443
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.345672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471326893619849:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:46.341525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:46.354055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471328619648764:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50443
2025-11-27T18:18:46.354962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471328619648750:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:46.349528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471326717842549:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.356266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471328619648745:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.356324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:46.356370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471328619648745:2108] [node 5] ICP22 created new session: [4:7577471328619648767:2108]
2025-11-27T18:18:46.356416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471328619648767:2108] [node 5] ICS09 handshake done sender: [4:7577471328619648750:2113] self: [4:7577471328619648749:0] peer: [5:7577471327631107657:0] socket: 571 qp: -1
2025-11-27T18:18:46.356445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471328619648767:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:46.356479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:46.357016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471328619648751:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:46.349561Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471326717842549:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.341582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471323182784839:2048] [node 1] ICP22 created new session: [2:7577471327477752235:2048]
2025-11-27T18:18:46.348292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471326893619842:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:46.360639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471328619648756:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:46.360059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471326717842560:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:46.341633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471327477752235:2048] [node 1] ICS09 handshake done sender: [2:7577471327477752218:2119] self: [2:7577471327477752228:0] peer: [1:7577471322598652541:0] socket: 582 qp: -1
2025-11-27T18:18:46.354736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471322598652480:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.354825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:46.341668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471327477752235:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:46.361002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471329434154226:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57329
2025-11-27T18:18:46.341708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:46.354875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471322598652480:2048] [node 2] ICP22 created new session: [1:7577471326893619872:2048]
2025-11-27T18:18:46.354926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471326893619872:2048] [node 2] ICS09 handshake done sender: [1:7577471322598652544:2110] self: [1:7577471322598652541:0] peer: [2:7577471327477752228:0] socket: 575 qp: -1
2025-11-27T18:18:46.348441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471327477752209:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.363798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471324324681385:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.363848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:46.363888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471324324681385:2050] [node 3] ICP22 created new session: [4:7577471328619648769:2050]
2025-11-27T18:18:46.363960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471328619648769:2050] [node 3] ICS09 handshake done sender: [4:7577471328619648751:2114] self: [4:7577471328619648760:0] peer: [3:7577471326717842542:0] socket: 557 qp: -1
2025-11-27T18:18:46.363992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471328619648769:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:46.364027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:46.354980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471326893619872:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:46.348470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471327477752209:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.364103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471327477752211:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:46.363890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471322422875178:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.355019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:46.361131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471326893619845:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.363961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:46.363999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471322422875178:2048] [node 1] ICP22 created new session: [3:7577471326717842575:2048]
2025-11-27T18:18:46.361220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:46.361749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471326893619845:2109] [node 5] ICP22 created new session: [1:7577471326893619874:2109]
2025-11-27T18:18:46.364036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471326717842575:2048] [node 1] ICS09 handshake done sender: [3:7577471326717842560:2117] self: [3:7577471326717842567:0] peer: [1:7577471326893619841:0] socket: 577 qp: -1
2025-11-27T18:18:46.364060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471326717842575:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:46.363095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471326893619874:2109] [node 5] ICS09 handshake done sender: [1:7577471326893619849:2113] self: [1:7577471326893619848:0] peer: [5:7577471327631107660:0] socket: 581 qp: -1
2025-11-27T18:18:46.365004Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471329434154224:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:46.363232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471326893619874:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:46.364104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:46.363279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:46.365482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471326717842549:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:46.365712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471322598652481:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.365567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471327477752201:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.365836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471328619648762:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.365786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:46.365662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:46.365904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471324324681384:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.365706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471327477752201:2110] [node 6] ICP22 created new session: [2:7577471327477752238:2110]
2025-11-27T18:18:46.365926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471322598652481:2049] [node 3] ICP22 created new session: [1:7577471326893619876:2049]
2025-11-27T18:18:46.365945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: 2025-11-27T18:18:46.365605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471325139186839:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.365746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471327477752238:2110] [node 6] ICS09 handshake done sender: [2:7577471327477752211:2115] self: [2:7577471327477752210:0] peer: [6:7577471329434154225:0] socket: 580 qp: -1
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.365849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471327477752238:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:46.365685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:46.365914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:46.365724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471325139186839:2049] [node 2] ICP22 created new session: [6:7577471329434154227:2049]
2025-11-27T18:18:46.365977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471324324681384:2049] [node 2] ICP22 created new session: [4:7577471328619648771:2049]
2025-11-27T18:18:46.365965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471326893619876:2049] [node 3] ICS09 handshake done sender: [1:7577471326893619842:2111] self: [1:7577471326893619841:0] peer: [3:7577471326717842567:0] socket: 576 qp: -1
2025-11-27T18:18:46.366144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471326893619876:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:46.366026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471328619648771:2049] [node 2] ICS09 handshake done sender: [4:7577471328619648756:2117] self: [4:7577471328619648761:0] peer: [2:7577471327477752206:0] socket: 568 qp: -1
2025-11-27T18:18:46.366177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:46.366111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471327631107667:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1616
2025-11-27T18:18:46.366206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471328619648771:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:46.366340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:46.365783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471329434154227:2049] [node 2] ICS09 handshake done sender: [6:7577471329434154224:2119] self: [6:7577471329434154225:0] peer: [2:7577471327477752210:0] socket: 600 qp: -1
2025-11-27T18:18:46.365823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471329434154227:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:46.366168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471326717842574:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25195
2025-11-27T18:18:46.366036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.368086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471327631107668:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1616
2025-11-27T18:18:46.368664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471326717842539:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.368732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:46.368906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471326717842539:2107] [node 5] ICP22 created new session: [3:7577471326717842577:2107]
2025-11-27T18:18:46.369202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471326717842577:2107] [node 5] ICS09 handshake done sender: [3:7577471326717842549:2114] self: [3:7577471326717842548:0] peer: [5:7577471327631107664:0] socket: 603 qp: -1
2025-11-27T18:18:46.369236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471326717842577:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:46.369281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:46.369866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471327631107652:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:46.370453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471327631107661:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.370520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471327477752209:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:46.370822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471329434154229:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57329
2025-11-27T18:18:46.371201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471327477752200:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.371250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471327631107642:2109] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:46.371285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
::ffff:127.1.0.0
2025-11-27T18:18:46.371308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:46.371333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471327477752200:2108] [node 5] ICP22 created new session: [2:7577471327477752240:2108]
2025-11-27T18:18:46.371354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471327631107642:2109] [node 4] ICP22 created new session: [5:7577471327631107671:2109]
2025-11-27T18:18:46.371373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471327477752240:2108] [node 5] ICS09 handshake done sender: [2:7577471327477752209:2114] self: [2:7577471327477752208:0] peer: [5:7577471327631107663:0] socket: 598 qp: -1
2025-11-27T18:18:46.371382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471328619648774:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50443
2025-11-27T18:18:46.371409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471327477752240:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:46.371451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:46.371393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471327631107671:2109] [node 4] ICS09 handshake done sender: [5:7577471327631107652:2116] self: [5:7577471327631107657:0] peer: [4:7577471328619648749:0] socket: 586 qp: -1
2025-11-27T18:18:46.371492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471327631107671:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:46.371525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:46.372458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471323336140282:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.372506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:46.372552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471323336140282:2050] [node 3] ICP22 created new session: [5:7577471327631107673:2050]
2025-11-27T18:18:46.372730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471327631107673:2050] [node 3] ICS09 handshake done sender: [5:7577471327631107661:2119] self: [5:7577471327631107664:0] peer: [3:7577471326717842548:0] socket: 604 qp: -1
2025-11-27T18:18:46.372775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471327631107673:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:46.372889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.373770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471329434154229:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.373788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471328619648776:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50443
2025-11-27T18:18:46.373930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471329434154207:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.375145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471328619648755:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.375180Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471328619648755:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.375925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471328619648755:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.376600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471328619648746:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.376510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471329434154231:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57329
2025-11-27T18:18:46.376608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471327631107670:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1616
2025-11-27T18:18:46.376690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:46.376741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471328619648746:2111] [node 6] ICP22 created new session: [4:7577471328619648778:2111]
2025-11-27T18:18:46.376790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471328619648778:2111] [node 6] ICS09 handshake done sender: [4:7577471328619648755:2116] self: [4:7577471328619648754:0] peer: [6:7577471329434154230:0] socket: 565 qp: -1
2025-11-27T18:18:46.376830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471328619648778:2111] [node 6] ICS10 traffic start
2025-11-27T18:18:46.376904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:46.377205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471329434154229:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:46.377319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471323336140282:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:46.377417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471323336140281:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8605) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471327631107659:2118] held: no
2025-11-27T18:18:46.377505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471329434154207:2113] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.377553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:46.377594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471329434154207:2113] [node 4] ICP22 created new session: [6:7577471329434154232:2113]
2025-11-27T18:18:46.377620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471327631107659:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:46.377632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471329434154232:2113] [node 4] ICS09 handshake done sender: [6:7577471329434154229:2121] self: [6:7577471329434154230:0] peer: [4:7577471328619648754:0] socket: 567 qp: -1
2025-11-27T18:18:46.377662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471329434154232:2113] [node 4] ICS10 traffic start
2025-11-27T18:18:46.377692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:46.377646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471328619648776:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.377911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471323336140281:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.378001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:46.378056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471323336140281:2049] [node 2] ICP22 created new session: [5:7577471327631107676:2049]
2025-11-27T18:18:46.378108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471327631107676:2049] [node 2] ICS09 handshake done sender: [5:7577471327631107659:2118] self: [5:7577471327631107663:0] peer: [2:7577471327477752208:0] socket: 599 qp: -1
2025-11-27T18:18:46.378139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471327631107676:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:46.378198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:46.378593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471324324681383:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.378881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471326893619854:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.378920Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471326893619854:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.380614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471329434154234:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57329
2025-11-27T18:18:46.380584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471326893619854:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.380800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471328619648781:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50443
2025-11-27T18:18:46.380909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471326893619843:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.380974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:46.381024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471326893619843:2107] [node 4] ICP22 created new session: [1:7577471326893619880:2107]
2025-11-27T18:18:46.381026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471329434154234:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.381070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471326893619880:2107] [node 4] ICS09 handshake done sender: [1:7577471326893619854:2117] self: [1:7577471326893619853:0] peer: [4:7577471328619648780:0] socket: 554 qp: -1
2025-11-27T18:18:46.381116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471326893619880:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:46.381162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:46.381211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471325139186898:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:46.381312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471328619648776:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:46.381471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471327631107651:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:46.381502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471327631107651:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:46.381704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471324324681383:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.381773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:46.381832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471324324681383:2048] [node 1] ICP22 created new session: [4:7577471328619648782:2048]
2025-11-27T18:18:46.381897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471328619648782:2048] [node 1] ICS09 handshake done sender: [4:7577471328619648776:2122] self: [4:7577471328619648780:0] peer: [1:7577471326893619853:0] socket: 564 qp: -1
2025-11-27T18:18:46.382002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471328619648782:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:46.382029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:46.382496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471327631107651:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:46.382752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471329434154236:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57329
2025-11-27T18:18:46.382892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471327631107643:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:46.382950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:46.383000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471327631107643:2110] [node 6] ICP22 created new session: [5:7577471327631107679:2110]
2025-11-27T18:18:46.383063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471327631107679:2110] [node 6] ICS09 handshake done sender: [5:7577471327631107651:2115] self: [5:7577471327631107650:0] peer: [6:7577471329434154235:0] socket: 574 qp: -1
2025-11-27T18:18:46.383376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471327631107679:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:46.383906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:46.384265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471329434154234:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:46.384514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471325139186898:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:46.384572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:46.384617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471325139186898:2105] [node 5] ICP22 created new session: [6:7577471329434154237:2105]
2025-11-27T18:18:46.384658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471329434154237:2105] [node 5] ICS09 handshake done sender: [6:7577471329434154234:2123] self: [6:7577471329434154235:0] peer: [5:7577471327631107650:0] socket: 566 qp: -1
2025-11-27T18:18:46.384695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471329434154237:2105] [node 5] ICS10 traffic start
2025-11-27T18:18:46.384727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:46.404881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:46.405184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471327477752235:2048] [node 1] ICS07 socket disconnect 582 reason# EndOfStream
2025-11-27T18:18:46.405224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471327477752235:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.405766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:46.405836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471327477752235:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:46.435222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:46.435468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471326717842575:2048] [node 1] ICS01 socket: 577 reason# EndOfStream
2025-11-27T18:18:46.435515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471322422875178:2048] [node 1] ICP30 unregister session Session# [3:7577471326717842575:2048] VirtualId# [3:7577471326717842567:0]
2025-11-27T18:18:46.435556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471326717842575:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.435675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:46.435741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:46.438401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471323182784839:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13660) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:46.438445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471327477752235:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:46.439334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:46.439662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471323182784839:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13660) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:46.439719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471327477752235:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:46.439749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471323182784839:2048] [node 1] ICP30 unregister session Session# [2:7577471327477752235:2048] VirtualId# [2:7577471327477752228:0]
2025-11-27T18:18:46.439551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471328619648782:2048] [node 1] ICS07 socket disconnect 564 reason# EndOfStream
2025-11-27T18:18:46.439798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:46.439594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471328619648782:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.439833Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471323182784839:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:46.439848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:46.439685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:46.439742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471328619648782:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:46.440828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471324324681383:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13660) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:46.440887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471328619648782:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:46.440919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471324324681383:2048] [node 1] ICP30 unregister session Session# [4:7577471328619648782:2048] VirtualId# [4:7577471328619648780:0]
2025-11-27T18:18:46.440966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:46.440990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471324324681383:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:46.441005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:46.441200Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:46.441505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471329434154221:2048] [node 1] ICS01 socket: 579 reason# EndOfStream
2025-11-27T18:18:46.441538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471325139186838:2048] [node 1] ICP30 unregister session Session# [6:7577471329434154221:2048] VirtualId# [6:7577471329434154219:0]
2025-11-27T18:18:46.441563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471329434154221:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.441662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:46.441701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:46.442121Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:46.442341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471327631107665:2048] [node 1] ICS01 socket: 595 reason# EndOfStream
2025-11-27T18:18:46.442376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471323336140280:2048] [node 1] ICP30 unregister session Session# [5:7577471327631107665:2048] VirtualId# [5:7577471327631107660:0]
2025-11-27T18:18:46.442405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471327631107665:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.442480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:46.442518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:46.450501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471324324681383:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:46.468863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:46.469112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471326717842565:2049] [node 2] ICS01 socket: 558 reason# EndOfStream
2025-11-27T18:18:46.469157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471322422875179:2049] [node 2] ICP30 unregister session Session# [3:7577471326717842565:2049] VirtualId# [3:7577471326717842561:0]
2025-11-27T18:18:46.469202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471326717842565:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.469304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:46.469354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:46.471679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:46.471912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471328619648771:2049] [node 2] ICS01 socket: 568 reason# EndOfStream
2025-11-27T18:18:46.471959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471324324681384:2049] [node 2] ICP30 unregister session Session# [4:7577471328619648771:2049] VirtualId# [4:7577471328619648761:0]
2025-11-27T18:18:46.471991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471328619648771:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.472077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:46.472124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:46.472661Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:46.472884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471327631107676:2049] [node 2] ICS01 socket: 599 reason# EndOfStream
2025-11-27T18:18:46.472938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471323336140281:2049] [node 2] ICP30 unregister session Session# [5:7577471327631107676:2049] VirtualId# [5:7577471327631107663:0]
2025-11-27T18:18:46.472965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471327631107676:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.473021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:46.473075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:46.473536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:46.473763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471329434154227:2049] [node 2] ICS01 socket: 600 reason# EndOfStream
2025-11-27T18:18:46.473801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471325139186839:2049] [node 2] ICP30 unregister session Session# [6:7577471329434154227:2049] VirtualId# [6:7577471329434154225:0]
2025-11-27T18:18:46.473831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471329434154227:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.473888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:46.473932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:46.526673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:46.526929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471327631107673:2050] [node 3] ICS07 socket disconnect 604 reason# EndOfStream
2025-11-27T18:18:46.526990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471327631107673:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.527068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:46.527108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471327631107673:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:46.534384Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471323336140282:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27490) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:46.534466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471327631107673:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:46.534506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471323336140282:2050] [node 3] ICP30 unregister session Session# [5:7577471327631107673:2050] VirtualId# [5:7577471327631107664:0]
2025-11-27T18:18:46.534555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:46.534589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471323336140282:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:46.534605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:46.536239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:46.536392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471329434154215:2050] [node 3] ICS01 socket: 556 reason# EndOfStream
2025-11-27T18:18:46.536426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471325139186840:2050] [node 3] ICP30 unregister session Session# [6:7577471329434154215:2050] VirtualId# [6:7577471329434154213:0]
2025-11-27T18:18:46.536455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471329434154215:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.536535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:46.536586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:46.534039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:46.534329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471328619648769:2050] [node 3] ICS01 socket: 557 reason# EndOfStream
2025-11-27T18:18:46.534397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471324324681385:2050] [node 3] ICP30 unregister session Session# [4:7577471328619648769:2050] VirtualId# [4:7577471328619648760:0]
2025-11-27T18:18:46.534438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471328619648769:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.534525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:46.534585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:46.546547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471323336140282:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:46.567872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:46.568114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471327631107671:2109] [node 4] ICS01 socket: 586 reason# EndOfStream
2025-11-27T18:18:46.568169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471327631107642:2109] [node 4] ICP30 unregister session Session# [5:7577471327631107671:2109] VirtualId# [5:7577471327631107657:0]
2025-11-27T18:18:46.568220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471327631107671:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.568316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:46.568358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:46.569102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:46.569372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471329434154232:2113] [node 4] ICS01 socket: 567 reason# EndOfStream
2025-11-27T18:18:46.569422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471329434154207:2113] [node 4] ICP30 unregister session Session# [6:7577471329434154232:2113] VirtualId# [6:7577471329434154230:0]
2025-11-27T18:18:46.569454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471329434154232:2113] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.569548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:46.569624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:46.634292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:46.634583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471329434154237:2105] [node 5] ICS07 socket disconnect 566 reason# EndOfStream
2025-11-27T18:18:46.634624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471329434154237:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:46.634724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:46.634778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471329434154237:2105] [node 5] ICS15 start handshake
2025-11-27T18:18:46.635880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471325139186898:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20486) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:46.635992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471329434154237:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:46.636065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471325139186898:2105] [node 5] ICP30 unregister session Session# [6:7577471329434154237:2105] VirtualId# [6:7577471329434154235:0]
2025-11-27T18:18:46.636135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:18:46.636172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471325139186898:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20486) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:46.636218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20486) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:46.723373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471325697052686:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:46.723382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471325697052687:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:46.753685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471327590055091:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:46.753958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471327590055090:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:46.776167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471329087549179:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:46.776322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471329087549180:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:46.801121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471329719717245:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:46.801168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471329719717246:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:46.801355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471329719717244:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:46.826506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471329357267824:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:46.826509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471329357267823:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:46.826555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471329357267825:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:46.848881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471329216193239:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:46.848946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471329216193241:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:46.849061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471329216193240:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:46.957666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471329216193298:2104] [node 5] ICP01 ready to work
2025-11-27T18:18:47.031220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471333652235182:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:47.015320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471331885022450:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:47.045836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471334014684603:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:47.053280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471325697052686:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(109475315ns) {Configure:104(109470269ns) {ProcessConfigured:135(73740ns) {StartInitialHandshake:157(68864ns) {DropHandshakes:470(3684ns) {DropIncomingHandshake:442(1186ns) DropOutgoingHandshake:457(452ns)} GenerateSessionVirtualId:988(720ns) PrepareNewSessionHandshake:477(1133ns) {SwitchToState:179(416ns)}}}}}
2025-11-27T18:18:47.052368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471333652235183:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:47.057331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471329992020048:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:47.057106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471334014684604:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:47.057717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471329992020049:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:47.057991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471329992020050:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:47.077296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471327590055091:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42438555ns) {Configure:104(42428321ns) {ProcessConfigured:135(78320ns) {StartInitialHandshake:157(75867ns) {DropHandshakes:470(3273ns) {DropIncomingHandshake:442(931ns) DropOutgoingHandshake:457(714ns)} GenerateSessionVirtualId:988(1379ns) PrepareNewSessionHandshake:477(2053ns) {SwitchToState:179(711ns)}}}}}
2025-11-27T18:18:47.077727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471331885022453:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:47.078023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471331885022454:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:47.085523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471325697052687:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(120380609ns) {Configure:104(120376617ns) {ProcessConfigured:135(18687697ns) {StartInitialHandshake:157(18684671ns) {DropHandshakes:470(3857ns) {DropIncomingHandshake:442(781ns) DropOutgoingHandshake:457(517ns)} GenerateSessionVirtualId:988(951ns) PrepareNewSessionHandshake:477(1746ns) {SwitchToState:179(108ns)}}}}}
2025-11-27T18:18:47.088808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471329216193239:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(111127554ns) {Configure:104(106403419ns) {ProcessConfigured:135(65298ns) {StartInitialHandshake:157(63068ns) {DropHandshakes:470(2132ns) {DropIncomingHandshake:442(673ns) DropOutgoingHandshake:457(481ns)} GenerateSessionVirtualId:988(523ns) PrepareNewSessionHandshake:477(1839ns) {SwitchToState:179(528ns)}}}}}
2025-11-27T18:18:47.099457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471333511160602:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:47.110388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471329719717244:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(138454728ns) {Configure:104(138449597ns) {ProcessConfigured:135(85579ns) {StartInitialHandshake:157(83197ns) {DropHandshakes:470(3741ns) {DropIncomingHandshake:442(989ns) DropOutgoingHandshake:457(696ns)} GenerateSessionVirtualId:988(1590ns) PrepareNewSessionHandshake:477(2085ns) {SwitchToState:179(684ns)}}}}}
2025-11-27T18:18:47.106860Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471329357267823:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(111691183ns) {Configure:104(111682628ns) {ProcessConfigured:135(4125704ns) {StartInitialHandshake:157(4123339ns) {DropHandshakes:470(4589ns) {DropIncomingHandshake:442(489ns) DropOutgoingHandshake:457(349ns)} GenerateSessionVirtualId:988(513ns) PrepareNewSessionHandshake:477(3698ns) {SwitchToState:179(2297ns)}}}}}
2025-11-27T18:18:47.121782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471329719717245:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(52363716ns) {Configure:104(52357631ns) {ProcessConfigured:135(55102ns) {StartInitialHandshake:157(52955ns) {DropHandshakes:470(1228ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(314ns)} GenerateSessionVirtualId:988(658ns) PrepareNewSessionHandshake:477(1827ns) {SwitchToState:179(597ns)}}}}}
2025-11-27T18:18:47.124117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471329087549179:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(150358295ns) {Configure:104(150348316ns) {ProcessConfigured:135(23282184ns) {StartInitialHandshake:157(23278534ns) {DropHandshakes:470(1944ns) {DropIncomingHandshake:442(599ns) DropOutgoingHandshake:457(375ns)} GenerateSessionVirtualId:988(613ns) PrepareNewSessionHandshake:477(3818ns) {SwitchToState:179(2072ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.128360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471334014684614:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28770
2025-11-27T18:18:47.124797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471327590055090:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(176126794ns) {Configure:104(176117863ns) {ProcessConfigured:135(3692272ns) {StartInitialHandshake:157(3690468ns) {DropHandshakes:470(1964ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(553ns) PrepareNewSessionHandshake:477(2788ns) {SwitchToState:179(1459ns)}}}}}
2025-11-27T18:18:47.127166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471333382516540:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.132788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471333382516543:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:47.133154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471333382516544:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.140118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471331885022464:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61435
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.144371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471331885022467:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61435
2025-11-27T18:18:47.130320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471334014684615:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28770
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.137289Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471334014684603:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17007203ns) {Configure:104(17002023ns) {ProcessConfigured:135(4479672ns) {StartInitialHandshake:157(4477734ns) {DropHandshakes:470(1202ns) {DropIncomingHandshake:442(577ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(248ns) PrepareNewSessionHandshake:477(3025ns) {SwitchToState:179(1179ns)}}}}}
2025-11-27T18:18:47.144616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471329087549180:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(160742721ns) {Configure:104(159922784ns) {ProcessConfigured:135(47766ns) {StartInitialHandshake:157(46233ns) {DropHandshakes:470(1485ns) {DropIncomingHandshake:442(366ns) DropOutgoingHandshake:457(263ns)} GenerateSessionVirtualId:988(367ns) PrepareNewSessionHandshake:477(993ns) {SwitchToState:179(295ns)}}}}}
2025-11-27T18:18:47.138015Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471329216193298:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(170443906ns) {Configure:104(170435491ns) {ProcessConfigured:135(6952904ns) {StartInitialHandshake:157(6950723ns) {DropHandshakes:470(2982ns) {DropIncomingHandshake:442(851ns) DropOutgoingHandshake:457(369ns)} GenerateSessionVirtualId:988(488ns) PrepareNewSessionHandshake:477(3288ns) {SwitchToState:179(1834ns)}}}}}
2025-11-27T18:18:47.137471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471334014684616:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28770
2025-11-27T18:18:47.147204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471329992020061:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31785
2025-11-27T18:18:47.147648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471329357267824:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(90390485ns) {Configure:104(90382459ns) {ProcessConfigured:135(9978587ns) {StartInitialHandshake:157(9976136ns) {DropHandshakes:470(2411ns) {DropIncomingHandshake:442(525ns) DropOutgoingHandshake:457(363ns)} GenerateSessionVirtualId:988(491ns) PrepareNewSessionHandshake:477(4488ns) {SwitchToState:179(2688ns)}}}}}
2025-11-27T18:18:47.147810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471334014684615:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.148008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471329719717244:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:47.148065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471334014684614:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.148077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471331885022464:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.148667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471327590055090:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:47.148892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471329719717245:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.149116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471329992020062:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31785
2025-11-27T18:18:47.152852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471334014684616:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.153086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471333652235182:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24989810ns) {Configure:104(24438857ns) {ProcessConfigured:135(307691ns) {StartInitialHandshake:157(305905ns) {DropHandshakes:470(1707ns) {DropIncomingHandshake:442(720ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(347ns) PrepareNewSessionHandshake:477(2277ns) {SwitchToState:179(1382ns)}}}}}
2025-11-27T18:18:47.155208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471331885022468:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61435
2025-11-27T18:18:47.155437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471329992020047:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.155621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471331885022456:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.155670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471331885022456:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.156054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471333382516545:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-27T18:18:47.155482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471329992020047:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.155494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471329992020052:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.156096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471331885022467:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.155531Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471329992020052:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.161898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471333382516545:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.161168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471331885022456:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:47.162396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471329087549180:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.160961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471329992020061:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.161134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471329992020047:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:47.165236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471329992020052:2111] [node 4] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:18:47.166109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471333382516550:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-27T18:18:47.167367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471331885022452:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.166892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471325697052686:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.166960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:47.167410Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471331885022452:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.171413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471334014684624:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28770
2025-11-27T18:18:47.167012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471325697052686:2048] [node 2] ICP22 created new session: [1:7577471329992020066:2048]
2025-11-27T18:18:47.171162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471333652235190:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.179342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471334014684625:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28770
2025-11-27T18:18:47.167467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471331885022450:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.179810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471334014684626:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28770
2025-11-27T18:18:47.167060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471329992020066:2048] [node 2] ICS09 handshake done sender: [1:7577471329992020047:2110] self: [1:7577471329992020046:0] peer: [2:7577471331885022469:0] socket: 552 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.167090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471329992020066:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:47.167126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:47.167345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471329992020065:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31785
2025-11-27T18:18:47.167441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471329992020048:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.167474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:47.168495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471329992020048:2107] [node 4] ICP22 created new session: [1:7577471329992020068:2107]
2025-11-27T18:18:47.170112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471329992020068:2107] [node 4] ICS09 handshake done sender: [1:7577471329992020052:2111] self: [1:7577471329992020051:0] peer: [4:7577471334014684621:0] socket: 555 qp: -1
2025-11-27T18:18:47.172803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471329992020068:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:47.172862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:47.182421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471329992020062:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.183456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471329992020048:2107] [node 4] ICP18 handshake (actor [1:7577471329992020062:2116]) is too old
2025-11-27T18:18:47.183540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471329992020071:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31785
2025-11-27T18:18:47.172933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471333652235191:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-11-27T18:18:47.168476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:47.180014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471334014684614:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:47.178764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471333382516550:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.193515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471334014684615:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:47.183823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471333652235190:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.193976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471329992020071:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.195503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471329992020065:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.203788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471329992020054:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.203825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471329992020054:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.170450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471331885022450:2107] [node 4] ICP22 created new session: [2:7577471331885022474:2107]
2025-11-27T18:18:47.186716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471329357267823:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.207745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471333382516558:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-27T18:18:47.208059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471329719717245:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.208126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:47.208187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471329719717245:2049] [node 2] ICP22 created new session: [4:7577471334014684628:2049]
2025-11-27T18:18:47.208236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471334014684628:2049] [node 2] ICS09 handshake done sender: [4:7577471334014684614:2114] self: [4:7577471334014684622:0] peer: [2:7577471331885022455:0] socket: 564 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.213355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471333511160614:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9237
2025-11-27T18:18:47.172578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471331885022474:2107] [node 4] ICS09 handshake done sender: [2:7577471331885022456:2112] self: [2:7577471331885022455:0] peer: [4:7577471334014684622:0] socket: 554 qp: -1
2025-11-27T18:18:47.172615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471331885022474:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:47.177790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:47.186544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471331885022472:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61435
2025-11-27T18:18:47.198836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471331885022450:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(31119268ns) {HandleHandshakeStatus:329(31116814ns) {DropIncomingHandshake:442(339ns) DropOutgoingHandshake:457(355ns) SwitchToState:179(320ns) ProcessPendingSessionEvents:495(9081702ns) {ForwardSessionEventToSession:612(9017294ns) ForwardSessionEventToSession:612(6893ns) ForwardSessionEventToSession:612(10459ns) ForwardSessionEventToSession:612(5324ns) ForwardSessionEventToSession:612(3290ns) ForwardSessionEventToSession:612(6032ns) ForwardSessionEventToSession:612(5735ns) ForwardSessionEventToSession:612(2479ns) ForwardSessionEventToSession:612(6829ns) ForwardSessionEventToSession:612(2527ns)}}}
2025-11-27T18:18:47.203712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471331885022468:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.210220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471331885022473:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61435
2025-11-27T18:18:47.210390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471331885022458:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.210412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471331885022458:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.211405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471331885022473:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.208271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471334014684628:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.226765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471333511160614:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.226923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471329216193239:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:47.229472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471329992020056:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.212989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471333382516559:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-27T18:18:47.211507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471331885022464:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:47.199339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471333652235191:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.208306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:47.229519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471329992020056:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.199495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471329357267824:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:47.212107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471331885022452:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.225482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471333382516558:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.234385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471333382516545:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:47.246517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471333382516547:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.246546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471333382516547:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.247241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471329087549179:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:47.247919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471333382516547:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:47.253095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471333382516540:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.253176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:47.253230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471333382516540:2107] [node 4] ICP22 created new session: [3:7577471333382516562:2107]
2025-11-27T18:18:47.253330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471333382516562:2107] [node 4] ICS09 handshake done sender: [3:7577471333382516547:2113] self: [3:7577471333382516546:0] peer: [4:7577471334014684634:0] socket: 587 qp: -1
2025-11-27T18:18:47.253364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471333382516562:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:47.253412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:47.254316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471329087549180:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.212470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471334014684626:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.248374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471333511160617:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9237
2025-11-27T18:18:47.213133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471331885022450:2107] [node 4] ICP18 handshake (actor [2:7577471331885022468:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.222537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471333652235201:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-11-27T18:18:47.230460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471329992020054:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:47.260639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:47.227205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471329719717244:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.258145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471333511160618:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9237
2025-11-27T18:18:47.222640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471333652235202:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-11-27T18:18:47.231226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471329992020049:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.227270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:47.263193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471329087549180:2049] [node 2] ICP22 created new session: [3:7577471333382516564:2049]
2025-11-27T18:18:47.231290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:47.213320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471331885022477:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61435
2025-11-27T18:18:47.263793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471333382516564:2049] [node 2] ICS09 handshake done sender: [3:7577471333382516545:2112] self: [3:7577471333382516556:0] peer: [2:7577471331885022451:0] socket: 567 qp: -1
2025-11-27T18:18:47.227321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471329719717244:2048] [node 1] ICP22 created new session: [4:7577471334014684630:2048]
2025-11-27T18:18:47.231334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471329992020049:2108] [node 5] ICP22 created new session: [1:7577471329992020078:2108]
2025-11-27T18:18:47.246245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471333652235190:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:47.263315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471333511160615:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9237
2025-11-27T18:18:47.213576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471327590055090:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.227373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471334014684630:2048] [node 1] ICS09 handshake done sender: [4:7577471334014684615:2115] self: [4:7577471334014684621:0] peer: [1:7577471329992020051:0] socket: 565 qp: -1
2025-11-27T18:18:47.231385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471329992020078:2108] [node 5] ICS09 handshake done sender: [1:7577471329992020054:2112] self: [1:7577471329992020053:0] peer: [5:7577471333652235199:0] socket: 557 qp: -1
2025-11-27T18:18:47.264357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471333511160618:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.264743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471333382516564:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:47.231421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471329992020078:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:47.258509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471333652235203:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-11-27T18:18:47.227411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471334014684630:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:47.213622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.264797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:47.264501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471333511160602:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:47.213675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471327590055090:2048] [node 1] ICP22 created new session: [2:7577471331885022480:2048]
2025-11-27T18:18:47.213723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471331885022480:2048] [node 1] ICS09 handshake done sender: [2:7577471331885022464:2115] self: [2:7577471331885022469:0] peer: [1:7577471329992020046:0] socket: 556 qp: -1
2025-11-27T18:18:47.213765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471331885022480:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:47.213808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:47.264960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471327590055091:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.265047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:47.265088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471327590055091:2049] [node 3] ICP22 created new session: [2:7577471331885022483:2049]
2025-11-27T18:18:47.265133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471331885022483:2049] [node 3] ICS09 handshake done sender: [2:7577471331885022452:2111] self: [2:7577471331885022451:0] peer: [3:7577471333382516556:0] socket: 553 qp: -1
2025-11-27T18:18:47.265165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471331885022483:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:47.265195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:47.265991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471327590055090:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(40624482ns) {HandleHandshakeStatus:329(40622491ns) {DropIncomingHandshake:442(908ns) DropOutgoingHandshake:457(4475ns) SwitchToState:179(523ns) ProcessPendingSessionEvents:495(40293468ns) {ForwardSessionEventToSession:612(18000473ns) ForwardSessionEventToSession:612(5282ns) ForwardSessionEventToSession:612(3513ns) ForwardSessionEventToSession:612(1988652ns) ForwardSessionEventToSession:612(3682ns) ForwardSessionEventToSession:612(3330ns) ForwardSessionEventToSession:612(12002128ns) ForwardSessionEventToSession:612(6381551ns) ForwardSessionEventToSession:612(1873602ns) ForwardSessionEventToSession:612(7331ns)}}}
2025-11-27T18:18:47.268959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471331885022477:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.261607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471329357267823:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.227458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:47.231450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:47.261674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:47.261727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471329357267823:2048] [node 1] ICP22 created new session: [5:7577471333652235204:2048]
2025-11-27T18:18:47.261777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471333652235204:2048] [node 1] ICS09 handshake done sender: [5:7577471333652235190:2113] self: [5:7577471333652235199:0] peer: [1:7577471329992020053:0] socket: 578 qp: -1
2025-11-27T18:18:47.261816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471333652235204:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:47.261851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:47.262121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471329357267823:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:47.279067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471333511160620:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9237
2025-11-27T18:18:47.261028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471329992020057:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.227700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471329719717244:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:47.286419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471331885022458:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:47.285997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471333511160620:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.285426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471333652235203:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.229118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471334014684627:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28770
2025-11-27T18:18:47.261056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471329992020057:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.267011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471329992020056:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:47.293181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471329992020050:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.293250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:47.300219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471329992020050:2109] [node 6] ICP22 created new session: [1:7577471329992020081:2109]
2025-11-27T18:18:47.294178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471333511160614:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:47.285633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471333652235182:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:47.296118Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471329087549180:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17602317ns) {HandleHandshakeStatus:329(17598536ns) {DropIncomingHandshake:442(1523ns) DropOutgoingHandshake:457(5027ns) SwitchToState:179(371ns) ProcessPendingSessionEvents:495(2263344ns) {ForwardSessionEventToSession:612(973076ns) ForwardSessionEventToSession:612(4622ns) ForwardSessionEventToSession:612(2671ns) ForwardSessionEventToSession:612(601961ns) ForwardSessionEventToSession:612(473225ns) ForwardSessionEventToSession:612(5834ns) ForwardSessionEventToSession:612(3545ns) ForwardSessionEventToSession:612(2937ns) ForwardSessionEventToSession:612(179260ns) ForwardSessionEventToSession:612(3582ns)}}}
2025-11-27T18:18:47.306428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471329992020081:2109] [node 6] ICS09 handshake done sender: [1:7577471329992020056:2113] self: [1:7577471329992020055:0] peer: [6:7577471333511160616:0] socket: 599 qp: -1
2025-11-27T18:18:47.234514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471334014684627:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.307737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471329216193298:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:47.306823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471329992020081:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:47.297589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471333652235191:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:47.309357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471329216193239:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.235320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471329719717246:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.298147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471329357267824:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.306887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:47.310196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:47.310511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471331885022453:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.310573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.311152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471331885022453:2109] [node 5] ICP22 created new session: [2:7577471331885022485:2109]
2025-11-27T18:18:47.298202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:47.252864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471334014684636:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28770
2025-11-27T18:18:47.310674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471329216193239:2048] [node 1] ICP22 created new session: [6:7577471333511160623:2048]
2025-11-27T18:18:47.267118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471334014684612:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.316833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471333511160623:2048] [node 1] ICS09 handshake done sender: [6:7577471333511160614:2116] self: [6:7577471333511160616:0] peer: [1:7577471329992020055:0] socket: 601 qp: -1
2025-11-27T18:18:47.315043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471331885022485:2109] [node 5] ICS09 handshake done sender: [2:7577471331885022458:2113] self: [2:7577471331885022457:0] peer: [5:7577471333652235200:0] socket: 561 qp: -1
2025-11-27T18:18:47.316885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471333511160623:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:47.300240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471329357267824:2049] [node 2] ICP22 created new session: [5:7577471333652235208:2049]
2025-11-27T18:18:47.267166Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471334014684612:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.316591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471331885022485:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:47.307581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471333652235208:2049] [node 2] ICS09 handshake done sender: [5:7577471333652235191:2114] self: [5:7577471333652235200:0] peer: [2:7577471331885022457:0] socket: 579 qp: -1
2025-11-27T18:18:47.320938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471329992020050:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(26755361ns) {HandleHandshakeStatus:329(26752305ns) {DropIncomingHandshake:442(17674ns) DropOutgoingHandshake:457(450ns) SwitchToState:179(835ns) ProcessPendingSessionEvents:495(9710458ns) {ForwardSessionEventToSession:612(1330774ns) ForwardSessionEventToSession:612(3728289ns) ForwardSessionEventToSession:612(6231ns) ForwardSessionEventToSession:612(1900142ns) ForwardSessionEventToSession:612(9694ns) ForwardSessionEventToSession:612(3325ns) ForwardSessionEventToSession:612(542673ns) ForwardSessionEventToSession:612(3905ns) ForwardSessionEventToSession:612(2848ns) ForwardSessionEventToSession:612(2158468ns)}}}
2025-11-27T18:18:47.268958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471329719717246:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11563) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471334014684626:2120] held: no
2025-11-27T18:18:47.308547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471333652235208:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:47.316940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:47.316882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:47.308599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:47.320497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471331885022453:2109] [node 5] ICP18 handshake (actor [2:7577471331885022477:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.318442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471333511160602:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25200) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471333511160618:2119] held: no
2025-11-27T18:18:47.278939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471334014684626:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:47.283276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471334014684612:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:47.287454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471329719717246:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.287495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:47.287530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471329719717246:2050] [node 3] ICP22 created new session: [4:7577471334014684638:2050]
2025-11-27T18:18:47.330482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471329992020057:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:47.287568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471334014684638:2050] [node 3] ICS09 handshake done sender: [4:7577471334014684626:2120] self: [4:7577471334014684634:0] peer: [3:7577471333382516546:0] socket: 591 qp: -1
2025-11-27T18:18:47.287592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471334014684638:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:47.287616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:47.330960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471333511160621:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9237
2025-11-27T18:18:47.288083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471334014684617:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.288108Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471334014684617:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.316226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471333652235210:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-11-27T18:18:47.288835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471334014684604:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.288871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:47.332249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471333511160618:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:47.320069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471333652235211:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.288904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471334014684604:2109] [node 6] ICP22 created new session: [4:7577471334014684640:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.335033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471325697052687:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.332687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471333511160602:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.333352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:47.333425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471333511160602:2107] [node 4] ICP22 created new session: [6:7577471333511160627:2107]
2025-11-27T18:18:47.336398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471333382516567:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-27T18:18:47.333482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471333511160627:2107] [node 4] ICS09 handshake done sender: [6:7577471333511160618:2119] self: [6:7577471333511160619:0] peer: [4:7577471334014684611:0] socket: 598 qp: -1
2025-11-27T18:18:47.333527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471333511160627:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:47.333560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:47.340156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471333382516568:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-27T18:18:47.288931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471334014684640:2109] [node 6] ICS09 handshake done sender: [4:7577471334014684612:2113] self: [4:7577471334014684611:0] peer: [6:7577471333511160619:0] socket: 611 qp: -1
2025-11-27T18:18:47.336593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471333652235193:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.335120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:47.344206Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471329087549179:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10620) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471333382516558:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.288961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471334014684640:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:47.341103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471333511160615:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.335166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471325697052687:2049] [node 3] ICP22 created new session: [1:7577471329992020083:2049]
2025-11-27T18:18:47.344373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471333382516570:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-27T18:18:47.335219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471329992020083:2049] [node 3] ICS09 handshake done sender: [1:7577471329992020057:2114] self: [1:7577471329992020045:0] peer: [3:7577471333382516560:0] socket: 600 qp: -1
2025-11-27T18:18:47.344551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471333382516558:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:47.336623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471333652235203:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:47.335266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471329992020083:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:47.336663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471333652235193:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.335309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:47.288984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:47.345528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471333652235182:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25200) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471333652235203:2119] held: no
2025-11-27T18:18:47.306143Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471334014684617:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:47.310062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471334014684603:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.310438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:47.310891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471334014684603:2108] [node 5] ICP22 created new session: [4:7577471334014684643:2108]
2025-11-27T18:18:47.345834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471329087549179:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.345911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:47.316501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471334014684643:2108] [node 5] ICS09 handshake done sender: [4:7577471334014684617:2117] self: [4:7577471334014684613:0] peer: [5:7577471333652235207:0] socket: 612 qp: -1
2025-11-27T18:18:47.316877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471334014684643:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:47.316945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:47.345834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471333652235182:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.346059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:47.346102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471333652235182:2108] [node 4] ICP22 created new session: [5:7577471333652235213:2108]
2025-11-27T18:18:47.346159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471333652235213:2108] [node 4] ICS09 handshake done sender: [5:7577471333652235203:2119] self: [5:7577471333652235207:0] peer: [4:7577471334014684613:0] socket: 613 qp: -1
2025-11-27T18:18:47.346849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471329087549179:2048] [node 1] ICP22 created new session: [3:7577471333382516571:2048]
2025-11-27T18:18:47.346196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471333652235213:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.347715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471333382516571:2048] [node 1] ICS09 handshake done sender: [3:7577471333382516558:2117] self: [3:7577471333382516560:0] peer: [1:7577471329992020045:0] socket: 602 qp: -1
2025-11-27T18:18:47.346275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:47.357817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471329216193240:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:47.347763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471333382516571:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:47.347846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:47.357963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471333382516570:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.358747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471331885022460:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.353605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471333652235214:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-11-27T18:18:47.358788Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471331885022460:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.353802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471333652235193:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:47.359702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471331885022460:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:47.360777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471331885022454:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.361174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:47.361480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471331885022454:2110] [node 6] ICP22 created new session: [2:7577471331885022488:2110]
2025-11-27T18:18:47.362533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471333652235211:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.362910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471331885022488:2110] [node 6] ICS09 handshake done sender: [2:7577471331885022460:2114] self: [2:7577471331885022459:0] peer: [6:7577471333511160631:0] socket: 606 qp: -1
2025-11-27T18:18:47.362720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471333652235183:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.363371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471331885022488:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:47.363448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:47.362809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471333511160629:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9237
2025-11-27T18:18:47.364370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:47.364942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471333652235214:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.364975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471333652235183:2109] [node 6] ICP22 created new session: [5:7577471333652235217:2109]
2025-11-27T18:18:47.365117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471333652235217:2109] [node 6] ICS09 handshake done sender: [5:7577471333652235193:2115] self: [5:7577471333652235192:0] peer: [6:7577471333511160622:0] socket: 575 qp: -1
2025-11-27T18:18:47.365154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471333652235217:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:47.365191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471329357267825:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:47.365236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:47.365368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471329216193240:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64495) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471333511160615:2117] held: no
2025-11-27T18:18:47.365426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471333382516568:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.365971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577471333652235183:2109] [node 6] ICP18 handshake (actor [5:7577471333652235214:2122]) is too old
2025-11-27T18:18:47.365901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471333511160630:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9237
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.366887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471333511160629:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.366975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471333511160632:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9237
2025-11-27T18:18:47.367610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471333511160620:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:47.368046Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471329216193298:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30129) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471333511160620:2120] held: no
2025-11-27T18:18:47.368184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471329216193298:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.368223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:47.368267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471333511160615:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:47.368289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471329216193298:2104] [node 5] ICP22 created new session: [6:7577471333511160633:2104]
2025-11-27T18:18:47.368329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471333511160633:2104] [node 5] ICS09 handshake done sender: [6:7577471333511160620:2120] self: [6:7577471333511160622:0] peer: [5:7577471333652235192:0] socket: 577 qp: -1
2025-11-27T18:18:47.368363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471333511160633:2104] [node 5] ICS10 traffic start
2025-11-27T18:18:47.368397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:47.368918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471329216193241:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:47.368962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471329216193240:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.368999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:47.368864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471333382516549:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.369314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471329216193240:2049] [node 2] ICP22 created new session: [6:7577471333511160636:2049]
2025-11-27T18:18:47.368904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471333382516549:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.369389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471333511160636:2049] [node 2] ICS09 handshake done sender: [6:7577471333511160615:2117] self: [6:7577471333511160631:0] peer: [2:7577471331885022459:0] socket: 607 qp: -1
2025-11-27T18:18:47.369453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471333511160636:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:47.369499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:47.369511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471333382516554:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:47.369552Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471333382516554:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:47.370298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471333382516549:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:47.370775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471333382516543:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.370870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:47.370933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471333382516543:2108] [node 5] ICP22 created new session: [3:7577471333382516577:2108]
No RDMA context found for GID: 2025-11-27T18:18:47.370988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471333382516577:2108] [node 5] ICS09 handshake done sender: [3:7577471333382516549:2114] self: [3:7577471333382516548:0] peer: [5:7577471333652235218:0] socket: 594 qp: -1
::ffff:127.1.0.0
2025-11-27T18:18:47.371032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471333382516577:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:47.371092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:47.371149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471333652235220:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-11-27T18:18:47.372349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471329357267825:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11563) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471333652235211:2121] held: no
2025-11-27T18:18:47.372829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471333652235211:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:47.372864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471333382516554:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:47.373035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471333382516544:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:47.373365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:47.373411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471333382516544:2109] [node 6] ICP22 created new session: [3:7577471333382516579:2109]
2025-11-27T18:18:47.373451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471333382516579:2109] [node 6] ICS09 handshake done sender: [3:7577471333382516554:2116] self: [3:7577471333382516553:0] peer: [6:7577471333511160635:0] socket: 597 qp: -1
2025-11-27T18:18:47.373417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471333511160638:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9237
2025-11-27T18:18:47.373491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471333382516579:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:47.373523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:47.375048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471329357267825:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.375155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:47.375437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471329357267825:2050] [node 3] ICP22 created new session: [5:7577471333652235221:2050]
2025-11-27T18:18:47.375655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471333652235221:2050] [node 3] ICS09 handshake done sender: [5:7577471333652235211:2121] self: [5:7577471333652235218:0] peer: [3:7577471333382516548:0] socket: 616 qp: -1
2025-11-27T18:18:47.375698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471333652235221:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:47.375734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:47.375757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471329216193241:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11563) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471333511160629:2122] held: no
2025-11-27T18:18:47.375957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471333511160629:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:47.381457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471329216193241:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:47.381533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:47.381587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471329216193241:2050] [node 3] ICP22 created new session: [6:7577471333511160639:2050]
2025-11-27T18:18:47.381648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471333511160639:2050] [node 3] ICS09 handshake done sender: [6:7577471333511160629:2122] self: [6:7577471333511160635:0] peer: [3:7577471333382516553:0] socket: 574 qp: -1
2025-11-27T18:18:47.381703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471333511160639:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:47.381768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:47.427491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:47.427727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471333382516571:2048] [node 1] ICS01 socket: 602 reason# EndOfStream
2025-11-27T18:18:47.427786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471329087549179:2048] [node 1] ICP30 unregister session Session# [3:7577471333382516571:2048] VirtualId# [3:7577471333382516560:0]
2025-11-27T18:18:47.427851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471333382516571:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.427967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:47.428046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:47.425809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:47.426134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471331885022480:2048] [node 1] ICS07 socket disconnect 556 reason# EndOfStream
2025-11-27T18:18:47.426167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471331885022480:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.426269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:47.426319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471331885022480:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:47.429028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471327590055090:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10620) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:47.429072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471331885022480:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:47.430809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471327590055090:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10620) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:47.430880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471331885022480:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:47.430919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471327590055090:2048] [node 1] ICP30 unregister session Session# [2:7577471331885022480:2048] VirtualId# [2:7577471331885022469:0]
2025-11-27T18:18:47.430968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:47.431005Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471327590055090:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10620) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:47.431027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10620) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:47.431270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:47.431499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471334014684630:2048] [node 1] ICS01 socket: 565 reason# EndOfStream
2025-11-27T18:18:47.431545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471329719717244:2048] [node 1] ICP30 unregister session Session# [4:7577471334014684630:2048] VirtualId# [4:7577471334014684621:0]
2025-11-27T18:18:47.431575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471334014684630:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.431647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:47.432277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:47.432674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:47.432924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471333652235204:2048] [node 1] ICS01 socket: 578 reason# EndOfStream
2025-11-27T18:18:47.432976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471329357267823:2048] [node 1] ICP30 unregister session Session# [5:7577471333652235204:2048] VirtualId# [5:7577471333652235199:0]
2025-11-27T18:18:47.433008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471333652235204:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.433078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:47.433126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:47.441781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471327590055090:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:47.440378Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:47.440796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471333511160623:2048] [node 1] ICS01 socket: 601 reason# EndOfStream
2025-11-27T18:18:47.440900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471329216193239:2048] [node 1] ICP30 unregister session Session# [6:7577471333511160623:2048] VirtualId# [6:7577471333511160616:0]
2025-11-27T18:18:47.440982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471333511160623:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.441129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:47.441217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:47.489563Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:47.489795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471333382516564:2049] [node 2] ICS07 socket disconnect 567 reason# EndOfStream
2025-11-27T18:18:47.489831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471333382516564:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.489924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:47.489966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471333382516564:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:47.491972Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471329087549180:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64495) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:47.492031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471333382516564:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:47.492814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:47.493134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471334014684628:2049] [node 2] ICS01 socket: 564 reason# EndOfStream
2025-11-27T18:18:47.493196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471329719717245:2049] [node 2] ICP30 unregister session Session# [4:7577471334014684628:2049] VirtualId# [4:7577471334014684622:0]
2025-11-27T18:18:47.493233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471334014684628:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.493329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:47.493344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471329087549180:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64495) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:47.493375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:47.493395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471333382516564:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:47.493468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471329087549180:2049] [node 2] ICP30 unregister session Session# [3:7577471333382516564:2049] VirtualId# [3:7577471333382516556:0]
2025-11-27T18:18:47.494271Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:47.493519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:47.493548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471329087549180:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64495) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:47.493562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64495) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:47.494470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471333652235208:2049] [node 2] ICS01 socket: 579 reason# EndOfStream
2025-11-27T18:18:47.494567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471329357267824:2049] [node 2] ICP30 unregister session Session# [5:7577471333652235208:2049] VirtualId# [5:7577471333652235200:0]
2025-11-27T18:18:47.494604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471333652235208:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.494683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:47.494733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:47.497394Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:47.497597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471333511160636:2049] [node 2] ICS01 socket: 607 reason# EndOfStream
2025-11-27T18:18:47.497650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471329216193240:2049] [node 2] ICP30 unregister session Session# [6:7577471333511160636:2049] VirtualId# [6:7577471333511160631:0]
2025-11-27T18:18:47.497721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471333511160636:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.497835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:47.497931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:47.503197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471329087549180:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:47.560330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:47.560528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471334014684638:2050] [node 3] ICS01 socket: 591 reason# EndOfStream
2025-11-27T18:18:47.560565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471329719717246:2050] [node 3] ICP30 unregister session Session# [4:7577471334014684638:2050] VirtualId# [4:7577471334014684634:0]
2025-11-27T18:18:47.560590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471334014684638:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.560673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:47.560724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:47.561472Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:47.561683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471333652235221:2050] [node 3] ICS01 socket: 616 reason# EndOfStream
2025-11-27T18:18:47.561745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471329357267825:2050] [node 3] ICP30 unregister session Session# [5:7577471333652235221:2050] VirtualId# [5:7577471333652235218:0]
2025-11-27T18:18:47.561781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471333652235221:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.561869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:47.561927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:47.562447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:47.562638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471333511160639:2050] [node 3] ICS01 socket: 574 reason# EndOfStream
2025-11-27T18:18:47.562675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471329216193241:2050] [node 3] ICP30 unregister session Session# [6:7577471333511160639:2050] VirtualId# [6:7577471333511160635:0]
2025-11-27T18:18:47.562718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471333511160639:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.562790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:47.562839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:47.644999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:47.645249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471333652235213:2108] [node 4] ICS01 socket: 613 reason# EndOfStream
2025-11-27T18:18:47.645327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471333652235182:2108] [node 4] ICP30 unregister session Session# [5:7577471333652235213:2108] VirtualId# [5:7577471333652235207:0]
2025-11-27T18:18:47.645366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471333652235213:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.645444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:47.645506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:47.646222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:47.646457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471333511160627:2107] [node 4] ICS01 socket: 598 reason# EndOfStream
2025-11-27T18:18:47.646507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471333511160602:2107] [node 4] ICP30 unregister session Session# [6:7577471333511160627:2107] VirtualId# [6:7577471333511160619:0]
2025-11-27T18:18:47.646544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471333511160627:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.646626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:47.646674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:47.676181Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:47.676424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471333511160633:2104] [node 5] ICS01 socket: 577 reason# EndOfStream
2025-11-27T18:18:47.676480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471329216193298:2104] [node 5] ICP30 unregister session Session# [6:7577471333511160633:2104] VirtualId# [6:7577471333511160622:0]
2025-11-27T18:18:47.676513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471333511160633:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:47.676586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:47.676650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:47.756583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471332280746656:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:47.756631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471332280746657:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:47.779830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471330319643894:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:47.779883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471330319643895:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:47.802413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471330216128626:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:47.802632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471330216128627:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:47.824553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471332813783752:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:47.824624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471332813783753:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:47.825385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471332813783754:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:47.848129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471330278393553:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:47.848182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471330278393554:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:47.848214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471330278393555:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:47.869775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471331577112436:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:47.869833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471331577112438:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:47.870353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471331577112437:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:47.986169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471331577112497:2105] [node 5] ICP01 ready to work
2025-11-27T18:18:48.015867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471336575714016:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:48.017013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471336575714017:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:48.023836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471336575714018:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:48.026215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471334614611256:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:48.077764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471332280746656:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(84662084ns) {Configure:104(84652675ns) {ProcessConfigured:135(81114ns) {StartInitialHandshake:157(78854ns) {DropHandshakes:470(6174ns) {DropIncomingHandshake:442(757ns) DropOutgoingHandshake:457(380ns)} GenerateSessionVirtualId:988(992ns) PrepareNewSessionHandshake:477(1760ns) {SwitchToState:179(817ns)}}}}}
2025-11-27T18:18:48.053602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471334614611257:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:48.082685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471334511095987:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:48.104627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471334511095988:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:48.090511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471334614611258:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:48.107379Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471331577112497:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(84455876ns) {Configure:104(84447610ns) {ProcessConfigured:135(27650440ns) {StartInitialHandshake:157(27648189ns) {DropHandshakes:470(2984ns) {DropIncomingHandshake:442(641ns) DropOutgoingHandshake:457(512ns)} GenerateSessionVirtualId:988(793ns) PrepareNewSessionHandshake:477(5382ns) {SwitchToState:179(2774ns)}}}}}
2025-11-27T18:18:48.117309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471332280746657:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(118698286ns) {Configure:104(118691513ns) {ProcessConfigured:135(9105357ns) {StartInitialHandshake:157(9101834ns) {DropHandshakes:470(3443ns) {DropIncomingHandshake:442(718ns) DropOutgoingHandshake:457(632ns)} GenerateSessionVirtualId:988(800ns) PrepareNewSessionHandshake:477(4556ns) {SwitchToState:179(2261ns)}}}}}
2025-11-27T18:18:48.120793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471330319643894:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(102581864ns) {Configure:104(102574887ns) {ProcessConfigured:135(16638660ns) {StartInitialHandshake:157(16635345ns) {DropHandshakes:470(3004ns) {DropIncomingHandshake:442(799ns) DropOutgoingHandshake:457(485ns)} GenerateSessionVirtualId:988(811ns) PrepareNewSessionHandshake:477(4632ns) {SwitchToState:179(2721ns)}}}}}
2025-11-27T18:18:48.146889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471331577112436:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23876149ns) {Configure:104(23872358ns) {ProcessConfigured:135(51216ns) {StartInitialHandshake:157(49469ns) {DropHandshakes:470(2304ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(552ns) PrepareNewSessionHandshake:477(1308ns) {SwitchToState:179(415ns)}}}}}
2025-11-27T18:18:48.147129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471335872079801:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:48.133567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471334511095989:2111] [node 6] ICP01 ready to work
2025-11-27T18:18:48.141867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471337108751112:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:48.162479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471334573360912:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:48.164525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471337108751113:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:48.164393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471330319643895:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(110174256ns) {Configure:104(109894252ns) {ProcessConfigured:135(5729141ns) {StartInitialHandshake:157(5725210ns) {DropHandshakes:470(2799ns) {DropIncomingHandshake:442(724ns) DropOutgoingHandshake:457(383ns)} GenerateSessionVirtualId:988(971ns) PrepareNewSessionHandshake:477(4338ns) {SwitchToState:179(1718ns)}}}}}
2025-11-27T18:18:48.165688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471334573360913:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:48.166936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471330216128627:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(132533553ns) {Configure:104(132523415ns) {ProcessConfigured:135(218291ns) {StartInitialHandshake:157(215709ns) {DropHandshakes:470(2009ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(767ns) PrepareNewSessionHandshake:477(1806ns) {SwitchToState:179(776ns)}}}}}
2025-11-27T18:18:48.167423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471336575714018:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(89434708ns) {Configure:104(81964877ns) {ProcessConfigured:135(186978ns) {StartInitialHandshake:157(185179ns) {DropHandshakes:470(1542ns) {DropIncomingHandshake:442(419ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(700ns) PrepareNewSessionHandshake:477(1299ns) {SwitchToState:179(592ns)}}}}}
2025-11-27T18:18:48.168962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471330216128626:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(130117187ns) {Configure:104(130108021ns) {ProcessConfigured:135(657154ns) {StartInitialHandshake:157(655091ns) {DropHandshakes:470(2386ns) {DropIncomingHandshake:442(657ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(1276ns) PrepareNewSessionHandshake:477(1483ns) {SwitchToState:179(145ns)}}}}}
2025-11-27T18:18:48.169743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471332813783754:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(99307565ns) {Configure:104(99301856ns) {ProcessConfigured:135(534182ns) {StartInitialHandshake:157(531986ns) {DropHandshakes:470(1332ns) {DropIncomingHandshake:442(366ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(660ns) PrepareNewSessionHandshake:477(1747ns) {SwitchToState:179(820ns)}}}}}
2025-11-27T18:18:48.169788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471336575714017:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(115145009ns) {Configure:104(115138074ns) {ProcessConfigured:135(1088547ns) {StartInitialHandshake:157(1083641ns) {DropHandshakes:470(1085ns) {DropIncomingHandshake:442(359ns) DropOutgoingHandshake:457(196ns)} GenerateSessionVirtualId:988(544ns) PrepareNewSessionHandshake:477(2006ns) {SwitchToState:179(888ns)}}}}}
2025-11-27T18:18:48.173134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471332813783752:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(109037430ns) {Configure:104(109028593ns) {ProcessConfigured:135(509911ns) {StartInitialHandshake:157(299855ns) {DropHandshakes:470(1796ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(1209ns) PrepareNewSessionHandshake:477(1751ns) {SwitchToState:179(864ns)}}}}}
2025-11-27T18:18:48.170202Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471336575714016:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(50644181ns) {Configure:104(50637166ns) {ProcessConfigured:135(389813ns) {StartInitialHandshake:157(387573ns) {DropHandshakes:470(1439ns) {DropIncomingHandshake:442(519ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(601ns) PrepareNewSessionHandshake:477(1388ns) {SwitchToState:179(603ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.174905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471334614611256:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(62692085ns) {Configure:104(62683384ns) {ProcessConfigured:135(661437ns) {StartInitialHandshake:157(456867ns) {DropHandshakes:470(1571ns) {DropIncomingHandshake:442(385ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(2195ns) PrepareNewSessionHandshake:477(2179ns) {SwitchToState:179(1009ns)}}}}}
2025-11-27T18:18:48.175737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471334614611257:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(43959791ns) {Configure:104(43953691ns) {ProcessConfigured:135(736041ns) {StartInitialHandshake:157(734402ns) {DropHandshakes:470(153400ns) {DropIncomingHandshake:442(1299ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(729ns) PrepareNewSessionHandshake:477(1094ns) {SwitchToState:179(465ns)}}}}}
2025-11-27T18:18:48.184805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471330278393553:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(64775030ns) {Configure:104(64767548ns) {ProcessConfigured:135(1917891ns) {StartInitialHandshake:157(1915931ns) {DropHandshakes:470(1576ns) {DropIncomingHandshake:442(412ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(659ns) PrepareNewSessionHandshake:477(2501ns) {SwitchToState:179(1455ns)}}}}}
2025-11-27T18:18:48.184552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471335872079806:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50957
2025-11-27T18:18:48.184900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471331577112437:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(30846353ns) {Configure:104(30839976ns) {ProcessConfigured:135(8966232ns) {StartInitialHandshake:157(8963321ns) {DropHandshakes:470(4148492ns) {DropIncomingHandshake:442(3310ns) DropOutgoingHandshake:457(344ns)} GenerateSessionVirtualId:988(2133ns) PrepareNewSessionHandshake:477(4266ns) {SwitchToState:179(2292ns)}}}}}
2025-11-27T18:18:48.184881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471330278393554:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(52487949ns) {Configure:104(52476637ns) {ProcessConfigured:135(1716108ns) {StartInitialHandshake:157(1713726ns) {DropHandshakes:470(1334ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(212ns)} GenerateSessionVirtualId:988(538ns) PrepareNewSessionHandshake:477(3502ns) {SwitchToState:179(1627ns)}}}}}
2025-11-27T18:18:48.185215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471331577112438:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(39372943ns) {Configure:104(39367530ns) {ProcessConfigured:135(3280030ns) {StartInitialHandshake:157(3278499ns) {DropHandshakes:470(952ns) {DropIncomingHandshake:442(243ns) DropOutgoingHandshake:457(161ns)} GenerateSessionVirtualId:988(411ns) PrepareNewSessionHandshake:477(2810ns) {SwitchToState:179(1279ns)}}}}}
2025-11-27T18:18:48.194672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471334511095989:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19819866ns) {Configure:104(19801848ns) {ProcessConfigured:135(9239213ns) {StartInitialHandshake:157(9236310ns) {DropHandshakes:470(1237ns) {DropIncomingHandshake:442(406ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:477(3556ns) {SwitchToState:179(2121ns)}}}}}
2025-11-27T18:18:48.199525Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471334614611258:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29772833ns) {Configure:104(29766011ns) {ProcessConfigured:135(2468207ns) {StartInitialHandshake:157(2465447ns) {DropHandshakes:470(2865ns) {DropIncomingHandshake:442(928ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(624ns) PrepareNewSessionHandshake:477(3008ns) {SwitchToState:179(1406ns)}}}}}
2025-11-27T18:18:48.199580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471334511095987:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32329611ns) {Configure:104(32322501ns) {ProcessConfigured:135(2556981ns) {StartInitialHandshake:157(2553784ns) {DropHandshakes:470(2376ns) {DropIncomingHandshake:442(765ns) DropOutgoingHandshake:457(164ns)} GenerateSessionVirtualId:988(633ns) PrepareNewSessionHandshake:477(2979ns) {SwitchToState:179(1012ns)}}}}}
2025-11-27T18:18:48.203523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471330278393555:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(30230537ns) {Configure:104(30225970ns) {ProcessConfigured:135(2484971ns) {StartInitialHandshake:157(2483159ns) {DropHandshakes:470(534098ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(1082ns)} GenerateSessionVirtualId:988(1606ns) PrepareNewSessionHandshake:477(2336ns) {SwitchToState:179(713ns)}}}}}
2025-11-27T18:18:48.204896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471334511095988:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31900626ns) {Configure:104(31896913ns) {ProcessConfigured:135(1361757ns) {StartInitialHandshake:157(1359546ns) {DropHandshakes:470(2032ns) {DropIncomingHandshake:442(850ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:477(1931ns) {SwitchToState:179(989ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.205762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471337108751113:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30279082ns) {Configure:104(30203011ns) {ProcessConfigured:135(3838303ns) {StartInitialHandshake:157(3835778ns) {DropHandshakes:470(1665ns) {DropIncomingHandshake:442(425ns) DropOutgoingHandshake:457(531ns)} GenerateSessionVirtualId:988(521ns) PrepareNewSessionHandshake:477(3238ns) {SwitchToState:179(2044ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:18:48.206686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471336575714033:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51318
::ffff:127.1.0.0
2025-11-27T18:18:48.206153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471335872079806:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.207138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471337108751112:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(36035947ns) {Configure:104(36031305ns) {ProcessConfigured:135(1004041ns) {StartInitialHandshake:157(1002083ns) {DropHandshakes:470(6074ns) {DropIncomingHandshake:442(577ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:477(2444ns) {SwitchToState:179(1169ns)}}}}}
2025-11-27T18:18:48.206714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471331577112436:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.207994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471334614611270:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17151
2025-11-27T18:18:48.208519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471336575714032:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51318
2025-11-27T18:18:48.208666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471336575714024:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.208692Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471336575714024:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.209003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471336575714033:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.210799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471334614611270:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.210558Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471334573360912:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19352405ns) {Configure:104(19344105ns) {ProcessConfigured:135(41891ns) {StartInitialHandshake:157(40530ns) {DropHandshakes:470(2142ns) {DropIncomingHandshake:442(492ns) DropOutgoingHandshake:457(568ns)} GenerateSessionVirtualId:988(746ns) PrepareNewSessionHandshake:477(1656ns) {SwitchToState:179(789ns)}}}}}
2025-11-27T18:18:48.210691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471334573360913:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24472416ns) {Configure:104(19772374ns) {ProcessConfigured:135(320173ns) {StartInitialHandshake:157(318368ns) {DropHandshakes:470(1356ns) {DropIncomingHandshake:442(387ns) DropOutgoingHandshake:457(309ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:477(1297ns) {SwitchToState:179(574ns)}}}}}
2025-11-27T18:18:48.209822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471336575714032:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.211052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471330319643894:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.211931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471336575714020:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:18:48.214154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471334573360914:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-27T18:18:48.211956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471336575714020:2111] [node 2] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-27T18:18:48.213108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471336575714024:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.214337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471336575714035:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51318
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.214574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471334614611274:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17151
2025-11-27T18:18:48.215064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471334573360914:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.214443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471336575714018:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.220004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471334614611275:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17151
2025-11-27T18:18:48.214819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471334511096000:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52293
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.214949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:48.218963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471335872079812:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50957
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.220513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471334511096002:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52293
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.222264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471335872079813:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50957
2025-11-27T18:18:48.218955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471336575714018:2110] [node 6] ICP22 created new session: [1:7577471336575714040:2110]
2025-11-27T18:18:48.223870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471334573360925:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-27T18:18:48.229059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471334614611276:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17151
2025-11-27T18:18:48.223774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471334511096001:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52293
No RDMA context found for GID: 2025-11-27T18:18:48.219701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471336575714038:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51318
::ffff:127.1.0.0
2025-11-27T18:18:48.236963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471331577112436:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30408) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471335872079806:2114] held: no
2025-11-27T18:18:48.234994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471334511096005:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52293
2025-11-27T18:18:48.237175Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471335872079806:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:48.235155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471334573360925:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.219855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471336575714020:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:48.241326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471337108751127:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33877
2025-11-27T18:18:48.237478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471331577112436:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.241445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471330278393554:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.243598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471334614611274:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.243362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471334511096001:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.220200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471336575714040:2110] [node 6] ICS09 handshake done sender: [1:7577471336575714024:2113] self: [1:7577471336575714023:0] peer: [6:7577471335872079811:0] socket: 561 qp: -1
2025-11-27T18:18:48.237530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:48.243513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471334511096002:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.244843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471334614611276:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.245776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471337108751127:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.238583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471331577112436:2048] [node 1] ICP22 created new session: [6:7577471335872079814:2048]
2025-11-27T18:18:48.221486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471336575714040:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:48.244808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471334511096005:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.223693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:48.244885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471334614611277:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17151
2025-11-27T18:18:48.242837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471335872079814:2048] [node 1] ICS09 handshake done sender: [6:7577471335872079806:2114] self: [6:7577471335872079811:0] peer: [1:7577471336575714023:0] socket: 563 qp: -1
2025-11-27T18:18:48.245982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471332280746656:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.245289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471334511096000:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.245240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471334614611266:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.243876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471335872079814:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:48.245295Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471334614611266:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.253847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:48.243933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:48.261778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471337108751128:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33877
2025-11-27T18:18:48.253705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471330216128627:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:18:48.257563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471330216128626:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.258591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471334511095995:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.257582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471335872079813:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.253726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471336575714018:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(29235959ns) {HandleHandshakeStatus:329(29233208ns) {DropIncomingHandshake:442(17501ns) DropOutgoingHandshake:457(367ns) SwitchToState:179(716ns) ProcessPendingSessionEvents:495(8712741ns) {ForwardSessionEventToSession:612(2691048ns) ForwardSessionEventToSession:612(921290ns) ForwardSessionEventToSession:612(4557ns) ForwardSessionEventToSession:612(2786ns) ForwardSessionEventToSession:612(3777405ns) ForwardSessionEventToSession:612(553114ns) ForwardSessionEventToSession:612(7598ns) ForwardSessionEventToSession:612(481190ns) ForwardSessionEventToSession:612(3112ns) ForwardSessionEventToSession:612(252429ns)}}}
2025-11-27T18:18:48.258326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471334614611262:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.258612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471334511095995:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.264607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471334573360932:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-27T18:18:48.259033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471331577112438:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.254358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471332280746656:2048] [node 2] ICP22 created new session: [1:7577471336575714044:2048]
2025-11-27T18:18:48.254753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471336575714044:2048] [node 2] ICS09 handshake done sender: [1:7577471336575714020:2111] self: [1:7577471336575714019:0] peer: [2:7577471334614611273:0] socket: 562 qp: -1
2025-11-27T18:18:48.262475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471337108751130:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33877
2025-11-27T18:18:48.258349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471334614611262:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.262387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471334511095995:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:48.262540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471337108751128:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.258497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471334614611275:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.256295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471336575714044:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:48.262048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471334614611270:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:48.256496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:48.262614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471330319643894:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30408) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471334614611270:2117] held: no
2025-11-27T18:18:48.257784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471336575714035:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:18:48.263574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471334614611266:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:48.261659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471336575714041:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51318
::ffff:127.1.0.0
2025-11-27T18:18:48.262138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471336575714038:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.265674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471330319643894:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.262513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471332280746656:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(16210473ns) {HandleHandshakeStatus:329(16208305ns) {DropIncomingHandshake:442(15100ns) DropOutgoingHandshake:457(2729ns) SwitchToState:179(320ns) ProcessPendingSessionEvents:495(3855695ns) {ForwardSessionEventToSession:612(60041ns) ForwardSessionEventToSession:612(9599ns) ForwardSessionEventToSession:612(344558ns) ForwardSessionEventToSession:612(3201ns) ForwardSessionEventToSession:612(2707ns) ForwardSessionEventToSession:612(3173412ns) ForwardSessionEventToSession:612(6524ns) ForwardSessionEventToSession:612(96876ns) ForwardSessionEventToSession:612(2984ns) ForwardSessionEventToSession:612(2654ns)}}}
2025-11-27T18:18:48.265825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:48.266146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471334511095989:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.266129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471330319643894:2048] [node 1] ICP22 created new session: [2:7577471334614611283:2048]
2025-11-27T18:18:48.266377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471334573360931:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-27T18:18:48.262650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471336575714022:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.266346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:48.266155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471337108751130:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.262670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471336575714022:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.266673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471332813783752:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.266401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471334511095989:2111] [node 6] ICP22 created new session: [3:7577471334511096014:2111]
2025-11-27T18:18:48.266627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471335872079818:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50957
2025-11-27T18:18:48.266193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471334614611283:2048] [node 1] ICS09 handshake done sender: [2:7577471334614611270:2117] self: [2:7577471334614611273:0] peer: [1:7577471336575714019:0] socket: 569 qp: -1
2025-11-27T18:18:48.267583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471334511096014:2111] [node 6] ICS09 handshake done sender: [3:7577471334511095995:2114] self: [3:7577471334511095994:0] peer: [6:7577471335872079817:0] socket: 600 qp: -1
2025-11-27T18:18:48.266351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471334614611283:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:48.263442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471336575714041:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.268506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471331577112438:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17868) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471335872079813:2117] held: no
2025-11-27T18:18:48.266731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471336575714022:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:48.267952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471334511096014:2111] [node 6] ICS10 traffic start
2025-11-27T18:18:48.268507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471332280746657:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.268720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:48.268042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:48.266387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:48.269673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471334573360925:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.269329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471332280746657:2049] [node 3] ICP22 created new session: [1:7577471336575714051:2049]
2025-11-27T18:18:48.270138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471335872079813:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:48.270304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471334511096012:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52293
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.267611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471334614611257:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.271222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471331577112438:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.269352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471336575714028:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.270880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471330278393554:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.267785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:48.272003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471337108751133:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33877
2025-11-27T18:18:48.269392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471336575714028:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.272182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:48.270942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:48.268025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471334614611257:2108] [node 5] ICP22 created new session: [2:7577471334614611285:2108]
2025-11-27T18:18:48.273161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471334511096016:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52293
2025-11-27T18:18:48.269577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471336575714051:2049] [node 3] ICS09 handshake done sender: [1:7577471336575714022:2112] self: [1:7577471336575714021:0] peer: [3:7577471334511096010:0] socket: 560 qp: -1
2025-11-27T18:18:48.271295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471330278393554:2049] [node 2] ICP22 created new session: [5:7577471334573360933:2049]
2025-11-27T18:18:48.272243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471331577112438:2050] [node 3] ICP22 created new session: [6:7577471335872079819:2050]
2025-11-27T18:18:48.269294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471334614611285:2108] [node 5] ICS09 handshake done sender: [2:7577471334614611266:2116] self: [2:7577471334614611264:0] peer: [5:7577471334573360930:0] socket: 567 qp: -1
2025-11-27T18:18:48.269781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471336575714051:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:48.269817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:48.276355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471336575714028:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:48.278636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471330216128626:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30408) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471334511096000:2117] held: no
2025-11-27T18:18:48.269327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471334614611285:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:48.273212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471334573360933:2049] [node 2] ICS09 handshake done sender: [5:7577471334573360925:2116] self: [5:7577471334573360930:0] peer: [2:7577471334614611264:0] socket: 580 qp: -1
2025-11-27T18:18:48.269375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:48.273858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471335872079819:2050] [node 3] ICS09 handshake done sender: [6:7577471335872079813:2117] self: [6:7577471335872079817:0] peer: [3:7577471334511095994:0] socket: 601 qp: -1
2025-11-27T18:18:48.270300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471334614611262:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.280283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471336575714026:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.280331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471336575714026:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.273890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471335872079819:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:48.279973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471334511096001:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:48.279840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471337108751133:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.273511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471334573360933:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:48.277856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471330319643895:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.277917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:48.280745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471336575714016:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.280597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471332813783754:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.274042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:48.280962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:48.275924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:48.282025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471334511096000:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:48.281203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471337108751134:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33877
2025-11-27T18:18:48.278903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471330319643895:2049] [node 3] ICP22 created new session: [2:7577471334614611289:2049]
2025-11-27T18:18:48.276031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471334573360932:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.281582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471336575714016:2107] [node 4] ICP22 created new session: [1:7577471336575714054:2107]
2025-11-27T18:18:48.283764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471330216128627:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.283712Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471337108751130:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:48.284007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:48.279501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471334614611289:2049] [node 3] ICS09 handshake done sender: [2:7577471334614611262:2114] self: [2:7577471334614611261:0] peer: [3:7577471334511096007:0] socket: 566 qp: -1
2025-11-27T18:18:48.278578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471330278393553:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.279785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471334614611289:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:48.282910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471336575714054:2107] [node 4] ICS09 handshake done sender: [1:7577471336575714028:2115] self: [1:7577471336575714027:0] peer: [4:7577471337108751132:0] socket: 604 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.284402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471330216128627:2049] [node 2] ICP22 created new session: [3:7577471334511096018:2049]
2025-11-27T18:18:48.285332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471332813783752:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30408) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471337108751130:2118] held: no
2025-11-27T18:18:48.279816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:48.283124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471336575714026:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.284456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471334511096018:2049] [node 2] ICS09 handshake done sender: [3:7577471334511096001:2118] self: [3:7577471334511096007:0] peer: [2:7577471334614611261:0] socket: 576 qp: -1
2025-11-27T18:18:48.289700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471334614611290:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17151
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.283285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471336575714054:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:48.289810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471332813783752:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.289875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:48.284527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471334511096018:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:48.285607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:48.290254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471332813783752:2048] [node 1] ICP22 created new session: [4:7577471337108751137:2048]
2025-11-27T18:18:48.283327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:48.290495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471337108751136:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33877
2025-11-27T18:18:48.293056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471334573360937:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-27T18:18:48.292704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471335872079822:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50957
2025-11-27T18:18:48.286923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471334511095997:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.285497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471336575714017:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.294191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471334614611290:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.297478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471330278393553:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30408) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471334573360932:2118] held: no
2025-11-27T18:18:48.291221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471337108751137:2048] [node 1] ICS09 handshake done sender: [4:7577471337108751130:2118] self: [4:7577471337108751132:0] peer: [1:7577471336575714027:0] socket: 605 qp: -1
2025-11-27T18:18:48.285618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:48.286975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471334511095997:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.287012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471336575714017:2109] [node 5] ICP22 created new session: [1:7577471336575714057:2109]
2025-11-27T18:18:48.298420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471335872079822:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.292300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471330216128626:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.290883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471336575714057:2109] [node 5] ICS09 handshake done sender: [1:7577471336575714026:2114] self: [1:7577471336575714025:0] peer: [5:7577471334573360935:0] socket: 609 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.291478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471337108751137:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:48.298516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471334573360932:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:48.292379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:48.291986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:48.299294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471331577112437:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.299352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471334614611265:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.290922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471336575714057:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:48.298327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471337108751136:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.292422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471330216128626:2048] [node 1] ICP22 created new session: [3:7577471334511096020:2048]
2025-11-27T18:18:48.299382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471334614611265:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.299674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471335872079823:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50957
2025-11-27T18:18:48.300635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471330278393553:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.292794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471334511096020:2048] [node 1] ICS09 handshake done sender: [3:7577471334511096000:2117] self: [3:7577471334511096010:0] peer: [1:7577471336575714021:0] socket: 575 qp: -1
2025-11-27T18:18:48.291083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:48.304953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471335872079823:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.300427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471334614611265:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:48.298483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471332813783753:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.300750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471334614611272:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.292825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471334511096020:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:48.300787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471334614611272:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.304385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471334614611256:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.301676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:48.305089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471330278393553:2048] [node 1] ICP22 created new session: [5:7577471334573360938:2048]
2025-11-27T18:18:48.293080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:48.304438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.305395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471334614611256:2107] [node 4] ICP22 created new session: [2:7577471334614611294:2107]
2025-11-27T18:18:48.293349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471330216128627:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:48.307797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471334614611294:2107] [node 4] ICS09 handshake done sender: [2:7577471334614611265:2115] self: [2:7577471334614611263:0] peer: [4:7577471337108751140:0] socket: 616 qp: -1
2025-11-27T18:18:48.304536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471334511096017:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52293
::ffff:127.1.0.0
2025-11-27T18:18:48.305342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471334573360938:2048] [node 1] ICS09 handshake done sender: [5:7577471334573360932:2118] self: [5:7577471334573360935:0] peer: [1:7577471336575714025:0] socket: 610 qp: -1
2025-11-27T18:18:48.309457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471337108751121:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.307708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471335872079801:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.309503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471337108751121:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.307762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471334573360938:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:48.304810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471334511095997:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:48.308201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471334614611294:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:48.310444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471337108751141:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33877
2025-11-27T18:18:48.309764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471334614611272:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:48.309781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: 2025-11-27T18:18:48.311246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471334614611258:2109] [node 6] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:18:48.308066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471334511095987:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.309443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:48.311347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:48.311168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471335872079826:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50957
2025-11-27T18:18:48.309504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471334573360939:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-27T18:18:48.308176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:48.311388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471334614611258:2109] [node 6] ICP22 created new session: [2:7577471334614611297:2109]
2025-11-27T18:18:48.312377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471335872079827:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50957
2025-11-27T18:18:48.309497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471334511095987:2107] [node 4] ICP22 created new session: [3:7577471334511096024:2107]
2025-11-27T18:18:48.317739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471332813783753:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471337108751136:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.318510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471335872079822:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:48.310507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471334511096024:2107] [node 4] ICS09 handshake done sender: [3:7577471334511095997:2115] self: [3:7577471334511095996:0] peer: [4:7577471337108751135:0] socket: 613 qp: -1
2025-11-27T18:18:48.312299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471334614611297:2109] [node 6] ICS09 handshake done sender: [2:7577471334614611272:2118] self: [2:7577471334614611271:0] peer: [6:7577471335872079824:0] socket: 588 qp: -1
2025-11-27T18:18:48.310550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471334511096024:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:48.318967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471337108751142:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33877
2025-11-27T18:18:48.319044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471332813783754:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17868) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471337108751133:2119] held: no
2025-11-27T18:18:48.312833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471334614611297:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:48.310897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:48.319547Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471331577112437:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471335872079822:2119] held: no
2025-11-27T18:18:48.319429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471334573360939:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.319867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471337108751133:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:48.317783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:48.311237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471334511096017:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.320101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471331577112437:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.320644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471330278393555:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.320282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:48.320622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471331577112437:2049] [node 2] ICP22 created new session: [6:7577471335872079828:2049]
2025-11-27T18:18:48.321522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471337108751121:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.320877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471335872079828:2049] [node 2] ICS09 handshake done sender: [6:7577471335872079822:2119] self: [6:7577471335872079824:0] peer: [2:7577471334614611271:0] socket: 590 qp: -1
2025-11-27T18:18:48.321609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471332813783754:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.320894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471335872079826:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.321899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:48.321303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471335872079828:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:48.323565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471334573360924:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.323596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471334573360924:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.323633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471332813783754:2050] [node 3] ICP22 created new session: [4:7577471337108751145:2050]
2025-11-27T18:18:48.321619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:48.323963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471337108751136:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:48.320891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471334511095999:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.325166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471334573360924:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:48.321705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471331577112497:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.324710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471337108751145:2050] [node 3] ICS09 handshake done sender: [4:7577471337108751133:2119] self: [4:7577471337108751135:0] peer: [3:7577471334511095996:0] socket: 577 qp: -1
2025-11-27T18:18:48.320931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471334511095999:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.324105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471335872079829:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50957
2025-11-27T18:18:48.324748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471337108751145:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:48.325722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471337108751113:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.327303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471334573360913:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.325756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:48.324404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471334511095999:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.326375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:48.329645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471335872079823:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:48.327639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:48.324972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471334511095988:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.326473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471332813783753:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.328448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471334573360913:2109] [node 6] ICP22 created new session: [5:7577471334573360946:2109]
2025-11-27T18:18:48.330477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471335872079832:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50957
2025-11-27T18:18:48.326508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:48.326085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.329389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471334573360946:2109] [node 6] ICS09 handshake done sender: [5:7577471334573360924:2115] self: [5:7577471334573360923:0] peer: [6:7577471335872079831:0] socket: 624 qp: -1
2025-11-27T18:18:48.326139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471334511095988:2109] [node 5] ICP22 created new session: [3:7577471334511096028:2109]
2025-11-27T18:18:48.326562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471337108751113:2109] [node 6] ICP22 created new session: [4:7577471337108751147:2109]
2025-11-27T18:18:48.330144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471334573360946:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:48.326182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471334511096028:2109] [node 5] ICS09 handshake done sender: [3:7577471334511095999:2116] self: [3:7577471334511095998:0] peer: [5:7577471334573360942:0] socket: 595 qp: -1
2025-11-27T18:18:48.326653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471332813783753:2049] [node 2] ICP22 created new session: [4:7577471337108751148:2049]
2025-11-27T18:18:48.330155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471330278393555:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17868) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471334573360939:2120] held: no
2025-11-27T18:18:48.326216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471334511096028:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:48.326247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.330391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:48.331885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471335872079801:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.326859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471337108751147:2109] [node 6] ICS09 handshake done sender: [4:7577471337108751121:2114] self: [4:7577471337108751120:0] peer: [6:7577471335872079825:0] socket: 579 qp: -1
2025-11-27T18:18:48.331930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:48.331868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471334573360944:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-27T18:18:48.327051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471337108751148:2049] [node 2] ICS09 handshake done sender: [4:7577471337108751136:2121] self: [4:7577471337108751140:0] peer: [2:7577471334614611263:0] socket: 617 qp: -1
2025-11-27T18:18:48.332141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471335872079801:2110] [node 4] ICP22 created new session: [6:7577471335872079833:2110]
2025-11-27T18:18:48.333050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471335872079833:2110] [node 4] ICS09 handshake done sender: [6:7577471335872079823:2120] self: [6:7577471335872079825:0] peer: [4:7577471337108751120:0] socket: 582 qp: -1
2025-11-27T18:18:48.327061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471337108751147:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:48.327092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:48.333238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471335872079833:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:48.333287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:48.333358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471334573360945:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-27T18:18:48.327306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471337108751148:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:48.334685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471334573360939:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:48.327809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:48.334911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471330278393555:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.335194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471335872079826:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:48.334992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:48.335352Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471331577112497:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19498) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471335872079826:2121] held: no
2025-11-27T18:18:48.335048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471330278393555:2050] [node 3] ICP22 created new session: [5:7577471334573360948:2050]
2025-11-27T18:18:48.335607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471331577112497:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.335209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471334573360948:2050] [node 3] ICS09 handshake done sender: [5:7577471334573360939:2120] self: [5:7577471334573360942:0] peer: [3:7577471334511095998:0] socket: 622 qp: -1
2025-11-27T18:18:48.335644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:48.335245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471334573360948:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:48.335307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:48.335731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471331577112497:2105] [node 5] ICP22 created new session: [6:7577471335872079835:2105]
2025-11-27T18:18:48.335435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471334573360945:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.335804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471335872079835:2105] [node 5] ICS09 handshake done sender: [6:7577471335872079826:2121] self: [6:7577471335872079831:0] peer: [5:7577471334573360923:0] socket: 625 qp: -1
2025-11-27T18:18:48.335623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471334573360912:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:48.335833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471335872079835:2105] [node 5] ICS10 traffic start
2025-11-27T18:18:48.335860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:48.335915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471337108751123:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:48.335945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471337108751123:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:48.336978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471337108751123:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:48.337107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471334573360951:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-27T18:18:48.337149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471337108751112:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:48.337221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:48.337286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471337108751112:2108] [node 5] ICP22 created new session: [4:7577471337108751152:2108]
2025-11-27T18:18:48.337335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471337108751152:2108] [node 5] ICS09 handshake done sender: [4:7577471337108751123:2115] self: [4:7577471337108751122:0] peer: [5:7577471334573360950:0] socket: 627 qp: -1
2025-11-27T18:18:48.337375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471337108751152:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:48.337384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471334573360945:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:48.337585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471334573360912:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:48.337406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:48.337631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:48.337963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471334573360912:2108] [node 4] ICP22 created new session: [5:7577471334573360952:2108]
2025-11-27T18:18:48.338011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471334573360952:2108] [node 4] ICS09 handshake done sender: [5:7577471334573360945:2122] self: [5:7577471334573360950:0] peer: [4:7577471337108751122:0] socket: 628 qp: -1
2025-11-27T18:18:48.338045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471334573360952:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:48.338173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:48.374313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:48.374568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471334614611283:2048] [node 1] ICS01 socket: 569 reason# EndOfStream
2025-11-27T18:18:48.374607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471330319643894:2048] [node 1] ICP30 unregister session Session# [2:7577471334614611283:2048] VirtualId# [2:7577471334614611273:0]
2025-11-27T18:18:48.374636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471334614611283:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.374729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:48.374778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:48.375547Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:48.375756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471334511096020:2048] [node 1] ICS01 socket: 575 reason# EndOfStream
2025-11-27T18:18:48.375801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471330216128626:2048] [node 1] ICP30 unregister session Session# [3:7577471334511096020:2048] VirtualId# [3:7577471334511096010:0]
2025-11-27T18:18:48.375835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471334511096020:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.375915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:48.375961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:48.378061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:48.378284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471337108751137:2048] [node 1] ICS01 socket: 605 reason# EndOfStream
2025-11-27T18:18:48.378321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471332813783752:2048] [node 1] ICP30 unregister session Session# [4:7577471337108751137:2048] VirtualId# [4:7577471337108751132:0]
2025-11-27T18:18:48.378351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471337108751137:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.378452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:48.378500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:48.378861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:48.379043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471334573360938:2048] [node 1] ICS01 socket: 610 reason# EndOfStream
2025-11-27T18:18:48.379086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471330278393553:2048] [node 1] ICP30 unregister session Session# [5:7577471334573360938:2048] VirtualId# [5:7577471334573360935:0]
2025-11-27T18:18:48.379123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471334573360938:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.379222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:48.379280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:48.379598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:48.379791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471335872079814:2048] [node 1] ICS01 socket: 563 reason# EndOfStream
2025-11-27T18:18:48.379836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471331577112436:2048] [node 1] ICP30 unregister session Session# [6:7577471335872079814:2048] VirtualId# [6:7577471335872079811:0]
2025-11-27T18:18:48.379903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471335872079814:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.379988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:48.380047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:48.445026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:48.445281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471334511096018:2049] [node 2] ICS07 socket disconnect 576 reason# EndOfStream
2025-11-27T18:18:48.445313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471334511096018:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.445393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:48.445443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471334511096018:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:48.447660Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471330216128627:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65346) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:48.447703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471334511096018:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:48.448840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471330216128627:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65346) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:48.448954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471334511096018:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:48.449000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471330216128627:2049] [node 2] ICP30 unregister session Session# [3:7577471334511096018:2049] VirtualId# [3:7577471334511096007:0]
2025-11-27T18:18:48.449058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:48.449107Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471330216128627:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:48.449125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:48.450530Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:48.450762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471337108751148:2049] [node 2] ICS01 socket: 617 reason# EndOfStream
2025-11-27T18:18:48.450805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471332813783753:2049] [node 2] ICP30 unregister session Session# [4:7577471337108751148:2049] VirtualId# [4:7577471337108751140:0]
2025-11-27T18:18:48.450834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471337108751148:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.450910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:48.450971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:48.451635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:48.451845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471334573360933:2049] [node 2] ICS07 socket disconnect 580 reason# EndOfStream
2025-11-27T18:18:48.451873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471334573360933:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.452860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:48.451931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:48.451973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471334573360933:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:48.452658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471330278393554:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65346) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:48.453076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471335872079828:2049] [node 2] ICS01 socket: 590 reason# EndOfStream
2025-11-27T18:18:48.452717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471334573360933:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:48.452799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471330278393554:2049] [node 2] ICP30 unregister session Session# [5:7577471334573360933:2049] VirtualId# [5:7577471334573360930:0]
2025-11-27T18:18:48.453127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471331577112437:2049] [node 2] ICP30 unregister session Session# [6:7577471335872079828:2049] VirtualId# [6:7577471335872079824:0]
2025-11-27T18:18:48.452862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:48.453161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471335872079828:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.452888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471330278393554:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:48.452918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:48.453241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:48.453311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:48.458599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471330216128627:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:48.462645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471330278393554:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:48.520886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:48.521124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471337108751145:2050] [node 3] ICS01 socket: 577 reason# EndOfStream
2025-11-27T18:18:48.521170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471332813783754:2050] [node 3] ICP30 unregister session Session# [4:7577471337108751145:2050] VirtualId# [4:7577471337108751135:0]
2025-11-27T18:18:48.521208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471337108751145:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.521341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:48.521419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:48.522164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:48.522426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471334573360948:2050] [node 3] ICS01 socket: 622 reason# EndOfStream
2025-11-27T18:18:48.522493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471330278393555:2050] [node 3] ICP30 unregister session Session# [5:7577471334573360948:2050] VirtualId# [5:7577471334573360942:0]
2025-11-27T18:18:48.522522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471334573360948:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.522619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:48.522677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:48.523166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:48.523359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471335872079819:2050] [node 3] ICS01 socket: 601 reason# EndOfStream
2025-11-27T18:18:48.523402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471331577112438:2050] [node 3] ICP30 unregister session Session# [6:7577471335872079819:2050] VirtualId# [6:7577471335872079817:0]
2025-11-27T18:18:48.523431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471335872079819:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.523538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:48.523602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:48.612412Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:48.612624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471335872079833:2110] [node 4] ICS07 socket disconnect 582 reason# EndOfStream
2025-11-27T18:18:48.612654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471335872079833:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.612748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:48.612873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471335872079833:2110] [node 4] ICS15 start handshake
2025-11-27T18:18:48.611172Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:48.611422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471334573360952:2108] [node 4] ICS07 socket disconnect 628 reason# EndOfStream
2025-11-27T18:18:48.611463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471334573360952:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.611540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:48.611601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471334573360952:2108] [node 4] ICS15 start handshake
2025-11-27T18:18:48.612721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471334573360912:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21892) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:48.612789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471334573360952:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:48.612867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471334573360912:2108] [node 4] ICP30 unregister session Session# [5:7577471334573360952:2108] VirtualId# [5:7577471334573360950:0]
2025-11-27T18:18:48.612916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:48.612947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471334573360912:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21892) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:48.612968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21892) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:48.613961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471335872079801:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21892) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:48.614015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471335872079833:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:48.614051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471335872079801:2110] [node 4] ICP30 unregister session Session# [6:7577471335872079833:2110] VirtualId# [6:7577471335872079825:0]
2025-11-27T18:18:48.614104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:48.614136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471335872079801:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21892) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:48.614150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21892) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:48.620471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471335872079801:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:18:48.657375Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:48.657618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471335872079835:2105] [node 5] ICS01 socket: 625 reason# EndOfStream
2025-11-27T18:18:48.657664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471331577112497:2105] [node 5] ICP30 unregister session Session# [6:7577471335872079835:2105] VirtualId# [6:7577471335872079831:0]
2025-11-27T18:18:48.657709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471335872079835:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:48.657793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:48.657841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:48.764968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471337784186139:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:48.766110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471337784186140:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:48.791009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471337761263466:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:48.791187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471337761263467:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:48.815310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471337211694892:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:48.815974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471337211694891:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:48.841547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471338173835340:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:48.841591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471338173835341:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:48.841615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471338173835342:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:48.880177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471334896130618:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:48.880231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471334896130619:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:48.885445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471334896130620:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:48.901923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471335650297382:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:48.901979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471335650297383:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:48.902059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471335650297384:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:49.072960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471339945264730:2099] [node 5] ICP01 ready to work
2025-11-27T18:18:49.074090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471339945264733:2096] [node 4] ICP01 ready to work
2025-11-27T18:18:49.077800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471342079153496:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:49.080816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471342079153500:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:49.080660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471342079153499:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:49.102863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471337784186140:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21650887ns) {Configure:104(17664467ns) {ProcessConfigured:135(56125ns) {StartInitialHandshake:157(53203ns) {DropHandshakes:470(2620ns) {DropIncomingHandshake:442(461ns) DropOutgoingHandshake:457(333ns)} GenerateSessionVirtualId:988(815ns) PrepareNewSessionHandshake:477(1591ns) {SwitchToState:179(545ns)}}}}}
2025-11-27T18:18:49.125021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471342468802699:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:49.134428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471342056230824:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:49.140690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471341506662251:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:49.147311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471339191097977:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:49.134469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471342056230827:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:49.149352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471341506662252:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:49.148621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471342468802700:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:49.145416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471342056230828:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:49.150820Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471337784186139:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(84711056ns) {Configure:104(84703200ns) {ProcessConfigured:135(1525630ns) {StartInitialHandshake:157(1522782ns) {DropHandshakes:470(2122ns) {DropIncomingHandshake:442(606ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(730ns) PrepareNewSessionHandshake:477(2058ns) {SwitchToState:179(955ns)}}}}}
2025-11-27T18:18:49.150360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471339191097978:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:49.151033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471341506662253:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:49.151927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471339945264733:2096] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(75158574ns) {Configure:104(75150754ns) {ProcessConfigured:135(358907ns) {StartInitialHandshake:157(355945ns) {DropHandshakes:470(2538ns) {DropIncomingHandshake:442(865ns) DropOutgoingHandshake:457(359ns)} GenerateSessionVirtualId:988(746ns) PrepareNewSessionHandshake:477(1719ns) {SwitchToState:179(849ns)}}}}}
2025-11-27T18:18:49.158133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471339945264730:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(78560210ns) {Configure:104(78555637ns) {ProcessConfigured:135(3064499ns) {StartInitialHandshake:157(3061857ns) {DropHandshakes:470(1775ns) {DropIncomingHandshake:442(515ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:477(2342ns) {SwitchToState:179(1079ns)}}}}}
2025-11-27T18:18:49.166858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471342079153500:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(59801215ns) {Configure:104(59794709ns) {ProcessConfigured:135(760333ns) {StartInitialHandshake:157(755888ns) {DropHandshakes:470(1185ns) {DropIncomingHandshake:442(356ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(383ns) PrepareNewSessionHandshake:477(2175ns) {SwitchToState:179(970ns)}}}}}
2025-11-27T18:18:49.169703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471342079153499:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(84481394ns) {Configure:104(84475289ns) {ProcessConfigured:135(515913ns) {StartInitialHandshake:157(513390ns) {DropHandshakes:470(1251ns) {DropIncomingHandshake:442(316ns) DropOutgoingHandshake:457(180ns)} GenerateSessionVirtualId:988(482ns) PrepareNewSessionHandshake:477(2098ns) {SwitchToState:179(1041ns)}}}}}
2025-11-27T18:18:49.170375Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471337761263466:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(85894377ns) {Configure:104(85884875ns) {ProcessConfigured:135(1236912ns) {StartInitialHandshake:157(1235431ns) {DropHandshakes:470(1074ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(651ns) PrepareNewSessionHandshake:477(1034ns) {SwitchToState:179(83ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.170715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471342079153496:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(84909679ns) {Configure:104(84525967ns) {ProcessConfigured:135(1054709ns) {StartInitialHandshake:157(1049088ns) {DropHandshakes:470(2167ns) {DropIncomingHandshake:442(900ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(684ns) PrepareNewSessionHandshake:477(2669ns) {SwitchToState:179(858ns)}}}}}
2025-11-27T18:18:49.171558Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471334896130619:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21134671ns) {Configure:104(21127982ns) {ProcessConfigured:135(591774ns) {StartInitialHandshake:157(589955ns) {DropHandshakes:470(886ns) {DropIncomingHandshake:442(232ns) DropOutgoingHandshake:457(173ns)} GenerateSessionVirtualId:988(764ns) PrepareNewSessionHandshake:477(1611ns) {SwitchToState:179(621ns)}}}}}
2025-11-27T18:18:49.170430Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471342056230824:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17263180ns) {Configure:104(17257419ns) {ProcessConfigured:135(52981ns) {StartInitialHandshake:157(51537ns) {DropHandshakes:470(840ns) {DropIncomingHandshake:442(238ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:477(936ns) {SwitchToState:179(516ns)}}}}}
2025-11-27T18:18:49.170864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471337761263467:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(85282123ns) {Configure:104(85277423ns) {ProcessConfigured:135(2430905ns) {StartInitialHandshake:157(2242523ns) {DropHandshakes:470(709ns) {DropIncomingHandshake:442(246ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(424ns) PrepareNewSessionHandshake:477(1922ns) {SwitchToState:179(699ns)}}}}}
2025-11-27T18:18:49.172191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471342079153508:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27639
2025-11-27T18:18:49.173695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471337211694892:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24212661ns) {Configure:104(24205258ns) {ProcessConfigured:135(351148ns) {StartInitialHandshake:157(349636ns) {DropHandshakes:470(857ns) {DropIncomingHandshake:442(306ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(252ns) PrepareNewSessionHandshake:477(1671ns) {SwitchToState:179(879ns)}}}}}
2025-11-27T18:18:49.171884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471334896130620:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20952074ns) {Configure:104(20945607ns) {ProcessConfigured:135(1088415ns) {StartInitialHandshake:157(1085311ns) {DropHandshakes:470(4470ns) {DropIncomingHandshake:442(282ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(480ns) PrepareNewSessionHandshake:477(3827ns) {SwitchToState:179(1753ns)}}}}}
2025-11-27T18:18:49.174598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471335650297384:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25863791ns) {Configure:104(25859558ns) {ProcessConfigured:135(1666961ns) {StartInitialHandshake:157(1665382ns) {DropHandshakes:470(1155ns) {DropIncomingHandshake:442(341ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(701ns) PrepareNewSessionHandshake:477(1435ns) {SwitchToState:179(537ns)}}}}}
2025-11-27T18:18:49.182188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471338173835340:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24101074ns) {Configure:104(24094969ns) {ProcessConfigured:135(1399902ns) {StartInitialHandshake:157(1398216ns) {DropHandshakes:470(540402ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(1677ns)} GenerateSessionVirtualId:988(1803ns) PrepareNewSessionHandshake:477(1163ns) {SwitchToState:179(529ns)}}}}}
2025-11-27T18:18:49.174436Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471337211694891:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(79881116ns) {Configure:104(79876387ns) {ProcessConfigured:135(642480ns) {StartInitialHandshake:157(640486ns) {DropHandshakes:470(170753ns) {DropIncomingHandshake:442(1040ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(571ns) PrepareNewSessionHandshake:477(1414ns) {SwitchToState:179(600ns)}}}}}
2025-11-27T18:18:49.171427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471342056230827:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20130806ns) {Configure:104(20124147ns) {ProcessConfigured:135(600785ns) {StartInitialHandshake:157(599079ns) {DropHandshakes:470(1165ns) {DropIncomingHandshake:442(325ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(489ns) PrepareNewSessionHandshake:477(1219ns) {SwitchToState:179(595ns)}}}}}
2025-11-27T18:18:49.174601Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471335650297382:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21155319ns) {Configure:104(21151340ns) {ProcessConfigured:135(1284079ns) {StartInitialHandshake:157(1282121ns) {DropHandshakes:470(755ns) {DropIncomingHandshake:442(118ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(442ns) PrepareNewSessionHandshake:477(1253ns) {SwitchToState:179(182ns)}}}}}
2025-11-27T18:18:49.174639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471341506662251:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22361326ns) {Configure:104(22357449ns) {ProcessConfigured:135(1968513ns) {StartInitialHandshake:157(1966339ns) {DropHandshakes:470(1407ns) {DropIncomingHandshake:442(545ns) DropOutgoingHandshake:457(327ns)} GenerateSessionVirtualId:988(543ns) PrepareNewSessionHandshake:477(2190ns) {SwitchToState:179(1019ns)}}}}}
2025-11-27T18:18:49.185677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471342468802699:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29486265ns) {Configure:104(29478035ns) {ProcessConfigured:135(1289760ns) {StartInitialHandshake:157(1288490ns) {DropHandshakes:470(525975ns) {DropIncomingHandshake:442(1430ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(1526ns) PrepareNewSessionHandshake:477(1658ns) {SwitchToState:179(672ns)}}}}}
2025-11-27T18:18:49.186180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471339191097977:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22933618ns) {Configure:104(22929186ns) {ProcessConfigured:135(1398419ns) {StartInitialHandshake:157(1396930ns) {DropHandshakes:470(1226ns) {DropIncomingHandshake:442(338ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(535ns) PrepareNewSessionHandshake:477(1890ns) {SwitchToState:179(897ns)}}}}}
2025-11-27T18:18:49.178137Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471341506662252:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21460535ns) {Configure:104(21456355ns) {ProcessConfigured:135(1507053ns) {StartInitialHandshake:157(1505401ns) {DropHandshakes:470(250741ns) {DropIncomingHandshake:442(356ns) DropOutgoingHandshake:457(482ns)} GenerateSessionVirtualId:988(337ns) PrepareNewSessionHandshake:477(1819ns) {SwitchToState:179(837ns)}}}}}
2025-11-27T18:18:49.181645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471335650297383:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27370502ns) {Configure:104(27362491ns) {ProcessConfigured:135(819043ns) {StartInitialHandshake:157(817046ns) {DropHandshakes:470(120589ns) {DropIncomingHandshake:442(414ns) DropOutgoingHandshake:457(2105ns)} GenerateSessionVirtualId:988(1374ns) PrepareNewSessionHandshake:477(2086ns) {SwitchToState:179(931ns)}}}}}
2025-11-27T18:18:49.186302Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471342079153508:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.186014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471338173835341:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(32656947ns) {Configure:104(32643231ns) {ProcessConfigured:135(3602995ns) {StartInitialHandshake:157(3601229ns) {DropHandshakes:470(1156ns) {DropIncomingHandshake:442(455ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:477(5553ns) {SwitchToState:179(4074ns)}}}}}
2025-11-27T18:18:49.186033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471338173835342:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29619235ns) {Configure:104(29612768ns) {ProcessConfigured:135(1519479ns) {StartInitialHandshake:157(1517879ns) {DropHandshakes:470(1354ns) {DropIncomingHandshake:442(284ns) DropOutgoingHandshake:457(248ns)} GenerateSessionVirtualId:988(388ns) PrepareNewSessionHandshake:477(1588ns) {SwitchToState:179(485ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.192855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471339191097993:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49272
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.197617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471342056230837:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39757
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.194220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471342079153517:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27639
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.198170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471342056230841:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39757
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.199846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471342468802709:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7666
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.200425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471339191097992:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49272
2025-11-27T18:18:49.199942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471339945264753:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33585
2025-11-27T18:18:49.200458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471341506662262:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7422
2025-11-27T18:18:49.205902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471342056230842:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39757
2025-11-27T18:18:49.206763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471342079153516:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27639
2025-11-27T18:18:49.202788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471342468802700:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16994024ns) {Configure:104(16986090ns) {ProcessConfigured:135(46470ns) {StartInitialHandshake:157(42657ns) {DropHandshakes:470(2431ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(592ns) PrepareNewSessionHandshake:477(960ns) {SwitchToState:179(363ns)}}}}}
2025-11-27T18:18:49.203083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471342468802709:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.206449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471339945264753:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.206980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471342056230828:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28634517ns) {Configure:104(28627992ns) {ProcessConfigured:135(2423931ns) {StartInitialHandshake:157(2420732ns) {DropHandshakes:470(2723ns) {DropIncomingHandshake:442(1257ns) DropOutgoingHandshake:457(244ns)} GenerateSessionVirtualId:988(384ns) PrepareNewSessionHandshake:477(1952ns) {SwitchToState:179(596ns)}}}}}
2025-11-27T18:18:49.206870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471339191097992:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.206914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471341506662262:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.209521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471342079153502:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.208499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471335650297382:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.209548Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471342079153502:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.203273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471342468802712:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7666
2025-11-27T18:18:49.208606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471337211694891:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.206807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471342468802715:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7666
2025-11-27T18:18:49.214307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471342056230837:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.213316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471342079153506:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.214656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471341506662266:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7422
2025-11-27T18:18:49.209107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471342468802715:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.216335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471339191097993:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.213348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471342079153506:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.214638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471342079153502:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:49.216416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471341506662253:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36215399ns) {Configure:104(36208547ns) {ProcessConfigured:135(4228106ns) {StartInitialHandshake:157(4225089ns) {DropHandshakes:470(1984ns) {DropIncomingHandshake:442(230ns) DropOutgoingHandshake:457(424ns)} GenerateSessionVirtualId:988(395ns) PrepareNewSessionHandshake:477(2882ns) {SwitchToState:179(1608ns)}}}}}
2025-11-27T18:18:49.214673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471342056230842:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.217930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471342079153518:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27639
2025-11-27T18:18:49.218043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471342468802712:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.214681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471342056230841:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.217969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471339191097995:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49272
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.217717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471341506662266:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.218162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471338173835341:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.221287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471342079153516:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.219664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471334896130620:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.217361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471342056230843:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39757
2025-11-27T18:18:49.222988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471342468802705:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.218795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471337211694892:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.221349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471342079153506:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:49.217701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471342056230843:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.220434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471339191097995:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.223019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471342468802705:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.222815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471341506662260:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.220698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471339191097978:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.217768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471342056230846:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39757
2025-11-27T18:18:49.219388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471337761263466:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.222455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471337784186140:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.222838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471341506662260:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.222495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:49.224465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471341506662271:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7422
2025-11-27T18:18:49.220605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471342056230832:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.228805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471341506662262:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:49.230376Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471341506662260:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:49.234509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471337211694891:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.234577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:49.234618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471337211694891:2048] [node 1] ICP22 created new session: [3:7577471341506662274:2048]
2025-11-27T18:18:49.230769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471339945264758:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33585
2025-11-27T18:18:49.220633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471342056230832:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.231407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471342468802705:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.222828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471342079153504:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.220641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471342056230846:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.236176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471341506662251:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.222860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471342079153504:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.221001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471342056230834:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.236235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471341506662274:2048] [node 1] ICS09 handshake done sender: [3:7577471341506662262:2115] self: [3:7577471341506662267:0] peer: [1:7577471342079153501:0] socket: 573 qp: -1
2025-11-27T18:18:49.242980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471342468802720:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7666
2025-11-27T18:18:49.223645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471337784186140:2049] [node 3] ICP22 created new session: [1:7577471342079153521:2049]
2025-11-27T18:18:49.245148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471339945264753:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.245902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471339191098001:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49272
2025-11-27T18:18:49.221026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471342056230834:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.224363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471342079153521:2049] [node 3] ICS09 handshake done sender: [1:7577471342079153502:2110] self: [1:7577471342079153501:0] peer: [3:7577471341506662267:0] socket: 568 qp: -1
2025-11-27T18:18:49.236276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471341506662274:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.251832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471335650297382:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.253540Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471338173835341:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19867) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471342468802712:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.253056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471339191097999:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49272
2025-11-27T18:18:49.222851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471342056230832:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:49.236304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:49.224500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471342079153517:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.252983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: 2025-11-27T18:18:49.226388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471342056230834:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:49.257023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471342468802699:2108] [node 5] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:18:49.264515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471339191097998:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49272
2025-11-27T18:18:49.225133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471342079153521:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:49.256599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471339945264760:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33585
2025-11-27T18:18:49.229227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471342056230824:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.238765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471341506662273:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7422
2025-11-27T18:18:49.257624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:49.243300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:49.256661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471339945264761:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33585
2025-11-27T18:18:49.270970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471339191098000:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49272
2025-11-27T18:18:49.225176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471342079153500:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.258765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471342468802712:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:49.271523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471339191097978:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61981) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471339191097995:2117] held: no
2025-11-27T18:18:49.230429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:49.225810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:49.257621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471335650297382:2048] [node 1] ICP22 created new session: [6:7577471339945264759:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.259102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471342468802699:2108] [node 5] ICP22 created new session: [4:7577471342468802723:2108]
2025-11-27T18:18:49.244793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471341506662251:2109] [node 5] ICP22 created new session: [3:7577471341506662276:2109]
2025-11-27T18:18:49.272790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471339191098001:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.230469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471342056230824:2107] [node 4] ICP22 created new session: [2:7577471342056230854:2107]
2025-11-27T18:18:49.245110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471341506662266:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:49.225874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:49.259115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471339945264759:2048] [node 1] ICS09 handshake done sender: [6:7577471339945264753:2115] self: [6:7577471339945264757:0] peer: [1:7577471342079153505:0] socket: 575 qp: -1
2025-11-27T18:18:49.260544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471338173835341:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.260579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:49.272840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471339191097995:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:49.262078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471339945264759:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.227448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471342079153500:2109] [node 6] ICP22 created new session: [1:7577471342079153523:2109]
2025-11-27T18:18:49.231340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471342056230854:2107] [node 4] ICS09 handshake done sender: [2:7577471342056230832:2111] self: [2:7577471342056230831:0] peer: [4:7577471342468802719:0] socket: 579 qp: -1
2025-11-27T18:18:49.246170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471341506662276:2109] [node 5] ICS09 handshake done sender: [3:7577471341506662260:2113] self: [3:7577471341506662254:0] peer: [5:7577471339191097996:0] socket: 591 qp: -1
2025-11-27T18:18:49.277529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471339191098000:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.234997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471337761263467:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.260619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471338173835341:2049] [node 2] ICP22 created new session: [4:7577471342468802725:2049]
No RDMA context found for GID: 2025-11-27T18:18:49.263391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
::ffff:127.1.0.0
2025-11-27T18:18:49.229294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471342079153518:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.277546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471339191097993:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:49.248499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471341506662276:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:49.270465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471339945264760:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.235931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471342056230854:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:49.230303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471342079153523:2109] [node 6] ICS09 handshake done sender: [1:7577471342079153506:2112] self: [1:7577471342079153505:0] peer: [6:7577471339945264757:0] socket: 570 qp: -1
2025-11-27T18:18:49.260659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471342468802725:2049] [node 2] ICS09 handshake done sender: [4:7577471342468802712:2115] self: [4:7577471342468802719:0] peer: [2:7577471342056230831:0] socket: 598 qp: -1
2025-11-27T18:18:49.253084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:49.289818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471334896130620:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.235971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:49.277320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471339945264761:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.230845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471342079153523:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:49.254250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471337211694892:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19867) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471341506662266:2116] held: no
2025-11-27T18:18:49.260687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471342468802725:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:49.291152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:49.230881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:49.236173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:49.277511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471335650297383:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.259676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471337211694892:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.231213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471342079153504:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:49.260738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:49.281688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471335650297384:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.239342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471337761263467:2049] [node 3] ICP22 created new session: [2:7577471342056230855:2049]
2025-11-27T18:18:49.259724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:49.239361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471337784186139:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.295239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471339191097978:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.261232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471342468802722:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7666
2025-11-27T18:18:49.244785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471342056230855:2049] [node 3] ICS09 handshake done sender: [2:7577471342056230834:2113] self: [2:7577471342056230830:0] peer: [3:7577471341506662270:0] socket: 593 qp: -1
2025-11-27T18:18:49.263356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471337211694892:2049] [node 2] ICP22 created new session: [3:7577471341506662280:2049]
2025-11-27T18:18:49.296720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471335650297382:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32898495ns) {HandleHandshakeStatus:329(32894633ns) {DropIncomingHandshake:442(822ns) DropOutgoingHandshake:457(15718ns) SwitchToState:179(426ns) ProcessPendingSessionEvents:495(14219541ns) {ForwardSessionEventToSession:612(1239582ns) ForwardSessionEventToSession:612(3831ns) ForwardSessionEventToSession:612(3168ns) ForwardSessionEventToSession:612(223282ns) ForwardSessionEventToSession:612(2511ns) ForwardSessionEventToSession:612(2908ns) ForwardSessionEventToSession:612(12716099ns) ForwardSessionEventToSession:612(4364ns) ForwardSessionEventToSession:612(8799ns) ForwardSessionEventToSession:612(3046ns)}}}
2025-11-27T18:18:49.243295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:49.265639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471341506662280:2049] [node 2] ICS09 handshake done sender: [3:7577471341506662266:2116] self: [3:7577471341506662270:0] peer: [2:7577471342056230830:0] socket: 594 qp: -1
2025-11-27T18:18:49.244837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471342056230855:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:49.295282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:49.268152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471341506662280:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:49.243347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471337784186139:2048] [node 2] ICP22 created new session: [1:7577471342079153530:2048]
2025-11-27T18:18:49.261506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471342468802724:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7666
2025-11-27T18:18:49.296784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471339945264764:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33585
2025-11-27T18:18:49.295321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471339191097978:2108] [node 4] ICP22 created new session: [5:7577471339191098003:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.268201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:49.300159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471339945264767:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33585
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.246132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:49.251895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471342079153530:2048] [node 2] ICS09 handshake done sender: [1:7577471342079153504:2111] self: [1:7577471342079153503:0] peer: [2:7577471342056230847:0] socket: 569 qp: -1
2025-11-27T18:18:49.263320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471342468802723:2108] [node 5] ICS09 handshake done sender: [4:7577471342468802705:2110] self: [4:7577471342468802703:0] peer: [5:7577471339191097997:0] socket: 600 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.295342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471334896130619:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.251666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471342056230857:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39757
2025-11-27T18:18:49.295373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471339191098003:2108] [node 4] ICS09 handshake done sender: [5:7577471339191097995:2117] self: [5:7577471339191097997:0] peer: [4:7577471342468802703:0] socket: 601 qp: -1
2025-11-27T18:18:49.259176Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471337761263466:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471342056230837:2115] held: no
2025-11-27T18:18:49.254564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471342079153530:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:49.263360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471342468802723:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.289578Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471341506662251:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(34999892ns) {HandleHandshakeStatus:329(34997386ns) {DropIncomingHandshake:442(479ns) DropOutgoingHandshake:457(382ns) SwitchToState:179(209ns) ProcessPendingSessionEvents:495(7576034ns) {ForwardSessionEventToSession:612(2074150ns) ForwardSessionEventToSession:612(4340ns) ForwardSessionEventToSession:612(576809ns) ForwardSessionEventToSession:612(1399883ns) ForwardSessionEventToSession:612(1191735ns) ForwardSessionEventToSession:612(6072ns) ForwardSessionEventToSession:612(3080ns) ForwardSessionEventToSession:612(563558ns) ForwardSessionEventToSession:612(3862ns) ForwardSessionEventToSession:612(3126ns)}}}
2025-11-27T18:18:49.300353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471339945264764:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.254604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:49.264108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:49.291014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471341506662269:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.264884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471337761263467:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(29698049ns) {HandleHandshakeStatus:329(29685611ns) {DropIncomingHandshake:442(12300ns) DropOutgoingHandshake:457(243ns) SwitchToState:179(229ns) ProcessPendingSessionEvents:495(7443631ns) {ForwardSessionEventToSession:612(982736ns) ForwardSessionEventToSession:612(4514ns) ForwardSessionEventToSession:612(3141ns) ForwardSessionEventToSession:612(1361513ns) ForwardSessionEventToSession:612(3023ns) ForwardSessionEventToSession:612(2380ns) ForwardSessionEventToSession:612(4000501ns) ForwardSessionEventToSession:612(4551ns) ForwardSessionEventToSession:612(1063261ns) ForwardSessionEventToSession:612(3942ns)}}}
2025-11-27T18:18:49.295402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471339191098003:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:49.301406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471339945264733:2096] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.257060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471337784186140:2049] [node 3] ICP18 handshake (actor [1:7577471342079153517:2117]) is too old
2025-11-27T18:18:49.264134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471342468802722:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.295425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.302490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471339945264760:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:49.291055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471341506662269:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.265617Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471342056230824:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(28340088ns) {HandleHandshakeStatus:329(27938992ns) {DropIncomingHandshake:442(12673ns) DropOutgoingHandshake:457(318ns) SwitchToState:179(2267ns) ProcessPendingSessionEvents:495(13945606ns) {ForwardSessionEventToSession:612(4648598ns) ForwardSessionEventToSession:612(5666ns) ForwardSessionEventToSession:612(3979953ns) ForwardSessionEventToSession:612(1199309ns) ForwardSessionEventToSession:612(5162ns) ForwardSessionEventToSession:612(1498055ns) ForwardSessionEventToSession:612(3152ns) ForwardSessionEventToSession:612(2681ns) ForwardSessionEventToSession:612(2584847ns) ForwardSessionEventToSession:612(3185ns)}}}
2025-11-27T18:18:49.267562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471338173835340:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.295656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471334896130620:2050] [node 3] ICP22 created new session: [5:7577471339191098002:2050]
2025-11-27T18:18:49.262894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471337784186139:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(25751082ns) {HandleHandshakeStatus:329(23347086ns) {DropIncomingHandshake:442(13094ns) DropOutgoingHandshake:457(201ns) SwitchToState:179(4608ns) ProcessPendingSessionEvents:495(3199953ns) {ForwardSessionEventToSession:612(3145946ns) ForwardSessionEventToSession:612(4344ns) ForwardSessionEventToSession:612(14817ns) ForwardSessionEventToSession:612(4702ns) ForwardSessionEventToSession:612(2759ns) ForwardSessionEventToSession:612(4673ns) ForwardSessionEventToSession:612(2317ns) ForwardSessionEventToSession:612(2218ns) ForwardSessionEventToSession:612(4449ns) ForwardSessionEventToSession:612(2654ns)}}}
2025-11-27T18:18:49.268904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471342056230837:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:49.295576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471341506662282:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7422
2025-11-27T18:18:49.304727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471335650297383:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.265585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471342079153500:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(29228023ns) {HandleHandshakeStatus:329(29226013ns) {DropIncomingHandshake:442(201ns) DropOutgoingHandshake:457(311ns) SwitchToState:179(211ns) ProcessPendingSessionEvents:495(17495436ns) {ForwardSessionEventToSession:612(9239242ns) ForwardSessionEventToSession:612(12552ns) ForwardSessionEventToSession:612(1762318ns) ForwardSessionEventToSession:612(4675ns) ForwardSessionEventToSession:612(2521ns) ForwardSessionEventToSession:612(3808124ns) ForwardSessionEventToSession:612(3583ns) ForwardSessionEventToSession:612(2153ns) ForwardSessionEventToSession:612(2623957ns) ForwardSessionEventToSession:612(9542ns)}}}
2025-11-27T18:18:49.277250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471342468802724:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.296733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471339191098002:2050] [node 3] ICS09 handshake done sender: [5:7577471339191097993:2116] self: [5:7577471339191097996:0] peer: [3:7577471341506662254:0] socket: 592 qp: -1
2025-11-27T18:18:49.271619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471337761263466:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.298323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471341506662261:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.271664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:49.307109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:49.281485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471338173835342:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.298349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471341506662261:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.271752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471342079153513:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.296840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471334896130618:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.298826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471339191098002:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:49.303434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471341506662283:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7422
2025-11-27T18:18:49.303461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471341506662282:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.298864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:49.303909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471341506662269:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:49.319684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471339191098010:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49272
2025-11-27T18:18:49.272202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471337761263466:2048] [node 1] ICP22 created new session: [2:7577471342056230861:2048]
2025-11-27T18:18:49.277578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471342056230861:2048] [node 1] ICS09 handshake done sender: [2:7577471342056230837:2115] self: [2:7577471342056230847:0] peer: [1:7577471342079153503:0] socket: 572 qp: -1
2025-11-27T18:18:49.319973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471334896130619:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19867) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471339191098001:2121] held: no
2025-11-27T18:18:49.285120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471342056230861:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:49.320066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471334896130618:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471339191098000:2120] held: no
2025-11-27T18:18:49.307822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471335650297383:2049] [node 2] ICP22 created new session: [6:7577471339945264771:2049]
2025-11-27T18:18:49.321456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471339191098009:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49272
2025-11-27T18:18:49.285172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:49.303689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471342468802714:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.271786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471342079153513:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.302316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471342079153513:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:49.303912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471341506662261:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:49.322195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471339191098000:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:49.295099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471342056230845:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.302360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471342079153512:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.324201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471339191098001:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:49.295146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471342056230845:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.311487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471339945264771:2049] [node 2] ICS09 handshake done sender: [6:7577471339945264760:2117] self: [6:7577471339945264765:0] peer: [2:7577471342056230844:0] socket: 580 qp: -1
2025-11-27T18:18:49.302384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471342079153512:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.297809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471342056230845:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:49.303717Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471342468802714:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.303990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471342079153496:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.324287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471334896130618:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.311835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471339945264771:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:49.299455Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471337761263466:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23969719ns) {HandleHandshakeStatus:329(23967521ns) {DropIncomingHandshake:442(541ns) DropOutgoingHandshake:457(271ns) SwitchToState:179(413ns) ProcessPendingSessionEvents:495(1506ns)}}
2025-11-27T18:18:49.304046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:49.311105Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471338173835340:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471342468802722:2119] held: no
2025-11-27T18:18:49.299501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471342056230836:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.304687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471342079153496:2107] [node 4] ICP22 created new session: [1:7577471342079153533:2107]
2025-11-27T18:18:49.312834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471342468802714:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:49.299537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471342056230836:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.305800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471342079153533:2107] [node 4] ICS09 handshake done sender: [1:7577471342079153513:2115] self: [1:7577471342079153507:0] peer: [4:7577471342468802730:0] socket: 620 qp: -1
2025-11-27T18:18:49.314220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471342468802732:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7666
2025-11-27T18:18:49.302145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471342056230828:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.306037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471342079153512:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:49.318082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471342468802700:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.302315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:49.311887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:49.318133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471342468802722:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:49.307262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471342079153533:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:49.303332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471342056230828:2110] [node 6] ICP22 created new session: [2:7577471342056230864:2110]
2025-11-27T18:18:49.318330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:49.317157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471339945264769:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33585
2025-11-27T18:18:49.303531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471342056230864:2110] [node 6] ICS09 handshake done sender: [2:7577471342056230845:2119] self: [2:7577471342056230844:0] peer: [6:7577471339945264765:0] socket: 584 qp: -1
2025-11-27T18:18:49.318309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471339945264770:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33585
2025-11-27T18:18:49.318652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471342468802700:2109] [node 6] ICP22 created new session: [4:7577471342468802735:2109]
2025-11-27T18:18:49.303570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471342056230864:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:49.320351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471335650297384:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65052) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471339945264761:2118] held: no
2025-11-27T18:18:49.303609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:49.318713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471342468802734:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7666
2025-11-27T18:18:49.307125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471342056230836:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:49.319968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471342468802735:2109] [node 6] ICS09 handshake done sender: [4:7577471342468802714:2116] self: [4:7577471342468802713:0] peer: [6:7577471339945264768:0] socket: 626 qp: -1
2025-11-27T18:18:49.310938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471342056230827:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.320309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471342468802735:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:49.310992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:49.320818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:49.311346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471342056230827:2108] [node 5] ICP22 created new session: [2:7577471342056230867:2108]
2025-11-27T18:18:49.322121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471338173835340:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.312609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471342056230867:2108] [node 5] ICS09 handshake done sender: [2:7577471342056230836:2114] self: [2:7577471342056230835:0] peer: [5:7577471339191098004:0] socket: 621 qp: -1
2025-11-27T18:18:49.323688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:49.313914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471342056230867:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:49.323989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471338173835340:2048] [node 1] ICP22 created new session: [4:7577471342468802737:2048]
2025-11-27T18:18:49.313962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:49.325415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471342468802724:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:49.321306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471335650297383:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:49.325682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471342468802737:2048] [node 1] ICS09 handshake done sender: [4:7577471342468802722:2119] self: [4:7577471342468802730:0] peer: [1:7577471342079153507:0] socket: 590 qp: -1
2025-11-27T18:18:49.322349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471339945264769:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.307309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:49.325412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:49.309405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471341506662283:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.327099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471342468802737:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:49.323733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471339945264761:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:49.325953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471339945264773:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33585
2025-11-27T18:18:49.325732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471334896130618:2048] [node 1] ICP22 created new session: [5:7577471339191098011:2048]
2025-11-27T18:18:49.310873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471341506662253:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.311402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:49.310875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471341506662252:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.311817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471341506662253:2110] [node 6] ICP22 created new session: [3:7577471341506662287:2110]
2025-11-27T18:18:49.311922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:49.312137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471341506662252:2107] [node 4] ICP22 created new session: [3:7577471341506662288:2107]
2025-11-27T18:18:49.312597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471341506662287:2110] [node 6] ICS09 handshake done sender: [3:7577471341506662269:2117] self: [3:7577471341506662268:0] peer: [6:7577471339945264766:0] socket: 624 qp: -1
2025-11-27T18:18:49.312841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471341506662288:2107] [node 4] ICS09 handshake done sender: [3:7577471341506662261:2114] self: [3:7577471341506662259:0] peer: [4:7577471342468802731:0] socket: 623 qp: -1
2025-11-27T18:18:49.312889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471341506662287:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:49.313873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471341506662288:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:49.313909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:49.314084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:49.321727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471341506662252:2107] [node 4] ICP18 handshake (actor [3:7577471341506662283:2121]) is too old
2025-11-27T18:18:49.308314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471342079153499:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.327887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471339945264733:2096] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61981) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471339945264764:2119] held: no
2025-11-27T18:18:49.308369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:49.330295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471335650297384:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.308427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471342079153499:2108] [node 5] ICP22 created new session: [1:7577471342079153536:2108]
2025-11-27T18:18:49.311859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471342079153536:2108] [node 5] ICS09 handshake done sender: [1:7577471342079153512:2114] self: [1:7577471342079153511:0] peer: [5:7577471339191098006:0] socket: 604 qp: -1
2025-11-27T18:18:49.330326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:49.311892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471342079153536:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:49.331633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471339945264764:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:49.312527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:49.331714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471335650297384:2050] [node 3] ICP22 created new session: [6:7577471339945264774:2050]
2025-11-27T18:18:49.332587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471339945264774:2050] [node 3] ICS09 handshake done sender: [6:7577471339945264761:2118] self: [6:7577471339945264766:0] peer: [3:7577471341506662268:0] socket: 597 qp: -1
2025-11-27T18:18:49.332893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471339945264774:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:49.332924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:49.327154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:49.333293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471339945264733:2096] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.328989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471338173835342:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65052) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471342468802724:2120] held: no
2025-11-27T18:18:49.327085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471339191098011:2048] [node 1] ICS09 handshake done sender: [5:7577471339191098000:2120] self: [5:7577471339191098006:0] peer: [1:7577471342079153511:0] socket: 588 qp: -1
2025-11-27T18:18:49.333336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:49.327165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471334896130619:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.330625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471338173835342:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.337665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471339945264733:2096] [node 4] ICP22 created new session: [6:7577471339945264776:2096]
2025-11-27T18:18:49.328099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471339191098011:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:49.339588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471339945264730:2099] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:49.330663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:49.328651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:49.339681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471339945264776:2096] [node 4] ICS09 handshake done sender: [6:7577471339945264764:2119] self: [6:7577471339945264768:0] peer: [4:7577471342468802713:0] socket: 627 qp: -1
2025-11-27T18:18:49.339733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471339945264776:2096] [node 4] ICS10 traffic start
2025-11-27T18:18:49.331409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471338173835342:2050] [node 3] ICP22 created new session: [4:7577471342468802739:2050]
2025-11-27T18:18:49.329244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:49.329430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471334896130619:2049] [node 2] ICP22 created new session: [5:7577471339191098013:2049]
2025-11-27T18:18:49.331847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471342468802739:2050] [node 3] ICS09 handshake done sender: [4:7577471342468802724:2120] self: [4:7577471342468802731:0] peer: [3:7577471341506662259:0] socket: 589 qp: -1
2025-11-27T18:18:49.339771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:49.330440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471339191098013:2049] [node 2] ICS09 handshake done sender: [5:7577471339191098001:2121] self: [5:7577471339191098004:0] peer: [2:7577471342056230835:0] socket: 622 qp: -1
2025-11-27T18:18:49.331890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471342468802739:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:49.331484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471339191098013:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:49.331918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:49.331793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:49.340650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471339191097990:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:49.340670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471339191097990:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:49.342916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471339191097990:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:49.343143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471339191097977:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:49.343217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:49.343273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471339191097977:2109] [node 6] ICP22 created new session: [5:7577471339191098016:2109]
2025-11-27T18:18:49.343323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471339191098016:2109] [node 6] ICS09 handshake done sender: [5:7577471339191097990:2114] self: [5:7577471339191097989:0] peer: [6:7577471339945264777:0] socket: 632 qp: -1
2025-11-27T18:18:49.343351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471339191098016:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:49.343413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:49.343100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471339945264779:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33585
2025-11-27T18:18:49.343434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471339945264769:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:49.343828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471339945264730:2099] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30912) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471339945264769:2121] held: no
2025-11-27T18:18:49.343992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471339945264730:2099] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:49.344040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:49.344086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471339945264730:2099] [node 5] ICP22 created new session: [6:7577471339945264780:2099]
2025-11-27T18:18:49.344124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471339945264780:2099] [node 5] ICS09 handshake done sender: [6:7577471339945264769:2121] self: [6:7577471339945264777:0] peer: [5:7577471339191097989:0] socket: 633 qp: -1
2025-11-27T18:18:49.344149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471339945264780:2099] [node 5] ICS10 traffic start
2025-11-27T18:18:49.344192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:49.391261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:49.391500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471342056230861:2048] [node 1] ICS01 socket: 572 reason# EndOfStream
2025-11-27T18:18:49.391560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471337761263466:2048] [node 1] ICP30 unregister session Session# [2:7577471342056230861:2048] VirtualId# [2:7577471342056230847:0]
2025-11-27T18:18:49.391595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471342056230861:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.391679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:49.391742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:49.407021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:49.407402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471341506662274:2048] [node 1] ICS01 socket: 573 reason# EndOfStream
2025-11-27T18:18:49.407457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471337211694891:2048] [node 1] ICP30 unregister session Session# [3:7577471341506662274:2048] VirtualId# [3:7577471341506662267:0]
2025-11-27T18:18:49.407487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471341506662274:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.407582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:49.407642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:49.408254Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:49.408427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471342468802737:2048] [node 1] ICS01 socket: 590 reason# EndOfStream
2025-11-27T18:18:49.408470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471338173835340:2048] [node 1] ICP30 unregister session Session# [4:7577471342468802737:2048] VirtualId# [4:7577471342468802730:0]
2025-11-27T18:18:49.408498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471342468802737:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.408567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:49.408613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:49.409395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:49.409593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471339191098011:2048] [node 1] ICS01 socket: 588 reason# EndOfStream
2025-11-27T18:18:49.409639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471334896130618:2048] [node 1] ICP30 unregister session Session# [5:7577471339191098011:2048] VirtualId# [5:7577471339191098006:0]
2025-11-27T18:18:49.409673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471339191098011:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.409751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:49.409783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:49.413491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:49.413879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471339945264759:2048] [node 1] ICS07 socket disconnect 575 reason# EndOfStream
2025-11-27T18:18:49.413961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471339945264759:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.414129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:49.414242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471339945264759:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:49.415465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471335650297382:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63339) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:49.415572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471339945264759:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:49.415775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471335650297382:2048] [node 1] ICP30 unregister session Session# [6:7577471339945264759:2048] VirtualId# [6:7577471339945264757:0]
2025-11-27T18:18:49.415825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:49.415872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471335650297382:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63339) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:49.415903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63339) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:49.425211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471335650297382:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:49.465977Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:49.466212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471341506662280:2049] [node 2] ICS01 socket: 594 reason# EndOfStream
2025-11-27T18:18:49.466259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471337211694892:2049] [node 2] ICP30 unregister session Session# [3:7577471341506662280:2049] VirtualId# [3:7577471341506662270:0]
2025-11-27T18:18:49.466296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471341506662280:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.466383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:49.466459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:49.469229Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:49.469476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471342468802725:2049] [node 2] ICS01 socket: 598 reason# EndOfStream
2025-11-27T18:18:49.469523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471338173835341:2049] [node 2] ICP30 unregister session Session# [4:7577471342468802725:2049] VirtualId# [4:7577471342468802719:0]
2025-11-27T18:18:49.469560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471342468802725:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.469637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:49.469696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:49.470277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:49.470505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471339191098013:2049] [node 2] ICS01 socket: 622 reason# EndOfStream
2025-11-27T18:18:49.470554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471334896130619:2049] [node 2] ICP30 unregister session Session# [5:7577471339191098013:2049] VirtualId# [5:7577471339191098004:0]
2025-11-27T18:18:49.470588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471339191098013:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.470658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:49.470730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:49.471127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:49.471347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471339945264771:2049] [node 2] ICS07 socket disconnect 580 reason# EndOfStream
2025-11-27T18:18:49.471378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471339945264771:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.471452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:49.471529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471339945264771:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:49.472304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471335650297383:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19867) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:49.472403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471339945264771:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:49.472460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471335650297383:2049] [node 2] ICP30 unregister session Session# [6:7577471339945264771:2049] VirtualId# [6:7577471339945264765:0]
2025-11-27T18:18:49.472503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:49.472548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471335650297383:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19867) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:49.472561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19867) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:49.482612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471335650297383:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:49.534512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:49.534744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471342468802739:2050] [node 3] ICS01 socket: 589 reason# EndOfStream
2025-11-27T18:18:49.534793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471338173835342:2050] [node 3] ICP30 unregister session Session# [4:7577471342468802739:2050] VirtualId# [4:7577471342468802731:0]
2025-11-27T18:18:49.534836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471342468802739:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.534923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:49.534980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:49.535534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:49.535731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471339191098002:2050] [node 3] ICS01 socket: 592 reason# EndOfStream
2025-11-27T18:18:49.535770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471334896130620:2050] [node 3] ICP30 unregister session Session# [5:7577471339191098002:2050] VirtualId# [5:7577471339191097996:0]
2025-11-27T18:18:49.535799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471339191098002:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.535866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:49.535920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:49.536636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:49.536835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471339945264774:2050] [node 3] ICS01 socket: 597 reason# EndOfStream
2025-11-27T18:18:49.536876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471335650297384:2050] [node 3] ICP30 unregister session Session# [6:7577471339945264774:2050] VirtualId# [6:7577471339945264766:0]
2025-11-27T18:18:49.536899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471339945264774:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.536952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:49.536982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:49.596435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:49.596669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471339191098003:2108] [node 4] ICS01 socket: 601 reason# EndOfStream
2025-11-27T18:18:49.596716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471339191097978:2108] [node 4] ICP30 unregister session Session# [5:7577471339191098003:2108] VirtualId# [5:7577471339191097997:0]
2025-11-27T18:18:49.596760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471339191098003:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.596845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:49.596889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:49.597597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:49.597850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471339945264776:2096] [node 4] ICS01 socket: 627 reason# EndOfStream
2025-11-27T18:18:49.597906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471339945264733:2096] [node 4] ICP30 unregister session Session# [6:7577471339945264776:2096] VirtualId# [6:7577471339945264768:0]
2025-11-27T18:18:49.597950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471339945264776:2096] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.598020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:49.598085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:49.655496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:49.655759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471339945264780:2099] [node 5] ICS01 socket: 633 reason# EndOfStream
2025-11-27T18:18:49.655810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471339945264730:2099] [node 5] ICP30 unregister session Session# [6:7577471339945264780:2099] VirtualId# [6:7577471339945264777:0]
2025-11-27T18:18:49.655842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471339945264780:2099] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:49.655929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:49.655987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:49.764095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471339089196701:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:49.764260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471339089196702:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:49.788762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471340794099314:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:49.789236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471340794099315:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:49.813110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471338912735079:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:49.813159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471338912735080:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:49.835927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471340228101942:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:49.836164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471340228101943:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:49.836536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471340228101944:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:49.860136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471338489123721:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:49.860364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471338489123723:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:49.860443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471338489123722:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:49.887925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471341898028977:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:49.887952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471341898028976:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:49.888203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471341898028978:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:50.006571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471341898029038:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:50.006508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471339089196764:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:50.038784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471344523069302:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:50.029928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471338489123784:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:50.038512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471343384164061:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:50.054526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471343384164062:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:50.054580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471343207702439:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:50.054523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471343207702440:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:50.050587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471346192996337:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:50.054648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471345089066674:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:50.050568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471342784091081:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:50.055859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471345089066676:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:50.057007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471345089066675:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:50.057755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471343207702441:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:50.057236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471344523069303:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:50.074416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471339089196702:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(55842148ns) {Configure:104(55834499ns) {ProcessConfigured:135(98826ns) {StartInitialHandshake:157(95972ns) {DropHandshakes:470(3099ns) {DropIncomingHandshake:442(992ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(1561ns) PrepareNewSessionHandshake:477(1416ns) {SwitchToState:179(481ns)}}}}}
2025-11-27T18:18:50.080536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471338912735080:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49804591ns) {Configure:104(49797035ns) {ProcessConfigured:135(838764ns) {StartInitialHandshake:157(836663ns) {DropHandshakes:470(1557ns) {DropIncomingHandshake:442(460ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(566ns) PrepareNewSessionHandshake:477(1426ns) {SwitchToState:179(634ns)}}}}}
2025-11-27T18:18:50.082524Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471338912735079:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(110969469ns) {Configure:104(110962457ns) {ProcessConfigured:135(587730ns) {StartInitialHandshake:157(584632ns) {DropHandshakes:470(3024ns) {DropIncomingHandshake:442(1526ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(1493ns) PrepareNewSessionHandshake:477(2387ns) {SwitchToState:179(1068ns)}}}}}
2025-11-27T18:18:50.081936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471339089196764:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(41613560ns) {Configure:104(41607939ns) {ProcessConfigured:135(987352ns) {StartInitialHandshake:157(754383ns) {DropHandshakes:470(1403ns) {DropIncomingHandshake:442(291ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(593ns) PrepareNewSessionHandshake:477(1492ns) {SwitchToState:179(230ns)}}}}}
2025-11-27T18:18:50.083217Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471338489123722:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44408500ns) {Configure:104(44403185ns) {ProcessConfigured:135(172480ns) {StartInitialHandshake:157(170324ns) {DropHandshakes:470(1320ns) {DropIncomingHandshake:442(463ns) DropOutgoingHandshake:457(304ns)} GenerateSessionVirtualId:988(528ns) PrepareNewSessionHandshake:477(1533ns) {SwitchToState:179(486ns)}}}}}
2025-11-27T18:18:50.083556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471344523069303:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23474907ns) {Configure:104(23469104ns) {ProcessConfigured:135(267664ns) {StartInitialHandshake:157(265776ns) {DropHandshakes:470(4495ns) {DropIncomingHandshake:442(367ns) DropOutgoingHandshake:457(212ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:477(2191ns) {SwitchToState:179(1295ns)}}}}}
2025-11-27T18:18:50.083601Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471339089196701:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(126671639ns) {Configure:104(126663901ns) {ProcessConfigured:135(2312519ns) {StartInitialHandshake:157(2310650ns) {DropHandshakes:470(943ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(213ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:477(1748ns) {SwitchToState:179(419ns)}}}}}
2025-11-27T18:18:50.083470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471343207702441:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23421633ns) {Configure:104(23415603ns) {ProcessConfigured:135(996811ns) {StartInitialHandshake:157(994888ns) {DropHandshakes:470(1304ns) {DropIncomingHandshake:442(477ns) DropOutgoingHandshake:457(448ns)} GenerateSessionVirtualId:988(527ns) PrepareNewSessionHandshake:477(1562ns) {SwitchToState:179(774ns)}}}}}
2025-11-27T18:18:50.083996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471341898028976:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(65125551ns) {Configure:104(65116935ns) {ProcessConfigured:135(2291707ns) {StartInitialHandshake:157(2290147ns) {DropHandshakes:470(1420ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(466ns) PrepareNewSessionHandshake:477(1166ns) {SwitchToState:179(382ns)}}}}}
2025-11-27T18:18:50.083759Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471342784091081:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25986394ns) {Configure:104(25979925ns) {ProcessConfigured:135(33948ns) {StartInitialHandshake:157(32955ns) {DropHandshakes:470(1116ns) {DropIncomingHandshake:442(368ns) DropOutgoingHandshake:457(155ns)} GenerateSessionVirtualId:988(282ns) PrepareNewSessionHandshake:477(597ns) {SwitchToState:179(316ns)}}}}}
2025-11-27T18:18:50.083485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471343207702440:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(26751268ns) {Configure:104(26747464ns) {ProcessConfigured:135(2869820ns) {StartInitialHandshake:157(2867778ns) {DropHandshakes:470(1087ns) {DropIncomingHandshake:442(272ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:477(1863ns) {SwitchToState:179(610ns)}}}}}
2025-11-27T18:18:50.085685Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471345089066676:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28150160ns) {Configure:104(28145447ns) {ProcessConfigured:135(523780ns) {StartInitialHandshake:157(522368ns) {DropHandshakes:470(1269ns) {DropIncomingHandshake:442(732ns) DropOutgoingHandshake:457(104ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(514ns) {SwitchToState:179(143ns)}}}}}
2025-11-27T18:18:50.083916Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471340228101942:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44613585ns) {Configure:104(44604808ns) {ProcessConfigured:135(571119ns) {StartInitialHandshake:157(569904ns) {DropHandshakes:470(994ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(151ns)} GenerateSessionVirtualId:988(281ns) PrepareNewSessionHandshake:477(195717ns) {SwitchToState:179(395ns)}}}}}
2025-11-27T18:18:50.084053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471343384164061:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(26863165ns) {Configure:104(26856710ns) {ProcessConfigured:135(2936059ns) {StartInitialHandshake:157(2934382ns) {DropHandshakes:470(1125ns) {DropIncomingHandshake:442(290ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(356ns) PrepareNewSessionHandshake:477(1681ns) {SwitchToState:179(527ns)}}}}}
2025-11-27T18:18:50.084245Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471338489123721:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(104374377ns) {Configure:104(104368721ns) {ProcessConfigured:135(166295ns) {StartInitialHandshake:157(164548ns) {DropHandshakes:470(11351ns) {DropIncomingHandshake:442(387ns) DropOutgoingHandshake:457(180ns)} GenerateSessionVirtualId:988(510ns) PrepareNewSessionHandshake:477(1119ns) {SwitchToState:179(485ns)}}}}}
2025-11-27T18:18:50.084001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471344523069302:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(26597241ns) {Configure:104(26590704ns) {ProcessConfigured:135(386584ns) {StartInitialHandshake:157(385006ns) {DropHandshakes:470(961ns) {DropIncomingHandshake:442(167ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(320ns) PrepareNewSessionHandshake:477(1749ns) {SwitchToState:179(899ns)}}}}}
2025-11-27T18:18:50.083566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471343207702439:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25321499ns) {Configure:104(25316630ns) {ProcessConfigured:135(2644586ns) {StartInitialHandshake:157(2642130ns) {DropHandshakes:470(1404ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(184ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(1688ns) {SwitchToState:179(740ns)}}}}}
2025-11-27T18:18:50.084207Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471346192996337:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(26376009ns) {Configure:104(26370190ns) {ProcessConfigured:135(139853ns) {StartInitialHandshake:157(138346ns) {DropHandshakes:470(936ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(568ns) PrepareNewSessionHandshake:477(728ns) {SwitchToState:179(123ns)}}}}}
2025-11-27T18:18:50.084073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471343384164062:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(26765787ns) {Configure:104(26762986ns) {ProcessConfigured:135(2659437ns) {StartInitialHandshake:157(2658094ns) {DropHandshakes:470(933ns) {DropIncomingHandshake:442(242ns) DropOutgoingHandshake:457(152ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:477(2081ns) {SwitchToState:179(773ns)}}}}}
2025-11-27T18:18:50.085818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471345089066674:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27477712ns) {Configure:104(27471559ns) {ProcessConfigured:135(504896ns) {StartInitialHandshake:157(503227ns) {DropHandshakes:470(1032ns) {DropIncomingHandshake:442(339ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(558ns) PrepareNewSessionHandshake:477(1154ns) {SwitchToState:179(558ns)}}}}}
2025-11-27T18:18:50.084787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471338489123784:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(35929531ns) {Configure:104(27788793ns) {ProcessConfigured:135(387362ns) {StartInitialHandshake:157(385722ns) {DropHandshakes:470(6932ns) {DropIncomingHandshake:442(292ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(500ns) PrepareNewSessionHandshake:477(1131ns) {SwitchToState:179(527ns)}}}}}
2025-11-27T18:18:50.086159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471338489123723:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(77054132ns) {Configure:104(77047711ns) {ProcessConfigured:135(636425ns) {StartInitialHandshake:157(634860ns) {DropHandshakes:470(683ns) {DropIncomingHandshake:442(170ns) DropOutgoingHandshake:457(104ns)} GenerateSessionVirtualId:988(294ns) PrepareNewSessionHandshake:477(185230ns) {SwitchToState:179(994ns)}}}}}
2025-11-27T18:18:50.084463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471341898028978:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(44747636ns) {Configure:104(35579391ns) {ProcessConfigured:135(452715ns) {StartInitialHandshake:157(451341ns) {DropHandshakes:470(1275ns) {DropIncomingHandshake:442(341ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(731ns) PrepareNewSessionHandshake:477(913ns) {SwitchToState:179(62ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.085833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471340794099314:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(120960769ns) {Configure:104(120953412ns) {ProcessConfigured:135(410215ns) {StartInitialHandshake:157(408661ns) {DropHandshakes:470(1241ns) {DropIncomingHandshake:442(272ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:477(1340ns) {SwitchToState:179(420ns)}}}}}
2025-11-27T18:18:50.084255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471340228101944:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35288653ns) {Configure:104(35282835ns) {ProcessConfigured:135(576483ns) {StartInitialHandshake:157(575160ns) {DropHandshakes:470(915ns) {DropIncomingHandshake:442(269ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(296ns) PrepareNewSessionHandshake:477(1170ns) {SwitchToState:179(119ns)}}}}}
2025-11-27T18:18:50.085901Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471345089066675:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25321168ns) {Configure:104(25317879ns) {ProcessConfigured:135(645084ns) {StartInitialHandshake:157(643596ns) {DropHandshakes:470(4961ns) {DropIncomingHandshake:442(313ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(414ns) PrepareNewSessionHandshake:477(1045ns) {SwitchToState:179(497ns)}}}}}
2025-11-27T18:18:50.084493Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471341898029038:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(35966658ns) {Configure:104(35957348ns) {ProcessConfigured:135(642691ns) {StartInitialHandshake:157(640518ns) {DropHandshakes:470(1596ns) {DropIncomingHandshake:442(660ns) DropOutgoingHandshake:457(186ns)} GenerateSessionVirtualId:988(494ns) PrepareNewSessionHandshake:477(1877ns) {SwitchToState:179(667ns)}}}}}
2025-11-27T18:18:50.084508Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471340228101943:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(105030463ns) {Configure:104(105021783ns) {ProcessConfigured:135(495974ns) {StartInitialHandshake:157(491807ns) {DropHandshakes:470(1026ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(169ns)} GenerateSessionVirtualId:988(649ns) PrepareNewSessionHandshake:477(617ns) {SwitchToState:179(192ns)}}}}}
2025-11-27T18:18:50.085817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471341898028977:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(29737574ns) {Configure:104(29728925ns) {ProcessConfigured:135(629065ns) {StartInitialHandshake:157(627441ns) {DropHandshakes:470(1302ns) {DropIncomingHandshake:442(411ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(704ns) PrepareNewSessionHandshake:477(1466ns) {SwitchToState:179(564ns)}}}}}
2025-11-27T18:18:50.086219Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471340794099315:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(114425292ns) {Configure:104(114418457ns) {ProcessConfigured:135(524596ns) {StartInitialHandshake:157(523020ns) {DropHandshakes:470(807ns) {DropIncomingHandshake:442(173ns) DropOutgoingHandshake:457(94ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(1521ns) {SwitchToState:179(686ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.099961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471346192996352:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51003
2025-11-27T18:18:50.089599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471343207702451:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31533
2025-11-27T18:18:50.110027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471343207702451:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.124658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471343384164076:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:347
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.113514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471338912735079:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.128396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471343384164064:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.136669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471343207702459:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31533
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.128426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471343384164079:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:347
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.128441Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471343384164064:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.156836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471345089066690:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49156
2025-11-27T18:18:50.156907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471346192996351:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51003
2025-11-27T18:18:50.157016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471344523069319:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36171
2025-11-27T18:18:50.157042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471343207702457:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31533
2025-11-27T18:18:50.157032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471344523069317:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36171
2025-11-27T18:18:50.158943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471346192996353:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51003
2025-11-27T18:18:50.157245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471343207702456:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31533
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.158896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471345089066691:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49156
2025-11-27T18:18:50.159448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471342784091095:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4849
2025-11-27T18:18:50.136884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471343384164064:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:50.159663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471346192996352:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:50.158939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471343207702457:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.163742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471342784091095:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.161024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471345089066690:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.157605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471344523069318:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36171
2025-11-27T18:18:50.137111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471339089196702:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.162973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471346192996337:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.165048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471338489123723:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.163254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471346192996351:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.137165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:50.159031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471343207702451:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:50.163783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471341898028976:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.158942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471344523069319:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.162942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471345089066691:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.141747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471339089196702:2049] [node 3] ICP22 created new session: [1:7577471343384164082:2049]
2025-11-27T18:18:50.162743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471338912735079:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.163931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471346192996353:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.165678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471345089066684:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.153045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471343384164082:2049] [node 3] ICS09 handshake done sender: [1:7577471343384164064:2110] self: [1:7577471343384164063:0] peer: [3:7577471343207702458:0] socket: 570 qp: -1
2025-11-27T18:18:50.166885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471342784091096:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4849
2025-11-27T18:18:50.159508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471344523069317:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.162818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:50.165718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471345089066684:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.165016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471346192996354:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51003
2025-11-27T18:18:50.167555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471342784091087:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.163412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471338912735079:2048] [node 1] ICP22 created new session: [3:7577471343207702461:2048]
2025-11-27T18:18:50.153098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471343384164076:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.162912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471344523069318:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.165606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471341898029038:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.167587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471342784091087:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.163441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471343207702456:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.168653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471345089066684:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:50.163237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471340228101943:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.153102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471343384164082:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:50.163493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471340228101942:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.169365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471342784091096:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.166186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471346192996354:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.157063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:50.169846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471345089066686:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.163811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471344523069310:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.163866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471343207702461:2048] [node 1] ICS09 handshake done sender: [3:7577471343207702451:2115] self: [3:7577471343207702458:0] peer: [1:7577471343384164063:0] socket: 572 qp: -1
2025-11-27T18:18:50.167704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471341898028977:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.169915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471338489123721:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.169872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471345089066686:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.157274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471343384164078:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:347
2025-11-27T18:18:50.163907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471343207702461:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:50.172049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471345089066686:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:50.163843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471344523069310:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.171178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471342784091087:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:50.179358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471346192996359:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51003
2025-11-27T18:18:50.157598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471343384164077:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:347
2025-11-27T18:18:50.165079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.168140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471343207702449:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.177503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471345089066674:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.167621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471344523069310:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:50.158962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471343384164079:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.181984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471342784091081:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.177541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:50.181932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471346192996361:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51003
2025-11-27T18:18:50.171515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471344523069303:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.168171Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471343207702449:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.161000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471343384164077:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.182808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:50.172969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471343207702449:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:50.189672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471346192996362:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51003
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.179400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471345089066674:2107] [node 4] ICP22 created new session: [2:7577471345089066696:2107]
2025-11-27T18:18:50.162661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471343384164078:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.171919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:50.189683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471342784091081:2109] [node 6] ICP22 created new session: [5:7577471342784091102:2109]
2025-11-27T18:18:50.163660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471343384164080:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:347
2025-11-27T18:18:50.179455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471345089066696:2107] [node 4] ICS09 handshake done sender: [2:7577471345089066684:2113] self: [2:7577471345089066679:0] peer: [4:7577471344523069320:0] socket: 599 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-27T18:18:50.171962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471344523069303:2110] [node 6] ICP22 created new session: [4:7577471344523069325:2110]
2025-11-27T18:18:50.180200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471343207702440:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.197973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471346192996337:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19341) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471346192996352:2117] held: no
2025-11-27T18:18:50.163856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471343384164069:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.179569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471345089066675:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.194098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471342784091102:2109] [node 6] ICS09 handshake done sender: [5:7577471342784091087:2112] self: [5:7577471342784091086:0] peer: [6:7577471346192996357:0] socket: 592 qp: -1
2025-11-27T18:18:50.172896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471344523069325:2110] [node 6] ICS09 handshake done sender: [4:7577471344523069310:2112] self: [4:7577471344523069307:0] peer: [6:7577471346192996355:0] socket: 586 qp: -1
2025-11-27T18:18:50.195626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471342784091102:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:50.179044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471344523069325:2110] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.181161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: 2025-11-27T18:18:50.198945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471346192996351:2116] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0No RDMA context found for GID: 
::ffff:127.1.0.0
2025-11-27T18:18:50.197962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:50.179464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:50.163897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471343384164069:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.179610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.199618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471346192996360:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51003
2025-11-27T18:18:50.182603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471343207702440:2109] [node 5] ICP22 created new session: [3:7577471343207702466:2109]
2025-11-27T18:18:50.181947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471344523069326:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36171
2025-11-27T18:18:50.203538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471342784091100:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4849
2025-11-27T18:18:50.179771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471345089066675:2110] [node 6] ICP22 created new session: [2:7577471345089066697:2110]
2025-11-27T18:18:50.165624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471343384164080:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.194067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471343207702466:2109] [node 5] ICS09 handshake done sender: [3:7577471343207702449:2113] self: [3:7577471343207702445:0] peer: [5:7577471342784091097:0] socket: 579 qp: -1
2025-11-27T18:18:50.199812Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471346192996353:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:50.179709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471345089066696:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:50.204470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471342784091101:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4849
2025-11-27T18:18:50.165752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471343384164071:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.181943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471344523069324:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36171
2025-11-27T18:18:50.207111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471342784091104:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4849
2025-11-27T18:18:50.201480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471341898028976:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23297) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471346192996351:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.194473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471343207702466:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:50.194497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471340228101942:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23297) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471344523069317:2116] held: no
2025-11-27T18:18:50.180194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:50.207123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471342784091095:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:50.207856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471342784091106:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4849
2025-11-27T18:18:50.182608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471345089066697:2110] [node 6] ICS09 handshake done sender: [2:7577471345089066686:2115] self: [2:7577471345089066681:0] peer: [6:7577471346192996358:0] socket: 606 qp: -1
2025-11-27T18:18:50.208164Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471342784091081:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(24910704ns) {HandleHandshakeStatus:329(24908452ns) {DropIncomingHandshake:442(1684ns) DropOutgoingHandshake:457(1241ns) SwitchToState:179(386ns) ProcessPendingSessionEvents:495(5181338ns) {ForwardSessionEventToSession:612(2839085ns) ForwardSessionEventToSession:612(4186ns) ForwardSessionEventToSession:612(3700ns) ForwardSessionEventToSession:612(906047ns) ForwardSessionEventToSession:612(356128ns) ForwardSessionEventToSession:612(692432ns) ForwardSessionEventToSession:612(6733ns) ForwardSessionEventToSession:612(2504ns) ForwardSessionEventToSession:612(347345ns) ForwardSessionEventToSession:612(4075ns)}}}
2025-11-27T18:18:50.182660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471345089066697:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:50.208230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471338489123723:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.193675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:50.208268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.205218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471345089066674:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22689097ns) {HandleHandshakeStatus:329(22685694ns) {DropIncomingHandshake:442(321ns) DropOutgoingHandshake:457(426ns) SwitchToState:179(900ns) ProcessPendingSessionEvents:495(14807680ns) {ForwardSessionEventToSession:612(8707555ns) ForwardSessionEventToSession:612(7536ns) ForwardSessionEventToSession:612(3517ns) ForwardSessionEventToSession:612(1506141ns) ForwardSessionEventToSession:612(4085ns) ForwardSessionEventToSession:612(3137ns) ForwardSessionEventToSession:612(3447870ns) ForwardSessionEventToSession:612(4492ns) ForwardSessionEventToSession:612(1081450ns) ForwardSessionEventToSession:612(8477ns)}}}
2025-11-27T18:18:50.194518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:50.165778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471343384164071:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.168342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471339089196702:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(26782628ns) {HandleHandshakeStatus:329(26778732ns) {DropIncomingHandshake:442(717ns) DropOutgoingHandshake:457(589ns) SwitchToState:179(973ns) ProcessPendingSessionEvents:495(4138793ns) {ForwardSessionEventToSession:612(3641833ns) ForwardSessionEventToSession:612(282227ns) ForwardSessionEventToSession:612(8118ns) ForwardSessionEventToSession:612(3775ns) ForwardSessionEventToSession:612(149361ns) ForwardSessionEventToSession:612(3393ns) ForwardSessionEventToSession:612(2848ns) ForwardSessionEventToSession:612(5687ns) ForwardSessionEventToSession:612(9043ns) ForwardSessionEventToSession:612(9701ns)}}}
2025-11-27T18:18:50.208881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471338489123721:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23297) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471342784091096:2116] held: no
2025-11-27T18:18:50.201552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471346192996352:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:50.207736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471343207702469:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31533
2025-11-27T18:18:50.207472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471345089066702:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49156
2025-11-27T18:18:50.210219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471338489123723:2050] [node 3] ICP22 created new session: [5:7577471342784091107:2050]
2025-11-27T18:18:50.200331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471344523069317:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:50.204486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471346192996363:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51003
2025-11-27T18:18:50.170856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471343384164071:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:50.209291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471343207702469:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:18:50.204931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471341898028976:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.171088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471343384164073:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:50.210388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471342784091107:2050] [node 3] ICS09 handshake done sender: [5:7577471342784091095:2115] self: [5:7577471342784091097:0] peer: [3:7577471343207702445:0] socket: 605 qp: -1
2025-11-27T18:18:50.200350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471344523069319:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:50.209880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471343207702440:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(25866892ns) {HandleHandshakeStatus:329(25865257ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(225ns) SwitchToState:179(122ns) ProcessPendingSessionEvents:495(4526865ns) {ForwardSessionEventToSession:612(2038127ns) ForwardSessionEventToSession:612(4024ns) ForwardSessionEventToSession:612(2991ns) ForwardSessionEventToSession:612(1270702ns) ForwardSessionEventToSession:612(380023ns) ForwardSessionEventToSession:612(4805ns) ForwardSessionEventToSession:612(347403ns) ForwardSessionEventToSession:612(2749ns) ForwardSessionEventToSession:612(456863ns) ForwardSessionEventToSession:612(3880ns)}}}
2025-11-27T18:18:50.207910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471345089066675:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(26511450ns) {HandleHandshakeStatus:329(26507944ns) {DropIncomingHandshake:442(398ns) DropOutgoingHandshake:457(307ns) SwitchToState:179(692ns) ProcessPendingSessionEvents:495(6184414ns) {ForwardSessionEventToSession:612(1618525ns) ForwardSessionEventToSession:612(540213ns) ForwardSessionEventToSession:612(2827ns) ForwardSessionEventToSession:612(2501ns) ForwardSessionEventToSession:612(556657ns) ForwardSessionEventToSession:612(3349ns) ForwardSessionEventToSession:612(1543340ns) ForwardSessionEventToSession:612(2711ns) ForwardSessionEventToSession:612(359995ns) ForwardSessionEventToSession:612(470534ns)}}}
2025-11-27T18:18:50.210863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471345089066683:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.171111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471343384164073:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.210526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471342784091101:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.209918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471338912735080:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.210905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471345089066683:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.170897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471343384164069:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:50.204986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:50.204474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471340228101943:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.212000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471345089066702:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.210798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471342784091107:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:50.171917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471339089196702:2049] [node 3] ICP18 handshake (actor [1:7577471343384164079:2118]) is too old
2025-11-27T18:18:50.205217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471340228101942:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.205044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471341898028976:2048] [node 1] ICP22 created new session: [6:7577471346192996365:2048]
2025-11-27T18:18:50.214119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471343207702452:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.205253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:50.213150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471345089066683:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:50.174647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471339089196764:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.210833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471342784091106:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.205101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471346192996365:2048] [node 1] ICS09 handshake done sender: [6:7577471346192996351:2116] self: [6:7577471346192996356:0] peer: [1:7577471343384164067:0] socket: 580 qp: -1
2025-11-27T18:18:50.205431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:50.214136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471343207702450:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.213385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471340794099314:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.210837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:50.174720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:50.214153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471343207702452:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.206126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471340228101943:2049] [node 2] ICP22 created new session: [4:7577471344523069332:2049]
2025-11-27T18:18:50.213659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471340794099315:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.205132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471346192996365:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:50.179038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471339089196764:2107] [node 4] ICP22 created new session: [1:7577471343384164091:2107]
2025-11-27T18:18:50.210892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471342784091096:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:50.205179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:50.206468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471346192996360:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.211565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471338489123721:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.179435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471343384164091:2107] [node 4] ICS09 handshake done sender: [1:7577471343384164069:2111] self: [1:7577471343384164066:0] peer: [4:7577471344523069322:0] socket: 574 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.214161Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471343207702450:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.206524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471341898029038:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.206421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471340228101942:2048] [node 1] ICP22 created new session: [4:7577471344523069331:2048]
2025-11-27T18:18:50.179764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471343384164091:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.213697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471345089066703:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49156
2025-11-27T18:18:50.211608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:50.206576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:50.217748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471343207702452:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:50.213704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:50.212358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471338489123721:2048] [node 1] ICP22 created new session: [5:7577471342784091109:2048]
2025-11-27T18:18:50.206710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471344523069332:2049] [node 2] ICS09 handshake done sender: [4:7577471344523069319:2118] self: [4:7577471344523069320:0] peer: [2:7577471345089066679:0] socket: 603 qp: -1
2025-11-27T18:18:50.179896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471343384164062:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.180245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:50.207179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471341898029038:2108] [node 5] ICP22 created new session: [6:7577471346192996367:2108]
2025-11-27T18:18:50.213741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471340794099315:2049] [node 3] ICP22 created new session: [2:7577471345089066706:2049]
2025-11-27T18:18:50.218558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471338912735080:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1216) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471343207702469:2120] held: no
2025-11-27T18:18:50.207174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471344523069332:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.212861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471342784091109:2048] [node 1] ICS09 handshake done sender: [5:7577471342784091096:2116] self: [5:7577471342784091098:0] peer: [1:7577471343384164068:0] socket: 615 qp: -1
2025-11-27T18:18:50.181186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:50.207316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471344523069303:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(28379309ns) {HandleHandshakeStatus:329(28375860ns) {DropIncomingHandshake:442(115739ns) DropOutgoingHandshake:457(514ns) SwitchToState:179(304ns) ProcessPendingSessionEvents:495(17289225ns) {ForwardSessionEventToSession:612(11099060ns) ForwardSessionEventToSession:612(765431ns) ForwardSessionEventToSession:612(4214ns) ForwardSessionEventToSession:612(2647ns) ForwardSessionEventToSession:612(1184237ns) ForwardSessionEventToSession:612(2292382ns) ForwardSessionEventToSession:612(1544332ns) ForwardSessionEventToSession:612(3544ns) ForwardSessionEventToSession:612(2139ns) ForwardSessionEventToSession:612(371866ns)}}}
2025-11-27T18:18:50.207199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471346192996337:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.213954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471345089066685:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.219684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471343207702441:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.213311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471338489123784:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.185354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471343384164062:2109] [node 6] ICP22 created new session: [1:7577471343384164094:2109]
2025-11-27T18:18:50.207234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:50.213974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471345089066685:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.219991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:50.207383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:50.213350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471342784091109:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:50.193614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471343384164094:2109] [node 6] ICS09 handshake done sender: [1:7577471343384164071:2112] self: [1:7577471343384164067:0] peer: [6:7577471346192996356:0] socket: 573 qp: -1
2025-11-27T18:18:50.207875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471346192996337:2109] [node 4] ICP22 created new session: [6:7577471346192996368:2109]
2025-11-27T18:18:50.207385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471344523069331:2048] [node 1] ICS09 handshake done sender: [4:7577471344523069317:2116] self: [4:7577471344523069322:0] peer: [1:7577471343384164066:0] socket: 591 qp: -1
2025-11-27T18:18:50.213386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:50.220331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471343207702450:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:50.214502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471345089066706:2049] [node 3] ICS09 handshake done sender: [2:7577471345089066683:2112] self: [2:7577471345089066678:0] peer: [3:7577471343207702470:0] socket: 635 qp: -1
2025-11-27T18:18:50.207805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471344523069331:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:50.214528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471345089066706:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:50.193669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471343384164094:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:50.220507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471343207702441:2107] [node 4] ICP22 created new session: [3:7577471343207702474:2107]
2025-11-27T18:18:50.215214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:50.213441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471338489123722:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.207882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471346192996367:2108] [node 5] ICS09 handshake done sender: [6:7577471346192996353:2118] self: [6:7577471346192996357:0] peer: [5:7577471342784091086:0] socket: 594 qp: -1
2025-11-27T18:18:50.207839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:50.193709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:50.217470Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471345089066675:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:50.221052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471343207702474:2107] [node 4] ICS09 handshake done sender: [3:7577471343207702452:2116] self: [3:7577471343207702448:0] peer: [4:7577471344523069336:0] socket: 593 qp: -1
2025-11-27T18:18:50.211296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471344523069329:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36171
2025-11-27T18:18:50.195635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471343384164073:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:50.217862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471345089066685:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:50.221090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471343207702474:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:50.208224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471346192996367:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:50.212027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471344523069330:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36171
2025-11-27T18:18:50.202654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471343384164061:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.219689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471345089066676:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.212058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471344523069329:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.221337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471343207702471:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31533
2025-11-27T18:18:50.208361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:50.219979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:50.212862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471340228101944:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.203266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:50.222267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:50.219854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471342784091113:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4849
2025-11-27T18:18:50.220022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471345089066676:2108] [node 5] ICP22 created new session: [2:7577471345089066710:2108]
2025-11-27T18:18:50.214017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471344523069330:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.209108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471346192996368:2109] [node 4] ICS09 handshake done sender: [6:7577471346192996352:2117] self: [6:7577471346192996355:0] peer: [4:7577471344523069307:0] socket: 587 qp: -1
2025-11-27T18:18:50.203323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471343384164061:2108] [node 5] ICP22 created new session: [1:7577471343384164097:2108]
2025-11-27T18:18:50.223236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471343207702439:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.215173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471344523069311:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.222133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471342784091106:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:50.220070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471345089066710:2108] [node 5] ICS09 handshake done sender: [2:7577471345089066685:2114] self: [2:7577471345089066680:0] peer: [5:7577471342784091111:0] socket: 632 qp: -1
2025-11-27T18:18:50.209868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471346192996368:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:50.223292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:50.226239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471343207702469:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:50.203721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471339089196764:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22697537ns) {HandleHandshakeStatus:329(22694109ns) {DropIncomingHandshake:442(24620ns) DropOutgoingHandshake:457(357ns) SwitchToState:179(348ns) ProcessPendingSessionEvents:495(13460713ns) {ForwardSessionEventToSession:612(7223241ns) ForwardSessionEventToSession:612(314599ns) ForwardSessionEventToSession:612(3861429ns) ForwardSessionEventToSession:612(495241ns) ForwardSessionEventToSession:612(3895ns) ForwardSessionEventToSession:612(2618ns) ForwardSessionEventToSession:612(382083ns) ForwardSessionEventToSession:612(4452ns) ForwardSessionEventToSession:612(2602ns) ForwardSessionEventToSession:612(1147606ns)}}}
2025-11-27T18:18:50.223064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471338489123722:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1216) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471342784091106:2120] held: no
2025-11-27T18:18:50.226260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471343207702439:2110] [node 6] ICP22 created new session: [3:7577471343207702476:2110]
2025-11-27T18:18:50.215198Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471344523069311:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.220705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471345089066710:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:50.224653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471338489123722:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.227336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471343207702476:2110] [node 6] ICS09 handshake done sender: [3:7577471343207702450:2114] self: [3:7577471343207702447:0] peer: [6:7577471346192996371:0] socket: 616 qp: -1
2025-11-27T18:18:50.205237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471343384164097:2108] [node 5] ICS09 handshake done sender: [1:7577471343384164073:2114] self: [1:7577471343384164068:0] peer: [5:7577471342784091098:0] socket: 614 qp: -1
2025-11-27T18:18:50.219824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471344523069311:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:50.225757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471338489123784:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19341) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471342784091101:2118] held: no
2025-11-27T18:18:50.220748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:50.227581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471338912735080:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.206152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471343384164097:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:50.220765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471344523069339:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36171
2025-11-27T18:18:50.226333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:50.209913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:50.206523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:50.227959Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471340794099314:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23297) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471345089066702:2118] held: no
2025-11-27T18:18:50.220818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471344523069302:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.227627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:50.226376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471338489123722:2049] [node 2] ICP22 created new session: [5:7577471342784091115:2049]
2025-11-27T18:18:50.232584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471345089066709:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49156
2025-11-27T18:18:50.210242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471346192996354:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:50.227751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471343207702476:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:50.220875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:50.233333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471345089066702:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:50.207358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471343384164062:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(23800920ns) {HandleHandshakeStatus:329(23798506ns) {DropIncomingHandshake:442(12658ns) DropOutgoingHandshake:457(374ns) SwitchToState:179(349ns) ProcessPendingSessionEvents:495(3872444ns) {ForwardSessionEventToSession:612(1785533ns) ForwardSessionEventToSession:612(4111ns) ForwardSessionEventToSession:612(1238643ns) ForwardSessionEventToSession:612(10729ns) ForwardSessionEventToSession:612(2656ns) ForwardSessionEventToSession:612(2151ns) ForwardSessionEventToSession:612(331465ns) ForwardSessionEventToSession:612(3221ns) ForwardSessionEventToSession:612(478396ns) ForwardSessionEventToSession:612(2754ns)}}}
2025-11-27T18:18:50.227726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471342784091114:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4849
2025-11-27T18:18:50.211336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471341898028978:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:50.221092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471344523069302:2108] [node 5] ICP22 created new session: [4:7577471344523069340:2108]
2025-11-27T18:18:50.227944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:50.214133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471343384164072:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:50.227750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471342784091115:2049] [node 2] ICS09 handshake done sender: [5:7577471342784091106:2120] self: [5:7577471342784091111:0] peer: [2:7577471345089066680:0] socket: 582 qp: -1
2025-11-27T18:18:50.212356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471341898028977:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.221784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471344523069340:2108] [node 5] ICS09 handshake done sender: [4:7577471344523069311:2113] self: [4:7577471344523069305:0] peer: [5:7577471342784091110:0] socket: 631 qp: -1
2025-11-27T18:18:50.214176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471343384164072:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:50.228098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471338912735080:2049] [node 2] ICP22 created new session: [3:7577471343207702477:2049]
2025-11-27T18:18:50.228523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471342784091115:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:50.212841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:50.217380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471343384164072:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:50.228576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:50.233993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471340794099314:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.219153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471339089196701:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:50.222257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471344523069340:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:50.228547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471343207702477:2049] [node 2] ICS09 handshake done sender: [3:7577471343207702469:2120] self: [3:7577471343207702470:0] peer: [2:7577471345089066678:0] socket: 585 qp: -1
2025-11-27T18:18:50.223182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471340228101944:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471344523069329:2121] held: no
2025-11-27T18:18:50.229740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471342784091101:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:50.212991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471341898028977:2049] [node 2] ICP22 created new session: [6:7577471346192996372:2049]
2025-11-27T18:18:50.229223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471343207702477:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:50.234041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:50.224578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:50.229856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471338489123784:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.225971Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471344523069329:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:50.219390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:50.231889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:50.213400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471346192996372:2049] [node 2] ICS09 handshake done sender: [6:7577471346192996354:2119] self: [6:7577471346192996358:0] peer: [2:7577471345089066681:0] socket: 607 qp: -1
2025-11-27T18:18:50.234094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471340794099314:2048] [node 1] ICP22 created new session: [2:7577471345089066712:2048]
2025-11-27T18:18:50.232024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:50.227364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471340228101944:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.234136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471345089066712:2048] [node 1] ICS09 handshake done sender: [2:7577471345089066702:2118] self: [2:7577471345089066705:0] peer: [1:7577471343384164070:0] socket: 638 qp: -1
2025-11-27T18:18:50.232340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471338489123784:2108] [node 4] ICP22 created new session: [5:7577471342784091117:2108]
2025-11-27T18:18:50.219887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471339089196701:2048] [node 2] ICP22 created new session: [1:7577471343384164100:2048]
2025-11-27T18:18:50.213528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471346192996372:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:50.232917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471342784091117:2108] [node 4] ICS09 handshake done sender: [5:7577471342784091101:2118] self: [5:7577471342784091110:0] peer: [4:7577471344523069305:0] socket: 595 qp: -1
2025-11-27T18:18:50.234307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471345089066712:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:50.233284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471342784091117:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:50.213694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:50.233331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:50.234367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:50.226725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471346192996374:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51003
2025-11-27T18:18:50.227777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:50.231869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471341898028978:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471346192996360:2121] held: no
2025-11-27T18:18:50.227841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471340228101944:2050] [node 3] ICP22 created new session: [4:7577471344523069342:2050]
2025-11-27T18:18:50.231887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471346192996360:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:50.231840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471344523069342:2050] [node 3] ICS09 handshake done sender: [4:7577471344523069329:2121] self: [4:7577471344523069336:0] peer: [3:7577471343207702448:0] socket: 637 qp: -1
2025-11-27T18:18:50.232421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471341898028978:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:50.232083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471344523069342:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:50.232132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:50.232567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:50.220721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471343384164100:2048] [node 2] ICS09 handshake done sender: [1:7577471343384164072:2113] self: [1:7577471343384164070:0] peer: [2:7577471345089066705:0] socket: 636 qp: -1
2025-11-27T18:18:50.233002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471341898028978:2050] [node 3] ICP22 created new session: [6:7577471346192996375:2050]
2025-11-27T18:18:50.221045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471343384164100:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:50.233474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471346192996375:2050] [node 3] ICS09 handshake done sender: [6:7577471346192996360:2121] self: [6:7577471346192996371:0] peer: [3:7577471343207702447:0] socket: 619 qp: -1
2025-11-27T18:18:50.221536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:50.233503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471346192996375:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:50.233671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:50.284097Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:50.284343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471345089066712:2048] [node 1] ICS01 socket: 638 reason# EndOfStream
2025-11-27T18:18:50.284379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471340794099314:2048] [node 1] ICP30 unregister session Session# [2:7577471345089066712:2048] VirtualId# [2:7577471345089066705:0]
2025-11-27T18:18:50.284414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471345089066712:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.284491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:50.284549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:50.285362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:50.285556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471343207702461:2048] [node 1] ICS07 socket disconnect 572 reason# EndOfStream
2025-11-27T18:18:50.285585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471343207702461:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.285670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:50.285772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471343207702461:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:50.286526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471338912735079:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23297) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:50.286632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471343207702461:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:50.286688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471338912735079:2048] [node 1] ICP30 unregister session Session# [3:7577471343207702461:2048] VirtualId# [3:7577471343207702458:0]
2025-11-27T18:18:50.286733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:50.286772Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471338912735079:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:50.286789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:50.287864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:50.288072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471344523069331:2048] [node 1] ICS01 socket: 591 reason# EndOfStream
2025-11-27T18:18:50.288111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471340228101942:2048] [node 1] ICP30 unregister session Session# [4:7577471344523069331:2048] VirtualId# [4:7577471344523069322:0]
2025-11-27T18:18:50.288135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471344523069331:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.288196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:50.288240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:50.288826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:50.289018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471342784091109:2048] [node 1] ICS01 socket: 615 reason# EndOfStream
2025-11-27T18:18:50.289059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471338489123721:2048] [node 1] ICP30 unregister session Session# [5:7577471342784091109:2048] VirtualId# [5:7577471342784091098:0]
2025-11-27T18:18:50.289083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471342784091109:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.289157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:50.289200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:50.289790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:50.289961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471346192996365:2048] [node 1] ICS01 socket: 580 reason# EndOfStream
2025-11-27T18:18:50.289998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471341898028976:2048] [node 1] ICP30 unregister session Session# [6:7577471346192996365:2048] VirtualId# [6:7577471346192996356:0]
2025-11-27T18:18:50.290021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471346192996365:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.290081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:50.290116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:50.296687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471338912735079:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:50.345882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:50.346071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471343207702477:2049] [node 2] ICS01 socket: 585 reason# EndOfStream
2025-11-27T18:18:50.346110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471338912735080:2049] [node 2] ICP30 unregister session Session# [3:7577471343207702477:2049] VirtualId# [3:7577471343207702470:0]
2025-11-27T18:18:50.346132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471343207702477:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.346192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:50.346224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:50.348477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:50.348692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471344523069332:2049] [node 2] ICS07 socket disconnect 603 reason# EndOfStream
2025-11-27T18:18:50.348731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471344523069332:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.348830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:50.348878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471344523069332:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:50.349531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471340228101943:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1216) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:50.349551Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:50.349587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471344523069332:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:50.349682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471340228101943:2049] [node 2] ICP30 unregister session Session# [4:7577471344523069332:2049] VirtualId# [4:7577471344523069320:0]
2025-11-27T18:18:50.349735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:50.349743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471342784091115:2049] [node 2] ICS01 socket: 582 reason# EndOfStream
2025-11-27T18:18:50.349769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471340228101943:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:50.349772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471338489123722:2049] [node 2] ICP30 unregister session Session# [5:7577471342784091115:2049] VirtualId# [5:7577471342784091111:0]
2025-11-27T18:18:50.349783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:50.349790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471342784091115:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.349866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:50.349900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:50.350683Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:50.350863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471346192996372:2049] [node 2] ICS07 socket disconnect 607 reason# EndOfStream
2025-11-27T18:18:50.350881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471346192996372:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.350932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:50.350962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471346192996372:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:50.351525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471341898028977:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1216) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:50.351622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471346192996372:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:50.351670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471341898028977:2049] [node 2] ICP30 unregister session Session# [6:7577471346192996372:2049] VirtualId# [6:7577471346192996358:0]
2025-11-27T18:18:50.351709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:50.351728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471341898028977:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:50.351736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:50.360074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471340228101943:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:50.361134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471341898028977:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:50.432099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:50.432338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471344523069342:2050] [node 3] ICS01 socket: 637 reason# EndOfStream
2025-11-27T18:18:50.432384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471340228101944:2050] [node 3] ICP30 unregister session Session# [4:7577471344523069342:2050] VirtualId# [4:7577471344523069336:0]
2025-11-27T18:18:50.432417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471344523069342:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.432486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:50.432535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:50.433157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:50.433342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471342784091107:2050] [node 3] ICS07 socket disconnect 605 reason# EndOfStream
2025-11-27T18:18:50.433385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471342784091107:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.433451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:50.433487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471342784091107:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:50.434150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:50.434234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471338489123723:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11643) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:50.434285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471342784091107:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:50.434306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471338489123723:2050] [node 3] ICP30 unregister session Session# [5:7577471342784091107:2050] VirtualId# [5:7577471342784091097:0]
2025-11-27T18:18:50.434371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471346192996375:2050] [node 3] ICS01 socket: 619 reason# EndOfStream
2025-11-27T18:18:50.434361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:50.434405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471338489123723:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11643) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:50.434447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471341898028978:2050] [node 3] ICP30 unregister session Session# [6:7577471346192996375:2050] VirtualId# [6:7577471346192996371:0]
2025-11-27T18:18:50.434427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11643) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:50.434484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471346192996375:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.434559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:50.434601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:50.444093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471338489123723:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:50.478000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:50.478246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471342784091117:2108] [node 4] ICS01 socket: 595 reason# EndOfStream
2025-11-27T18:18:50.478298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471338489123784:2108] [node 4] ICP30 unregister session Session# [5:7577471342784091117:2108] VirtualId# [5:7577471342784091110:0]
2025-11-27T18:18:50.478334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471342784091117:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.478421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:50.478478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:50.479028Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:50.479244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471346192996368:2109] [node 4] ICS01 socket: 587 reason# EndOfStream
2025-11-27T18:18:50.479283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471346192996337:2109] [node 4] ICP30 unregister session Session# [6:7577471346192996368:2109] VirtualId# [6:7577471346192996355:0]
2025-11-27T18:18:50.479315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471346192996368:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.479393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:50.479443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:50.574918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:50.575152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471346192996367:2108] [node 5] ICS07 socket disconnect 594 reason# EndOfStream
2025-11-27T18:18:50.575189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471346192996367:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:50.575257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:50.575292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471346192996367:2108] [node 5] ICS15 start handshake
2025-11-27T18:18:50.576053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471341898029038:2108] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61714) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:50.576156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471346192996367:2108] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:50.576235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471341898029038:2108] [node 5] ICP30 unregister session Session# [6:7577471346192996367:2108] VirtualId# [6:7577471346192996357:0]
2025-11-27T18:18:50.576286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:18:50.576319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471341898029038:2108] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61714) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:50.576344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61714) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:50.586443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471341898029038:2108] [node 5] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:50.660639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471344624655023:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:50.660758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471344624655024:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:50.690131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471343032223955:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:50.690199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471343032223956:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:50.714121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471346583387940:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:50.714168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471346583387941:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:50.737586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471343953724645:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:50.737643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471343953724646:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:50.737689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471343953724647:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:50.755100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471345484675805:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:50.755143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471345484675806:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:50.755172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471345484675807:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:50.776614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471343214931638:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:50.776687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471343214931639:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:50.776728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471343214931640:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:50.857593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471343214931697:2104] [node 5] ICP01 ready to work
2025-11-27T18:18:50.965550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471345484675869:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:50.948615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471346583388003:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:50.965550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471343032224018:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:50.966010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471343953724709:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:50.969928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471346583388004:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:50.973956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471343032224019:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:50.974133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471343953724710:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:50.973950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471345484675870:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:50.974821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471343032224020:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:50.974876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471346583388005:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:50.975994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471344624655023:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(128841824ns) {Configure:104(128824362ns) {ProcessConfigured:135(305615ns) {StartInitialHandshake:157(302771ns) {DropHandshakes:470(3739ns) {DropIncomingHandshake:442(937ns) DropOutgoingHandshake:457(457ns)} GenerateSessionVirtualId:988(665ns) PrepareNewSessionHandshake:477(1442ns) {SwitchToState:179(559ns)}}}}}
2025-11-27T18:18:50.976676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471344624655089:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:50.977639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471344624655090:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:50.977754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471343214931697:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(106807636ns) {Configure:104(106800227ns) {ProcessConfigured:135(650366ns) {StartInitialHandshake:157(648488ns) {DropHandshakes:470(1813ns) {DropIncomingHandshake:442(546ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(578ns) PrepareNewSessionHandshake:477(1543ns) {SwitchToState:179(669ns)}}}}}
2025-11-27T18:18:50.977959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471344624655091:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:50.978515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471343214931705:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.000609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471344624655103:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26466
2025-11-27T18:18:51.002303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471343953724721:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46351
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.002691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350878355312:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50799
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.002838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350878355313:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50799
2025-11-27T18:18:51.002701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471344624655104:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26466
2025-11-27T18:18:51.003627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471344624655103:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.003556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471343953724721:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.003233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350878355312:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.003673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471346583387940:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.004051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471344624655093:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.004084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471344624655093:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.004338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349779643177:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50247
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.004956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471348919622401:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26466
2025-11-27T18:18:51.006499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471344624655093:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:51.006642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471349779643177:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.010559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471344624655024:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.010532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471343032224031:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40254
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.010845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:51.010916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471347327191328:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40254
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.010975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471344624655024:2049] [node 3] ICP22 created new session: [1:7577471348919622405:2049]
2025-11-27T18:18:51.011052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471347327191329:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40254
2025-11-27T18:18:51.011054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471348919622405:2049] [node 3] ICS09 handshake done sender: [1:7577471344624655093:2112] self: [1:7577471344624655092:0] peer: [3:7577471350878355317:0] socket: 577 qp: -1
2025-11-27T18:18:51.011096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471348919622405:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.011290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471347327191330:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40254
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.011400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:51.011459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471347327191334:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40254
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.013066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471348919622404:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26466
2025-11-27T18:18:51.013183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347327191328:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.014999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350878355318:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50799
2025-11-27T18:18:51.015088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350878355319:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50799
2025-11-27T18:18:51.015170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471343953724645:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.015223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471348919622401:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.015358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348248692021:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46351
2025-11-27T18:18:51.015359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471343032224031:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.015477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471348919622407:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26466
2025-11-27T18:18:51.015462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347327191330:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.015606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471344624655024:2049] [node 3] ICP18 handshake (actor [1:7577471348919622401:2118]) is too old
2025-11-27T18:18:51.015701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471344624655104:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.016998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350878355320:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50799
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.018054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471348248692021:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.021453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471350878355312:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:51.023476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347327191329:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.023729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471343032223955:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.024872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471344624655095:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.023050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348248692023:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46351
2025-11-27T18:18:51.024912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471344624655095:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.025642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471348919622404:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.025735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471344624655088:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.025753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471344624655088:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.029078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471344624655095:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:51.029908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471346583387940:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25191) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471350878355312:2115] held: no
No RDMA context found for GID: 2025-11-27T18:18:51.030044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471346583387940:2048] [node 1] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:18:51.030096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:51.030072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471343953724647:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.030192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347327191334:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.030298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471346583387940:2048] [node 1] ICP22 created new session: [3:7577471350878355321:2048]
2025-11-27T18:18:51.030508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471344624655088:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.030162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349779643182:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50247
2025-11-27T18:18:51.030947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471347327191339:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40254
2025-11-27T18:18:51.030358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471350878355321:2048] [node 1] ICS09 handshake done sender: [3:7577471350878355312:2115] self: [3:7577471350878355317:0] peer: [1:7577471344624655092:0] socket: 594 qp: -1
2025-11-27T18:18:51.031107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471348248692023:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.030389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471350878355321:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:51.030603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:51.031301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348248692024:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46351
2025-11-27T18:18:51.031312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471344624655089:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.033057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350878355313:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.031405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:51.032680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471349779643182:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.033460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471345484675805:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.033760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471346583388011:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.033791Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471346583388011:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.031895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471344624655089:2107] [node 4] ICP22 created new session: [1:7577471348919622413:2107]
2025-11-27T18:18:51.031933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471348919622413:2107] [node 4] ICS09 handshake done sender: [1:7577471344624655095:2113] self: [1:7577471344624655094:0] peer: [4:7577471348248692022:0] socket: 579 qp: -1
2025-11-27T18:18:51.031959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471348919622413:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:51.031996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:51.032432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471348919622407:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.032549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471344624655023:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.032574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:51.032609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471344624655023:2048] [node 2] ICP22 created new session: [1:7577471348919622415:2048]
2025-11-27T18:18:51.032646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471348919622415:2048] [node 2] ICS09 handshake done sender: [1:7577471344624655088:2111] self: [1:7577471344624655087:0] peer: [2:7577471347327191336:0] socket: 576 qp: -1
2025-11-27T18:18:51.032667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471348919622415:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:51.032685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:51.034871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471343032224031:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:51.036704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350878355320:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.037269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471343953724721:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:51.038118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471344624655097:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.038151Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471344624655097:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.039077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471344624655023:2048] [node 2] ICP18 handshake (actor [1:7577471348919622407:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.039853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349779643184:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50247
2025-11-27T18:18:51.037664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471343953724646:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.040618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471347509899013:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56423
No RDMA context found for GID: 2025-11-27T18:18:51.037452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350878355318:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:18:51.040783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471346583388011:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:51.041086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349779643185:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50247
2025-11-27T18:18:51.041106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471343953724645:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.041167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:51.041134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471343032223955:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.041191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:51.041381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471344624655097:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:51.041222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471343032223955:2048] [node 1] ICP22 created new session: [2:7577471347327191341:2048]
2025-11-27T18:18:51.041268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471347327191341:2048] [node 1] ICS09 handshake done sender: [2:7577471343032224031:2115] self: [2:7577471347327191336:0] peer: [1:7577471344624655087:0] socket: 588 qp: -1
2025-11-27T18:18:51.041220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471343953724645:2048] [node 1] ICP22 created new session: [4:7577471348248692028:2048]
2025-11-27T18:18:51.041377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471348248692028:2048] [node 1] ICS09 handshake done sender: [4:7577471343953724721:2116] self: [4:7577471348248692022:0] peer: [1:7577471344624655094:0] socket: 583 qp: -1
2025-11-27T18:18:51.043279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471348248692028:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:51.043322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:51.046200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471346583387941:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.041439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471347327191341:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:51.047113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471346583388003:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.041487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:51.047857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:51.043252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471343032223955:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:51.048037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471346583388003:2107] [node 4] ICP22 created new session: [3:7577471350878355327:2107]
2025-11-27T18:18:51.046178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471343032224026:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.046206Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471343032224026:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.048103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471350878355327:2107] [node 4] ICS09 handshake done sender: [3:7577471346583388011:2112] self: [3:7577471346583388010:0] peer: [4:7577471348248692025:0] socket: 598 qp: -1
2025-11-27T18:18:51.048442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471343953724645:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:51.048140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471350878355327:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:51.048948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471349779643182:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:51.048914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:51.049514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471344624655090:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.049572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:51.049610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471344624655090:2108] [node 5] ICP22 created new session: [1:7577471348919622419:2108]
2025-11-27T18:18:51.049652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471348919622419:2108] [node 5] ICS09 handshake done sender: [1:7577471344624655097:2114] self: [1:7577471344624655096:0] peer: [5:7577471349779643183:0] socket: 620 qp: -1
2025-11-27T18:18:51.049685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471348919622419:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:51.051811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348248692027:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46351
2025-11-27T18:18:51.049715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:51.051886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471345484675805:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.051947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:51.051973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471343032224026:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:51.052002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471345484675805:2048] [node 1] ICP22 created new session: [5:7577471349779643187:2048]
2025-11-27T18:18:51.052039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471349779643187:2048] [node 1] ICS09 handshake done sender: [5:7577471349779643182:2117] self: [5:7577471349779643183:0] peer: [1:7577471344624655096:0] socket: 596 qp: -1
2025-11-27T18:18:51.052066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471349779643187:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:51.052096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:51.052139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471347509899013:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.052490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471343214931638:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.053114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471343032224022:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.053159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471343032224022:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.054175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349779643186:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50247
2025-11-27T18:18:51.054323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471344624655099:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.053952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471343032224018:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.054363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471344624655099:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.054012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:51.054059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471343032224018:2107] [node 4] ICP22 created new session: [2:7577471347327191345:2107]
2025-11-27T18:18:51.054104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471347327191345:2107] [node 4] ICS09 handshake done sender: [2:7577471343032224026:2112] self: [2:7577471343032224025:0] peer: [4:7577471348248692026:0] socket: 614 qp: -1
2025-11-27T18:18:51.054145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471347327191345:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:51.054170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:51.057994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471344624655099:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:51.058822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471348248692021:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.062266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471347509899015:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56423
2025-11-27T18:18:51.064407Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471343953724647:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28614) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471348248692021:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.064555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471343953724647:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.064603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:51.064650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471343953724647:2050] [node 3] ICP22 created new session: [4:7577471348248692032:2050]
2025-11-27T18:18:51.064696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471348248692032:2050] [node 3] ICS09 handshake done sender: [4:7577471348248692021:2117] self: [4:7577471348248692025:0] peer: [3:7577471346583388010:0] socket: 600 qp: -1
2025-11-27T18:18:51.064749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471348248692032:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:51.064782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:51.065625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471344624655091:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.065706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:51.065922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471349779643185:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.065769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471344624655091:2109] [node 6] ICP22 created new session: [1:7577471348919622422:2109]
2025-11-27T18:18:51.066175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349779643190:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50247
2025-11-27T18:18:51.065813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471348919622422:2109] [node 6] ICS09 handshake done sender: [1:7577471344624655099:2115] self: [1:7577471344624655098:0] peer: [6:7577471347509899014:0] socket: 592 qp: -1
2025-11-27T18:18:51.066261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471345484675807:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.065848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471348919622422:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.065379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348248692030:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46351
2025-11-27T18:18:51.068985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471348248692023:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:51.069211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471346583388013:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.065885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:51.069269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471346583388013:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.077393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471349779643186:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.073430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471347509899016:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56423
2025-11-27T18:18:51.077846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471345484675806:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.077461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471347509899013:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:51.078182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471343214931638:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.078322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:51.078395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471343214931638:2048] [node 1] ICP22 created new session: [6:7577471347509899017:2048]
2025-11-27T18:18:51.078439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471347509899017:2048] [node 1] ICS09 handshake done sender: [6:7577471347509899013:2116] self: [6:7577471347509899014:0] peer: [1:7577471344624655098:0] socket: 625 qp: -1
2025-11-27T18:18:51.078482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471347509899017:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:51.078520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:51.080975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471343953724646:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.081044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:51.081098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471343953724646:2049] [node 2] ICP22 created new session: [4:7577471348248692036:2049]
2025-11-27T18:18:51.081146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471348248692036:2049] [node 2] ICS09 handshake done sender: [4:7577471348248692023:2118] self: [4:7577471348248692026:0] peer: [2:7577471343032224025:0] socket: 619 qp: -1
2025-11-27T18:18:51.081118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349779643193:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50247
2025-11-27T18:18:51.081197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471348248692036:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:51.081144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471346583388013:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:51.081302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:51.078465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471343032224028:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.078492Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471343032224028:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.083436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471346583388004:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.083666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:51.083712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471346583388004:2108] [node 5] ICP22 created new session: [3:7577471350878355333:2108]
2025-11-27T18:18:51.083947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471343953724646:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:51.081127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471343032224022:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:51.083999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471350878355333:2108] [node 5] ICS09 handshake done sender: [3:7577471346583388013:2113] self: [3:7577471346583388012:0] peer: [5:7577471349779643191:0] socket: 629 qp: -1
2025-11-27T18:18:51.082167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471343032223956:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.084061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471350878355333:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:51.082226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:51.084098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:51.082270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471343032223956:2049] [node 3] ICP22 created new session: [2:7577471347327191350:2049]
2025-11-27T18:18:51.082316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471347327191350:2049] [node 3] ICS09 handshake done sender: [2:7577471343032224022:2110] self: [2:7577471343032224021:0] peer: [3:7577471350878355326:0] socket: 609 qp: -1
2025-11-27T18:18:51.082346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471347327191350:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:51.084265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471349779643185:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:51.082374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:51.084145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471343032224028:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.085698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350878355332:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50799
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.087200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471347509899019:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56423
2025-11-27T18:18:51.086015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471345484675807:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.087291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471347509899020:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56423
2025-11-27T18:18:51.086920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471343032224019:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.087072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348248692034:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46351
2025-11-27T18:18:51.086066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:51.086999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:51.086119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471345484675807:2050] [node 3] ICP22 created new session: [5:7577471349779643195:2050]
2025-11-27T18:18:51.087083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471343032224019:2108] [node 5] ICP22 created new session: [2:7577471347327191352:2108]
2025-11-27T18:18:51.087128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471347327191352:2108] [node 5] ICS09 handshake done sender: [2:7577471343032224028:2113] self: [2:7577471343032224027:0] peer: [5:7577471349779643192:0] socket: 613 qp: -1
2025-11-27T18:18:51.086275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471349779643195:2050] [node 3] ICS09 handshake done sender: [5:7577471349779643185:2119] self: [5:7577471349779643191:0] peer: [3:7577471346583388012:0] socket: 630 qp: -1
2025-11-27T18:18:51.086306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471349779643195:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:51.086341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.091795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349779643194:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50247
2025-11-27T18:18:51.091793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471346583387941:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16029) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471350878355320:2119] held: no
2025-11-27T18:18:51.091538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471347509899020:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.091685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471343214931639:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.087158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471347327191352:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:51.087191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:51.094143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471347509899016:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.094543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471343032224030:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.095372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471343214931640:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.094588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471343032224030:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.095869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471350878355320:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:51.095983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471347509899019:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.096092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471343032224030:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:51.096110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471343214931705:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.096469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471348248692034:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.096659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471343032224020:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.096936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471343953724720:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.099107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471347509899025:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56423
2025-11-27T18:18:51.096732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:51.096993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471343032224020:2109] [node 6] ICP22 created new session: [2:7577471347327191355:2109]
2025-11-27T18:18:51.096974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471343953724720:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.099187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471347327191355:2109] [node 6] ICS09 handshake done sender: [2:7577471343032224030:2114] self: [2:7577471343032224029:0] peer: [6:7577471347509899022:0] socket: 587 qp: -1
2025-11-27T18:18:51.099547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471347327191355:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:51.100215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471345484675806:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16029) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471349779643186:2120] held: no
2025-11-27T18:18:51.099602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:51.100711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471347509899020:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:51.100780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471349779643190:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.100835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471349779643186:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:51.100939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471346583388015:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.100979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471346583388015:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.101222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471343953724720:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:51.101432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471343214931639:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.101486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:51.101534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471343214931639:2049] [node 2] ICP22 created new session: [6:7577471347509899027:2049]
2025-11-27T18:18:51.101575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471347509899027:2049] [node 2] ICS09 handshake done sender: [6:7577471347509899020:2120] self: [6:7577471347509899022:0] peer: [2:7577471343032224029:0] socket: 640 qp: -1
2025-11-27T18:18:51.101533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471345484675806:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.101603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471347509899027:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:51.101577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:51.102117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471345484675806:2049] [node 2] ICP22 created new session: [5:7577471349779643198:2049]
2025-11-27T18:18:51.101641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:51.102177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471349779643198:2049] [node 2] ICS09 handshake done sender: [5:7577471349779643186:2120] self: [5:7577471349779643192:0] peer: [2:7577471343032224027:0] socket: 586 qp: -1
2025-11-27T18:18:51.102148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471346583387941:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.102222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471349779643198:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:51.102271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:51.102216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:51.102309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471346583387941:2049] [node 2] ICP22 created new session: [3:7577471350878355336:2049]
2025-11-27T18:18:51.102574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471350878355336:2049] [node 2] ICS09 handshake done sender: [3:7577471350878355320:2119] self: [3:7577471350878355326:0] peer: [2:7577471343032224021:0] socket: 612 qp: -1
2025-11-27T18:18:51.102610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471350878355336:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:51.102645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:51.102248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471343953724710:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.102459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:51.102571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471343953724710:2110] [node 6] ICP22 created new session: [4:7577471348248692040:2110]
2025-11-27T18:18:51.102828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471348248692040:2110] [node 6] ICS09 handshake done sender: [4:7577471343953724720:2115] self: [4:7577471343953724719:0] peer: [6:7577471347509899024:0] socket: 636 qp: -1
2025-11-27T18:18:51.102861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471348248692040:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:51.102907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.104406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471345484675869:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.104417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350878355335:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50799
2025-11-27T18:18:51.104828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471343953724718:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.104859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471343953724718:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.107049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471343953724718:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:51.104917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471346583388015:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:51.107775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471346583388005:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.107869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:51.107921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471346583388005:2109] [node 6] ICP22 created new session: [3:7577471350878355339:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.107975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471350878355339:2109] [node 6] ICS09 handshake done sender: [3:7577471346583388015:2114] self: [3:7577471346583388014:0] peer: [6:7577471347509899023:0] socket: 603 qp: -1
2025-11-27T18:18:51.107978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471343953724709:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.108028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349779643201:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50247
2025-11-27T18:18:51.108069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:51.108048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471350878355339:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:51.108126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471343953724709:2108] [node 5] ICP22 created new session: [4:7577471348248692043:2108]
2025-11-27T18:18:51.108092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:51.108189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471348248692043:2108] [node 5] ICS09 handshake done sender: [4:7577471343953724718:2114] self: [4:7577471343953724717:0] peer: [5:7577471349779643200:0] socket: 633 qp: -1
2025-11-27T18:18:51.108223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471348248692043:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:51.108254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.108576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471347509899026:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56423
2025-11-27T18:18:51.109172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350878355335:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.108821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471349779643190:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:51.109239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471345484675869:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.109355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:51.109565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471346583388005:2109] [node 6] ICP18 handshake (actor [3:7577471350878355335:2121]) is too old
2025-11-27T18:18:51.109476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471345484675869:2107] [node 4] ICP22 created new session: [5:7577471349779643202:2107]
2025-11-27T18:18:51.109615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471349779643202:2107] [node 4] ICS09 handshake done sender: [5:7577471349779643190:2121] self: [5:7577471349779643200:0] peer: [4:7577471343953724717:0] socket: 634 qp: -1
2025-11-27T18:18:51.109654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471349779643202:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:51.109691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:51.110098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471347509899019:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.110444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471347509899029:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56423
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.110617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471347509899030:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56423
2025-11-27T18:18:51.111295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471343214931705:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.111662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:51.111703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471343214931705:2107] [node 4] ICP22 created new session: [6:7577471347509899032:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.111916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471347509899032:2107] [node 4] ICS09 handshake done sender: [6:7577471347509899019:2119] self: [6:7577471347509899024:0] peer: [4:7577471343953724719:0] socket: 580 qp: -1
2025-11-27T18:18:51.111949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471347509899032:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:51.112001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348248692045:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46351
2025-11-27T18:18:51.111986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:51.113317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471347509899029:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.113481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471347509899016:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:51.114798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471343214931640:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.114845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:51.114886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471343214931640:2050] [node 3] ICP22 created new session: [6:7577471347509899034:2050]
2025-11-27T18:18:51.114926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471347509899034:2050] [node 3] ICS09 handshake done sender: [6:7577471347509899016:2118] self: [6:7577471347509899023:0] peer: [3:7577471346583388014:0] socket: 608 qp: -1
2025-11-27T18:18:51.114960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471347509899034:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:51.114999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:51.115212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471343214931640:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:51.115695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471343214931697:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.116035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471345484675880:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.116067Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471345484675880:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.117016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471345484675880:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.117240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471347509899037:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56423
2025-11-27T18:18:51.117338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471345484675870:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.117428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:51.117469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471345484675870:2109] [node 6] ICP22 created new session: [5:7577471349779643205:2109]
2025-11-27T18:18:51.117509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471349779643205:2109] [node 6] ICS09 handshake done sender: [5:7577471345484675880:2115] self: [5:7577471345484675879:0] peer: [6:7577471347509899036:0] socket: 590 qp: -1
2025-11-27T18:18:51.117544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471349779643205:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:51.117613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:51.117716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471347509899029:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:51.118028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471343214931697:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.118077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:51.118115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471343214931697:2104] [node 5] ICP22 created new session: [6:7577471347509899038:2104]
2025-11-27T18:18:51.118152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471347509899038:2104] [node 5] ICS09 handshake done sender: [6:7577471347509899029:2123] self: [6:7577471347509899036:0] peer: [5:7577471345484675879:0] socket: 582 qp: -1
2025-11-27T18:18:51.118188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471347509899038:2104] [node 5] ICS10 traffic start
2025-11-27T18:18:51.118216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:51.146364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:51.146603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471347327191341:2048] [node 1] ICS01 socket: 588 reason# EndOfStream
2025-11-27T18:18:51.146667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471343032223955:2048] [node 1] ICP30 unregister session Session# [2:7577471347327191341:2048] VirtualId# [2:7577471347327191336:0]
2025-11-27T18:18:51.146709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471347327191341:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.146799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:51.146843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:51.149449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:51.149681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471350878355321:2048] [node 1] ICS01 socket: 594 reason# EndOfStream
2025-11-27T18:18:51.149724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471346583387940:2048] [node 1] ICP30 unregister session Session# [3:7577471350878355321:2048] VirtualId# [3:7577471350878355317:0]
2025-11-27T18:18:51.149769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471350878355321:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.149839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:51.149886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:51.153052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:51.153276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471348248692028:2048] [node 1] ICS01 socket: 583 reason# EndOfStream
2025-11-27T18:18:51.153321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471343953724645:2048] [node 1] ICP30 unregister session Session# [4:7577471348248692028:2048] VirtualId# [4:7577471348248692022:0]
2025-11-27T18:18:51.153347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471348248692028:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.153411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:51.153464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:51.154142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:51.154320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471349779643187:2048] [node 1] ICS01 socket: 596 reason# EndOfStream
2025-11-27T18:18:51.154361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471345484675805:2048] [node 1] ICP30 unregister session Session# [5:7577471349779643187:2048] VirtualId# [5:7577471349779643183:0]
2025-11-27T18:18:51.154388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471349779643187:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.154465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:51.154520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:51.155058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:51.155255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471347509899017:2048] [node 1] ICS01 socket: 625 reason# EndOfStream
2025-11-27T18:18:51.155294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471343214931638:2048] [node 1] ICP30 unregister session Session# [6:7577471347509899017:2048] VirtualId# [6:7577471347509899014:0]
2025-11-27T18:18:51.155323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471347509899017:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.155401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:51.155495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:51.212427Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:51.212655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471350878355336:2049] [node 2] ICS01 socket: 612 reason# EndOfStream
2025-11-27T18:18:51.212697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471346583387941:2049] [node 2] ICP30 unregister session Session# [3:7577471350878355336:2049] VirtualId# [3:7577471350878355326:0]
2025-11-27T18:18:51.212730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471350878355336:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.212825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:51.212898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:51.215554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:51.215756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471348248692036:2049] [node 2] ICS07 socket disconnect 619 reason# EndOfStream
2025-11-27T18:18:51.215794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471348248692036:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.215876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:51.215919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471348248692036:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:51.216724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:51.216801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471343953724646:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16029) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:51.216870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471348248692036:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:51.216908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471343953724646:2049] [node 2] ICP30 unregister session Session# [4:7577471348248692036:2049] VirtualId# [4:7577471348248692026:0]
2025-11-27T18:18:51.216925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471349779643198:2049] [node 2] ICS01 socket: 586 reason# EndOfStream
2025-11-27T18:18:51.216968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471345484675806:2049] [node 2] ICP30 unregister session Session# [5:7577471349779643198:2049] VirtualId# [5:7577471349779643192:0]
2025-11-27T18:18:51.216965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:51.217001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471343953724646:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.217000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471349779643198:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.217015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.217079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:51.217126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:51.217748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:51.217935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471347509899027:2049] [node 2] ICS07 socket disconnect 640 reason# EndOfStream
2025-11-27T18:18:51.217969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471347509899027:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.218024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:51.218064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471347509899027:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:51.218866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471343214931639:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16029) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:51.218907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471347509899027:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:51.218938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471343214931639:2049] [node 2] ICP30 unregister session Session# [6:7577471347509899027:2049] VirtualId# [6:7577471347509899022:0]
2025-11-27T18:18:51.218983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:51.219007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471343214931639:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.219019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.226869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471343953724646:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:51.228485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471343214931639:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:51.256010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:51.256245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471348248692032:2050] [node 3] ICS01 socket: 600 reason# EndOfStream
2025-11-27T18:18:51.256292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471343953724647:2050] [node 3] ICP30 unregister session Session# [4:7577471348248692032:2050] VirtualId# [4:7577471348248692025:0]
2025-11-27T18:18:51.256320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471348248692032:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.256416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:51.256462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:51.257016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:51.257233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471349779643195:2050] [node 3] ICS01 socket: 630 reason# EndOfStream
2025-11-27T18:18:51.257291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471345484675807:2050] [node 3] ICP30 unregister session Session# [5:7577471349779643195:2050] VirtualId# [5:7577471349779643191:0]
2025-11-27T18:18:51.257319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471349779643195:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.257395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:51.257487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:51.258008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:51.258248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471347509899034:2050] [node 3] ICS07 socket disconnect 608 reason# EndOfStream
2025-11-27T18:18:51.258293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471347509899034:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.258433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:51.258507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471347509899034:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:51.259377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471343214931640:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28614) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:51.259442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471347509899034:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:51.259485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471343214931640:2050] [node 3] ICP30 unregister session Session# [6:7577471347509899034:2050] VirtualId# [6:7577471347509899023:0]
2025-11-27T18:18:51.259533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:51.259572Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471343214931640:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.259592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.269460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471343214931640:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:51.282575Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:51.282800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471349779643202:2107] [node 4] ICS07 socket disconnect 634 reason# EndOfStream
2025-11-27T18:18:51.282833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471349779643202:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.282918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:51.282965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471349779643202:2107] [node 4] ICS15 start handshake
2025-11-27T18:18:51.283673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:51.283821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471345484675869:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4021) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:51.283874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471349779643202:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:51.283883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471347509899032:2107] [node 4] ICS07 socket disconnect 580 reason# EndOfStream
2025-11-27T18:18:51.283910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471345484675869:2107] [node 4] ICP30 unregister session Session# [5:7577471349779643202:2107] VirtualId# [5:7577471349779643200:0]
2025-11-27T18:18:51.283953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:51.283915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471347509899032:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.283983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471345484675869:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4021) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.283998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4021) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.283989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:51.284048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471347509899032:2107] [node 4] ICS15 start handshake
2025-11-27T18:18:51.284912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471343214931705:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4021) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:51.284987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471347509899032:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:51.285034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471343214931705:2107] [node 4] ICP30 unregister session Session# [6:7577471347509899032:2107] VirtualId# [6:7577471347509899024:0]
2025-11-27T18:18:51.285077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:51.285110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471343214931705:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4021) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.285123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4021) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.294025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471345484675869:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:18:51.295074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471343214931705:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:18:51.333602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:51.333812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471347509899038:2104] [node 5] ICS01 socket: 582 reason# EndOfStream
2025-11-27T18:18:51.333873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471343214931697:2104] [node 5] ICP30 unregister session Session# [6:7577471347509899038:2104] VirtualId# [6:7577471347509899036:0]
2025-11-27T18:18:51.333908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471347509899038:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.333973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:51.334008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:51.413531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471349639147503:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:51.413595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471349639147504:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:51.433323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471347597576259:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:51.433388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471347597576260:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:51.452235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471350036813895:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:51.452360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471350036813896:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:51.473546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471348410709565:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:51.473596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471348410709566:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:51.473644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471348410709567:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:51.494428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471349748944322:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:51.494499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471349748944323:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:51.494575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471349748944324:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:51.515156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471351087782957:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:51.515211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471351087782958:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:51.515210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471351087782956:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:51.586423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471351087783018:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:51.680788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471349639147593:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:51.686775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471348410709629:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:51.685282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471347597576325:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:51.694600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471348410709628:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:51.693962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471351087783018:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(57984286ns) {Configure:104(57976643ns) {ProcessConfigured:135(40973324ns) {StartInitialHandshake:157(40969774ns) {DropHandshakes:470(2578ns) {DropIncomingHandshake:442(579ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(769ns) PrepareNewSessionHandshake:477(5816ns) {SwitchToState:179(3801ns)}}}}}
2025-11-27T18:18:51.685446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471347597576324:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:51.695730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471349639147594:2115] [node 5] ICP01 ready to work
2025-11-27T18:18:51.695669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471347597576326:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:51.696200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471351087783023:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:51.696256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471349639147597:2135] [node 6] ICP01 ready to work
2025-11-27T18:18:51.693784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471349748944387:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:51.695076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471349748944388:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:51.697610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471349639147503:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(108636065ns) {Configure:104(108631108ns) {ProcessConfigured:135(179893ns) {StartInitialHandshake:157(176608ns) {DropHandshakes:470(2500ns) {DropIncomingHandshake:442(501ns) DropOutgoingHandshake:457(330ns)} GenerateSessionVirtualId:988(863ns) PrepareNewSessionHandshake:477(5036ns) {SwitchToState:179(1168ns)}}}}}
2025-11-27T18:18:51.698533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471351087782956:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(101391332ns) {Configure:104(94386614ns) {ProcessConfigured:135(507726ns) {StartInitialHandshake:157(504805ns) {DropHandshakes:470(7367ns) {DropIncomingHandshake:442(712ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(827ns) PrepareNewSessionHandshake:477(2118ns) {SwitchToState:179(1023ns)}}}}}
2025-11-27T18:18:51.700039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471350036813895:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(95184020ns) {Configure:104(95163517ns) {ProcessConfigured:135(118753ns) {StartInitialHandshake:157(116389ns) {DropHandshakes:470(1146ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(532ns) PrepareNewSessionHandshake:477(2746ns) {SwitchToState:179(856ns)}}}}}
2025-11-27T18:18:51.700351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471350036813896:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(96119268ns) {Configure:104(96105533ns) {ProcessConfigured:135(1145387ns) {StartInitialHandshake:157(1142702ns) {DropHandshakes:470(1327ns) {DropIncomingHandshake:442(383ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(357ns) PrepareNewSessionHandshake:477(2171ns) {SwitchToState:179(1082ns)}}}}}
2025-11-27T18:18:51.700963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471350036813989:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:51.701521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471350036813990:2131] [node 6] ICP01 ready to work
2025-11-27T18:18:51.702242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471350036813991:2135] [node 5] ICP01 ready to work
2025-11-27T18:18:51.745877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471347597576326:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29071891ns) {Configure:104(29066344ns) {ProcessConfigured:135(253178ns) {StartInitialHandshake:157(249465ns) {DropHandshakes:470(2901ns) {DropIncomingHandshake:442(606ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(675ns) PrepareNewSessionHandshake:477(1879ns) {SwitchToState:179(927ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.751311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471349748944388:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(38919934ns) {Configure:104(38307015ns) {ProcessConfigured:135(447058ns) {StartInitialHandshake:157(444213ns) {DropHandshakes:470(2639ns) {DropIncomingHandshake:442(672ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(1120ns) PrepareNewSessionHandshake:477(1787ns) {SwitchToState:179(700ns)}}}}}
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.753729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348410709643:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23311
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.754389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471347597576337:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62238
2025-11-27T18:18:51.754745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350036814001:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30508
2025-11-27T18:18:51.754389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471347597576338:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62238
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.760544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350036814002:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30508
2025-11-27T18:18:51.759897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348410709644:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23311
2025-11-27T18:18:51.760038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471347597576339:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62238
2025-11-27T18:18:51.761206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471349639147609:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23415
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.766935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349748944402:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.764791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350036814004:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30508
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.764426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471349639147610:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23415
2025-11-27T18:18:51.760047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471347597576340:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62238
2025-11-27T18:18:51.776310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471349639147609:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.773967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471349748944402:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.763800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347597576338:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.776474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471349639147611:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23415
2025-11-27T18:18:51.777617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347597576339:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.776510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350036814003:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30508
2025-11-27T18:18:51.777672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350036814001:2142] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.779031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471350036813895:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.779954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350036814004:2145] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.777930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347597576337:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.783850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471347597576344:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62238
2025-11-27T18:18:51.797349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471349639147611:2144] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.784025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471347597576259:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.793142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347597576344:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.799100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348410709645:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23311
2025-11-27T18:18:51.799439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471348410709644:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.799940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471348410709643:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.799759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471349639147599:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.799795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471349639147599:2138] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.799944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471349639147596:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.801508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347597576340:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.800083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350036814003:2144] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.801641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350036814002:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.801719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471348410709567:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.801760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471348410709645:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.801764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471350036813896:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.802171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471348410709566:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.802723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347597576328:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.802768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471347597576328:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.803416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347597576332:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.799971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471349639147596:2137] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.802513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471348410709565:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.801728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471349639147610:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.802501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350036813993:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.803452Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471347597576332:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.802656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471349639147612:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23415
2025-11-27T18:18:51.802540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471350036813993:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.804348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471347597576332:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:51.805278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471349639147613:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23415
2025-11-27T18:18:51.806482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471349639147601:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.806524Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471349639147601:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.806528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471349639147613:2146] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.808486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471349639147599:2138] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:51.805242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471347597576328:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.808771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348410709649:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23311
2025-11-27T18:18:51.809226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471349639147612:2145] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.808709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471347597576260:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.808785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: 2025-11-27T18:18:51.809311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350036814009:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30508
::ffff:127.1.0.0
2025-11-27T18:18:51.808826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471347597576260:2049] [node 3] ICP22 created new session: [2:7577471347597576352:2049]
2025-11-27T18:18:51.809280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471347597576324:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.811231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471349639147596:2137] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:51.809304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471347597576352:2049] [node 3] ICS09 handshake done sender: [2:7577471347597576328:2112] self: [2:7577471347597576327:0] peer: [3:7577471350036814007:0] socket: 609 qp: -1
2025-11-27T18:18:51.809353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471347597576352:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:51.813659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471348410709645:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:51.809359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:51.809405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:51.809416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471347597576324:2107] [node 4] ICP22 created new session: [2:7577471347597576353:2107]
2025-11-27T18:18:51.811183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471347597576353:2107] [node 4] ICS09 handshake done sender: [2:7577471347597576332:2114] self: [2:7577471347597576331:0] peer: [4:7577471348410709647:0] socket: 616 qp: -1
2025-11-27T18:18:51.811190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350036814010:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30508
2025-11-27T18:18:51.811221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471347597576353:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:51.813815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471349639147504:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.816978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348410709650:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23311
2025-11-27T18:18:51.816895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471350036813993:2139] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:51.813644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:51.813881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:51.818167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471350036814003:2144] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:51.814342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471349639147504:2049] [node 3] ICP22 created new session: [1:7577471349639147621:2049]
2025-11-27T18:18:51.814420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471349639147621:2049] [node 3] ICS09 handshake done sender: [1:7577471349639147599:2138] self: [1:7577471349639147598:0] peer: [3:7577471350036814005:0] socket: 584 qp: -1
2025-11-27T18:18:51.814456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471349639147621:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:51.817984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471348410709566:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.814485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:51.818176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:51.819510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471348410709566:2049] [node 2] ICP22 created new session: [4:7577471348410709651:2049]
2025-11-27T18:18:51.819051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471349639147503:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.820041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471348410709651:2049] [node 2] ICS09 handshake done sender: [4:7577471348410709645:2117] self: [4:7577471348410709647:0] peer: [2:7577471347597576331:0] socket: 619 qp: -1
2025-11-27T18:18:51.819114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:51.820325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471348410709651:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:51.820719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471350036813989:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:18:51.820364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
::ffff:127.1.0.0
2025-11-27T18:18:51.819534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471349639147503:2048] [node 2] ICP22 created new session: [1:7577471349639147624:2048]
2025-11-27T18:18:51.820317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471349639147624:2048] [node 2] ICS09 handshake done sender: [1:7577471349639147596:2137] self: [1:7577471349639147595:0] peer: [2:7577471347597576347:0] socket: 583 qp: -1
2025-11-27T18:18:51.820786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:51.820980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471350036813989:2107] [node 4] ICP22 created new session: [3:7577471350036814012:2107]
2025-11-27T18:18:51.820444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471349639147624:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:51.821089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471347597576356:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62238
2025-11-27T18:18:51.820496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:51.821033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471350036814012:2107] [node 4] ICS09 handshake done sender: [3:7577471350036813993:2139] self: [3:7577471350036813992:0] peer: [4:7577471348410709646:0] socket: 599 qp: -1
2025-11-27T18:18:51.821090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471350036813896:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7923) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471350036814003:2144] held: no
2025-11-27T18:18:51.821691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471348410709566:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:51.822463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471347597576337:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:51.822919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471348410709644:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.821105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471350036814012:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:51.821144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:51.821385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471350036813896:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.824093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349748944404:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-27T18:18:51.821420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:51.824250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471348410709567:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.824886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471349639147601:2139] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:51.824946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471347597576259:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.824600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:51.824727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471348410709567:2050] [node 3] ICP22 created new session: [4:7577471348410709654:2050]
2025-11-27T18:18:51.825225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471348410709654:2050] [node 3] ICS09 handshake done sender: [4:7577471348410709644:2116] self: [4:7577471348410709646:0] peer: [3:7577471350036813992:0] socket: 603 qp: -1
2025-11-27T18:18:51.825250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471348410709654:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:51.825295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:51.821468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471350036813896:2049] [node 2] ICP22 created new session: [3:7577471350036814014:2049]
2025-11-27T18:18:51.829811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471351087783035:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22387
2025-11-27T18:18:51.825192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.825239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471347597576259:2048] [node 1] ICP22 created new session: [2:7577471347597576359:2048]
2025-11-27T18:18:51.821524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471350036814014:2049] [node 2] ICS09 handshake done sender: [3:7577471350036814003:2144] self: [3:7577471350036814007:0] peer: [2:7577471347597576327:0] socket: 617 qp: -1
2025-11-27T18:18:51.829703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471347597576359:2048] [node 1] ICS09 handshake done sender: [2:7577471347597576337:2117] self: [2:7577471347597576347:0] peer: [1:7577471349639147595:0] socket: 602 qp: -1
2025-11-27T18:18:51.821559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471350036814014:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:51.833763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471347597576359:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:51.822342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:51.833933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:51.838326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348410709652:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23311
2025-11-27T18:18:51.823510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471350036814001:2142] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:51.838388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471350036813895:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.838433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:51.838445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471349639147593:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.838485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471350036813895:2048] [node 1] ICP22 created new session: [3:7577471350036814016:2048]
2025-11-27T18:18:51.838516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:51.838547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471350036814016:2048] [node 1] ICS09 handshake done sender: [3:7577471350036814001:2142] self: [3:7577471350036814005:0] peer: [1:7577471349639147598:0] socket: 594 qp: -1
2025-11-27T18:18:51.838571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471349639147593:2107] [node 4] ICP22 created new session: [1:7577471349639147628:2107]
2025-11-27T18:18:51.838602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471350036814016:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:51.838635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:51.838640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471349639147628:2107] [node 4] ICS09 handshake done sender: [1:7577471349639147601:2139] self: [1:7577471349639147600:0] peer: [4:7577471348410709648:0] socket: 585 qp: -1
2025-11-27T18:18:51.838689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471349639147628:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:51.839539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471351087783035:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.839559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471349748944404:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.838722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.840134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471349748944323:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.840496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471347597576259:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:51.840478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471350036813895:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:51.840113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471351087782957:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.841067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347597576336:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.840902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349748944405:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-27T18:18:51.841115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471347597576336:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.842187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471348410709567:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:51.841610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471351087783037:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22387
2025-11-27T18:18:51.842374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471347597576334:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.842414Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471347597576334:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.843682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471347597576336:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:51.845711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471348410709643:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.846653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471351087783038:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22387
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.846227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471348410709565:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471348410709643:2115] held: no
2025-11-27T18:18:51.848589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471348410709565:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.848039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471347597576334:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:51.851551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471347597576326:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.847252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471351087783039:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22387
2025-11-27T18:18:51.849451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:51.851617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:51.852363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471347597576326:2110] [node 6] ICP22 created new session: [2:7577471347597576363:2110]
2025-11-27T18:18:51.851548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471348410709565:2048] [node 1] ICP22 created new session: [4:7577471348410709657:2048]
2025-11-27T18:18:51.852650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471347597576363:2110] [node 6] ICS09 handshake done sender: [2:7577471347597576336:2116] self: [2:7577471347597576335:0] peer: [6:7577471351087783036:0] socket: 607 qp: -1
2025-11-27T18:18:51.852127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471351087783035:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:51.852472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471348410709657:2048] [node 1] ICS09 handshake done sender: [4:7577471348410709643:2115] self: [4:7577471348410709648:0] peer: [1:7577471349639147600:0] socket: 596 qp: -1
2025-11-27T18:18:51.853670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471351087783039:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.852695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471347597576363:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:51.852506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471348410709657:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:51.854876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471351087782957:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.853727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:51.852642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:51.854936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.857638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471347597576325:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.856214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471349748944405:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.857705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:51.855370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471351087782957:2049] [node 2] ICP22 created new session: [6:7577471351087783040:2049]
2025-11-27T18:18:51.857900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471347597576325:2108] [node 5] ICP22 created new session: [2:7577471347597576365:2108]
2025-11-27T18:18:51.855461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471351087783040:2049] [node 2] ICS09 handshake done sender: [6:7577471351087783035:2116] self: [6:7577471351087783036:0] peer: [2:7577471347597576335:0] socket: 635 qp: -1
2025-11-27T18:18:51.858082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349748944407:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-27T18:18:51.858192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471347597576365:2108] [node 5] ICS09 handshake done sender: [2:7577471347597576334:2115] self: [2:7577471347597576333:0] peer: [5:7577471349748944406:0] socket: 587 qp: -1
2025-11-27T18:18:51.855569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471351087783040:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:51.861304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471347597576365:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:51.857603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:51.858198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349748944408:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-27T18:18:51.861347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:51.858432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349748944409:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-27T18:18:51.863274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471351087782957:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:51.863360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471349639147603:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.861253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471349748944322:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.862321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471349748944407:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.863114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471349748944404:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:51.863410Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471349639147603:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.863509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471349748944408:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.863551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471349748944323:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.863595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:51.864377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471349748944323:2049] [node 2] ICP22 created new session: [5:7577471349748944411:2049]
2025-11-27T18:18:51.864980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471349748944411:2049] [node 2] ICS09 handshake done sender: [5:7577471349748944404:2118] self: [5:7577471349748944406:0] peer: [2:7577471347597576333:0] socket: 590 qp: -1
2025-11-27T18:18:51.864864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471349639147603:2140] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:51.865047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471349748944411:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:51.865251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471351087783037:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.865115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471349639147594:2115] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.865103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:51.865240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:51.865839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471351087782958:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.865513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471349639147594:2115] [node 5] ICP22 created new session: [1:7577471349639147631:2115]
2025-11-27T18:18:51.865799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471349639147631:2115] [node 5] ICS09 handshake done sender: [1:7577471349639147603:2140] self: [1:7577471349639147602:0] peer: [5:7577471349748944410:0] socket: 600 qp: -1
2025-11-27T18:18:51.866033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471349639147631:2115] [node 5] ICS10 traffic start
2025-11-27T18:18:51.866215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:51.866274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471351087783041:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22387
2025-11-27T18:18:51.866583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471351087782956:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.866640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350036813999:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.866614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471349748944324:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.866691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471350036813999:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.866971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471351087783041:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.867353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471351087783023:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.868133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471350036813999:2141] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:51.868492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471348410709635:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.868534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471348410709635:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.868459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471350036814020:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30508
2025-11-27T18:18:51.871091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471350036813990:2131] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.870911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471349748944387:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.871129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:51.871179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471350036813990:2131] [node 6] ICP22 created new session: [3:7577471350036814022:2131]
2025-11-27T18:18:51.871309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471348410709635:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.867317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471349639147607:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.871720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471349748944322:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471349748944405:2119] held: no
2025-11-27T18:18:51.871322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471350036814022:2131] [node 6] ICS09 handshake done sender: [3:7577471350036813999:2141] self: [3:7577471350036813997:0] peer: [6:7577471351087783043:0] socket: 598 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.871772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471348410709629:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.867351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471349639147607:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.872726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471351087783047:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22387
2025-11-27T18:18:51.872484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:51.868965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471349639147607:2141] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:51.871356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471350036814022:2131] [node 6] ICS10 traffic start
2025-11-27T18:18:51.871418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:51.871616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350036813995:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.873032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349748944413:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.872627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471348410709629:2109] [node 6] ICP22 created new session: [4:7577471348410709661:2109]
2025-11-27T18:18:51.871293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471349639147597:2135] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.871523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:51.871652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471349639147597:2135] [node 6] ICP22 created new session: [1:7577471349639147634:2135]
2025-11-27T18:18:51.871706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471349639147634:2135] [node 6] ICS09 handshake done sender: [1:7577471349639147607:2141] self: [1:7577471349639147606:0] peer: [6:7577471351087783045:0] socket: 601 qp: -1
2025-11-27T18:18:51.871640Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471350036813995:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.872685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471348410709661:2109] [node 6] ICS09 handshake done sender: [4:7577471348410709635:2112] self: [4:7577471348410709634:0] peer: [6:7577471351087783046:0] socket: 642 qp: -1
2025-11-27T18:18:51.872727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471349639147634:2135] [node 6] ICS10 traffic start
2025-11-27T18:18:51.874017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471350036813995:2140] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:51.881836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349748944418:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-27T18:18:51.872789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:51.872917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471348410709661:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:51.872950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:51.883806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471348410709639:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.883840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471348410709639:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.882898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471349748944405:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:51.889686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471350036813991:2135] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.889806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:51.890384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348410709663:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23311
2025-11-27T18:18:51.889955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471350036813991:2135] [node 5] ICP22 created new session: [3:7577471350036814025:2135]
2025-11-27T18:18:51.883129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471349748944407:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:51.890298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471350036814025:2135] [node 5] ICS09 handshake done sender: [3:7577471350036813995:2140] self: [3:7577471350036813994:0] peer: [5:7577471349748944414:0] socket: 592 qp: -1
2025-11-27T18:18:51.883350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471349748944322:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.890496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471350036814025:2135] [node 5] ICS10 traffic start
2025-11-27T18:18:51.883397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:51.890563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:51.883440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471349748944322:2048] [node 1] ICP22 created new session: [5:7577471349748944419:2048]
2025-11-27T18:18:51.883481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471349748944419:2048] [node 1] ICS09 handshake done sender: [5:7577471349748944405:2119] self: [5:7577471349748944410:0] peer: [1:7577471349639147602:0] socket: 610 qp: -1
2025-11-27T18:18:51.883512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471349748944419:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:51.883548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:51.886145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471349748944324:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.890636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471351087782956:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471351087783037:2117] held: no
2025-11-27T18:18:51.886207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:51.890638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471348410709639:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:51.886247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471349748944324:2050] [node 3] ICP22 created new session: [5:7577471349748944421:2050]
2025-11-27T18:18:51.891100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471350036814020:2148] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.891004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471348410709628:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.886291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471349748944421:2050] [node 3] ICS09 handshake done sender: [5:7577471349748944407:2120] self: [5:7577471349748944414:0] peer: [3:7577471350036813994:0] socket: 589 qp: -1
2025-11-27T18:18:51.891038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:51.886325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471349748944421:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:51.886364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:51.891373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471348410709628:2108] [node 5] ICP22 created new session: [4:7577471348410709665:2108]
2025-11-27T18:18:51.892941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471351087783039:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:51.892528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471348410709665:2108] [node 5] ICS09 handshake done sender: [4:7577471348410709639:2114] self: [4:7577471348410709638:0] peer: [5:7577471349748944415:0] socket: 637 qp: -1
2025-11-27T18:18:51.892788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471348410709665:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:18:51.892825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
::ffff:127.1.0.0
2025-11-27T18:18:51.894591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471351087783048:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22387
2025-11-27T18:18:51.894664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471351087783049:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22387
2025-11-27T18:18:51.894672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471350036813990:2131] [node 6] ICP18 handshake (actor [3:7577471350036814020:2148]) is too old
2025-11-27T18:18:51.895489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471351087782958:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.895380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471348410709663:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.895630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.895679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471351087782958:2050] [node 3] ICP22 created new session: [6:7577471351087783051:2050]
2025-11-27T18:18:51.895684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471348410709628:2108] [node 5] ICP18 handshake (actor [4:7577471348410709663:2121]) is too old
2025-11-27T18:18:51.895726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471351087783051:2050] [node 3] ICS09 handshake done sender: [6:7577471351087783039:2119] self: [6:7577471351087783043:0] peer: [3:7577471350036813997:0] socket: 639 qp: -1
2025-11-27T18:18:51.895757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471351087783051:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:51.895783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:51.895767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471349748944423:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8807
2025-11-27T18:18:51.895980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471349748944408:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:51.896405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471349748944387:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.896449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:51.896502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471349748944387:2108] [node 4] ICP22 created new session: [5:7577471349748944424:2108]
2025-11-27T18:18:51.896559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471349748944424:2108] [node 4] ICS09 handshake done sender: [5:7577471349748944408:2121] self: [5:7577471349748944415:0] peer: [4:7577471348410709638:0] socket: 640 qp: -1
2025-11-27T18:18:51.896601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471349748944424:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:51.896634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:51.896609Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471351087782958:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:51.897347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471349748944387:2108] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.897891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471351087783050:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22387
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.898320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471348410709667:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23311
2025-11-27T18:18:51.897971Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471351087783037:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:51.898055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471351087783041:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:51.898316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471351087782956:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.898345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:51.898385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471351087782956:2048] [node 1] ICP22 created new session: [6:7577471351087783054:2048]
2025-11-27T18:18:51.898419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471351087783054:2048] [node 1] ICS09 handshake done sender: [6:7577471351087783037:2117] self: [6:7577471351087783045:0] peer: [1:7577471349639147606:0] socket: 622 qp: -1
2025-11-27T18:18:51.898440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471351087783054:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:51.898470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:51.898968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471351087783023:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.899012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:51.899041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471351087783023:2109] [node 4] ICP22 created new session: [6:7577471351087783056:2109]
2025-11-27T18:18:51.899086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471351087783056:2109] [node 4] ICS09 handshake done sender: [6:7577471351087783041:2120] self: [6:7577471351087783046:0] peer: [4:7577471348410709634:0] socket: 605 qp: -1
2025-11-27T18:18:51.899120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471351087783056:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:51.899149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:51.900267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471351087783050:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.900398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471351087783018:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:51.900690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471349748944398:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:51.900723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471349748944398:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:51.901742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471349748944398:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:51.902001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471349748944388:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:51.902100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:51.902169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471349748944388:2109] [node 6] ICP22 created new session: [5:7577471349748944427:2109]
2025-11-27T18:18:51.902214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471349748944427:2109] [node 6] ICS09 handshake done sender: [5:7577471349748944398:2116] self: [5:7577471349748944397:0] peer: [6:7577471351087783058:0] socket: 649 qp: -1
2025-11-27T18:18:51.902249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471349748944427:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:51.902278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:51.902249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471351087783059:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22387
2025-11-27T18:18:51.902775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471351087783018:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26402) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471351087783050:2124] held: no
2025-11-27T18:18:51.902933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471351087783050:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:51.903515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471351087783018:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:51.903553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:51.903590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471351087783018:2107] [node 5] ICP22 created new session: [6:7577471351087783060:2107]
2025-11-27T18:18:51.903747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471351087783060:2107] [node 5] ICS09 handshake done sender: [6:7577471351087783050:2124] self: [6:7577471351087783058:0] peer: [5:7577471349748944397:0] socket: 652 qp: -1
2025-11-27T18:18:51.903787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471351087783060:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:51.903822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:51.923302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:51.923589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471347597576359:2048] [node 1] ICS07 socket disconnect 602 reason# EndOfStream
2025-11-27T18:18:51.923630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471347597576359:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.923740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:51.923798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471347597576359:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:51.924758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:51.925006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471350036814016:2048] [node 1] ICS01 socket: 594 reason# EndOfStream
2025-11-27T18:18:51.925054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471350036813895:2048] [node 1] ICP30 unregister session Session# [3:7577471350036814016:2048] VirtualId# [3:7577471350036814005:0]
2025-11-27T18:18:51.925092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471350036814016:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.925193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:51.925249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:51.926362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471347597576259:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30555) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:51.926405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471347597576359:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:51.927708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471347597576259:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30555) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:51.927764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471347597576359:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:51.928285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471347597576259:2048] [node 1] ICP30 unregister session Session# [2:7577471347597576359:2048] VirtualId# [2:7577471347597576347:0]
2025-11-27T18:18:51.928339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:51.928503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471347597576259:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.928520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.928515Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:51.928887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471348410709657:2048] [node 1] ICS01 socket: 596 reason# EndOfStream
2025-11-27T18:18:51.929009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471348410709565:2048] [node 1] ICP30 unregister session Session# [4:7577471348410709657:2048] VirtualId# [4:7577471348410709648:0]
2025-11-27T18:18:51.929053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471348410709657:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.929160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:51.929222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:51.930061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:51.930256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471349748944419:2048] [node 1] ICS01 socket: 610 reason# EndOfStream
2025-11-27T18:18:51.930302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471349748944322:2048] [node 1] ICP30 unregister session Session# [5:7577471349748944419:2048] VirtualId# [5:7577471349748944410:0]
2025-11-27T18:18:51.930334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471349748944419:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.930414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:51.930477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:51.931162Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:51.931362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471351087783054:2048] [node 1] ICS01 socket: 622 reason# EndOfStream
2025-11-27T18:18:51.931433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471351087782956:2048] [node 1] ICP30 unregister session Session# [6:7577471351087783054:2048] VirtualId# [6:7577471351087783045:0]
2025-11-27T18:18:51.931459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471351087783054:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.931515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:51.931549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:51.958251Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:51.958479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471350036814014:2049] [node 2] ICS01 socket: 617 reason# EndOfStream
2025-11-27T18:18:51.958520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471350036813896:2049] [node 2] ICP30 unregister session Session# [3:7577471350036814014:2049] VirtualId# [3:7577471350036814007:0]
2025-11-27T18:18:51.958546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471350036814014:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.958604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:51.958666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:51.960808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:51.961055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471348410709651:2049] [node 2] ICS07 socket disconnect 619 reason# EndOfStream
2025-11-27T18:18:51.961120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471348410709651:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.961199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:51.961232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471348410709651:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:51.961762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:51.961967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471349748944411:2049] [node 2] ICS07 socket disconnect 590 reason# EndOfStream
2025-11-27T18:18:51.962032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471349748944411:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.962109Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471348410709566:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7923) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:51.962113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:51.962214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471348410709651:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:51.962173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471349748944411:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:51.962266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471348410709566:2049] [node 2] ICP30 unregister session Session# [4:7577471348410709651:2049] VirtualId# [4:7577471348410709647:0]
2025-11-27T18:18:51.962330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:51.962364Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471348410709566:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7923) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.962398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7923) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.962560Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:51.962795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471351087783040:2049] [node 2] ICS01 socket: 635 reason# EndOfStream
2025-11-27T18:18:51.962848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471351087782957:2049] [node 2] ICP30 unregister session Session# [6:7577471351087783040:2049] VirtualId# [6:7577471351087783036:0]
2025-11-27T18:18:51.962903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471351087783040:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:51.962861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471349748944323:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7923) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:51.963023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:51.962903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471349748944411:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:51.962995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471349748944323:2049] [node 2] ICP30 unregister session Session# [5:7577471349748944411:2049] VirtualId# [5:7577471349748944406:0]
2025-11-27T18:18:51.963085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:51.963038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:51.963075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471349748944323:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7923) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.963089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7923) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:51.971916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471348410709566:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:51.972950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471349748944323:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:52.003962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:52.004167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471348410709654:2050] [node 3] ICS07 socket disconnect 603 reason# EndOfStream
2025-11-27T18:18:52.004210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471348410709654:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:52.004277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:52.004319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471348410709654:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:52.005164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471348410709567:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11383) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:52.005297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471348410709654:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:52.005331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471348410709567:2050] [node 3] ICP30 unregister session Session# [4:7577471348410709654:2050] VirtualId# [4:7577471348410709646:0]
2025-11-27T18:18:52.005383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:52.005439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471348410709567:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.005454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.005166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:52.005414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471351087783051:2050] [node 3] ICS07 socket disconnect 639 reason# EndOfStream
2025-11-27T18:18:52.005460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471351087783051:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:52.005534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:52.005585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471351087783051:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:52.006315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:52.006378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471351087782958:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11383) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:52.006420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471351087783051:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:52.006509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471349748944421:2050] [node 3] ICS07 socket disconnect 589 reason# EndOfStream
2025-11-27T18:18:52.006479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471351087782958:2050] [node 3] ICP30 unregister session Session# [6:7577471351087783051:2050] VirtualId# [6:7577471351087783043:0]
2025-11-27T18:18:52.006543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471349748944421:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:52.006519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:52.006551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471351087782958:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.006562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.006640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:52.006690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471349748944421:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:52.007416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471349748944324:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11383) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:52.007541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471349748944421:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:52.007570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471349748944324:2050] [node 3] ICP30 unregister session Session# [5:7577471349748944421:2050] VirtualId# [5:7577471349748944414:0]
2025-11-27T18:18:52.007614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:52.007668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471349748944324:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.007690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.015925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471351087782958:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:52.018032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471349748944324:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:52.047992Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:52.048254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471349748944424:2108] [node 4] ICS07 socket disconnect 640 reason# EndOfStream
2025-11-27T18:18:52.048298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471349748944424:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:52.048368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:52.048407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471349748944424:2108] [node 4] ICS15 start handshake
2025-11-27T18:18:52.049246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471349748944387:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3931) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:52.049388Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:52.049392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471349748944424:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:52.049625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471351087783056:2109] [node 4] ICS07 socket disconnect 605 reason# EndOfStream
2025-11-27T18:18:52.049650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471351087783056:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:52.049756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471349748944387:2108] [node 4] ICP30 unregister session Session# [5:7577471349748944424:2108] VirtualId# [5:7577471349748944415:0]
2025-11-27T18:18:52.049744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:52.049834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:52.049787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471351087783056:2109] [node 4] ICS15 start handshake
2025-11-27T18:18:52.049912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471349748944387:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3931) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.049927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3931) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.050558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471351087783023:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3931) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:52.050630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471351087783056:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:52.050677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471351087783023:2109] [node 4] ICP30 unregister session Session# [6:7577471351087783056:2109] VirtualId# [6:7577471351087783046:0]
2025-11-27T18:18:52.050720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:52.050759Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471351087783023:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3931) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.050791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3931) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.060003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471349748944387:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:18:52.061630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471351087783023:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:18:52.101103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:52.101326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471351087783060:2107] [node 5] ICS01 socket: 652 reason# EndOfStream
2025-11-27T18:18:52.101372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471351087783018:2107] [node 5] ICP30 unregister session Session# [6:7577471351087783060:2107] VirtualId# [6:7577471351087783058:0]
2025-11-27T18:18:52.101404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471351087783060:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:52.101488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:52.101533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:52.207684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471354400940932:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:52.207762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471354400940933:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:52.235048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471354174119923:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:52.235256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471354174119924:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:52.257819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471351503875552:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:52.257865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471351503875553:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:52.282843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471351665618370:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:52.282902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471351665618372:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:52.283312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471351665618371:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:52.304842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471351489101789:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:52.304897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471351489101791:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:52.305004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471351489101790:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:52.327766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471353071441001:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:52.327812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471353071441003:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:52.328089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471353071441002:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:52.431347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471353071441061:2106] [node 5] ICP01 ready to work
2025-11-27T18:18:52.460301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471354174119986:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:52.450709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471354400941002:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:52.526604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471354174119988:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:52.535651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471351665618434:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:52.510384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471354400941003:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:52.539681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471351665618435:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:52.541382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471351489101871:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:52.540640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471353071441066:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:52.541851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471351489101870:2126] [node 6] ICP01 ready to work
2025-11-27T18:18:52.540562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471354400941004:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:52.541884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471351503875615:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:52.532410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471354174119987:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:52.540721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471351503875616:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:52.543417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471351503875617:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:52.547486Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471354400940932:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(118867943ns) {Configure:104(118860415ns) {ProcessConfigured:135(1764465ns) {StartInitialHandshake:157(1760391ns) {DropHandshakes:470(9267ns) {DropIncomingHandshake:442(594ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(814ns) PrepareNewSessionHandshake:477(3052ns) {SwitchToState:179(1181ns)}}}}}
2025-11-27T18:18:52.548414Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471354400940933:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(117885931ns) {Configure:104(117881528ns) {ProcessConfigured:135(399165ns) {StartInitialHandshake:157(396600ns) {DropHandshakes:470(9260ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(532ns)} GenerateSessionVirtualId:988(881ns) PrepareNewSessionHandshake:477(2036ns) {SwitchToState:179(973ns)}}}}}
2025-11-27T18:18:52.549992Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471351489101789:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(101953164ns) {Configure:104(101946958ns) {ProcessConfigured:135(886646ns) {StartInitialHandshake:157(884565ns) {DropHandshakes:470(1514ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(527ns)} GenerateSessionVirtualId:988(350ns) PrepareNewSessionHandshake:477(1927ns) {SwitchToState:179(869ns)}}}}}
2025-11-27T18:18:52.566229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471351503875615:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(21056774ns) {Configure:104(21051852ns) {ProcessConfigured:135(71193ns) {StartInitialHandshake:157(68629ns) {DropHandshakes:470(2984ns) {DropIncomingHandshake:442(791ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(773ns) PrepareNewSessionHandshake:477(2287ns) {SwitchToState:179(908ns)}}}}}
2025-11-27T18:18:52.572252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471351665618372:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31837325ns) {Configure:104(31830770ns) {ProcessConfigured:135(86965ns) {StartInitialHandshake:157(84315ns) {DropHandshakes:470(12059ns) {DropIncomingHandshake:442(663ns) DropOutgoingHandshake:457(300ns)} GenerateSessionVirtualId:988(1103ns) PrepareNewSessionHandshake:477(1487ns) {SwitchToState:179(542ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.575959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471351665618443:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13574
2025-11-27T18:18:52.557411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471353071441061:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(118783084ns) {Configure:104(118777796ns) {ProcessConfigured:135(969694ns) {StartInitialHandshake:157(968428ns) {DropHandshakes:470(506065ns) {DropIncomingHandshake:442(415ns) DropOutgoingHandshake:457(1360ns)} GenerateSessionVirtualId:988(1413ns) PrepareNewSessionHandshake:477(4769ns) {SwitchToState:179(1350ns)}}}}}
2025-11-27T18:18:52.576877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471354174119923:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(113462946ns) {Configure:104(113454811ns) {ProcessConfigured:135(762104ns) {StartInitialHandshake:157(760120ns) {DropHandshakes:470(1434ns) {DropIncomingHandshake:442(411ns) DropOutgoingHandshake:457(458ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(1818ns) {SwitchToState:179(753ns)}}}}}
2025-11-27T18:18:52.589873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471353071441002:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45440724ns) {Configure:104(45436230ns) {ProcessConfigured:135(84452ns) {StartInitialHandshake:157(82081ns) {DropHandshakes:470(1601ns) {DropIncomingHandshake:442(434ns) DropOutgoingHandshake:457(373ns)} GenerateSessionVirtualId:988(611ns) PrepareNewSessionHandshake:477(1257ns) {SwitchToState:179(405ns)}}}}}
2025-11-27T18:18:52.585883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471351489101790:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(43738645ns) {Configure:104(43733167ns) {ProcessConfigured:135(4774795ns) {StartInitialHandshake:157(4773062ns) {DropHandshakes:470(1220ns) {DropIncomingHandshake:442(306ns) DropOutgoingHandshake:457(276ns)} GenerateSessionVirtualId:988(501ns) PrepareNewSessionHandshake:477(2233ns) {SwitchToState:179(1220ns)}}}}}
2025-11-27T18:18:52.594126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471354174119986:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20914889ns) {Configure:104(20908359ns) {ProcessConfigured:135(10060785ns) {StartInitialHandshake:157(10058151ns) {DropHandshakes:470(1277ns) {DropIncomingHandshake:442(511ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(712ns) PrepareNewSessionHandshake:477(3718ns) {SwitchToState:179(1349ns)}}}}}
2025-11-27T18:18:52.597322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471351665618371:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40597301ns) {Configure:104(40194412ns) {ProcessConfigured:135(19151269ns) {StartInitialHandshake:157(19149316ns) {DropHandshakes:470(3012ns) {DropIncomingHandshake:442(709ns) DropOutgoingHandshake:457(311ns)} GenerateSessionVirtualId:988(730ns) PrepareNewSessionHandshake:477(5283ns) {SwitchToState:179(1062ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.595369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471351489101791:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(50545727ns) {Configure:104(50541135ns) {ProcessConfigured:135(14248412ns) {StartInitialHandshake:157(14245283ns) {DropHandshakes:470(1216ns) {DropIncomingHandshake:442(355ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(409ns) PrepareNewSessionHandshake:477(3468ns) {SwitchToState:179(1177ns)}}}}}
2025-11-27T18:18:52.600261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471354400941003:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(51221678ns) {Configure:104(51204046ns) {ProcessConfigured:135(39017ns) {StartInitialHandshake:157(37626ns) {DropHandshakes:470(1374ns) {DropIncomingHandshake:442(490ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(612ns) {SwitchToState:179(236ns)}}}}}
2025-11-27T18:18:52.600104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471353071441001:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29275435ns) {Configure:104(29269495ns) {ProcessConfigured:135(9667362ns) {StartInitialHandshake:157(9664546ns) {DropHandshakes:470(1026ns) {DropIncomingHandshake:442(422ns) DropOutgoingHandshake:457(158ns)} GenerateSessionVirtualId:988(340ns) PrepareNewSessionHandshake:477(3499ns) {SwitchToState:179(1718ns)}}}}}
2025-11-27T18:18:52.599971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471351503875552:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(134931786ns) {Configure:104(134923243ns) {ProcessConfigured:135(4730362ns) {StartInitialHandshake:157(4727052ns) {DropHandshakes:470(1990ns) {DropIncomingHandshake:442(467ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(942ns) PrepareNewSessionHandshake:477(3238ns) {SwitchToState:179(1514ns)}}}}}
2025-11-27T18:18:52.600623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471354400941002:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(56543994ns) {Configure:104(56537558ns) {ProcessConfigured:135(1152851ns) {StartInitialHandshake:157(1150899ns) {DropHandshakes:470(2403ns) {DropIncomingHandshake:442(911ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(635ns) PrepareNewSessionHandshake:477(1921ns) {SwitchToState:179(516ns)}}}}}
2025-11-27T18:18:52.611283Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471351665618370:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(66819881ns) {Configure:104(63794705ns) {ProcessConfigured:135(11531879ns) {StartInitialHandshake:157(11529014ns) {DropHandshakes:470(2508ns) {DropIncomingHandshake:442(564ns) DropOutgoingHandshake:457(404ns)} GenerateSessionVirtualId:988(913ns) PrepareNewSessionHandshake:477(3423ns) {SwitchToState:179(455ns)}}}}}
2025-11-27T18:18:52.602910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471354400941004:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(54257216ns) {Configure:104(53406398ns) {ProcessConfigured:135(3349544ns) {StartInitialHandshake:157(2835243ns) {DropHandshakes:470(1392ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(173ns)} GenerateSessionVirtualId:988(1086ns) PrepareNewSessionHandshake:477(2376ns) {SwitchToState:179(1081ns)}}}}}
2025-11-27T18:18:52.603718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471354400941015:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47921
2025-11-27T18:18:52.603848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471354400941016:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47921
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.613343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471351489101886:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36198
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.613723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471351489101885:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36198
2025-11-27T18:18:52.611175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471351503875553:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(62251410ns) {Configure:104(62245892ns) {ProcessConfigured:135(8053577ns) {StartInitialHandshake:157(8050570ns) {DropHandshakes:470(2474ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:477(2341ns) {SwitchToState:179(1075ns)}}}}}
2025-11-27T18:18:52.616892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471354174119987:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(35868191ns) {Configure:104(35860516ns) {ProcessConfigured:135(45358ns) {StartInitialHandshake:157(43457ns) {DropHandshakes:470(2186ns) {DropIncomingHandshake:442(516ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(728ns) PrepareNewSessionHandshake:477(2205ns) {SwitchToState:179(1055ns)}}}}}
2025-11-27T18:18:52.620767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471351489101886:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.620924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471351489101789:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.620936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471351665618443:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.613728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471353071441066:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23612883ns) {Configure:104(23607796ns) {ProcessConfigured:135(1429273ns) {StartInitialHandshake:157(1426059ns) {DropHandshakes:470(2033ns) {DropIncomingHandshake:442(483ns) DropOutgoingHandshake:457(321ns)} GenerateSessionVirtualId:988(669ns) PrepareNewSessionHandshake:477(2328ns) {SwitchToState:179(1244ns)}}}}}
2025-11-27T18:18:52.624558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471354400941015:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.629435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471354400941011:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.631246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471351665618451:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13574
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.638419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471351665618451:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.629470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471354400941011:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.638841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471354174119988:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(81255425ns) {Configure:104(81250260ns) {ProcessConfigured:135(3885185ns) {StartInitialHandshake:157(3882763ns) {DropHandshakes:470(3149ns) {DropIncomingHandshake:442(780ns) DropOutgoingHandshake:457(394ns)} GenerateSessionVirtualId:988(1286ns) PrepareNewSessionHandshake:477(2964ns) {SwitchToState:179(1540ns)}}}}}
2025-11-27T18:18:52.638857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471353071441003:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(71794737ns) {Configure:104(71778444ns) {ProcessConfigured:135(4399417ns) {StartInitialHandshake:157(4397425ns) {DropHandshakes:470(1516ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(294ns)} GenerateSessionVirtualId:988(548ns) PrepareNewSessionHandshake:477(1605ns) {SwitchToState:179(398ns)}}}}}
2025-11-27T18:18:52.639359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471351665618450:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13574
2025-11-27T18:18:52.634108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471354400941011:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:52.637757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471354400941016:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.639978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471351665618370:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.643537Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471351503875617:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(32119939ns) {Configure:104(31088227ns) {ProcessConfigured:135(3147653ns) {StartInitialHandshake:157(3144932ns) {DropHandshakes:470(1624ns) {DropIncomingHandshake:442(246ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(529ns) PrepareNewSessionHandshake:477(2209ns) {SwitchToState:179(1104ns)}}}}}
2025-11-27T18:18:52.640515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471354400941013:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.640019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471351665618450:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.644271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471351503875624:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36147
2025-11-27T18:18:52.640543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471354400941013:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.641714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471351665618454:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13574
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.646504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471351489101888:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36198
2025-11-27T18:18:52.645015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471354400941003:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.644425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471351503875626:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36147
2025-11-27T18:18:52.651708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471353071441078:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34365
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.646154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471351665618371:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.651927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471354174119994:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.652371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471351489101885:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.647464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471351503875624:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.658155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471353071441076:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34365
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.645797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471354400941013:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:52.651972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471354174119994:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.677753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471351665618456:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13574
2025-11-27T18:18:52.652494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471351489101886:2134] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.652512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471354174119991:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.646642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:52.685567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471353071441079:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34365
2025-11-27T18:18:52.647653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471351503875626:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.678225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471351665618450:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.678839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471351489101871:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:52.705488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471353071441076:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.678552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471351665618371:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.678596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:52.678641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471351665618371:2049] [node 2] ICP22 created new session: [4:7577471351665618457:2049]
2025-11-27T18:18:52.678776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471351665618451:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:52.690779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471351665618457:2049] [node 2] ICS09 handshake done sender: [4:7577471351665618450:2117] self: [4:7577471351665618455:0] peer: [2:7577471354174119993:0] socket: 605 qp: -1
2025-11-27T18:18:52.693500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471351665618457:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:52.709333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:52.720670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471351665618440:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.720702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471351665618440:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.651904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471351503875553:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:52.652534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471354174119991:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.647571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471354400941003:2108] [node 5] ICP22 created new session: [1:7577471354400941024:2108]
2025-11-27T18:18:52.681593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471351489101789:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.651317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471354400941024:2108] [node 5] ICS09 handshake done sender: [1:7577471354400941011:2119] self: [1:7577471354400941010:0] peer: [5:7577471351489101887:0] socket: 601 qp: -1
2025-11-27T18:18:52.653557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471354174119991:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:52.681653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:52.665551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471351503875616:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(46176100ns) {Configure:104(43371328ns) {ProcessConfigured:135(4044892ns) {StartInitialHandshake:157(4041548ns) {DropHandshakes:470(1607ns) {DropIncomingHandshake:442(500ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(647ns) PrepareNewSessionHandshake:477(3162ns) {SwitchToState:179(1132ns)}}}}}
2025-11-27T18:18:52.651592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471354400941024:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:52.717832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471353071441061:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:52.689600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471351503875634:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36147
2025-11-27T18:18:52.717980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471353071441078:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.673972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471354174119924:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.681706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471351489101789:2048] [node 1] ICP22 created new session: [5:7577471351489101891:2048]
2025-11-27T18:18:52.741412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471351665618370:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.652120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.674046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:52.693404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471351503875624:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:52.681767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471351489101891:2048] [node 1] ICS09 handshake done sender: [5:7577471351489101886:2134] self: [5:7577471351489101887:0] peer: [1:7577471354400941010:0] socket: 609 qp: -1
2025-11-27T18:18:52.741472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:52.715811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471351503875553:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.674102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471354174119924:2049] [node 3] ICP22 created new session: [2:7577471354174120005:2049]
2025-11-27T18:18:52.658427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471354400941002:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.674151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471354174120005:2049] [node 3] ICS09 handshake done sender: [2:7577471354174119991:2110] self: [2:7577471354174119989:0] peer: [3:7577471351503875633:0] socket: 593 qp: -1
2025-11-27T18:18:52.681806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471351489101891:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:52.720593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:52.665443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:52.674196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471354174120005:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:52.681845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:52.721587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471351503875553:2049] [node 2] ICP22 created new session: [3:7577471351503875636:2049]
2025-11-27T18:18:52.674235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:52.718426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471353071441001:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:52.675292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471354174120004:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60271
2025-11-27T18:18:52.675565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471354174119994:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:52.676783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471354400941002:2107] [node 4] ICP22 created new session: [1:7577471354400941028:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.685335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471351489101889:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36198
2025-11-27T18:18:52.718501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471353071441084:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34365
2025-11-27T18:18:52.736753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471351503875636:2049] [node 2] ICS09 handshake done sender: [3:7577471351503875624:2113] self: [3:7577471351503875633:0] peer: [2:7577471354174119989:0] socket: 616 qp: -1
2025-11-27T18:18:52.749397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471351665618370:2048] [node 1] ICP22 created new session: [4:7577471351665618460:2048]
2025-11-27T18:18:52.685509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471354400941028:2107] [node 4] ICS09 handshake done sender: [1:7577471354400941013:2120] self: [1:7577471354400941009:0] peer: [4:7577471351665618453:0] socket: 602 qp: -1
2025-11-27T18:18:52.740371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471353071441079:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.701925Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471351489101789:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:52.758482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471351665618460:2048] [node 1] ICS09 handshake done sender: [4:7577471351665618451:2118] self: [4:7577471351665618453:0] peer: [1:7577471354400941009:0] socket: 608 qp: -1
2025-11-27T18:18:52.736798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471351503875636:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:52.675978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471354174119986:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.759060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471353071441084:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.733683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471351489101893:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36198
2025-11-27T18:18:52.736840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:52.693424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471354400941028:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:52.758983Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471351665618371:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(56065253ns) {HandleHandshakeStatus:329(56054210ns) {DropIncomingHandshake:442(469ns) DropOutgoingHandshake:457(5126ns) SwitchToState:179(839ns) ProcessPendingSessionEvents:495(25068078ns) {ForwardSessionEventToSession:612(3823806ns) ForwardSessionEventToSession:612(3976255ns) ForwardSessionEventToSession:612(3854ns) ForwardSessionEventToSession:612(2226763ns) ForwardSessionEventToSession:612(4982ns) ForwardSessionEventToSession:612(1897227ns) ForwardSessionEventToSession:612(12610545ns) ForwardSessionEventToSession:612(5363ns) ForwardSessionEventToSession:612(3321ns) ForwardSessionEventToSession:612(493405ns)}}}
2025-11-27T18:18:52.676037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:52.734201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471351489101894:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36198
2025-11-27T18:18:52.759455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471351665618460:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:52.760130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471351503875637:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36147
2025-11-27T18:18:52.679012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471354174119986:2107] [node 4] ICP22 created new session: [2:7577471354174120009:2107]
2025-11-27T18:18:52.739201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471351489101877:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.759505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:52.739240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471351489101877:2130] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.701697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471354174120009:2107] [node 4] ICS09 handshake done sender: [2:7577471354174119994:2112] self: [2:7577471354174119993:0] peer: [4:7577471351665618455:0] socket: 595 qp: -1
2025-11-27T18:18:52.770544Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471351503875553:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(54485098ns) {HandleHandshakeStatus:329(54483062ns) {DropIncomingHandshake:442(688ns) DropOutgoingHandshake:457(4591ns) SwitchToState:179(510ns) ProcessPendingSessionEvents:495(11254785ns) {ForwardSessionEventToSession:612(1946429ns) ForwardSessionEventToSession:612(3695ns) ForwardSessionEventToSession:612(5856888ns) ForwardSessionEventToSession:612(3413941ns) ForwardSessionEventToSession:612(3577ns) ForwardSessionEventToSession:612(2689ns) ForwardSessionEventToSession:612(4826ns) ForwardSessionEventToSession:612(2264ns) ForwardSessionEventToSession:612(5452ns) ForwardSessionEventToSession:612(2014ns)}}}
2025-11-27T18:18:52.760457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471351665618440:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:52.759498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471353071441066:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:52.746450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471351489101877:2130] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:52.774218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471351665618442:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.709519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471354174120002:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60271
2025-11-27T18:18:52.765665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471353071441002:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:52.746924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471351489101870:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.774347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471351665618442:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.709819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471354174120009:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:52.765956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471353071441087:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34365
2025-11-27T18:18:52.749479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:52.774382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471351665618434:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.766530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471353071441076:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:52.709882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:52.774448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:52.737412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471354174120004:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.749515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471351489101870:2126] [node 6] ICP22 created new session: [5:7577471351489101897:2126]
2025-11-27T18:18:52.776195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471353071441061:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.775114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471351665618434:2108] [node 5] ICP22 created new session: [4:7577471351665618463:2108]
2025-11-27T18:18:52.758505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471351489101897:2126] [node 6] ICS09 handshake done sender: [5:7577471351489101877:2130] self: [5:7577471351489101876:0] peer: [6:7577471353071441083:0] socket: 590 qp: -1
2025-11-27T18:18:52.701683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.776800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:52.737485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471354174120012:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60271
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.759001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471351489101889:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.713988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471354400941003:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(71571167ns) {HandleHandshakeStatus:329(71567860ns) {DropIncomingHandshake:442(24735ns) DropOutgoingHandshake:457(660ns) SwitchToState:179(429ns) ProcessPendingSessionEvents:495(37567823ns) {ForwardSessionEventToSession:612(17318357ns) ForwardSessionEventToSession:612(6438ns) ForwardSessionEventToSession:612(7942830ns) ForwardSessionEventToSession:612(10175ns) ForwardSessionEventToSession:612(3096ns) ForwardSessionEventToSession:612(4114062ns) ForwardSessionEventToSession:612(3477ns) ForwardSessionEventToSession:612(8134454ns) ForwardSessionEventToSession:612(11272ns) ForwardSessionEventToSession:612(2658ns)}}}
2025-11-27T18:18:52.776273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471351665618370:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19821485ns) {HandleHandshakeStatus:329(19819392ns) {DropIncomingHandshake:442(747ns) DropOutgoingHandshake:457(5428ns) SwitchToState:179(882ns) ProcessPendingSessionEvents:495(1025734ns) {ForwardSessionEventToSession:612(973415ns) ForwardSessionEventToSession:612(8849ns) ForwardSessionEventToSession:612(3127ns) ForwardSessionEventToSession:612(7558ns) ForwardSessionEventToSession:612(2354ns) ForwardSessionEventToSession:612(2591ns) ForwardSessionEventToSession:612(7504ns) ForwardSessionEventToSession:612(2544ns) ForwardSessionEventToSession:612(2871ns) ForwardSessionEventToSession:612(7788ns)}}}
2025-11-27T18:18:52.753414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471354174119924:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:52.777878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471353071441061:2106] [node 5] ICP22 created new session: [6:7577471353071441092:2106]
2025-11-27T18:18:52.715294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471354400941026:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47921
2025-11-27T18:18:52.778664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471351503875637:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.779176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471353071441092:2106] [node 5] ICS09 handshake done sender: [6:7577471353071441076:2115] self: [6:7577471353071441083:0] peer: [5:7577471351489101876:0] socket: 611 qp: -1
2025-11-27T18:18:52.759986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471351489101897:2126] [node 6] ICS10 traffic start
2025-11-27T18:18:52.776457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471351665618463:2108] [node 5] ICS09 handshake done sender: [4:7577471351665618440:2112] self: [4:7577471351665618439:0] peer: [5:7577471351489101890:0] socket: 597 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.767018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471354174119986:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(82529332ns) {HandleHandshakeStatus:329(82526759ns) {DropIncomingHandshake:442(454ns) DropOutgoingHandshake:457(383ns) SwitchToState:179(631ns) ProcessPendingSessionEvents:495(23564562ns) {ForwardSessionEventToSession:612(4404977ns) ForwardSessionEventToSession:612(7740152ns) ForwardSessionEventToSession:612(24573ns) ForwardSessionEventToSession:612(7992752ns) ForwardSessionEventToSession:612(7896ns) ForwardSessionEventToSession:612(3810ns) ForwardSessionEventToSession:612(283467ns) ForwardSessionEventToSession:612(4267ns) ForwardSessionEventToSession:612(2519ns) ForwardSessionEventToSession:612(796774ns)}}}
2025-11-27T18:18:52.780593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471351503875619:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.767185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471354174120001:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.778097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471351665618463:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:52.767222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471354174120001:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.768404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471354174120001:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:52.778785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.715359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471354400941027:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47921
2025-11-27T18:18:52.760541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.774460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471354174119988:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.780626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471351503875619:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.779469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471353071441088:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34365
2025-11-27T18:18:52.774013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471351489101893:2137] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.775021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:52.735981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471354400941014:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.779570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471353071441092:2106] [node 5] ICS10 traffic start
2025-11-27T18:18:52.780625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471351503875552:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:52.779604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:52.775017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471351489101894:2138] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.784708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471351503875640:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36147
2025-11-27T18:18:52.776383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471354174119988:2109] [node 6] ICP22 created new session: [2:7577471354174120016:2109]
2025-11-27T18:18:52.736009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471354400941014:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.784671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471353071441090:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34365
2025-11-27T18:18:52.784743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471351503875619:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:52.777599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471354174120016:2109] [node 6] ICS09 handshake done sender: [2:7577471354174120001:2114] self: [2:7577471354174120000:0] peer: [6:7577471353071441089:0] socket: 639 qp: -1
2025-11-27T18:18:52.758895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471354400941027:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.787011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471351503875615:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.785425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471353071441091:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34365
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.777780Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471351489101870:2126] [node 6] ICP35 event processing took too much time PendingConnection:279(28075963ns) {HandleHandshakeStatus:329(28069499ns) {DropIncomingHandshake:442(848ns) DropOutgoingHandshake:457(575ns) SwitchToState:179(417ns) ProcessPendingSessionEvents:495(2091407ns) {ForwardSessionEventToSession:612(844956ns) ForwardSessionEventToSession:612(301798ns) ForwardSessionEventToSession:612(5997ns) ForwardSessionEventToSession:612(2562ns) ForwardSessionEventToSession:612(438570ns) ForwardSessionEventToSession:612(443217ns) ForwardSessionEventToSession:612(4329ns) ForwardSessionEventToSession:612(5736ns) ForwardSessionEventToSession:612(2802ns) ForwardSessionEventToSession:612(11298ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.799559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471351665618442:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:52.777653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471354174120016:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:52.778751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471351489101791:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.787069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:52.799708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471353071441002:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28651) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471353071441084:2119] held: no
2025-11-27T18:18:52.760180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471354400941002:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(81940350ns) {HandleHandshakeStatus:329(81936511ns) {DropIncomingHandshake:442(279ns) DropOutgoingHandshake:457(509ns) SwitchToState:179(6601ns) ProcessPendingSessionEvents:495(6764705ns) {ForwardSessionEventToSession:612(1450470ns) ForwardSessionEventToSession:612(3829ns) ForwardSessionEventToSession:612(702965ns) ForwardSessionEventToSession:612(23791ns) ForwardSessionEventToSession:612(2896ns) ForwardSessionEventToSession:612(2556005ns) ForwardSessionEventToSession:612(4348ns) ForwardSessionEventToSession:612(7892ns) ForwardSessionEventToSession:612(1108677ns) ForwardSessionEventToSession:612(3036ns)}}}
2025-11-27T18:18:52.777716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.799348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471351503875615:2108] [node 5] ICP22 created new session: [3:7577471351503875644:2108]
2025-11-27T18:18:52.801030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471353071441088:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.760334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471354400941026:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.779367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471351489101900:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36198
2025-11-27T18:18:52.807234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471351665618435:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.784419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471354174120012:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.801137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471351503875644:2108] [node 5] ICS09 handshake done sender: [3:7577471351503875619:2110] self: [3:7577471351503875618:0] peer: [5:7577471351489101901:0] socket: 620 qp: -1
2025-11-27T18:18:52.808680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:52.785545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471354174120014:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60271
2025-11-27T18:18:52.797698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471354174119923:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:52.799949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471354174119999:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.799977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471354174119999:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.800837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471354174119999:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:52.801456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471354174119987:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.802298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:52.805040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471354174119987:2108] [node 5] ICP22 created new session: [2:7577471354174120020:2108]
2025-11-27T18:18:52.805890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471354174120014:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.809653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471354174120020:2108] [node 5] ICS09 handshake done sender: [2:7577471354174119999:2113] self: [2:7577471354174119998:0] peer: [5:7577471351489101902:0] socket: 634 qp: -1
2025-11-27T18:18:52.809696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471354174120020:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:52.813251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:52.815190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471354174120021:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60271
2025-11-27T18:18:52.801849Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471353071441001:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12731) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471353071441078:2117] held: no
2025-11-27T18:18:52.782534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471351489101790:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:52.801179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471351503875644:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:52.802248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:52.805934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471351503875643:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36147
2025-11-27T18:18:52.813163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471351503875640:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.820248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471351503875627:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.820273Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471351503875627:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.822555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471351503875647:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36147
2025-11-27T18:18:52.825204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471351503875627:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:52.825218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471351503875552:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12731) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471351503875637:2118] held: no
2025-11-27T18:18:52.784862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577471351489101870:2126] [node 6] ICP18 handshake (actor [5:7577471351489101893:2137]) is too old
2025-11-27T18:18:52.808748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471351665618435:2109] [node 6] ICP22 created new session: [4:7577471351665618466:2109]
2025-11-27T18:18:52.760679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471354400941014:2121] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:52.810702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471353071441084:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:52.842582Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471354174119987:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(40859820ns) {HandleHandshakeStatus:329(40857462ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(234ns) SwitchToState:179(770ns) ProcessPendingSessionEvents:495(23550978ns) {ForwardSessionEventToSession:612(10250121ns) ForwardSessionEventToSession:612(4966ns) ForwardSessionEventToSession:612(5685454ns) ForwardSessionEventToSession:612(7560290ns) ForwardSessionEventToSession:612(4353ns) ForwardSessionEventToSession:612(7120ns) ForwardSessionEventToSession:612(3339ns) ForwardSessionEventToSession:612(3028ns) ForwardSessionEventToSession:612(9495ns) ForwardSessionEventToSession:612(2417ns)}}}
2025-11-27T18:18:52.785138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471351489101871:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1589) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471351489101885:2133] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.813495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471353071441078:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:52.813803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471351665618466:2109] [node 6] ICS09 handshake done sender: [4:7577471351665618442:2113] self: [4:7577471351665618441:0] peer: [6:7577471353071441086:0] socket: 604 qp: -1
2025-11-27T18:18:52.774048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471354400941004:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.798762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471351489101885:2133] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:52.846373Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471354174119923:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12731) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471354174120012:2117] held: no
2025-11-27T18:18:52.816142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471353071441002:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.774103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:52.822838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471351665618466:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:52.846505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471354174119987:2108] [node 5] ICP18 handshake (actor [2:7577471354174120014:2118]) is too old
2025-11-27T18:18:52.803610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471351489101871:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.849776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471351503875615:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(42042484ns) {HandleHandshakeStatus:329(42039958ns) {DropIncomingHandshake:442(257ns) DropOutgoingHandshake:457(380ns) SwitchToState:179(833ns) ProcessPendingSessionEvents:495(23336683ns) {ForwardSessionEventToSession:612(6882497ns) ForwardSessionEventToSession:612(822185ns) ForwardSessionEventToSession:612(2871ns) ForwardSessionEventToSession:612(554011ns) ForwardSessionEventToSession:612(12631ns) ForwardSessionEventToSession:612(3334ns) ForwardSessionEventToSession:612(4683583ns) ForwardSessionEventToSession:612(4203ns) ForwardSessionEventToSession:612(4235836ns) ForwardSessionEventToSession:612(665692ns)}}}
2025-11-27T18:18:52.774160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471354400941004:2110] [node 6] ICP22 created new session: [1:7577471354400941035:2110]
2025-11-27T18:18:52.816175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:52.822893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:52.846738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471354174120012:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:52.849508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471351665618469:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13574
2025-11-27T18:18:52.816214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471353071441002:2049] [node 2] ICP22 created new session: [6:7577471353071441097:2049]
2025-11-27T18:18:52.803650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:52.857513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471351503875617:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.816289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471353071441097:2049] [node 2] ICS09 handshake done sender: [6:7577471353071441084:2119] self: [6:7577471353071441089:0] peer: [2:7577471354174120000:0] socket: 640 qp: -1
2025-11-27T18:18:52.855189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471351665618468:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13574
2025-11-27T18:18:52.816342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471353071441097:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:52.866852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471351665618468:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.774999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471354400941035:2110] [node 6] ICS09 handshake done sender: [1:7577471354400941014:2121] self: [1:7577471354400941012:0] peer: [6:7577471353071441085:0] socket: 603 qp: -1
2025-11-27T18:18:52.803692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471351489101871:2108] [node 4] ICP22 created new session: [5:7577471351489101905:2108]
2025-11-27T18:18:52.803734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471351489101905:2108] [node 4] ICS09 handshake done sender: [5:7577471351489101885:2133] self: [5:7577471351489101890:0] peer: [4:7577471351665618439:0] socket: 606 qp: -1
2025-11-27T18:18:52.803761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471351489101905:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:52.857568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:52.866376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471354174119923:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.878355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471351665618372:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:52.816372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:52.867393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:52.857642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471351503875617:2109] [node 6] ICP22 created new session: [3:7577471351503875649:2109]
2025-11-27T18:18:52.776187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471354400941035:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:52.803789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:52.857689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471351503875649:2109] [node 6] ICS09 handshake done sender: [3:7577471351503875627:2115] self: [3:7577471351503875625:0] peer: [6:7577471353071441099:0] socket: 649 qp: -1
2025-11-27T18:18:52.877978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471354174119923:2048] [node 1] ICP22 created new session: [2:7577471354174120023:2048]
2025-11-27T18:18:52.880449Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471351665618435:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(61901678ns) {HandleHandshakeStatus:329(61899094ns) {DropIncomingHandshake:442(898ns) DropOutgoingHandshake:457(330ns) SwitchToState:179(521ns) ProcessPendingSessionEvents:495(32582776ns) {ForwardSessionEventToSession:612(14031483ns) ForwardSessionEventToSession:612(4992ns) ForwardSessionEventToSession:612(15376ns) ForwardSessionEventToSession:612(1129814ns) ForwardSessionEventToSession:612(7504176ns) ForwardSessionEventToSession:612(5523911ns) ForwardSessionEventToSession:612(4535ns) ForwardSessionEventToSession:612(3309ns) ForwardSessionEventToSession:612(3059401ns) ForwardSessionEventToSession:612(3844ns)}}}
2025-11-27T18:18:52.804591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471351489101903:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36198
2025-11-27T18:18:52.816766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471353071441003:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:52.806145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471351489101889:2136] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:52.776786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471354400941032:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47921
2025-11-27T18:18:52.809678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471351489101904:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36198
2025-11-27T18:18:52.818562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471353071441094:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34365
2025-11-27T18:18:52.878508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471354174120023:2048] [node 1] ICS09 handshake done sender: [2:7577471354174120012:2117] self: [2:7577471354174120018:0] peer: [1:7577471354400941005:0] socket: 641 qp: -1
2025-11-27T18:18:52.776824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:52.810814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471351489101791:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.823154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471353071441001:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.810863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:52.810912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471351489101791:2050] [node 3] ICP22 created new session: [5:7577471351489101907:2050]
2025-11-27T18:18:52.823198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:52.783315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471354400941008:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.879057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471354174120023:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:52.825118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471353071441001:2048] [node 1] ICP22 created new session: [6:7577471353071441100:2048]
2025-11-27T18:18:52.857724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471351503875649:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:52.783347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471354400941008:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.810963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471351489101907:2050] [node 3] ICS09 handshake done sender: [5:7577471351489101889:2136] self: [5:7577471351489101901:0] peer: [3:7577471351503875618:0] socket: 622 qp: -1
2025-11-27T18:18:52.879109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:52.857757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:52.834658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471353071441100:2048] [node 1] ICS09 handshake done sender: [6:7577471353071441078:2117] self: [6:7577471353071441085:0] peer: [1:7577471354400941012:0] socket: 614 qp: -1
2025-11-27T18:18:52.885140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471351665618471:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13574
2025-11-27T18:18:52.810998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471351489101907:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:52.811028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:52.785358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471354400941008:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:52.864283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471351503875615:2108] [node 5] ICP18 handshake (actor [3:7577471351503875640:2119]) is too old
2025-11-27T18:18:52.841891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471353071441100:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:52.842911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471353071441079:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:52.887381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471351665618372:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13197) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471351665618468:2121] held: no
2025-11-27T18:18:52.798101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471354400940933:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.845376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:52.887512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471351665618468:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:52.892136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471351665618435:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:52.895715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471351665618372:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.895774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:52.895821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471351665618372:2050] [node 3] ICP22 created new session: [4:7577471351665618472:2050]
2025-11-27T18:18:52.828865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471351489101894:2138] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:52.865460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471351503875552:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19489879ns) {HandleHandshakeStatus:395(19487403ns) {LogHandshakeFail:472(14027ns) {UpdateErrorStateLog:545(4372ns)} DropOutgoingHandshake:457(1572ns) ProcessPendingSessionEvents:495(9434025ns) {DropSessionEvent:512(10524ns) DropSessionEvent:512(16091ns) DropSessionEvent:512(4598ns) DropSessionEvent:512(11266ns) DropSessionEvent:512(5160ns) DropSessionEvent:512(5024ns) DropSessionEvent:512(6136ns) DropSessionEvent:512(4716ns) DropSessionEvent:512(4059ns) DropSessionEvent:512(18799ns)}}}
2025-11-27T18:18:52.863047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471353071441066:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.798187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:52.863097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:52.863146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471353071441066:2109] [node 4] ICP22 created new session: [6:7577471353071441103:2109]
2025-11-27T18:18:52.863183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471353071441103:2109] [node 4] ICS09 handshake done sender: [6:7577471353071441079:2118] self: [6:7577471353071441086:0] peer: [4:7577471351665618441:0] socket: 615 qp: -1
2025-11-27T18:18:52.863210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471353071441103:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:52.863243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:52.863801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471353071441102:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34365
2025-11-27T18:18:52.878818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471353071441001:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(41021813ns) {HandleHandshakeStatus:329(41018861ns) {DropIncomingHandshake:442(429ns) DropOutgoingHandshake:457(483ns) SwitchToState:179(550ns) ProcessPendingSessionEvents:495(671ns)}}
2025-11-27T18:18:52.879673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471353071441088:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:52.884264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471353071441003:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13197) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471353071441088:2121] held: no
2025-11-27T18:18:52.884365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471353071441003:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.884434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:52.884738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471353071441003:2050] [node 3] ICP22 created new session: [6:7577471353071441105:2050]
2025-11-27T18:18:52.885055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471353071441105:2050] [node 3] ICS09 handshake done sender: [6:7577471353071441088:2121] self: [6:7577471353071441099:0] peer: [3:7577471351503875625:0] socket: 650 qp: -1
2025-11-27T18:18:52.885959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471353071441105:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:52.886179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:52.871425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471351503875647:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.853651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471351489101790:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.905773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471351665618472:2050] [node 3] ICS09 handshake done sender: [4:7577471351665618468:2121] self: [4:7577471351665618470:0] peer: [3:7577471351503875631:0] socket: 655 qp: -1
2025-11-27T18:18:52.853704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:52.798233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471354400940933:2049] [node 3] ICP22 created new session: [1:7577471354400941039:2049]
2025-11-27T18:18:52.853764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471351489101790:2049] [node 2] ICP22 created new session: [5:7577471351489101909:2049]
2025-11-27T18:18:52.798278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471354400941039:2049] [node 3] ICS09 handshake done sender: [1:7577471354400941008:2118] self: [1:7577471354400941006:0] peer: [3:7577471351503875641:0] socket: 642 qp: -1
2025-11-27T18:18:52.853814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471351489101909:2049] [node 2] ICS09 handshake done sender: [5:7577471351489101894:2138] self: [5:7577471351489101902:0] peer: [2:7577471354174119998:0] socket: 638 qp: -1
2025-11-27T18:18:52.871484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471351503875637:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:52.798312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471354400941039:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:52.853849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471351489101909:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:52.871749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471351503875552:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:52.798351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:52.853882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:52.805359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471354400941007:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.871787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:52.805392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471354400941007:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.871826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471351503875552:2048] [node 1] ICP22 created new session: [3:7577471351503875651:2048]
2025-11-27T18:18:52.807079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471354400941007:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:52.871868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471351503875651:2048] [node 1] ICS09 handshake done sender: [3:7577471351503875637:2118] self: [3:7577471351503875641:0] peer: [1:7577471354400941006:0] socket: 643 qp: -1
2025-11-27T18:18:52.871901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471351503875651:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:52.813392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471354400940932:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.871929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:52.813865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:52.913950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471351665618472:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:52.818655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471354400940932:2048] [node 2] ICP22 created new session: [1:7577471354400941042:2048]
2025-11-27T18:18:52.872403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471351503875617:2109] [node 6] ICP18 handshake (actor [3:7577471351503875647:2121]) is too old
2025-11-27T18:18:52.825194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471354400941042:2048] [node 2] ICS09 handshake done sender: [1:7577471354400941007:2117] self: [1:7577471354400941005:0] peer: [2:7577471354174120018:0] socket: 588 qp: -1
2025-11-27T18:18:52.914023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:52.880439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471351503875632:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:52.829354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471354400941042:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:52.880468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471351503875632:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:52.829427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:52.883481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471351503875632:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:52.884632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471351503875616:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:52.881722Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471354400940932:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(65652892ns) {HandleHandshakeStatus:329(65650887ns) {DropIncomingHandshake:442(11989ns) DropOutgoingHandshake:457(457ns) SwitchToState:179(353ns) ProcessPendingSessionEvents:495(31673199ns) {ForwardSessionEventToSession:612(16824886ns) ForwardSessionEventToSession:612(3712868ns) ForwardSessionEventToSession:612(11621ns) ForwardSessionEventToSession:612(5009572ns) ForwardSessionEventToSession:612(2959ns) ForwardSessionEventToSession:612(2298ns) ForwardSessionEventToSession:612(5186519ns) ForwardSessionEventToSession:612(4416ns) ForwardSessionEventToSession:612(644905ns) ForwardSessionEventToSession:612(246693ns)}}}
2025-11-27T18:18:52.884679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:52.884999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471351503875616:2107] [node 4] ICP22 created new session: [3:7577471351503875654:2107]
2025-11-27T18:18:52.885971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471351503875654:2107] [node 4] ICS09 handshake done sender: [3:7577471351503875632:2116] self: [3:7577471351503875631:0] peer: [4:7577471351665618470:0] socket: 621 qp: -1
2025-11-27T18:18:52.886021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471351503875654:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:52.886371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:52.928338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471351665618372:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(35766999ns) {HandleHandshakeStatus:329(35764939ns) {DropIncomingHandshake:442(1494ns) DropOutgoingHandshake:457(299ns) SwitchToState:179(1370ns) ProcessPendingSessionEvents:495(789ns)}}
2025-11-27T18:18:52.967761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:52.968011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471354174120023:2048] [node 1] ICS01 socket: 641 reason# EndOfStream
2025-11-27T18:18:52.968052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471354174119923:2048] [node 1] ICP30 unregister session Session# [2:7577471354174120023:2048] VirtualId# [2:7577471354174120018:0]
2025-11-27T18:18:52.968084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471354174120023:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:52.969271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:52.969370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:52.971265Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:52.971502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471351503875651:2048] [node 1] ICS01 socket: 643 reason# EndOfStream
2025-11-27T18:18:52.971559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471351503875552:2048] [node 1] ICP30 unregister session Session# [3:7577471351503875651:2048] VirtualId# [3:7577471351503875641:0]
2025-11-27T18:18:52.971590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471351503875651:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:52.971672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:52.971722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:52.974306Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:52.974537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471351665618460:2048] [node 1] ICS07 socket disconnect 608 reason# EndOfStream
2025-11-27T18:18:52.974578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471351665618460:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:52.974659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:52.974707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471351665618460:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:52.975638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471351665618370:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12731) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:52.975702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471351665618460:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:52.975778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471351665618370:2048] [node 1] ICP30 unregister session Session# [4:7577471351665618460:2048] VirtualId# [4:7577471351665618453:0]
2025-11-27T18:18:52.976019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:52.976056Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471351665618370:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12731) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.976067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12731) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:52.977119Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:52.977356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471353071441100:2048] [node 1] ICS01 socket: 614 reason# EndOfStream
2025-11-27T18:18:52.977402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471353071441001:2048] [node 1] ICP30 unregister session Session# [6:7577471353071441100:2048] VirtualId# [6:7577471353071441085:0]
2025-11-27T18:18:52.977433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471353071441100:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:52.977514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:52.977555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:52.977712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:52.977914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471351489101891:2048] [node 1] ICS01 socket: 609 reason# EndOfStream
2025-11-27T18:18:52.977963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471351489101789:2048] [node 1] ICP30 unregister session Session# [5:7577471351489101891:2048] VirtualId# [5:7577471351489101887:0]
2025-11-27T18:18:52.977990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471351489101891:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:52.978075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:52.978130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:52.984846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471351665618370:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:53.004719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:53.005004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471351503875636:2049] [node 2] ICS07 socket disconnect 616 reason# EndOfStream
2025-11-27T18:18:53.005041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471351503875636:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:53.005160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:53.005214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471351503875636:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:53.007677Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:53.007756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471351503875553:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28651) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:18:53.007800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471351503875636:2049] [node 2] ICS15 start handshake
2025-11-27T18:18:53.007979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471351665618457:2049] [node 2] ICS01 socket: 605 reason# EndOfStream
2025-11-27T18:18:53.008031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471351665618371:2049] [node 2] ICP30 unregister session Session# [4:7577471351665618457:2049] VirtualId# [4:7577471351665618455:0]
2025-11-27T18:18:53.010711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471351503875553:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28651) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:53.008071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471351665618457:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:53.010874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471351503875636:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:53.008138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:53.010833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:53.008194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:53.010931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471351503875553:2049] [node 2] ICP30 unregister session Session# [3:7577471351503875636:2049] VirtualId# [3:7577471351503875633:0]
2025-11-27T18:18:53.011026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:53.011087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471351489101909:2049] [node 2] ICS01 socket: 638 reason# EndOfStream
2025-11-27T18:18:53.011117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471351503875553:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28651) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:53.011145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471351489101790:2049] [node 2] ICP30 unregister session Session# [5:7577471351489101909:2049] VirtualId# [5:7577471351489101902:0]
2025-11-27T18:18:53.011139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28651) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:53.011209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471351489101909:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:53.011294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:53.011365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:53.012270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:53.012498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471353071441097:2049] [node 2] ICS01 socket: 640 reason# EndOfStream
2025-11-27T18:18:53.012549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471353071441002:2049] [node 2] ICP30 unregister session Session# [6:7577471353071441097:2049] VirtualId# [6:7577471353071441089:0]
2025-11-27T18:18:53.012574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471353071441097:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:53.012635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:53.012682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:53.020791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471351503875553:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:18:53.095371Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:53.095589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471351665618472:2050] [node 3] ICS01 socket: 655 reason# EndOfStream
2025-11-27T18:18:53.095631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471351665618372:2050] [node 3] ICP30 unregister session Session# [4:7577471351665618472:2050] VirtualId# [4:7577471351665618470:0]
2025-11-27T18:18:53.095679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471351665618472:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:53.095754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:53.095799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:53.096759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:53.096907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471351489101907:2050] [node 3] ICS01 socket: 622 reason# EndOfStream
2025-11-27T18:18:53.096946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471351489101791:2050] [node 3] ICP30 unregister session Session# [5:7577471351489101907:2050] VirtualId# [5:7577471351489101901:0]
2025-11-27T18:18:53.096978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471351489101907:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:53.097030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:53.097064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:53.097780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:53.097959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471353071441105:2050] [node 3] ICS01 socket: 650 reason# EndOfStream
2025-11-27T18:18:53.097994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471353071441003:2050] [node 3] ICP30 unregister session Session# [6:7577471353071441105:2050] VirtualId# [6:7577471353071441099:0]
2025-11-27T18:18:53.098026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471353071441105:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:53.098084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:53.098124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:53.156680Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:53.156947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471351489101905:2108] [node 4] ICS01 socket: 606 reason# EndOfStream
2025-11-27T18:18:53.157000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471351489101871:2108] [node 4] ICP30 unregister session Session# [5:7577471351489101905:2108] VirtualId# [5:7577471351489101890:0]
2025-11-27T18:18:53.157046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471351489101905:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:53.157153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:53.157219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:53.157617Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:53.157850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471353071441103:2109] [node 4] ICS01 socket: 615 reason# EndOfStream
2025-11-27T18:18:53.157897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471353071441066:2109] [node 4] ICP30 unregister session Session# [6:7577471353071441103:2109] VirtualId# [6:7577471353071441086:0]
2025-11-27T18:18:53.157925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471353071441103:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:53.157999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:53.158058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:53.240651Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:53.240860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471353071441092:2106] [node 5] ICS07 socket disconnect 611 reason# EndOfStream
2025-11-27T18:18:53.240898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471353071441092:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:53.240998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:53.241065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471353071441092:2106] [node 5] ICS15 start handshake
2025-11-27T18:18:53.241912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471353071441061:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26253) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:53.242067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471353071441092:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:53.242117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471353071441061:2106] [node 5] ICP30 unregister session Session# [6:7577471353071441092:2106] VirtualId# [6:7577471353071441083:0]
2025-11-27T18:18:53.242195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:18:53.242234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471353071441061:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26253) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:53.242262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26253) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:53.331613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471357303083871:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:53.331742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471357303083872:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:53.355562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471355852267198:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:53.355618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471355852267199:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:53.379026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471359069797106:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:53.379084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471359069797107:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:53.400054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471357725202375:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:53.400096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471357725202376:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:53.400149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471357725202377:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:53.424674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471355711658384:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:53.424705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471355711658385:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:53.424796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471355711658386:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:53.448228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471357119112683:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:53.448246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471357119112684:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:53.448262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471357119112685:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:53.522487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471357303083950:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:53.531760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471359069797168:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:53.531771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471359069797169:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:53.541993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471357119112741:2105] [node 5] ICP01 ready to work
2025-11-27T18:18:53.554163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471357303083952:2117] [node 6] ICP01 ready to work
2025-11-27T18:18:53.564153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471355711658447:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:53.593539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471357119112745:2106] [node 4] ICP01 ready to work
2025-11-27T18:18:53.542223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471357303083951:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:53.610305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471357725202456:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:53.601477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471355711658448:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:53.610352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471355852267288:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:53.616661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471359069797172:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:53.633451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471355852267289:2117] [node 5] ICP01 ready to work
2025-11-27T18:18:53.618478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471357725202457:2116] [node 6] ICP01 ready to work
2025-11-27T18:18:53.645633Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471357725202377:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28790734ns) {Configure:104(28768901ns) {ProcessConfigured:135(63457ns) {StartInitialHandshake:157(60602ns) {DropHandshakes:470(2522ns) {DropIncomingHandshake:442(494ns) DropOutgoingHandshake:457(297ns)} GenerateSessionVirtualId:988(912ns) PrepareNewSessionHandshake:477(1493ns) {SwitchToState:179(388ns)}}}}}
2025-11-27T18:18:53.637815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471355852267290:2123] [node 6] ICP01 ready to work
2025-11-27T18:18:53.646418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471357303083871:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(124566793ns) {Configure:104(124561555ns) {ProcessConfigured:135(9063644ns) {StartInitialHandshake:157(9060585ns) {DropHandshakes:470(477134ns) {DropIncomingHandshake:442(1171ns) DropOutgoingHandshake:457(248ns)} GenerateSessionVirtualId:988(1187ns) PrepareNewSessionHandshake:477(4382ns) {SwitchToState:179(1487ns)}}}}}
2025-11-27T18:18:53.658590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471355711658385:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(58782900ns) {Configure:104(58775992ns) {ProcessConfigured:135(64649ns) {StartInitialHandshake:157(62013ns) {DropHandshakes:470(2477ns) {DropIncomingHandshake:442(367ns) DropOutgoingHandshake:457(602ns)} GenerateSessionVirtualId:988(897ns) PrepareNewSessionHandshake:477(1165ns) {SwitchToState:179(477ns)}}}}}
2025-11-27T18:18:53.663960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471357119112684:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(122845705ns) {Configure:104(122841702ns) {ProcessConfigured:135(54276ns) {StartInitialHandshake:157(52681ns) {DropHandshakes:470(817ns) {DropIncomingHandshake:442(101ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:477(1709ns) {SwitchToState:179(826ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.666280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471357119112753:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58960
2025-11-27T18:18:53.654891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471359069797169:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(119706658ns) {Configure:104(119702739ns) {ProcessConfigured:135(49227ns) {StartInitialHandshake:157(46046ns) {DropHandshakes:470(1182ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(516ns) PrepareNewSessionHandshake:477(1558ns) {SwitchToState:179(657ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.680848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471355711658453:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17470
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.681061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471355711658454:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17470
2025-11-27T18:18:53.669560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471357725202375:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(58427172ns) {Configure:104(58423202ns) {ProcessConfigured:135(1411719ns) {StartInitialHandshake:157(1409276ns) {DropHandshakes:470(2234ns) {DropIncomingHandshake:442(649ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(942ns) PrepareNewSessionHandshake:477(1217ns) {SwitchToState:179(264ns)}}}}}
2025-11-27T18:18:53.672765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471359069797106:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(54082170ns) {Configure:104(54070784ns) {ProcessConfigured:135(68246ns) {StartInitialHandshake:157(64703ns) {DropHandshakes:470(2264ns) {DropIncomingHandshake:442(576ns) DropOutgoingHandshake:457(346ns)} GenerateSessionVirtualId:988(857ns) PrepareNewSessionHandshake:477(1515ns) {SwitchToState:179(540ns)}}}}}
2025-11-27T18:18:53.677369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471357303083872:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(129959000ns) {Configure:104(129946735ns) {ProcessConfigured:135(1555819ns) {StartInitialHandshake:157(1553378ns) {DropHandshakes:470(1126ns) {DropIncomingHandshake:442(267ns) DropOutgoingHandshake:457(121ns)} GenerateSessionVirtualId:988(238ns) PrepareNewSessionHandshake:477(2083ns) {SwitchToState:179(670ns)}}}}}
2025-11-27T18:18:53.679026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471357119112753:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.689806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471355852267289:2117] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(43664115ns) {Configure:104(43657827ns) {ProcessConfigured:135(4864204ns) {StartInitialHandshake:157(4861324ns) {DropHandshakes:470(2302ns) {DropIncomingHandshake:442(663ns) DropOutgoingHandshake:457(288ns)} GenerateSessionVirtualId:988(981ns) PrepareNewSessionHandshake:477(3912ns) {SwitchToState:179(1598ns)}}}}}
2025-11-27T18:18:53.681713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471357725202456:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28122514ns) {Configure:104(28116083ns) {ProcessConfigured:135(2249683ns) {StartInitialHandshake:157(2247569ns) {DropHandshakes:470(837887ns) {DropIncomingHandshake:442(1573ns) DropOutgoingHandshake:457(185ns)} GenerateSessionVirtualId:988(1593ns) PrepareNewSessionHandshake:477(1762ns) {SwitchToState:179(729ns)}}}}}
2025-11-27T18:18:53.684715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471357119112685:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(135575455ns) {Configure:104(134714752ns) {ProcessConfigured:135(13342519ns) {StartInitialHandshake:157(13338522ns) {DropHandshakes:470(1788ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(311ns)} GenerateSessionVirtualId:988(634ns) PrepareNewSessionHandshake:477(4001ns) {SwitchToState:179(1680ns)}}}}}
2025-11-27T18:18:53.689470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471359069797168:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(149800517ns) {Configure:104(149797154ns) {ProcessConfigured:135(11935170ns) {StartInitialHandshake:157(11932799ns) {DropHandshakes:470(5269516ns) {DropIncomingHandshake:442(574ns) DropOutgoingHandshake:457(2053ns)} GenerateSessionVirtualId:988(2207ns) PrepareNewSessionHandshake:477(3569ns) {SwitchToState:179(1371ns)}}}}}
2025-11-27T18:18:53.691473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471357303083950:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(124632265ns) {Configure:104(124627832ns) {ProcessConfigured:135(3302905ns) {StartInitialHandshake:157(3299700ns) {DropHandshakes:470(1549ns) {DropIncomingHandshake:442(490ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(521ns) PrepareNewSessionHandshake:477(2550ns) {SwitchToState:179(1007ns)}}}}}
2025-11-27T18:18:53.698824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471355852267199:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(124735908ns) {Configure:104(113606469ns) {ProcessConfigured:135(7806436ns) {StartInitialHandshake:157(7804460ns) {DropHandshakes:470(1784ns) {DropIncomingHandshake:442(474ns) DropOutgoingHandshake:457(427ns)} GenerateSessionVirtualId:988(1042ns) PrepareNewSessionHandshake:477(2296ns) {SwitchToState:179(966ns)}}}}}
2025-11-27T18:18:53.709591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471355711658453:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.713331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471357725202376:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(62306480ns) {Configure:104(62289889ns) {ProcessConfigured:135(6751932ns) {StartInitialHandshake:157(6749187ns) {DropHandshakes:470(1100ns) {DropIncomingHandshake:442(268ns) DropOutgoingHandshake:457(149ns)} GenerateSessionVirtualId:988(489ns) PrepareNewSessionHandshake:477(2501ns) {SwitchToState:179(1145ns)}}}}}
2025-11-27T18:18:53.715323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471355852267198:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(150029718ns) {Configure:104(150019736ns) {ProcessConfigured:135(5969566ns) {StartInitialHandshake:157(5966294ns) {DropHandshakes:470(2799723ns) {DropIncomingHandshake:442(2832ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(1540ns) PrepareNewSessionHandshake:477(2861ns) {SwitchToState:179(1072ns)}}}}}
2025-11-27T18:18:53.709672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471357119112683:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(158729040ns) {Configure:104(158719940ns) {ProcessConfigured:135(12977611ns) {StartInitialHandshake:157(12974759ns) {DropHandshakes:470(1141ns) {DropIncomingHandshake:442(367ns) DropOutgoingHandshake:457(186ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:477(6955144ns) {SwitchToState:179(4607ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.713503Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471355711658448:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(83134805ns) {Configure:104(83130381ns) {ProcessConfigured:135(15980044ns) {StartInitialHandshake:157(15978899ns) {DropHandshakes:470(5486908ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(1293ns)} GenerateSessionVirtualId:988(2175ns) PrepareNewSessionHandshake:477(3267ns) {SwitchToState:179(1584ns)}}}}}
2025-11-27T18:18:53.737803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471357303083956:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.717687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471355852267288:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(55571630ns) {Configure:104(55552493ns) {ProcessConfigured:135(16613766ns) {StartInitialHandshake:157(16610395ns) {DropHandshakes:470(1412ns) {DropIncomingHandshake:442(352ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(569ns) PrepareNewSessionHandshake:477(4615ns) {SwitchToState:179(2579ns)}}}}}
2025-11-27T18:18:53.716453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471357119112745:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(73119307ns) {Configure:104(73109873ns) {ProcessConfigured:135(5963251ns) {StartInitialHandshake:157(5960632ns) {DropHandshakes:470(593ns) {DropIncomingHandshake:442(149ns) DropOutgoingHandshake:457(91ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(2845ns) {SwitchToState:179(1417ns)}}}}}
2025-11-27T18:18:53.741843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471357725202470:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64895
2025-11-27T18:18:53.725684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471355711658384:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(120882395ns) {Configure:104(120877645ns) {ProcessConfigured:135(11028382ns) {StartInitialHandshake:157(6331444ns) {DropHandshakes:470(3822ns) {DropIncomingHandshake:442(851ns) DropOutgoingHandshake:457(396ns)} GenerateSessionVirtualId:988(1696ns) PrepareNewSessionHandshake:477(3724ns) {SwitchToState:179(2096ns)}}}}}
2025-11-27T18:18:53.721842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471357119112683:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.737843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471357303083956:2127] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:53.752624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471359069797107:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(116962139ns) {Configure:104(116951963ns) {ProcessConfigured:135(16555602ns) {StartInitialHandshake:157(16552426ns) {DropHandshakes:470(1916ns) {DropIncomingHandshake:442(407ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(730ns) PrepareNewSessionHandshake:477(3956ns) {SwitchToState:179(1525ns)}}}}}
2025-11-27T18:18:53.733982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471355711658386:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(96198404ns) {Configure:104(96192392ns) {ProcessConfigured:135(24822692ns) {StartInitialHandshake:157(24819659ns) {DropHandshakes:470(1494ns) {DropIncomingHandshake:442(400ns) DropOutgoingHandshake:457(266ns)} GenerateSessionVirtualId:988(280523ns) PrepareNewSessionHandshake:477(6225ns) {SwitchToState:179(3862ns)}}}}}
2025-11-27T18:18:53.751108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471357303083966:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15178
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:18:53.755292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471357303083966:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.760095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471359069797186:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.760514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471357119112765:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58960
2025-11-27T18:18:53.755311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471357725202470:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.755298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471357303083956:2127] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.752916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471355711658384:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.761946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471357119112761:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58960
2025-11-27T18:18:53.755332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471357303083954:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.765856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471359069797186:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.755362Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471357303083954:2126] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.761986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471357119112763:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58960
2025-11-27T18:18:53.756059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471355711658454:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:18:53.764252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471355711658463:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17470
::ffff:127.1.0.0
2025-11-27T18:18:53.765388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471357303083954:2126] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:53.765596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471357303083967:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15178
2025-11-27T18:18:53.766998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471357303083952:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:53.767038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:53.767107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471357303083967:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.771107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471355852267301:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64603
2025-11-27T18:18:53.771598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471355852267300:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64603
2025-11-27T18:18:53.771768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471355852267299:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64603
2025-11-27T18:18:53.771136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471355711658463:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.770838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471357303083952:2117] [node 6] ICP22 created new session: [1:7577471357303083971:2117]
2025-11-27T18:18:53.770875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471357303083951:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:53.772225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:53.772503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471357303083971:2117] [node 6] ICS09 handshake done sender: [1:7577471357303083956:2127] self: [1:7577471357303083955:0] peer: [6:7577471357119112762:0] socket: 595 qp: -1
2025-11-27T18:18:53.777871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471357303083971:2117] [node 6] ICS10 traffic start
2025-11-27T18:18:53.782016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471357303083951:2108] [node 5] ICP22 created new session: [1:7577471357303083974:2108]
2025-11-27T18:18:53.783606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:53.767283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471357119112761:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.781961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471359069797187:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-27T18:18:53.777398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471357725202472:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64895
2025-11-27T18:18:53.772207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471355711658467:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17470
2025-11-27T18:18:53.772268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471357119112753:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:53.790262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471355852267290:2123] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(76651440ns) {Configure:104(76641679ns) {ProcessConfigured:135(3122915ns) {StartInitialHandshake:157(3119825ns) {DropHandshakes:470(2234ns) {DropIncomingHandshake:442(662ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(999ns) PrepareNewSessionHandshake:477(2597ns) {SwitchToState:179(1173ns)}}}}}
2025-11-27T18:18:53.794626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471357303083974:2108] [node 5] ICS09 handshake done sender: [1:7577471357303083954:2126] self: [1:7577471357303083953:0] peer: [5:7577471355711658464:0] socket: 594 qp: -1
2025-11-27T18:18:53.794686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471357303083974:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.795847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:53.790516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471355711658386:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.792223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471359069797178:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.793470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471355852267300:2142] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.772282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471357119112685:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.795609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471357725202472:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.792251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471359069797178:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.805649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471355711658453:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:53.797675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471357725202474:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64895
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.793711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471355852267299:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.772307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471357119112763:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.794710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471359069797178:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:53.807395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471355711658470:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17470
2025-11-27T18:18:53.781689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471357119112745:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.799583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471357725202463:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.799612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471357725202463:2130] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.797088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471355852267301:2143] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.808142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471357725202463:2130] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:53.795645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471359069797189:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-27T18:18:53.813378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471357303083960:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.793503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471357119112683:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:53.799746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471355852267198:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.809970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471355711658468:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17470
2025-11-27T18:18:53.826233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471357725202457:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:53.795772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471359069797180:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.813425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471357303083960:2128] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:53.793563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.819911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471355852267297:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.810088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471355711658384:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.795790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471359069797180:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:53.847894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.819944Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471355852267297:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:53.794800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471357119112683:2048] [node 1] ICP22 created new session: [6:7577471357119112768:2048]
2025-11-27T18:18:53.810158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:53.815627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471359069797180:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:53.815205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471357303083977:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15178
2025-11-27T18:18:53.853539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471357725202457:2116] [node 6] ICP22 created new session: [4:7577471357725202478:2116]
2025-11-27T18:18:53.796251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471357119112768:2048] [node 1] ICS09 handshake done sender: [6:7577471357119112753:2112] self: [6:7577471357119112762:0] peer: [1:7577471357303083955:0] socket: 598 qp: -1
2025-11-27T18:18:53.816403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471359069797187:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.810219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471355711658384:2048] [node 1] ICP22 created new session: [5:7577471355711658471:2048]
2025-11-27T18:18:53.848006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471355852267297:2139] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:53.815869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471357303083952:2117] [node 6] ICP35 event processing took too much time PendingConnection:279(41119547ns) {HandleHandshakeStatus:329(41116967ns) {DropIncomingHandshake:442(373ns) DropOutgoingHandshake:457(447ns) SwitchToState:179(456ns) ProcessPendingSessionEvents:495(12486166ns) {ForwardSessionEventToSession:612(3044967ns) ForwardSessionEventToSession:612(4675ns) ForwardSessionEventToSession:612(923090ns) ForwardSessionEventToSession:612(3046ns) ForwardSessionEventToSession:612(2956231ns) ForwardSessionEventToSession:612(4055ns) ForwardSessionEventToSession:612(2719ns) ForwardSessionEventToSession:612(1260433ns) ForwardSessionEventToSession:612(3903421ns) ForwardSessionEventToSession:612(362761ns)}}}
2025-11-27T18:18:53.810316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471355711658471:2048] [node 1] ICS09 handshake done sender: [5:7577471355711658453:2112] self: [5:7577471355711658464:0] peer: [1:7577471357303083953:0] socket: 597 qp: -1
2025-11-27T18:18:53.854167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471357725202479:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64895
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.801784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471357119112768:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:53.817935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471359069797192:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-27T18:18:53.863371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471357725202474:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.816864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471357303083960:2128] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:53.863929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471357725202478:2116] [node 6] ICS09 handshake done sender: [4:7577471357725202463:2130] self: [4:7577471357725202462:0] peer: [6:7577471357119112767:0] socket: 618 qp: -1
2025-11-27T18:18:53.873364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471357725202375:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.877368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471357725202478:2116] [node 6] ICS10 traffic start
2025-11-27T18:18:53.881483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:53.905702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471357725202479:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.913726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471357725202482:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64895
2025-11-27T18:18:53.918173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471357725202377:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.818045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471359069797193:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-27T18:18:53.810365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471355711658471:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.906853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471355852267292:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.801849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:53.818894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471357303083977:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.931173Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471357725202375:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19003) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471357725202474:2135] held: no
2025-11-27T18:18:53.906910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471355852267292:2137] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:53.818892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471359069797172:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:53.810409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.838594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471357303083871:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:53.812708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471357119112772:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58960
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.913360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471355852267198:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19003) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471355852267300:2142] held: no
2025-11-27T18:18:53.823070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.857538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:53.937432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471357725202474:2135] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:53.811215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471355711658468:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:18:53.814369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471357119112773:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58960
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.913520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471355852267289:2117] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.828681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471359069797168:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:53.825933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471357119112763:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:53.811408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471355711658385:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.949669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471357725202375:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:53.828258Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471357119112683:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(37665580ns) {HandleHandshakeStatus:329(37661625ns) {DropIncomingHandshake:442(1404ns) DropOutgoingHandshake:457(14013ns) SwitchToState:179(472ns) ProcessPendingSessionEvents:495(14696724ns) {ForwardSessionEventToSession:612(2897614ns) ForwardSessionEventToSession:612(408178ns) ForwardSessionEventToSession:612(3872ns) ForwardSessionEventToSession:612(2193148ns) ForwardSessionEventToSession:612(4042ns) ForwardSessionEventToSession:612(2711ns) ForwardSessionEventToSession:612(1803115ns) ForwardSessionEventToSession:612(4411ns) ForwardSessionEventToSession:612(7354091ns) ForwardSessionEventToSession:612(6460ns)}}}
2025-11-27T18:18:53.820578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471355711658463:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:53.858246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471357303083871:2048] [node 2] ICP22 created new session: [1:7577471357303083981:2048]
2025-11-27T18:18:53.913601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:53.828684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471359069797172:2109] [node 6] ICP22 created new session: [3:7577471359069797195:2109]
2025-11-27T18:18:53.949681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471357725202465:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.913640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471355852267289:2117] [node 5] ICP22 created new session: [2:7577471355852267313:2117]
2025-11-27T18:18:53.834066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471355711658386:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:53.877667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471357303083981:2048] [node 2] ICS09 handshake done sender: [1:7577471357303083960:2128] self: [1:7577471357303083959:0] peer: [2:7577471355852267308:0] socket: 602 qp: -1
2025-11-27T18:18:53.832157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471357119112745:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:53.949719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471357725202465:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:53.925464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471355852267313:2117] [node 5] ICS09 handshake done sender: [2:7577471355852267297:2139] self: [2:7577471355852267294:0] peer: [5:7577471355711658474:0] socket: 636 qp: -1
2025-11-27T18:18:53.831575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471359069797193:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.834108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:53.832207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:53.884870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471357303083981:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:53.925514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471355852267313:2117] [node 5] ICS10 traffic start
2025-11-27T18:18:53.928518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:53.953751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:53.884927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:53.954034Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471357725202457:2116] [node 6] ICP35 event processing took too much time PendingConnection:279(107179272ns) {HandleHandshakeStatus:329(107176485ns) {DropIncomingHandshake:442(21574810ns) DropOutgoingHandshake:457(438ns) SwitchToState:179(1346ns) ProcessPendingSessionEvents:495(30063813ns) {ForwardSessionEventToSession:612(7865457ns) ForwardSessionEventToSession:612(3993873ns) ForwardSessionEventToSession:612(4619ns) ForwardSessionEventToSession:612(2565ns) ForwardSessionEventToSession:612(8573622ns) ForwardSessionEventToSession:612(5990ns) ForwardSessionEventToSession:612(3353543ns) ForwardSessionEventToSession:612(3118ns) ForwardSessionEventToSession:612(2833ns) ForwardSessionEventToSession:612(6234029ns)}}}
2025-11-27T18:18:53.941309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471355852267312:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64603
2025-11-27T18:18:53.953396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471355852267310:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64603
2025-11-27T18:18:53.885372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471357303083871:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(51008351ns) {HandleHandshakeStatus:329(46572603ns) {DropIncomingHandshake:442(15023480ns) DropOutgoingHandshake:457(3028ns) SwitchToState:179(933ns) ProcessPendingSessionEvents:495(100482ns) {ForwardSessionEventToSession:612(52961ns) ForwardSessionEventToSession:612(2856ns) ForwardSessionEventToSession:612(5951ns) ForwardSessionEventToSession:612(2766ns) ForwardSessionEventToSession:612(2296ns) ForwardSessionEventToSession:612(4514ns) ForwardSessionEventToSession:612(3264ns) ForwardSessionEventToSession:612(13688ns) ForwardSessionEventToSession:612(2190ns) ForwardSessionEventToSession:612(2308ns)}}}
2025-11-27T18:18:53.956883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471355852267312:2146] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.962573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471357725202485:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64895
2025-11-27T18:18:53.885525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471357303083962:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.957232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471355852267300:2142] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:53.963363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471357725202375:2048] [node 1] ICP22 created new session: [4:7577471357725202484:2048]
2025-11-27T18:18:53.963173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471355852267292:2137] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.885548Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471357303083962:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:53.966750Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471355852267289:2117] [node 5] ICP35 event processing took too much time PendingConnection:279(53041275ns) {HandleHandshakeStatus:329(53037789ns) {DropIncomingHandshake:442(15697ns) DropOutgoingHandshake:457(266ns) SwitchToState:179(384ns) ProcessPendingSessionEvents:495(8894975ns) {ForwardSessionEventToSession:612(8839300ns) ForwardSessionEventToSession:612(8585ns) ForwardSessionEventToSession:612(5007ns) ForwardSessionEventToSession:612(2122ns) ForwardSessionEventToSession:612(2834ns) ForwardSessionEventToSession:612(5184ns) ForwardSessionEventToSession:612(3440ns) ForwardSessionEventToSession:612(8459ns) ForwardSessionEventToSession:612(2253ns) ForwardSessionEventToSession:612(5129ns)}}}
2025-11-27T18:18:53.973602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471357725202484:2048] [node 1] ICS09 handshake done sender: [4:7577471357725202474:2135] self: [4:7577471357725202480:0] peer: [1:7577471357303083964:0] socket: 643 qp: -1
2025-11-27T18:18:53.968868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471355852267319:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64603
2025-11-27T18:18:53.886155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471357303083965:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.986575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471357725202484:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:53.977608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471355852267198:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:53.986619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:53.985588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:53.997703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471357725202486:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64895
2025-11-27T18:18:53.989928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471355852267198:2048] [node 1] ICP22 created new session: [2:7577471355852267320:2048]
2025-11-27T18:18:53.991357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471355852267199:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:54.009424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471357725202479:2136] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:53.991440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:53.886172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471357303083965:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:53.991488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471355852267199:2049] [node 3] ICP22 created new session: [2:7577471355852267321:2049]
2025-11-27T18:18:53.889507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471357303083965:2130] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:53.991526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471355852267321:2049] [node 3] ICS09 handshake done sender: [2:7577471355852267292:2137] self: [2:7577471355852267291:0] peer: [3:7577471359069797202:0] socket: 633 qp: -1
2025-11-27T18:18:53.897323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471357303083962:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:53.991556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471355852267321:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:54.033287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471357725202375:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(67767484ns) {HandleHandshakeStatus:329(67762716ns) {DropIncomingHandshake:442(957ns) DropOutgoingHandshake:457(373ns) SwitchToState:179(389ns) ProcessPendingSessionEvents:495(1680ns)}}
2025-11-27T18:18:53.905316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471357303083951:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(62809876ns) {HandleHandshakeStatus:329(62806847ns) {DropIncomingHandshake:442(494ns) DropOutgoingHandshake:457(430ns) SwitchToState:179(753ns) ProcessPendingSessionEvents:495(27983217ns) {ForwardSessionEventToSession:612(8406177ns) ForwardSessionEventToSession:612(4592ns) ForwardSessionEventToSession:612(3259ns) ForwardSessionEventToSession:612(1755531ns) ForwardSessionEventToSession:612(3595ns) ForwardSessionEventToSession:612(2972939ns) ForwardSessionEventToSession:612(1871016ns) ForwardSessionEventToSession:612(7952165ns) ForwardSessionEventToSession:612(4988261ns) ForwardSessionEventToSession:612(3784ns)}}}
2025-11-27T18:18:53.991585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:54.039385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471357725202377:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27727) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471357725202479:2136] held: no
2025-11-27T18:18:53.909403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471357303083950:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:54.006030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471355852267320:2048] [node 1] ICS09 handshake done sender: [2:7577471355852267300:2142] self: [2:7577471355852267308:0] peer: [1:7577471357303083959:0] socket: 614 qp: -1
2025-11-27T18:18:53.909501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:53.921554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471357303083950:2107] [node 4] ICP22 created new session: [1:7577471357303083985:2107]
2025-11-27T18:18:54.021501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471355852267320:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:54.021565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:53.928365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471357303083872:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:53.928400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471357303083985:2107] [node 4] ICS09 handshake done sender: [1:7577471357303083965:2130] self: [1:7577471357303083964:0] peer: [4:7577471357725202480:0] socket: 642 qp: -1
2025-11-27T18:18:53.930383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471357303083985:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:53.930422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:53.933368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:53.933427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471357303083872:2049] [node 3] ICP22 created new session: [1:7577471357303083987:2049]
2025-11-27T18:18:54.040469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471357725202377:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:53.933467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471357303083987:2049] [node 3] ICS09 handshake done sender: [1:7577471357303083962:2129] self: [1:7577471357303083961:0] peer: [3:7577471359069797200:0] socket: 641 qp: -1
2025-11-27T18:18:54.040520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:53.933501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471357303083987:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:54.040561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471357725202377:2050] [node 3] ICP22 created new session: [4:7577471362020169786:2050]
2025-11-27T18:18:53.833491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:53.850883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471355711658476:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17470
2025-11-27T18:18:53.933537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:54.040625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471362020169786:2050] [node 3] ICS09 handshake done sender: [4:7577471357725202479:2136] self: [4:7577471357725202483:0] peer: [3:7577471359069797173:0] socket: 659 qp: -1
2025-11-27T18:18:53.832251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471357119112745:2106] [node 4] ICP22 created new session: [6:7577471357119112774:2106]
2025-11-27T18:18:53.966132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471357303083950:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(56507312ns) {HandleHandshakeStatus:329(56504201ns) {DropIncomingHandshake:442(24755ns) DropOutgoingHandshake:457(964ns) SwitchToState:179(369ns) ProcessPendingSessionEvents:495(20309360ns) {ForwardSessionEventToSession:612(20257497ns) ForwardSessionEventToSession:612(6818ns) ForwardSessionEventToSession:612(6663ns) ForwardSessionEventToSession:612(3376ns) ForwardSessionEventToSession:612(2619ns) ForwardSessionEventToSession:612(6005ns) ForwardSessionEventToSession:612(2779ns) ForwardSessionEventToSession:612(3100ns) ForwardSessionEventToSession:612(8032ns) ForwardSessionEventToSession:612(2813ns)}}}
2025-11-27T18:18:53.832305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471357119112774:2106] [node 4] ICS09 handshake done sender: [6:7577471357119112763:2117] self: [6:7577471357119112767:0] peer: [4:7577471357725202462:0] socket: 620 qp: -1
2025-11-27T18:18:53.853575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471355711658386:2050] [node 3] ICP22 created new session: [5:7577471355711658475:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:54.041216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471355852267198:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(65765712ns) {HandleHandshakeStatus:329(65763027ns) {DropIncomingHandshake:442(1836ns) DropOutgoingHandshake:457(1947ns) SwitchToState:179(3225ns) ProcessPendingSessionEvents:495(1706ns)}}
2025-11-27T18:18:53.857333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471355711658468:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:53.838565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471359069797168:2108] [node 5] ICP22 created new session: [3:7577471359069797196:2108]
2025-11-27T18:18:53.832345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471357119112774:2106] [node 4] ICS10 traffic start
2025-11-27T18:18:54.040657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471362020169786:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:53.841192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471359069797195:2109] [node 6] ICS09 handshake done sender: [3:7577471359069797178:2113] self: [3:7577471359069797177:0] peer: [6:7577471357119112766:0] socket: 609 qp: -1
2025-11-27T18:18:54.043530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471355852267298:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:54.043560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471355852267298:2140] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:53.863773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471355711658475:2050] [node 3] ICS09 handshake done sender: [5:7577471355711658463:2117] self: [5:7577471355711658469:0] peer: [3:7577471359069797179:0] socket: 615 qp: -1
2025-11-27T18:18:53.832384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:53.841239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471359069797195:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:54.040686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:54.044445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471355852267305:2144] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.890302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471357119112685:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27727) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471357119112761:2116] held: no
2025-11-27T18:18:53.841311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:54.040807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471357725202486:2139] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.863812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471355711658475:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:53.893339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471357119112770:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58960
2025-11-27T18:18:54.044469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471355852267319:2147] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.841769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471359069797172:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(22682374ns) {HandleHandshakeStatus:329(22680076ns) {DropIncomingHandshake:442(464ns) DropOutgoingHandshake:457(554ns) SwitchToState:179(423ns) ProcessPendingSessionEvents:495(110879ns) {ForwardSessionEventToSession:612(58804ns) ForwardSessionEventToSession:612(6559ns) ForwardSessionEventToSession:612(3336ns) ForwardSessionEventToSession:612(7666ns) ForwardSessionEventToSession:612(2931ns) ForwardSessionEventToSession:612(5510ns) ForwardSessionEventToSession:612(2334ns) ForwardSessionEventToSession:612(2581ns) ForwardSessionEventToSession:612(5295ns) ForwardSessionEventToSession:612(5332ns)}}}
2025-11-27T18:18:53.869738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471355711658477:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17470
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:54.044510Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471355852267305:2144] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:53.902690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471357119112773:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.842074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471359069797192:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:54.041657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471357725202376:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.877579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:54.045230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471355852267298:2140] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:53.842167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471359069797189:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:53.913352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471357119112741:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:54.042885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471357725202465:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:53.877839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471355711658385:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:53.851283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471359069797172:2109] [node 6] ICP18 handshake (actor [3:7577471359069797193:2120]) is too old
2025-11-27T18:18:54.048949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471355852267288:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:53.914289Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471357119112761:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:53.877913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:54.044088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471357725202456:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:54.049588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471355852267305:2144] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:53.927266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471357119112685:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:53.858089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471359069797196:2108] [node 5] ICS09 handshake done sender: [3:7577471359069797180:2114] self: [3:7577471359069797179:0] peer: [5:7577471355711658469:0] socket: 612 qp: -1
2025-11-27T18:18:54.045474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:53.877973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471355711658385:2049] [node 2] ICP22 created new session: [5:7577471355711658479:2049]
2025-11-27T18:18:53.927333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:54.049683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:54.045509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471357725202456:2108] [node 5] ICP22 created new session: [4:7577471362020169789:2108]
2025-11-27T18:18:53.930320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471357119112685:2050] [node 3] ICP22 created new session: [6:7577471357119112777:2050]
2025-11-27T18:18:53.861519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471359069797106:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.878016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471355711658479:2049] [node 2] ICS09 handshake done sender: [5:7577471355711658468:2119] self: [5:7577471355711658474:0] peer: [2:7577471355852267294:0] socket: 637 qp: -1
2025-11-27T18:18:53.930526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471357119112745:2106] [node 4] ICP35 event processing took too much time PendingConnection:279(65898244ns) {HandleHandshakeStatus:329(65895328ns) {DropIncomingHandshake:442(774ns) DropOutgoingHandshake:457(5104ns) SwitchToState:179(431ns) ProcessPendingSessionEvents:495(32662942ns) {ForwardSessionEventToSession:612(14303364ns) ForwardSessionEventToSession:612(4426ns) ForwardSessionEventToSession:612(7841110ns) ForwardSessionEventToSession:612(3048ns) ForwardSessionEventToSession:612(6225710ns) ForwardSessionEventToSession:612(699452ns) ForwardSessionEventToSession:612(4037ns) ForwardSessionEventToSession:612(3556186ns) ForwardSessionEventToSession:612(4693ns) ForwardSessionEventToSession:612(2558ns)}}}
2025-11-27T18:18:54.050181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471355852267288:2107] [node 4] ICP22 created new session: [2:7577471360147234623:2107]
2025-11-27T18:18:53.863518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471359069797196:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:53.878050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471355711658479:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:53.863586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:53.878082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:53.879651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471359069797107:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.914260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471355711658477:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:54.048956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471362020169789:2108] [node 5] ICS09 handshake done sender: [4:7577471357725202465:2131] self: [4:7577471357725202464:0] peer: [5:7577471355711658481:0] socket: 626 qp: -1
2025-11-27T18:18:54.051195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471360147234623:2107] [node 4] ICS09 handshake done sender: [2:7577471355852267298:2140] self: [2:7577471355852267296:0] peer: [4:7577471362020169788:0] socket: 621 qp: -1
2025-11-27T18:18:53.945463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471357119112777:2050] [node 3] ICS09 handshake done sender: [6:7577471357119112761:2116] self: [6:7577471357119112766:0] peer: [3:7577471359069797177:0] socket: 616 qp: -1
2025-11-27T18:18:53.945506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471357119112777:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:53.963374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:53.989706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471357119112685:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(58198642ns) {HandleHandshakeStatus:329(58196079ns) {DropIncomingHandshake:442(754ns) DropOutgoingHandshake:457(666ns) SwitchToState:179(524ns) ProcessPendingSessionEvents:495(781ns)}}
2025-11-27T18:18:53.997566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471357119112778:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58960
2025-11-27T18:18:54.030185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471357119112741:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15940) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471357119112773:2121] held: no
2025-11-27T18:18:54.030811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471357119112773:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:54.038395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471357119112741:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:54.038446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:54.039532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471357119112741:2105] [node 5] ICP22 created new session: [6:7577471361414080077:2105]
2025-11-27T18:18:54.039847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471361414080077:2105] [node 5] ICS09 handshake done sender: [6:7577471357119112773:2121] self: [6:7577471357119112776:0] peer: [5:7577471355711658458:0] socket: 651 qp: -1
2025-11-27T18:18:54.039888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471361414080077:2105] [node 5] ICS10 traffic start
2025-11-27T18:18:54.039964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:54.040280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471357119112780:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58960
2025-11-27T18:18:54.041223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471357119112780:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:54.042975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471357119112684:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:54.048056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471361414080080:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58960
2025-11-27T18:18:54.050966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471357119112780:2123] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:54.052666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471357119112684:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:54.052715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:54.053499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471357119112684:2049] [node 2] ICP22 created new session: [6:7577471361414080081:2049]
2025-11-27T18:18:54.054840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471361414080081:2049] [node 2] ICS09 handshake done sender: [6:7577471357119112780:2123] self: [6:7577471361414080079:0] peer: [2:7577471355852267302:0] socket: 607 qp: -1
2025-11-27T18:18:53.935492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471359069797174:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:54.055848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471361414080081:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:53.935533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471359069797174:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:53.949629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471359069797174:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:54.055883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:53.969467Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471359069797168:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(94872127ns) {HandleHandshakeStatus:329(94870224ns) {DropIncomingHandshake:442(14701ns) DropOutgoingHandshake:457(4518ns) SwitchToState:179(386ns) ProcessPendingSessionEvents:495(28042236ns) {ForwardSessionEventToSession:612(7608045ns) ForwardSessionEventToSession:612(7228ns) ForwardSessionEventToSession:612(4982522ns) ForwardSessionEventToSession:612(3635ns) ForwardSessionEventToSession:612(2826ns) ForwardSessionEventToSession:612(2626206ns) ForwardSessionEventToSession:612(3327ns) ForwardSessionEventToSession:612(8548700ns) ForwardSessionEventToSession:612(3417813ns) ForwardSessionEventToSession:612(6560ns)}}}
2025-11-27T18:18:53.928272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471355711658459:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:54.049724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471362020169789:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:53.973642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471359069797106:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19003) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471359069797192:2119] held: no
2025-11-27T18:18:53.928310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471355711658459:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:54.050164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:53.982551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471359069797169:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:53.928347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471355711658447:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:53.982623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:54.051924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471362020169790:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64895
2025-11-27T18:18:53.982666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471359069797169:2107] [node 4] ICP22 created new session: [3:7577471359069797205:2107]
2025-11-27T18:18:53.941221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471355711658459:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:54.054865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471357725202376:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23548) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471357725202486:2139] held: no
2025-11-27T18:18:53.989537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471359069797203:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-27T18:18:54.005649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471359069797205:2107] [node 4] ICS09 handshake done sender: [3:7577471359069797174:2111] self: [3:7577471359069797173:0] peer: [4:7577471357725202483:0] socket: 629 qp: -1
2025-11-27T18:18:54.057006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471357725202486:2139] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:54.057400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471357725202376:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:54.017406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471359069797205:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:53.953816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471355711658386:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(96298438ns) {HandleHandshakeStatus:329(96295766ns) {DropIncomingHandshake:442(424ns) DropOutgoingHandshake:457(3949ns) SwitchToState:179(397ns) ProcessPendingSessionEvents:495(24389439ns) {ForwardSessionEventToSession:612(11947882ns) ForwardSessionEventToSession:612(403007ns) ForwardSessionEventToSession:612(3944ns) ForwardSessionEventToSession:612(2951ns) ForwardSessionEventToSession:612(3172798ns) ForwardSessionEventToSession:612(3532ns) ForwardSessionEventToSession:612(3086ns) ForwardSessionEventToSession:612(3913739ns) ForwardSessionEventToSession:612(3051400ns) ForwardSessionEventToSession:612(1859271ns)}}}
2025-11-27T18:18:54.057436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:54.017467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:53.957505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471355711658386:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:54.057488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471357725202376:2049] [node 2] ICP22 created new session: [4:7577471362020169792:2049]
2025-11-27T18:18:54.040244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471359069797169:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(57373069ns) {HandleHandshakeStatus:329(57370514ns) {DropIncomingHandshake:442(16898ns) DropOutgoingHandshake:457(425ns) SwitchToState:179(1019ns) ProcessPendingSessionEvents:495(6474502ns) {ForwardSessionEventToSession:612(6414269ns) ForwardSessionEventToSession:612(4555ns) ForwardSessionEventToSession:612(3252ns) ForwardSessionEventToSession:612(8133ns) ForwardSessionEventToSession:612(2447ns) ForwardSessionEventToSession:612(5541ns) ForwardSessionEventToSession:612(13896ns) ForwardSessionEventToSession:612(2105ns) ForwardSessionEventToSession:612(5594ns) ForwardSessionEventToSession:612(2996ns)}}}
2025-11-27T18:18:53.973661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471355711658448:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:54.057531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471362020169792:2049] [node 2] ICS09 handshake done sender: [4:7577471357725202486:2139] self: [4:7577471362020169788:0] peer: [2:7577471355852267296:0] socket: 661 qp: -1
2025-11-27T18:18:53.977547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:54.040348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471359069797107:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23548) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471359069797189:2118] held: no
2025-11-27T18:18:54.057562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471362020169792:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:54.040479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471359069797192:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:54.057601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:53.986720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471355711658448:2109] [node 6] ICP22 created new session: [5:7577471355711658483:2109]
2025-11-27T18:18:54.041584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471359069797106:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:53.990025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471355711658483:2109] [node 6] ICS09 handshake done sender: [5:7577471355711658459:2114] self: [5:7577471355711658458:0] peer: [6:7577471357119112776:0] socket: 649 qp: -1
2025-11-27T18:18:54.041618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:53.990067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471355711658483:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:54.043059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471359069797106:2048] [node 1] ICP22 created new session: [3:7577471363364764504:2048]
2025-11-27T18:18:53.990132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:54.040125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471360006625781:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17470
2025-11-27T18:18:54.043488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471359069797206:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-27T18:18:54.043670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471363364764504:2048] [node 1] ICS09 handshake done sender: [3:7577471359069797192:2119] self: [3:7577471359069797200:0] peer: [1:7577471357303083961:0] socket: 652 qp: -1
2025-11-27T18:18:54.044478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471355711658448:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(74331499ns) {HandleHandshakeStatus:329(74327644ns) {DropIncomingHandshake:442(19223ns) DropOutgoingHandshake:457(250ns) SwitchToState:179(746ns) ProcessPendingSessionEvents:495(29641513ns) {ForwardSessionEventToSession:612(11458386ns) ForwardSessionEventToSession:612(6024ns) ForwardSessionEventToSession:612(4861949ns) ForwardSessionEventToSession:612(3880ns) ForwardSessionEventToSession:612(4834059ns) ForwardSessionEventToSession:612(6824ns) ForwardSessionEventToSession:612(3411ns) ForwardSessionEventToSession:612(7447524ns) ForwardSessionEventToSession:612(992799ns) ForwardSessionEventToSession:612(4843ns)}}}
2025-11-27T18:18:54.051225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471360147234623:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:54.051381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:54.044242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471363364764504:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:54.044533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471355711658477:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:54.052624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471355852267290:2123] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:54.044274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:54.052673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:54.049990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471355711658447:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:54.045789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471359069797189:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:54.053226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471355852267290:2123] [node 6] ICP22 created new session: [2:7577471360147234625:2123]
2025-11-27T18:18:54.050527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:54.048955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471359069797107:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:54.050745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471355711658447:2108] [node 4] ICP22 created new session: [5:7577471360006625782:2108]
2025-11-27T18:18:54.056953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471360147234625:2123] [node 6] ICS09 handshake done sender: [2:7577471355852267305:2144] self: [2:7577471355852267302:0] peer: [6:7577471361414080079:0] socket: 663 qp: -1
2025-11-27T18:18:54.049659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:54.051416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471360006625782:2108] [node 4] ICS09 handshake done sender: [5:7577471355711658477:2122] self: [5:7577471355711658481:0] peer: [4:7577471357725202464:0] socket: 628 qp: -1
2025-11-27T18:18:54.057096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471360147234625:2123] [node 6] ICS10 traffic start
2025-11-27T18:18:54.050598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471359069797107:2049] [node 2] ICP22 created new session: [3:7577471363364764506:2049]
2025-11-27T18:18:54.057132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:54.051930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471360006625782:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:54.050922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471363364764506:2049] [node 2] ICS09 handshake done sender: [3:7577471359069797189:2118] self: [3:7577471359069797202:0] peer: [2:7577471355852267291:0] socket: 634 qp: -1
2025-11-27T18:18:54.052129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:54.051921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471363364764506:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:54.057345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471355711658447:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:54.052232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:54.098949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:54.099170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471355852267320:2048] [node 1] ICS01 socket: 614 reason# EndOfStream
2025-11-27T18:18:54.099211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471355852267198:2048] [node 1] ICP30 unregister session Session# [2:7577471355852267320:2048] VirtualId# [2:7577471355852267308:0]
2025-11-27T18:18:54.099248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471355852267320:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.099322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:54.099374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:54.100163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:54.100401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471363364764504:2048] [node 1] ICS01 socket: 652 reason# EndOfStream
2025-11-27T18:18:54.100441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471359069797106:2048] [node 1] ICP30 unregister session Session# [3:7577471363364764504:2048] VirtualId# [3:7577471359069797200:0]
2025-11-27T18:18:54.100472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471363364764504:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.100551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:54.100607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:54.103326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:54.103554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471355711658471:2048] [node 1] ICS01 socket: 597 reason# EndOfStream
2025-11-27T18:18:54.103598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471355711658384:2048] [node 1] ICP30 unregister session Session# [5:7577471355711658471:2048] VirtualId# [5:7577471355711658464:0]
2025-11-27T18:18:54.103640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471355711658471:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.103717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:54.103763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:54.104208Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:54.104390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471357119112768:2048] [node 1] ICS07 socket disconnect 598 reason# EndOfStream
2025-11-27T18:18:54.104422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471357119112768:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.104486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:54.104526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471357119112768:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:54.105208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471357119112683:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19003) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:54.105322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471357119112768:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:54.105366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471357119112683:2048] [node 1] ICP30 unregister session Session# [6:7577471357119112768:2048] VirtualId# [6:7577471357119112762:0]
2025-11-27T18:18:54.105420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:54.105449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471357119112683:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19003) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:54.105462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19003) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:54.105613Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:54.105811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471357725202484:2048] [node 1] ICS01 socket: 643 reason# EndOfStream
2025-11-27T18:18:54.105844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471357725202375:2048] [node 1] ICP30 unregister session Session# [4:7577471357725202484:2048] VirtualId# [4:7577471357725202480:0]
2025-11-27T18:18:54.105867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471357725202484:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.105971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:54.106016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:54.115145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471357119112683:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:54.158764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:54.159061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471363364764506:2049] [node 2] ICS01 socket: 634 reason# EndOfStream
2025-11-27T18:18:54.159106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471359069797107:2049] [node 2] ICP30 unregister session Session# [3:7577471363364764506:2049] VirtualId# [3:7577471359069797202:0]
2025-11-27T18:18:54.159136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471363364764506:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.159214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:54.159293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:54.160568Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:54.160785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471362020169792:2049] [node 2] ICS01 socket: 661 reason# EndOfStream
2025-11-27T18:18:54.160832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471357725202376:2049] [node 2] ICP30 unregister session Session# [4:7577471362020169792:2049] VirtualId# [4:7577471362020169788:0]
2025-11-27T18:18:54.160859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471362020169792:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.160929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:54.160968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:54.162546Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:54.162749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471361414080081:2049] [node 2] ICS01 socket: 607 reason# EndOfStream
2025-11-27T18:18:54.162800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471357119112684:2049] [node 2] ICP30 unregister session Session# [6:7577471361414080081:2049] VirtualId# [6:7577471361414080079:0]
2025-11-27T18:18:54.162827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471361414080081:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.162896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:54.162940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:54.165503Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:54.165914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471355711658479:2049] [node 2] ICS01 socket: 637 reason# EndOfStream
2025-11-27T18:18:54.165989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471355711658385:2049] [node 2] ICP30 unregister session Session# [5:7577471355711658479:2049] VirtualId# [5:7577471355711658474:0]
2025-11-27T18:18:54.166059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471355711658479:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.166189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:54.166308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:54.226990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:54.227204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471362020169786:2050] [node 3] ICS01 socket: 659 reason# EndOfStream
2025-11-27T18:18:54.227244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471357725202377:2050] [node 3] ICP30 unregister session Session# [4:7577471362020169786:2050] VirtualId# [4:7577471357725202483:0]
2025-11-27T18:18:54.227284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471362020169786:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.227365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:54.227424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:54.228625Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:54.228833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471357119112777:2050] [node 3] ICS01 socket: 616 reason# EndOfStream
2025-11-27T18:18:54.228876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471357119112685:2050] [node 3] ICP30 unregister session Session# [6:7577471357119112777:2050] VirtualId# [6:7577471357119112766:0]
2025-11-27T18:18:54.228900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471357119112777:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.228977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:54.229040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:54.229522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:54.229728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471355711658475:2050] [node 3] ICS01 socket: 615 reason# EndOfStream
2025-11-27T18:18:54.229766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471355711658386:2050] [node 3] ICP30 unregister session Session# [5:7577471355711658475:2050] VirtualId# [5:7577471355711658469:0]
2025-11-27T18:18:54.229791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471355711658475:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.229859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:54.229898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:54.305902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:54.306153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471360006625782:2108] [node 4] ICS07 socket disconnect 628 reason# EndOfStream
2025-11-27T18:18:54.306179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471360006625782:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.306256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:54.306292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471360006625782:2108] [node 4] ICS15 start handshake
2025-11-27T18:18:54.314170Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:54.314207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471355711658447:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32765) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:54.314515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471360006625782:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:54.314914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471355711658447:2108] [node 4] ICP30 unregister session Session# [5:7577471360006625782:2108] VirtualId# [5:7577471355711658481:0]
2025-11-27T18:18:54.314988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:54.314513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471357119112774:2106] [node 4] ICS07 socket disconnect 620 reason# EndOfStream
2025-11-27T18:18:54.315026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471355711658447:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32765) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:54.314545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471357119112774:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.314611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:54.315040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32765) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:54.314661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471357119112774:2106] [node 4] ICS15 start handshake
2025-11-27T18:18:54.315433Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471357119112745:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32765) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:54.320659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471357119112774:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:54.321096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471357119112745:2106] [node 4] ICP30 unregister session Session# [6:7577471357119112774:2106] VirtualId# [6:7577471357119112767:0]
2025-11-27T18:18:54.321171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:54.321241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471357119112745:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32765) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:54.321271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32765) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:54.331228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471357119112745:2106] [node 4] ICP33 wake up from error state
2025-11-27T18:18:54.365600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:54.365926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471361414080077:2105] [node 5] ICS01 socket: 651 reason# EndOfStream
2025-11-27T18:18:54.366275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471357119112741:2105] [node 5] ICP30 unregister session Session# [6:7577471361414080077:2105] VirtualId# [6:7577471357119112776:0]
2025-11-27T18:18:54.366322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471361414080077:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:54.366435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:54.366501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:54.491320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471364184364051:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:54.491674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471364184364050:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:54.514199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471361735515778:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:54.514733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471361735515779:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:54.538430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471362689579047:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:54.538467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471362689579048:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:54.562099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471361181391992:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:54.562158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471361181391994:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:54.562402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471361181391993:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:54.590878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471363634840692:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:54.590922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471363634840694:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:54.591193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471363634840693:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:54.615277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471360774351139:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:54.615320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471360774351140:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:54.615351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471360774351141:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:54.761113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471360774351199:2106] [node 5] ICP01 ready to work
2025-11-27T18:18:54.853789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471361735515842:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:54.801962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471364184364112:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:54.850802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471361181392054:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:54.805305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471364184364115:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:54.865531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471361735515841:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:54.853896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471360774351202:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:54.859239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471362689579110:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:54.877429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471362689579111:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:54.885464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471362689579112:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:54.875478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471361735515843:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:54.882414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471361181392057:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:54.865630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471364184364116:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:54.877504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471363634840759:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:54.906858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471364184364050:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(146605375ns) {Configure:104(146600692ns) {ProcessConfigured:135(82594ns) {StartInitialHandshake:157(80149ns) {DropHandshakes:470(3544ns) {DropIncomingHandshake:442(867ns) DropOutgoingHandshake:457(733ns)} GenerateSessionVirtualId:988(954ns) PrepareNewSessionHandshake:477(2107ns) {SwitchToState:179(766ns)}}}}}
2025-11-27T18:18:54.907330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471364184364119:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56851
2025-11-27T18:18:54.879876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471363634840760:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:54.921889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471360774351202:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(52614479ns) {Configure:104(52607767ns) {ProcessConfigured:135(74847ns) {StartInitialHandshake:157(72601ns) {DropHandshakes:470(3479ns) {DropIncomingHandshake:442(881ns) DropOutgoingHandshake:457(637ns)} GenerateSessionVirtualId:988(932ns) PrepareNewSessionHandshake:477(1886ns) {SwitchToState:179(691ns)}}}}}
2025-11-27T18:18:54.922255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471360774351206:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1084
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:54.943391Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471361181392057:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48696407ns) {Configure:104(48690028ns) {ProcessConfigured:135(89793ns) {StartInitialHandshake:157(87529ns) {DropHandshakes:470(3715ns) {DropIncomingHandshake:442(941ns) DropOutgoingHandshake:457(360ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:477(1969ns) {SwitchToState:179(616ns)}}}}}
2025-11-27T18:18:54.943936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471361181392060:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43097
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:54.964652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471363634840761:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-11-27T18:18:54.948938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471360774351199:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(186435412ns) {Configure:104(186431131ns) {ProcessConfigured:135(68679ns) {StartInitialHandshake:157(67245ns) {DropHandshakes:470(2299ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(266ns)} GenerateSessionVirtualId:988(1916ns) PrepareNewSessionHandshake:477(1541ns) {SwitchToState:179(465ns)}}}}}
2025-11-27T18:18:54.956799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471361735515778:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(124124654ns) {Configure:104(124115896ns) {ProcessConfigured:135(38839875ns) {StartInitialHandshake:157(38837269ns) {DropHandshakes:470(3658397ns) {DropIncomingHandshake:442(1855ns) DropOutgoingHandshake:457(150ns)} GenerateSessionVirtualId:988(1230ns) PrepareNewSessionHandshake:477(5162ns) {SwitchToState:179(2136ns)}}}}}
2025-11-27T18:18:54.970428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471361181391993:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(100883263ns) {Configure:104(100875310ns) {ProcessConfigured:135(77448ns) {StartInitialHandshake:157(75286ns) {DropHandshakes:470(3753ns) {DropIncomingHandshake:442(708ns) DropOutgoingHandshake:457(532ns)} GenerateSessionVirtualId:988(968ns) PrepareNewSessionHandshake:477(2212ns) {SwitchToState:179(667ns)}}}}}
2025-11-27T18:18:54.949345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471360774351210:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1084
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:54.991082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471361735515846:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15433
2025-11-27T18:18:54.991170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471361735515847:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15433
2025-11-27T18:18:54.997680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471361181392060:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:54.998557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471362689579112:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(103598717ns) {Configure:104(103592906ns) {ProcessConfigured:135(52896ns) {StartInitialHandshake:157(51673ns) {DropHandshakes:470(1083ns) {DropIncomingHandshake:442(363ns) DropOutgoingHandshake:457(321ns)} GenerateSessionVirtualId:988(349ns) PrepareNewSessionHandshake:477(1802ns) {SwitchToState:179(1099ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.001765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471365476359361:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43097
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.009714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471361181391992:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(190928642ns) {Configure:104(190923727ns) {ProcessConfigured:135(36288ns) {StartInitialHandshake:157(34230ns) {DropHandshakes:470(1132ns) {DropIncomingHandshake:442(195ns) DropOutgoingHandshake:457(244ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:477(686ns) {SwitchToState:179(281ns)}}}}}
2025-11-27T18:18:55.003024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471364184364119:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.007867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471361735515849:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15433
2025-11-27T18:18:55.015009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471361735515842:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(145772123ns) {Configure:104(145768630ns) {ProcessConfigured:135(19850796ns) {StartInitialHandshake:157(19848966ns) {DropHandshakes:470(13469127ns) {DropIncomingHandshake:442(189ns) DropOutgoingHandshake:457(1398ns)} GenerateSessionVirtualId:988(1086ns) PrepareNewSessionHandshake:477(1766ns) {SwitchToState:179(855ns)}}}}}
2025-11-27T18:18:55.013774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471363634840761:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.016580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471360774351210:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.014055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471367929808059:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.032478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471367929808062:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-11-27T18:18:55.004883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471364184364115:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(145656650ns) {Configure:104(145645805ns) {ProcessConfigured:135(18763227ns) {StartInitialHandshake:157(18760852ns) {DropHandshakes:470(2990ns) {DropIncomingHandshake:442(706ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(506ns) PrepareNewSessionHandshake:477(3690ns) {SwitchToState:179(1842ns)}}}}}
2025-11-27T18:18:55.015563Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471361181391994:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(149485768ns) {Configure:104(149479440ns) {ProcessConfigured:135(6820353ns) {StartInitialHandshake:157(6818987ns) {DropHandshakes:470(1501ns) {DropIncomingHandshake:442(471ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(2768ns) {SwitchToState:179(92ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.035494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471361735515779:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(176218407ns) {Configure:104(176210828ns) {ProcessConfigured:135(8704915ns) {StartInitialHandshake:157(8701999ns) {DropHandshakes:470(2917ns) {DropIncomingHandshake:442(679ns) DropOutgoingHandshake:457(537ns)} GenerateSessionVirtualId:988(751ns) PrepareNewSessionHandshake:477(2461ns) {SwitchToState:179(932ns)}}}}}
2025-11-27T18:18:55.035701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471361735515846:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.035818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471361735515849:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.037177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471361735515847:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.050158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471363634840759:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(162552449ns) {Configure:104(162546315ns) {ProcessConfigured:135(32262847ns) {StartInitialHandshake:157(32260199ns) {DropHandshakes:470(1417ns) {DropIncomingHandshake:442(740ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(492ns) PrepareNewSessionHandshake:477(3519ns) {SwitchToState:179(1664ns)}}}}}
2025-11-27T18:18:55.016883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471360774351202:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.037031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471362689579119:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.056323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471363634840693:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(160809146ns) {Configure:104(160788699ns) {ProcessConfigured:135(31624858ns) {StartInitialHandshake:157(31622341ns) {DropHandshakes:470(1530ns) {DropIncomingHandshake:442(445ns) DropOutgoingHandshake:457(161ns)} GenerateSessionVirtualId:988(455ns) PrepareNewSessionHandshake:477(3231ns) {SwitchToState:179(751ns)}}}}}
2025-11-27T18:18:55.029823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471360774351140:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(160355967ns) {Configure:104(160351522ns) {ProcessConfigured:135(12587873ns) {StartInitialHandshake:157(12584697ns) {DropHandshakes:470(1895ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(344ns)} GenerateSessionVirtualId:988(1235ns) PrepareNewSessionHandshake:477(3818ns) {SwitchToState:179(2093ns)}}}}}
2025-11-27T18:18:55.005199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471368479331421:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56851
2025-11-27T18:18:55.015686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471365476359361:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.037077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471362689579119:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.058713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471361735515841:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(177876181ns) {Configure:104(177863704ns) {ProcessConfigured:135(19448432ns) {StartInitialHandshake:157(19446870ns) {DropHandshakes:470(7278730ns) {DropIncomingHandshake:442(2252ns) DropOutgoingHandshake:457(663ns)} GenerateSessionVirtualId:988(1351ns) PrepareNewSessionHandshake:477(3183ns) {SwitchToState:179(1232ns)}}}}}
2025-11-27T18:18:55.057721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471363634840694:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(178848749ns) {Configure:104(178845192ns) {ProcessConfigured:135(18984939ns) {StartInitialHandshake:157(10844405ns) {DropHandshakes:470(2325ns) {DropIncomingHandshake:442(881ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(1366ns) PrepareNewSessionHandshake:477(3563ns) {SwitchToState:179(1634ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.013691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471364184364051:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(234713259ns) {Configure:104(234707609ns) {ProcessConfigured:135(19972031ns) {StartInitialHandshake:157(19968792ns) {DropHandshakes:470(2344ns) {DropIncomingHandshake:442(624ns) DropOutgoingHandshake:457(358ns)} GenerateSessionVirtualId:988(490ns) PrepareNewSessionHandshake:477(3173ns) {SwitchToState:179(1106ns)}}}}}
2025-11-27T18:18:55.016113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471361181391994:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.058996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471361735515778:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.036477Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471360774351141:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(161015415ns) {Configure:104(161006872ns) {ProcessConfigured:135(9040043ns) {StartInitialHandshake:157(9037433ns) {DropHandshakes:470(1239ns) {DropIncomingHandshake:442(509ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(1257ns) PrepareNewSessionHandshake:477(3163ns) {SwitchToState:179(1984ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.037281Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471362689579048:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(194061112ns) {Configure:104(194054920ns) {ProcessConfigured:135(10429852ns) {StartInitialHandshake:157(10427845ns) {DropHandshakes:470(2002ns) {DropIncomingHandshake:442(516ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(633ns) PrepareNewSessionHandshake:477(4232ns) {SwitchToState:179(1980ns)}}}}}
2025-11-27T18:18:55.014316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471364184364112:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(154982548ns) {Configure:104(154973390ns) {ProcessConfigured:135(19188686ns) {StartInitialHandshake:157(19185516ns) {DropHandshakes:470(950ns) {DropIncomingHandshake:442(175ns) DropOutgoingHandshake:457(106ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:477(2761ns) {SwitchToState:179(1032ns)}}}}}
2025-11-27T18:18:55.075184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471367929808068:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.070006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471365069318516:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1084
2025-11-27T18:18:55.016189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471365476359367:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43097
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.063869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471366030483151:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15433
2025-11-27T18:18:55.042477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471366984546419:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52509
2025-11-27T18:18:55.039756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471361181392059:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.091598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471363634840760:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(200450280ns) {Configure:104(200445167ns) {ProcessConfigured:135(1671910ns) {StartInitialHandshake:157(1669201ns) {DropHandshakes:470(3079ns) {DropIncomingHandshake:442(661ns) DropOutgoingHandshake:457(440ns)} GenerateSessionVirtualId:988(790ns) PrepareNewSessionHandshake:477(2263ns) {SwitchToState:179(1242ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.016841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471364184364116:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(133931337ns) {Configure:104(133924030ns) {ProcessConfigured:135(19131158ns) {StartInitialHandshake:157(19129397ns) {DropHandshakes:470(11909083ns) {DropIncomingHandshake:442(493ns) DropOutgoingHandshake:457(1114ns)} GenerateSessionVirtualId:988(1098ns) PrepareNewSessionHandshake:477(1547ns) {SwitchToState:179(163ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.077158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471360774351139:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(151935428ns) {Configure:104(151924962ns) {ProcessConfigured:135(24277535ns) {StartInitialHandshake:157(24274800ns) {DropHandshakes:470(2876ns) {DropIncomingHandshake:442(867ns) DropOutgoingHandshake:457(425ns)} GenerateSessionVirtualId:988(1052ns) PrepareNewSessionHandshake:477(2130ns) {SwitchToState:179(623ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.068543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471366030483151:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.055552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471368479331421:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.066573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471364184364118:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.066601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471364184364118:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.068923Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471364184364118:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:55.070759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471364184364050:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.081652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:55.090929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471364184364050:2048] [node 2] ICP22 created new session: [1:7577471368479331430:2048]
2025-11-27T18:18:55.098600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471368479331426:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.098625Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471368479331426:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.100466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471368479331430:2048] [node 2] ICS09 handshake done sender: [1:7577471364184364118:2111] self: [1:7577471364184364117:0] peer: [2:7577471366030483155:0] socket: 603 qp: -1
2025-11-27T18:18:55.100506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471368479331430:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:55.102778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471368479331426:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:55.106418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:55.121856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471364184364112:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.129699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:55.129761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471364184364112:2107] [node 4] ICP22 created new session: [1:7577471368479331434:2107]
2025-11-27T18:18:55.129821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471368479331434:2107] [node 4] ICS09 handshake done sender: [1:7577471368479331426:2116] self: [1:7577471364184364123:0] peer: [4:7577471365476359377:0] socket: 628 qp: -1
2025-11-27T18:18:55.129855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471368479331434:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:55.129898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:55.130458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471368479331422:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.130480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471368479331422:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.137483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471364184364124:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.137528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471364184364124:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.138248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471368479331422:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:55.138928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471364184364115:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.138986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:55.139052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471364184364115:2108] [node 5] ICP22 created new session: [1:7577471368479331439:2108]
2025-11-27T18:18:55.139101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471368479331439:2108] [node 5] ICS09 handshake done sender: [1:7577471368479331422:2115] self: [1:7577471364184364122:0] peer: [5:7577471367929808073:0] socket: 625 qp: -1
2025-11-27T18:18:55.139136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471368479331439:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:55.139178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:55.145745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471364184364124:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:55.146015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471364184364051:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.146062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:55.146115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471364184364051:2049] [node 3] ICP22 created new session: [1:7577471368479331442:2049]
2025-11-27T18:18:55.146157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471368479331442:2049] [node 3] ICS09 handshake done sender: [1:7577471364184364124:2113] self: [1:7577471364184364121:0] peer: [3:7577471366984546431:0] socket: 650 qp: -1
2025-11-27T18:18:55.146207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471368479331442:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:55.146248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:55.156556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471368479331441:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56851
2025-11-27T18:18:55.058150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471362689579111:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(170674977ns) {Configure:104(170666035ns) {ProcessConfigured:135(20640294ns) {StartInitialHandshake:157(20638361ns) {DropHandshakes:470(8782937ns) {DropIncomingHandshake:442(1922ns) DropOutgoingHandshake:457(687ns)} GenerateSessionVirtualId:988(1324ns) PrepareNewSessionHandshake:477(3141ns) {SwitchToState:179(1066ns)}}}}}
2025-11-27T18:18:55.039796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471361181392059:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.082010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471360774351206:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.097551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471367929808068:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.070071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471366030483152:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15433
2025-11-27T18:18:55.082184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471360774351141:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.098336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471363634840694:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.059881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471366984546424:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52509
2025-11-27T18:18:55.045835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471361181392059:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:55.070244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471366030483158:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15433
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.102447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471367929808059:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.111246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471363634840692:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.126241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471367929808062:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.133360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471363634840693:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.149989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471367929808075:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-11-27T18:18:55.065722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471362689579119:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:55.080984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471365476359367:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.073783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471362689579112:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.086721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471360774351210:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:55.223739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471368479331428:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.154892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471363634840692:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5086) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471367929808059:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.102308Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471361735515843:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(212464117ns) {Configure:104(212454753ns) {ProcessConfigured:135(10176399ns) {StartInitialHandshake:157(10174025ns) {DropHandshakes:470(642ns) {DropIncomingHandshake:442(195ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(264ns) PrepareNewSessionHandshake:477(2388ns) {SwitchToState:179(1139ns)}}}}}
2025-11-27T18:18:55.085535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471361181392057:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.100097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471360774351202:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.223804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471368479331428:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.113132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471361735515846:2111] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:55.073835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:55.214518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471367929808059:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:55.102401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:55.085623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:55.281436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471368479331428:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.073886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471362689579112:2107] [node 4] ICP22 created new session: [3:7577471366984546425:2107]
2025-11-27T18:18:55.119069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471366030483152:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.229522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471363634840692:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.085677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471361181392057:2109] [node 6] ICP22 created new session: [4:7577471365476359374:2109]
2025-11-27T18:18:55.111145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471360774351202:2109] [node 4] ICP22 created new session: [6:7577471365069318518:2109]
2025-11-27T18:18:55.123056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471361735515778:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.229578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:55.073937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471366984546425:2107] [node 4] ICS09 handshake done sender: [3:7577471362689579119:2112] self: [3:7577471362689579118:0] peer: [4:7577471365476359368:0] socket: 622 qp: -1
2025-11-27T18:18:55.297414Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471364184364050:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(142997014ns) {HandleHandshakeStatus:329(142994247ns) {DropIncomingHandshake:442(540ns) DropOutgoingHandshake:457(569ns) SwitchToState:179(1206ns) ProcessPendingSessionEvents:495(80449574ns) {ForwardSessionEventToSession:612(39535418ns) ForwardSessionEventToSession:612(7670ns) ForwardSessionEventToSession:612(3718ns) ForwardSessionEventToSession:612(21111221ns) ForwardSessionEventToSession:612(4639ns) ForwardSessionEventToSession:612(3157ns) ForwardSessionEventToSession:612(12782329ns) ForwardSessionEventToSession:612(3711ns) ForwardSessionEventToSession:612(6965032ns) ForwardSessionEventToSession:612(6773ns)}}}
2025-11-27T18:18:55.113716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471365069318518:2109] [node 4] ICS09 handshake done sender: [6:7577471360774351210:2113] self: [6:7577471365069318510:0] peer: [4:7577471361181392058:0] socket: 611 qp: -1
2025-11-27T18:18:55.085728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471365476359374:2109] [node 6] ICS09 handshake done sender: [4:7577471361181392059:2111] self: [4:7577471361181392058:0] peer: [6:7577471365069318510:0] socket: 610 qp: -1
2025-11-27T18:18:55.073977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471366984546425:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:55.229634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471363634840692:2048] [node 1] ICP22 created new session: [5:7577471367929808078:2048]
2025-11-27T18:18:55.304321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471364184364116:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.123112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:55.085767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471365476359374:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.126159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471365069318518:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:55.305469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471364184364050:2048] [node 2] ICP18 handshake (actor [1:7577471368479331421:2114]) is too old
2025-11-27T18:18:55.074019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:55.229680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471367929808078:2048] [node 1] ICS09 handshake done sender: [5:7577471367929808059:2113] self: [5:7577471367929808073:0] peer: [1:7577471364184364122:0] socket: 627 qp: -1
2025-11-27T18:18:55.123166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471361735515778:2048] [node 1] ICP22 created new session: [2:7577471366030483164:2048]
2025-11-27T18:18:55.085802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:55.133140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:55.306263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:55.089424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471362689579047:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(194792771ns) {Configure:104(186603669ns) {ProcessConfigured:135(12044334ns) {StartInitialHandshake:157(12042601ns) {DropHandshakes:470(2321ns) {DropIncomingHandshake:442(560ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(687ns) PrepareNewSessionHandshake:477(1930ns) {SwitchToState:179(798ns)}}}}}
2025-11-27T18:18:55.229718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471367929808078:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.123218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471366030483164:2048] [node 1] ICS09 handshake done sender: [2:7577471361735515846:2111] self: [2:7577471366030483155:0] peer: [1:7577471364184364117:0] socket: 616 qp: -1
2025-11-27T18:18:55.150671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471365069318522:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1084
2025-11-27T18:18:55.308912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471364184364116:2110] [node 6] ICP22 created new session: [1:7577471368479331446:2110]
2025-11-27T18:18:55.097856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471361181391992:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.123256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471366030483164:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:55.123295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:55.126084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471366030483150:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.126123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471366030483150:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.151721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471366030483147:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.151748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471366030483147:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.168991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471366030483150:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:55.172891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471366030483147:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:55.189857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471366030483154:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.189888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471366030483154:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.202156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471366030483154:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:55.213971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471361735515842:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.214035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:55.214147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471361735515779:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.214199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:55.214247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471361735515779:2049] [node 3] ICP22 created new session: [2:7577471366030483171:2049]
2025-11-27T18:18:55.214295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471366030483171:2049] [node 3] ICS09 handshake done sender: [2:7577471366030483150:2115] self: [2:7577471366030483149:0] peer: [3:7577471366984546428:0] socket: 642 qp: -1
2025-11-27T18:18:55.214327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471366030483171:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:55.214374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:55.218535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471361735515842:2108] [node 5] ICP22 created new session: [2:7577471366030483170:2108]
2025-11-27T18:18:55.233824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471361735515841:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.233900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:55.233957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471361735515841:2109] [node 6] ICP22 created new session: [2:7577471366030483173:2109]
2025-11-27T18:18:55.234013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471366030483173:2109] [node 6] ICS09 handshake done sender: [2:7577471366030483154:2118] self: [2:7577471366030483153:0] peer: [6:7577471365069318524:0] socket: 619 qp: -1
2025-11-27T18:18:55.234048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471366030483173:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:55.234088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:55.246689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471366030483170:2108] [node 5] ICS09 handshake done sender: [2:7577471366030483147:2114] self: [2:7577471366030483146:0] peer: [5:7577471367929808074:0] socket: 629 qp: -1
2025-11-27T18:18:55.258644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471366030483170:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:55.258706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:55.288082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471366030483161:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.288107Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471366030483161:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.289176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471366030483161:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:55.298110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471361735515843:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.298165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:55.305984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471361735515842:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(98657162ns) {HandleHandshakeStatus:329(98654683ns) {DropIncomingHandshake:442(18968ns) DropOutgoingHandshake:457(311ns) SwitchToState:179(239ns) ProcessPendingSessionEvents:495(11004049ns) {ForwardSessionEventToSession:612(10948841ns) ForwardSessionEventToSession:612(9926ns) ForwardSessionEventToSession:612(2900ns) ForwardSessionEventToSession:612(4509ns) ForwardSessionEventToSession:612(3074ns) ForwardSessionEventToSession:612(2923ns) ForwardSessionEventToSession:612(5965ns) ForwardSessionEventToSession:612(2785ns) ForwardSessionEventToSession:612(5095ns) ForwardSessionEventToSession:612(4645ns)}}}
2025-11-27T18:18:55.310242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471361735515843:2107] [node 4] ICP22 created new session: [2:7577471366030483178:2107]
2025-11-27T18:18:55.310803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471366030483178:2107] [node 4] ICS09 handshake done sender: [2:7577471366030483161:2120] self: [2:7577471366030483160:0] peer: [4:7577471365476359387:0] socket: 636 qp: -1
2025-11-27T18:18:55.311650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471366030483178:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:55.311699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:55.333623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471361735515779:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(109296361ns) {HandleHandshakeStatus:329(109293704ns) {DropIncomingHandshake:442(13298ns) DropOutgoingHandshake:457(195ns) SwitchToState:179(104ns) ProcessPendingSessionEvents:495(35366398ns) {ForwardSessionEventToSession:612(27924006ns) ForwardSessionEventToSession:612(5080ns) ForwardSessionEventToSession:612(3276ns) ForwardSessionEventToSession:612(1305754ns) ForwardSessionEventToSession:612(943590ns) ForwardSessionEventToSession:612(5968ns) ForwardSessionEventToSession:612(167432ns) ForwardSessionEventToSession:612(2742ns) ForwardSessionEventToSession:612(4982648ns) ForwardSessionEventToSession:612(6370ns)}}}
2025-11-27T18:18:55.090619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471366984546424:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.229757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:55.311501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471368479331446:2110] [node 6] ICS09 handshake done sender: [1:7577471368479331428:2117] self: [1:7577471368479331425:0] peer: [6:7577471365069318525:0] socket: 658 qp: -1
2025-11-27T18:18:55.313134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471368479331446:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:55.313174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:55.322345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471364184364112:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:55.330963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471368479331444:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56851
2025-11-27T18:18:55.152672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471365069318520:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1084
2025-11-27T18:18:55.092051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471362689579048:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.100314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471361181392054:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(180523071ns) {Configure:104(180514320ns) {ProcessConfigured:135(28498596ns) {StartInitialHandshake:157(28495395ns) {DropHandshakes:470(590ns) {DropIncomingHandshake:442(307ns) DropOutgoingHandshake:457(34ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(5952ns) {SwitchToState:179(3483ns)}}}}}
2025-11-27T18:18:55.357725Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471361735515843:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(39644057ns) {HandleHandshakeStatus:329(39641089ns) {DropIncomingHandshake:442(18391ns) DropOutgoingHandshake:457(327ns) SwitchToState:179(302ns) ProcessPendingSessionEvents:495(14903188ns) {ForwardSessionEventToSession:612(6644239ns) ForwardSessionEventToSession:612(28095ns) ForwardSessionEventToSession:612(3532ns) ForwardSessionEventToSession:612(1802137ns) ForwardSessionEventToSession:612(3254ns) ForwardSessionEventToSession:612(3446ns) ForwardSessionEventToSession:612(2375565ns) ForwardSessionEventToSession:612(3032ns) ForwardSessionEventToSession:612(4012158ns) ForwardSessionEventToSession:612(7370ns)}}}
2025-11-27T18:18:55.230347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471367929808076:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:55.168592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471360774351206:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:55.101976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471365476359378:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43097
2025-11-27T18:18:55.099843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471366984546429:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52509
2025-11-27T18:18:55.230485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471367929808077:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-11-27T18:18:55.169324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471365069318520:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.109565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471365476359376:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43097
2025-11-27T18:18:55.385703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471364184364116:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(40315368ns) {HandleHandshakeStatus:329(40312205ns) {DropIncomingHandshake:442(1314ns) DropOutgoingHandshake:457(794ns) SwitchToState:179(537ns) ProcessPendingSessionEvents:495(14603502ns) {ForwardSessionEventToSession:612(2047966ns) ForwardSessionEventToSession:612(3741ns) ForwardSessionEventToSession:612(5686666ns) ForwardSessionEventToSession:612(2639089ns) ForwardSessionEventToSession:612(3677ns) ForwardSessionEventToSession:612(3833844ns) ForwardSessionEventToSession:612(4008ns) ForwardSessionEventToSession:612(3598ns) ForwardSessionEventToSession:612(354121ns) ForwardSessionEventToSession:612(3358ns)}}}
2025-11-27T18:18:55.126120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471365476359373:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43097
2025-11-27T18:18:55.170326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471360774351140:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.126474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471361181391994:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7629) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471365476359361:2114] held: no
2025-11-27T18:18:55.173031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471365069318523:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1084
2025-11-27T18:18:55.191666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471360774351202:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(91344106ns) {HandleHandshakeStatus:329(91340474ns) {DropIncomingHandshake:442(638ns) DropOutgoingHandshake:457(5387ns) SwitchToState:179(464ns) ProcessPendingSessionEvents:495(22381234ns) {ForwardSessionEventToSession:612(22320515ns) ForwardSessionEventToSession:612(6691ns) ForwardSessionEventToSession:612(3380ns) ForwardSessionEventToSession:612(7535ns) ForwardSessionEventToSession:612(2757ns) ForwardSessionEventToSession:612(8495ns) ForwardSessionEventToSession:612(5960ns) ForwardSessionEventToSession:612(2873ns) ForwardSessionEventToSession:612(5027ns) ForwardSessionEventToSession:612(2904ns)}}}
2025-11-27T18:18:55.191751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471360774351202:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:55.198247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471365069318523:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.198410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471360774351139:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.132113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471365476359361:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:55.206533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471360774351141:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.132267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471365476359367:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:55.213459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:55.155053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471361181391992:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.213560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471360774351141:2050] [node 3] ICP22 created new session: [6:7577471365069318527:2050]
2025-11-27T18:18:55.155112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:55.230721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471365069318527:2050] [node 3] ICS09 handshake done sender: [6:7577471360774351206:2111] self: [6:7577471365069318517:0] peer: [3:7577471362689579113:0] socket: 604 qp: -1
2025-11-27T18:18:55.155163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471361181391992:2048] [node 1] ICP22 created new session: [4:7577471365476359380:2048]
2025-11-27T18:18:55.237053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471360774351140:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18748) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471365069318520:2118] held: no
2025-11-27T18:18:55.246060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471365069318527:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:55.155213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471365476359380:2048] [node 1] ICS09 handshake done sender: [4:7577471365476359367:2117] self: [4:7577471365476359377:0] peer: [1:7577471364184364123:0] socket: 630 qp: -1
2025-11-27T18:18:55.250773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471365069318530:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1084
2025-11-27T18:18:55.155251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471365476359380:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:55.251419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:55.155283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:55.269590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471365069318526:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1084
2025-11-27T18:18:55.166831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471365476359373:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.270159Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471365069318520:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:55.285566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471360774351140:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.206647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471361181391994:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(30591332ns) {HandleHandshakeStatus:395(30589674ns) {LogHandshakeFail:472(4249131ns) {UpdateErrorStateLog:545(6010ns)} DropOutgoingHandshake:457(293ns) ProcessPendingSessionEvents:495(26285924ns) {DropSessionEvent:512(18998ns) DropSessionEvent:512(4962ns) DropSessionEvent:512(5090ns) DropSessionEvent:512(4879ns) DropSessionEvent:512(6429ns) DropSessionEvent:512(5372ns) DropSessionEvent:512(5914ns) DropSessionEvent:512(4659ns) DropSessionEvent:512(21329ns) DropSessionEvent:512(5983ns)}}}
2025-11-27T18:18:55.213847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471361181391994:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.285618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:55.213902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:55.285674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471360774351140:2049] [node 2] ICP22 created new session: [6:7577471365069318531:2049]
2025-11-27T18:18:55.218161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471361181391994:2050] [node 3] ICP22 created new session: [4:7577471365476359384:2050]
2025-11-27T18:18:55.285723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471365069318531:2049] [node 2] ICS09 handshake done sender: [6:7577471365069318520:2118] self: [6:7577471365069318524:0] peer: [2:7577471366030483153:0] socket: 634 qp: -1
2025-11-27T18:18:55.245845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471365476359384:2050] [node 3] ICS09 handshake done sender: [4:7577471365476359361:2114] self: [4:7577471365476359368:0] peer: [3:7577471362689579118:0] socket: 623 qp: -1
2025-11-27T18:18:55.285761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471365069318531:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:55.251206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471365476359385:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43097
2025-11-27T18:18:55.285804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:55.300692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471365069318523:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:55.300938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471360774351139:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.301006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:55.301063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471360774351139:2048] [node 1] ICP22 created new session: [6:7577471365069318533:2048]
2025-11-27T18:18:55.301121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471365069318533:2048] [node 1] ICS09 handshake done sender: [6:7577471365069318523:2120] self: [6:7577471365069318525:0] peer: [1:7577471368479331425:0] socket: 662 qp: -1
2025-11-27T18:18:55.301165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471365069318533:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:55.301202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:55.318120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471360774351141:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(119776689ns) {HandleHandshakeStatus:329(103953003ns) {DropIncomingHandshake:442(4723ns) DropOutgoingHandshake:457(16589ns) SwitchToState:179(1139ns) ProcessPendingSessionEvents:495(16490320ns) {ForwardSessionEventToSession:612(16431371ns) ForwardSessionEventToSession:612(5535ns) ForwardSessionEventToSession:612(2865ns) ForwardSessionEventToSession:612(12589ns) ForwardSessionEventToSession:612(3257ns) ForwardSessionEventToSession:612(2670ns) ForwardSessionEventToSession:612(5977ns) ForwardSessionEventToSession:612(2964ns) ForwardSessionEventToSession:612(10137ns) ForwardSessionEventToSession:612(2091ns)}}}
2025-11-27T18:18:55.251474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471365476359384:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:55.265763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:55.287591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471365476359385:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.287736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471361181391993:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.333415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471360774351139:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18331917ns) {HandleHandshakeStatus:329(18330243ns) {DropIncomingHandshake:442(715ns) DropOutgoingHandshake:457(9497ns) SwitchToState:179(545ns) ProcessPendingSessionEvents:495(17913810ns) {ForwardSessionEventToSession:612(4884176ns) ForwardSessionEventToSession:612(2679495ns) ForwardSessionEventToSession:612(1333370ns) ForwardSessionEventToSession:612(5746ns) ForwardSessionEventToSession:612(540823ns) ForwardSessionEventToSession:612(2606ns) ForwardSessionEventToSession:612(2162065ns) ForwardSessionEventToSession:612(20286ns) ForwardSessionEventToSession:612(2932ns) ForwardSessionEventToSession:612(982278ns)}}}
2025-11-27T18:18:55.290755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471365476359372:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.338326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471365069318535:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1084
2025-11-27T18:18:55.290787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471365476359372:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.346366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471365069318535:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.352851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471365069318537:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1084
2025-11-27T18:18:55.309836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471365476359388:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43097
2025-11-27T18:18:55.351503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471360774351199:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.310398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471361181391993:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18748) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471365476359385:2122] held: no
2025-11-27T18:18:55.372005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471365069318535:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:55.313357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471361181391994:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(103418482ns) {HandleHandshakeStatus:329(96307425ns) {DropIncomingHandshake:442(769ns) DropOutgoingHandshake:457(577ns) SwitchToState:179(669ns) ProcessPendingSessionEvents:495(856ns)}}
2025-11-27T18:18:55.377937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471360774351199:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.317817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471365476359372:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:55.378020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:55.317975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471361181392054:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.378074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471360774351199:2106] [node 5] ICP22 created new session: [6:7577471365069318538:2106]
2025-11-27T18:18:55.320992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:55.378134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471365069318538:2106] [node 5] ICS09 handshake done sender: [6:7577471365069318535:2123] self: [6:7577471365069318536:0] peer: [5:7577471367929808069:0] socket: 655 qp: -1
2025-11-27T18:18:55.414237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471364184364116:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:55.321058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471361181392054:2108] [node 5] ICP22 created new session: [4:7577471365476359390:2108]
2025-11-27T18:18:55.378173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471365069318538:2106] [node 5] ICS10 traffic start
2025-11-27T18:18:55.321110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471365476359390:2108] [node 5] ICS09 handshake done sender: [4:7577471365476359372:2118] self: [4:7577471365476359370:0] peer: [5:7577471367929808087:0] socket: 631 qp: -1
2025-11-27T18:18:55.378219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:55.413818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471360774351199:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(35623119ns) {HandleHandshakeStatus:329(35618728ns) {DropIncomingHandshake:442(1195ns) DropOutgoingHandshake:457(19624ns) SwitchToState:179(1050ns) ProcessPendingSessionEvents:495(11902585ns) {ForwardSessionEventToSession:612(11838622ns) ForwardSessionEventToSession:612(11232ns) ForwardSessionEventToSession:612(9387ns) ForwardSessionEventToSession:612(2542ns) ForwardSessionEventToSession:612(4985ns) ForwardSessionEventToSession:612(2813ns) ForwardSessionEventToSession:612(3284ns) ForwardSessionEventToSession:612(5081ns) ForwardSessionEventToSession:612(5368ns) ForwardSessionEventToSession:612(2386ns)}}}
2025-11-27T18:18:55.321162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471365476359390:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:55.321200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:55.241610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471363634840693:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18748) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471367929808062:2115] held: no
2025-11-27T18:18:55.321632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471365476359385:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:55.329998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471361181391993:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.253744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471367929808062:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:55.330057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:55.253832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471367929808080:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-11-27T18:18:55.330108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471361181391993:2049] [node 2] ICP22 created new session: [4:7577471365476359392:2049]
2025-11-27T18:18:55.254700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471367929808068:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:55.330151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471365476359392:2049] [node 2] ICS09 handshake done sender: [4:7577471365476359385:2122] self: [4:7577471365476359387:0] peer: [2:7577471366030483160:0] socket: 669 qp: -1
2025-11-27T18:18:55.254919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471363634840693:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.330187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471365476359392:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:55.254976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:55.330216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:55.255027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471363634840693:2049] [node 2] ICP22 created new session: [5:7577471367929808083:2049]
2025-11-27T18:18:55.255075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471367929808083:2049] [node 2] ICS09 handshake done sender: [5:7577471367929808062:2115] self: [5:7577471367929808074:0] peer: [2:7577471366030483146:0] socket: 632 qp: -1
2025-11-27T18:18:55.255110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471367929808083:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:55.255144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:55.255660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471363634840694:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.255703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:55.255744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471363634840694:2050] [node 3] ICP22 created new session: [5:7577471367929808085:2050]
2025-11-27T18:18:55.255782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471367929808085:2050] [node 3] ICS09 handshake done sender: [5:7577471367929808068:2118] self: [5:7577471367929808072:0] peer: [3:7577471366984546421:0] socket: 648 qp: -1
2025-11-27T18:18:55.255807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471367929808085:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:55.255836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:55.263022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471367929808076:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.263124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471363634840759:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.337344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471363634840759:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471367929808076:2121] held: no
2025-11-27T18:18:55.351779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471367929808070:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.351805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471367929808070:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.353081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471367929808070:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:55.361995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471363634840760:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.371866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:55.381822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471363634840760:2109] [node 6] ICP22 created new session: [5:7577471367929808090:2109]
2025-11-27T18:18:55.393517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471367929808088:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-11-27T18:18:55.393588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471367929808090:2109] [node 6] ICS09 handshake done sender: [5:7577471367929808070:2119] self: [5:7577471367929808069:0] peer: [6:7577471365069318536:0] socket: 671 qp: -1
2025-11-27T18:18:55.401484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471367929808090:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:55.408589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:55.409725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471367929808076:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:55.099953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471362689579114:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.100001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471362689579114:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.101491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471366984546419:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.113788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471366984546423:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.113823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471366984546423:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:55.117820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471362689579112:2107] [node 4] ICP18 handshake (actor [3:7577471366984546419:2113]) is too old
2025-11-27T18:18:55.118391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471366984546429:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.118656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471366984546430:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52509
2025-11-27T18:18:55.118756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471362689579047:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:55.133501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471362689579114:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:55.160338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471366984546433:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52509
2025-11-27T18:18:55.161098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471366984546429:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:55.161708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471362689579110:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.161756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:55.161806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471362689579110:2109] [node 6] ICP22 created new session: [3:7577471366984546434:2109]
2025-11-27T18:18:55.161859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471366984546434:2109] [node 6] ICS09 handshake done sender: [3:7577471362689579114:2110] self: [3:7577471362689579113:0] peer: [6:7577471365069318517:0] socket: 601 qp: -1
2025-11-27T18:18:55.161902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471366984546434:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:55.161945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:55.162266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471362689579047:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.164812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:55.177774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471366984546430:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:55.178221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471366984546438:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52509
2025-11-27T18:18:55.178831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471366984546423:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:55.178968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471366984546424:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:55.179311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471362689579048:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.179365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:55.179415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471362689579048:2049] [node 2] ICP22 created new session: [3:7577471366984546439:2049]
2025-11-27T18:18:55.179478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471366984546439:2049] [node 2] ICS09 handshake done sender: [3:7577471366984546424:2116] self: [3:7577471366984546428:0] peer: [2:7577471366030483149:0] socket: 643 qp: -1
2025-11-27T18:18:55.179517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471366984546439:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:55.179557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:55.185936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471362689579047:2048] [node 1] ICP22 created new session: [3:7577471366984546437:2048]
2025-11-27T18:18:55.193415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471362689579111:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:55.205524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:55.205612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471362689579111:2108] [node 5] ICP22 created new session: [3:7577471366984546441:2108]
2025-11-27T18:18:55.205670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471366984546441:2108] [node 5] ICS09 handshake done sender: [3:7577471366984546423:2115] self: [3:7577471366984546421:0] peer: [5:7577471367929808072:0] socket: 647 qp: -1
2025-11-27T18:18:55.205722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471366984546441:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:55.205776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:55.213964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471366984546437:2048] [node 1] ICS09 handshake done sender: [3:7577471366984546429:2117] self: [3:7577471366984546431:0] peer: [1:7577471364184364121:0] socket: 609 qp: -1
2025-11-27T18:18:55.223471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471366984546437:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:55.223557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:55.254150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471366984546444:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52509
2025-11-27T18:18:55.265517Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471362689579110:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(44898063ns) {HandleHandshakeStatus:329(44895314ns) {DropIncomingHandshake:442(763ns) DropOutgoingHandshake:457(723ns) SwitchToState:179(663ns) ProcessPendingSessionEvents:495(44495371ns) {ForwardSessionEventToSession:612(2021582ns) ForwardSessionEventToSession:612(5084ns) ForwardSessionEventToSession:612(5302631ns) ForwardSessionEventToSession:612(3956ns) ForwardSessionEventToSession:612(23278060ns) ForwardSessionEventToSession:612(28093ns) ForwardSessionEventToSession:612(4510ns) ForwardSessionEventToSession:612(4779535ns) ForwardSessionEventToSession:612(3917ns) ForwardSessionEventToSession:612(3303ns)}}}
2025-11-27T18:18:55.293988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471362689579047:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(131453032ns) {HandleHandshakeStatus:329(131450491ns) {DropIncomingHandshake:442(681ns) DropOutgoingHandshake:457(4531ns) SwitchToState:179(388ns) ProcessPendingSessionEvents:495(19891762ns) {ForwardSessionEventToSession:612(19834155ns) ForwardSessionEventToSession:612(6926ns) ForwardSessionEventToSession:612(3670ns) ForwardSessionEventToSession:612(5183ns) ForwardSessionEventToSession:612(2349ns) ForwardSessionEventToSession:612(3359ns) ForwardSessionEventToSession:612(5848ns) ForwardSessionEventToSession:612(2081ns) ForwardSessionEventToSession:612(9624ns) ForwardSessionEventToSession:612(3549ns)}}}
2025-11-27T18:18:55.319161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471362689579110:2109] [node 6] ICP18 handshake (actor [3:7577471366984546430:2118]) is too old
2025-11-27T18:18:55.417873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471363634840760:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(55669298ns) {HandleHandshakeStatus:329(55666208ns) {DropIncomingHandshake:442(17570ns) DropOutgoingHandshake:457(532ns) SwitchToState:179(1091ns) ProcessPendingSessionEvents:495(229837ns) {ForwardSessionEventToSession:612(176888ns) ForwardSessionEventToSession:612(3802ns) ForwardSessionEventToSession:612(2297ns) ForwardSessionEventToSession:612(5576ns) ForwardSessionEventToSession:612(2686ns) ForwardSessionEventToSession:612(2399ns) ForwardSessionEventToSession:612(8686ns) ForwardSessionEventToSession:612(4679ns) ForwardSessionEventToSession:612(9612ns) ForwardSessionEventToSession:612(2219ns)}}}
2025-11-27T18:18:55.421755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471363634840759:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:55.421822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:55.421881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471363634840759:2108] [node 4] ICP22 created new session: [5:7577471367929808092:2108]
2025-11-27T18:18:55.421932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471367929808092:2108] [node 4] ICS09 handshake done sender: [5:7577471367929808076:2121] self: [5:7577471367929808087:0] peer: [4:7577471365476359370:0] socket: 621 qp: -1
2025-11-27T18:18:55.421969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471367929808092:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:55.422011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:55.509092Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:55.508105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:55.508401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471366030483164:2048] [node 1] ICS01 socket: 616 reason# EndOfStream
2025-11-27T18:18:55.508440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471361735515778:2048] [node 1] ICP30 unregister session Session# [2:7577471366030483164:2048] VirtualId# [2:7577471366030483155:0]
2025-11-27T18:18:55.508486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471366030483164:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.508574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:55.508628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:55.509462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471366984546437:2048] [node 1] ICS07 socket disconnect 609 reason# EndOfStream
2025-11-27T18:18:55.509509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471366984546437:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.509586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:55.509625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471366984546437:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:55.511169Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:55.511339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471365476359380:2048] [node 1] ICS01 socket: 630 reason# EndOfStream
2025-11-27T18:18:55.511374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471361181391992:2048] [node 1] ICP30 unregister session Session# [4:7577471365476359380:2048] VirtualId# [4:7577471365476359377:0]
2025-11-27T18:18:55.511408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471365476359380:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.511470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:55.511506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:55.513555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:55.513848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471367929808078:2048] [node 1] ICS01 socket: 627 reason# EndOfStream
2025-11-27T18:18:55.513878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471363634840692:2048] [node 1] ICP30 unregister session Session# [5:7577471367929808078:2048] VirtualId# [5:7577471367929808073:0]
2025-11-27T18:18:55.513914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471367929808078:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.513972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:55.514009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:55.525354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:55.533439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471362689579047:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5086) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:55.533729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471366984546437:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:55.534494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471365069318533:2048] [node 1] ICS01 socket: 662 reason# EndOfStream
2025-11-27T18:18:55.534548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471360774351139:2048] [node 1] ICP30 unregister session Session# [6:7577471365069318533:2048] VirtualId# [6:7577471365069318525:0]
2025-11-27T18:18:55.534584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471365069318533:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.534664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:55.534712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:55.534100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471362689579047:2048] [node 1] ICP30 unregister session Session# [3:7577471366984546437:2048] VirtualId# [3:7577471366984546431:0]
2025-11-27T18:18:55.534166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:55.534207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471362689579047:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5086) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:55.534226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5086) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:55.545556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471362689579047:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:55.561422Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:55.561658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471366984546439:2049] [node 2] ICS01 socket: 643 reason# EndOfStream
2025-11-27T18:18:55.561705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471362689579048:2049] [node 2] ICP30 unregister session Session# [3:7577471366984546439:2049] VirtualId# [3:7577471366984546428:0]
2025-11-27T18:18:55.561744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471366984546439:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.561827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:55.561871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:55.566846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:55.567080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471365069318531:2049] [node 2] ICS01 socket: 634 reason# EndOfStream
2025-11-27T18:18:55.567118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471360774351140:2049] [node 2] ICP30 unregister session Session# [6:7577471365069318531:2049] VirtualId# [6:7577471365069318524:0]
2025-11-27T18:18:55.567172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471365069318531:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.567248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:55.567294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:55.569361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:55.569549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471367929808083:2049] [node 2] ICS01 socket: 632 reason# EndOfStream
2025-11-27T18:18:55.569595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471363634840693:2049] [node 2] ICP30 unregister session Session# [5:7577471367929808083:2049] VirtualId# [5:7577471367929808074:0]
2025-11-27T18:18:55.569627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471367929808083:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.569710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:55.569752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:55.570027Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:55.570242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471365476359392:2049] [node 2] ICS01 socket: 669 reason# EndOfStream
2025-11-27T18:18:55.570261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471361181391993:2049] [node 2] ICP30 unregister session Session# [4:7577471365476359392:2049] VirtualId# [4:7577471365476359387:0]
2025-11-27T18:18:55.570277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471365476359392:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.570329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:55.570363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:55.653449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:55.653686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471365476359384:2050] [node 3] ICS01 socket: 623 reason# EndOfStream
2025-11-27T18:18:55.653729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471361181391994:2050] [node 3] ICP30 unregister session Session# [4:7577471365476359384:2050] VirtualId# [4:7577471365476359368:0]
2025-11-27T18:18:55.653769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471365476359384:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.653843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:55.653885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:55.656766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:55.657027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471365069318527:2050] [node 3] ICS07 socket disconnect 604 reason# EndOfStream
2025-11-27T18:18:55.657059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471365069318527:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.657198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:55.657269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471365069318527:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:55.658320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471360774351141:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7629) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:55.658380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471365069318527:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:55.658422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471360774351141:2050] [node 3] ICP30 unregister session Session# [6:7577471365069318527:2050] VirtualId# [6:7577471365069318517:0]
2025-11-27T18:18:55.658476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:55.658514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471360774351141:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:55.658530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:55.661590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:55.661802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471367929808085:2050] [node 3] ICS07 socket disconnect 648 reason# EndOfStream
2025-11-27T18:18:55.661834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471367929808085:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.661940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:55.661982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471367929808085:2050] [node 3] ICS15 start handshake
2025-11-27T18:18:55.669836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471363634840694:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7629) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:55.669901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471367929808085:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:55.669929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471363634840694:2050] [node 3] ICP30 unregister session Session# [5:7577471367929808085:2050] VirtualId# [5:7577471367929808072:0]
2025-11-27T18:18:55.665789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471360774351141:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:55.669981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:55.670020Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471363634840694:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:55.670035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:55.681364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471363634840694:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:18:55.727319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:55.727523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471367929808092:2108] [node 4] ICS01 socket: 621 reason# EndOfStream
2025-11-27T18:18:55.727557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471363634840759:2108] [node 4] ICP30 unregister session Session# [5:7577471367929808092:2108] VirtualId# [5:7577471367929808087:0]
2025-11-27T18:18:55.727583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471367929808092:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.727633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:55.727668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:55.732040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:55.732353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471365069318518:2109] [node 4] ICS01 socket: 611 reason# EndOfStream
2025-11-27T18:18:55.732451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471360774351202:2109] [node 4] ICP30 unregister session Session# [6:7577471365069318518:2109] VirtualId# [6:7577471365069318510:0]
2025-11-27T18:18:55.732486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471365069318518:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.732562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:55.732606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:55.773641Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:55.773878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471365069318538:2106] [node 5] ICS01 socket: 655 reason# EndOfStream
2025-11-27T18:18:55.773927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471360774351199:2106] [node 5] ICP30 unregister session Session# [6:7577471365069318538:2106] VirtualId# [6:7577471365069318536:0]
2025-11-27T18:18:55.773964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471365069318538:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:55.774058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:55.774106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:55.967708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471366308564129:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:55.967749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471366308564130:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:55.991535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471367664421930:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:55.991565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471367664421931:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:56.014182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471369212260212:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:56.014257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471369212260213:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:56.035724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471369253244466:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:56.035766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471369253244467:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:56.035799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471369253244468:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:56.060404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471372239178739:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:56.060456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471372239178740:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:56.060503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471372239178741:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:56.173949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471372523095068:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:56.174070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471372523095069:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:56.177829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471372523095070:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:56.389801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471372523095129:2105] [node 5] ICP01 ready to work
2025-11-27T18:18:56.537838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471371959389315:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:56.538204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471371959389316:2116] [node 5] ICP01 ready to work
2025-11-27T18:18:56.538508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471371959389317:2124] [node 6] ICP01 ready to work
2025-11-27T18:18:56.557308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471369212260275:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:56.557832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471369212260276:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:56.558093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471369212260277:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:56.543908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471369253244538:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:56.567464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471372239178802:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:56.585747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471372239178803:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:56.610250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471366308564129:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(212526303ns) {Configure:104(212522506ns) {ProcessConfigured:135(19071098ns) {StartInitialHandshake:157(19068303ns) {DropHandshakes:470(3024ns) {DropIncomingHandshake:442(631ns) DropOutgoingHandshake:457(354ns)} GenerateSessionVirtualId:988(1266ns) PrepareNewSessionHandshake:477(3685ns) {SwitchToState:179(2188ns)}}}}}
2025-11-27T18:18:56.610839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471370603531492:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:56.611104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471370603531493:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:56.611373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471370603531494:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:56.585653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471372523095135:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:56.601476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471367664421931:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42397443ns) {Configure:104(42389728ns) {ProcessConfigured:135(11479180ns) {StartInitialHandshake:157(11476565ns) {DropHandshakes:470(3253ns) {DropIncomingHandshake:442(689ns) DropOutgoingHandshake:457(593ns)} GenerateSessionVirtualId:988(898ns) PrepareNewSessionHandshake:477(2530ns) {SwitchToState:179(152ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.631333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471369212260291:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65331
2025-11-27T18:18:56.640636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471369253244466:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(75035787ns) {Configure:104(75024037ns) {ProcessConfigured:135(79095ns) {StartInitialHandshake:157(77003ns) {DropHandshakes:470(3248ns) {DropIncomingHandshake:442(633ns) DropOutgoingHandshake:457(514ns)} GenerateSessionVirtualId:988(1478ns) PrepareNewSessionHandshake:477(1821ns) {SwitchToState:179(640ns)}}}}}
2025-11-27T18:18:56.641057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471369253244541:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:56.646433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471372523095068:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40609794ns) {Configure:104(40603395ns) {ProcessConfigured:135(47151ns) {StartInitialHandshake:157(45615ns) {DropHandshakes:470(1595ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(317ns) PrepareNewSessionHandshake:477(632ns) {SwitchToState:179(195ns)}}}}}
2025-11-27T18:18:56.650744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471370603531493:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(33432443ns) {Configure:104(33428418ns) {ProcessConfigured:135(45141ns) {StartInitialHandshake:157(42518ns) {DropHandshakes:470(1203ns) {DropIncomingHandshake:442(240ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(420ns) PrepareNewSessionHandshake:477(831ns) {SwitchToState:179(462ns)}}}}}
2025-11-27T18:18:56.653405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471372239178802:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17239047ns) {Configure:104(17235471ns) {ProcessConfigured:135(15668053ns) {StartInitialHandshake:157(15665838ns) {DropHandshakes:470(849ns) {DropIncomingHandshake:442(207ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(444ns) PrepareNewSessionHandshake:477(3285ns) {SwitchToState:179(1289ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.669994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471369253244548:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7276
2025-11-27T18:18:56.670145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471369253244549:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7276
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.686618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471370603531495:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.686178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471369212260292:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65331
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.693064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471370603531496:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3599
2025-11-27T18:18:56.693593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471370603531495:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.694177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471370603531505:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3599
2025-11-27T18:18:56.681556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471372239178806:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39244
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.699703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471371959389331:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35399
2025-11-27T18:18:56.695534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471369253244548:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.698740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471369253244468:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.698835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471369253244555:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7276
2025-11-27T18:18:56.698992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471369253244556:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7276
2025-11-27T18:18:56.699050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471369253244557:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7276
2025-11-27T18:18:56.702559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471369253244555:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.703652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471370603531505:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.704640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471369212260292:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.704916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471369212260283:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.704941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471369212260283:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.704827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471372239178806:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.714064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471369212260293:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65331
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.715024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471369253244560:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7276
2025-11-27T18:18:56.714507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471372523095146:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-11-27T18:18:56.717781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471372523095146:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.717920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471372523095129:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.719155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471369212260283:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:56.725752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471372523095148:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.718226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471372239178810:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.732557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471370603531508:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3599
2025-11-27T18:18:56.729792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471371959389332:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35399
2025-11-27T18:18:56.730222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471371959389333:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35399
2025-11-27T18:18:56.730352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471371959389334:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35399
2025-11-27T18:18:56.730483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471371959389331:2141] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.734795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471371959389333:2143] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.718260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471372239178810:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.742057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471370603531496:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.742196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471371959389334:2144] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.721742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471372239178810:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:56.742492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471369253244557:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.733844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471372239178803:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.733907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:56.742853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471369253244548:2123] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:56.737246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471369212260275:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.733957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471372239178803:2109] [node 6] ICP22 created new session: [5:7577471372239178820:2109]
2025-11-27T18:18:56.734018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471372239178820:2109] [node 6] ICS09 handshake done sender: [5:7577471372239178810:2113] self: [5:7577471372239178809:0] peer: [6:7577471372523095147:0] socket: 628 qp: -1
2025-11-27T18:18:56.734058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471372239178820:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:56.734092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:56.737329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:56.737377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471369212260275:2107] [node 4] ICP22 created new session: [3:7577471369212260296:2107]
2025-11-27T18:18:56.737421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471369212260296:2107] [node 4] ICS09 handshake done sender: [3:7577471369212260283:2112] self: [3:7577471369212260282:0] peer: [4:7577471369253244558:0] socket: 609 qp: -1
2025-11-27T18:18:56.737455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471369212260296:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:56.737487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:56.741878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471369212260291:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.759555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471369253244549:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.769534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471371959389332:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.776826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471370603531508:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.759617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471369253244468:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.759688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:56.759741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471369253244468:2050] [node 3] ICP22 created new session: [4:7577471369253244561:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.794795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471372239178823:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39244
2025-11-27T18:18:56.787865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471370603531498:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.759794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471369253244561:2050] [node 3] ICS09 handshake done sender: [4:7577471369253244548:2123] self: [4:7577471369253244558:0] peer: [3:7577471369212260282:0] socket: 619 qp: -1
2025-11-27T18:18:56.794395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471372523095129:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19609) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471372523095146:2115] held: no
2025-11-27T18:18:56.794567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471372523095146:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:56.781398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471371959389323:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.759837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471369253244561:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:56.759874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:56.765488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471369253244468:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:56.766450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471369253244467:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.780236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471369253244556:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.780415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471369253244466:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.794023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471369253244567:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7276
2025-11-27T18:18:56.803390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471369253244568:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7276
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.817835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471372239178824:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39244
2025-11-27T18:18:56.787897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471370603531498:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.781442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471371959389323:2138] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.814793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471372523095129:2105] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.788754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471370603531491:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.782091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471367664421930:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.788774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471370603531491:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.838521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471372239178823:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.793524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471370603531498:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:56.795467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471370603531491:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:56.795754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471370603531492:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.795817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:56.795870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471370603531492:2108] [node 4] ICP22 created new session: [1:7577471370603531514:2108]
2025-11-27T18:18:56.795927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471370603531514:2108] [node 4] ICS09 handshake done sender: [1:7577471370603531498:2114] self: [1:7577471370603531497:0] peer: [4:7577471369253244566:0] socket: 635 qp: -1
2025-11-27T18:18:56.795969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471370603531514:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:56.796007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:56.796412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471366308564129:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.796471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:56.796512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471366308564129:2048] [node 2] ICP22 created new session: [1:7577471370603531516:2048]
2025-11-27T18:18:56.796565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471370603531516:2048] [node 2] ICS09 handshake done sender: [1:7577471370603531491:2111] self: [1:7577471370603531490:0] peer: [2:7577471371959389338:0] socket: 633 qp: -1
2025-11-27T18:18:56.796604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471370603531516:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:56.796630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.814842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:56.840746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471369212260293:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.841460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471369212260213:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.841576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471369212260298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65331
2025-11-27T18:18:56.843008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471369212260212:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.841433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471372239178802:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.789948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471371959389323:2138] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:56.814885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471372523095129:2105] [node 5] ICP22 created new session: [6:7577471372523095149:2105]
2025-11-27T18:18:56.814927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471372523095149:2105] [node 5] ICS09 handshake done sender: [6:7577471372523095146:2115] self: [6:7577471372523095147:0] peer: [5:7577471372239178809:0] socket: 639 qp: -1
2025-11-27T18:18:56.814965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471372523095149:2105] [node 5] ICS10 traffic start
2025-11-27T18:18:56.814998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:56.837886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471372523095152:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-11-27T18:18:56.850275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471372523095152:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.866641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471372239178824:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.867010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471372239178826:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39244
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.867727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471372239178827:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39244
2025-11-27T18:18:56.867828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471372239178740:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.801966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471371959389315:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.865983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471369212260302:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65331
2025-11-27T18:18:56.868416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471369212260298:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.813430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:56.813508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471371959389315:2107] [node 4] ICP22 created new session: [2:7577471371959389341:2107]
2025-11-27T18:18:56.869472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471372239178826:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.869602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471372239178741:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.858279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471372523095069:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.813562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471371959389341:2107] [node 4] ICS09 handshake done sender: [2:7577471371959389323:2138] self: [2:7577471371959389322:0] peer: [4:7577471369253244565:0] socket: 612 qp: -1
2025-11-27T18:18:56.813610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471371959389341:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:56.813652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:56.815860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471371959389340:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35399
2025-11-27T18:18:56.816495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471371959389334:2144] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:56.816890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471367664421930:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:56.816943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:56.816990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471367664421930:2048] [node 1] ICP22 created new session: [2:7577471371959389344:2048]
2025-11-27T18:18:56.817030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471371959389344:2048] [node 1] ICS09 handshake done sender: [2:7577471371959389334:2144] self: [2:7577471371959389338:0] peer: [1:7577471370603531490:0] socket: 645 qp: -1
2025-11-27T18:18:56.817063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471371959389344:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:56.817089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:56.836561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471371959389346:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35399
2025-11-27T18:18:56.842106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471371959389346:2146] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.843090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471371959389319:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.843109Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471371959389319:2136] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.859138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471371959389327:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.859167Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471371959389327:2140] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.868102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471371959389319:2136] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:56.869311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471371959389327:2140] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:56.885607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471369212260285:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.885648Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471369212260285:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.858403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471372523095153:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-11-27T18:18:56.871440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471371959389325:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.871476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471371959389325:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.871987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471367664421931:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.872051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:56.872103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471367664421931:2049] [node 3] ICP22 created new session: [2:7577471371959389351:2049]
2025-11-27T18:18:56.872155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471371959389351:2049] [node 3] ICS09 handshake done sender: [2:7577471371959389319:2136] self: [2:7577471371959389318:0] peer: [3:7577471369212260300:0] socket: 610 qp: -1
2025-11-27T18:18:56.872190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471371959389351:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:56.872226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:56.872693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471371959389317:2124] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.872757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:56.872805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471371959389317:2124] [node 6] ICP22 created new session: [2:7577471371959389353:2124]
2025-11-27T18:18:56.872845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471371959389353:2124] [node 6] ICS09 handshake done sender: [2:7577471371959389327:2140] self: [2:7577471371959389326:0] peer: [6:7577471372523095154:0] socket: 658 qp: -1
2025-11-27T18:18:56.872880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471371959389353:2124] [node 6] ICS10 traffic start
2025-11-27T18:18:56.872910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:56.865687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471372523095155:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-11-27T18:18:56.873770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471372523095152:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:56.877555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471372523095069:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:56.877607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:56.877659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471372523095069:2049] [node 2] ICP22 created new session: [6:7577471372523095156:2049]
2025-11-27T18:18:56.877701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471372523095156:2049] [node 2] ICS09 handshake done sender: [6:7577471372523095152:2117] self: [6:7577471372523095154:0] peer: [2:7577471371959389326:0] socket: 659 qp: -1
2025-11-27T18:18:56.877741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471372523095156:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:56.877787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:56.888963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471370603531486:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.887530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471369253244549:2124] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:56.893515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471372239178827:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.893663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471372239178739:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.890620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471369253244556:2127] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:56.888999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471370603531486:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.890107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471370603531486:2107] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:56.917769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471366308564130:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.917860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:56.917914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471366308564130:2049] [node 3] ICP22 created new session: [1:7577471370603531521:2049]
2025-11-27T18:18:56.917963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471370603531521:2049] [node 3] ICS09 handshake done sender: [1:7577471370603531486:2107] self: [1:7577471370603531485:0] peer: [3:7577471369212260301:0] socket: 632 qp: -1
2025-11-27T18:18:56.918005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471370603531521:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:56.918039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:56.918682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471370603531500:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.918708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471370603531500:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.929705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471371959389325:2139] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.891038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471369253244466:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3854) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471369253244556:2127] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.925664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471369212260291:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:56.930256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471372239178831:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39244
2025-11-27T18:18:56.933643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471371959389316:2116] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.933733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:56.933795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471371959389316:2116] [node 5] ICP22 created new session: [2:7577471371959389356:2116]
2025-11-27T18:18:56.933846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471371959389356:2116] [node 5] ICS09 handshake done sender: [2:7577471371959389325:2139] self: [2:7577471371959389324:0] peer: [5:7577471372239178828:0] socket: 631 qp: -1
2025-11-27T18:18:56.933884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471371959389356:2116] [node 5] ICS10 traffic start
2025-11-27T18:18:56.933928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:56.891144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471369253244466:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:56.937458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471372523095153:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.937774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471372523095159:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-11-27T18:18:56.937908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471372523095068:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.891185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:56.891239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471369253244466:2048] [node 1] ICP22 created new session: [4:7577471369253244569:2048]
2025-11-27T18:18:56.891284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471369253244569:2048] [node 1] ICS09 handshake done sender: [4:7577471369253244556:2127] self: [4:7577471369253244566:0] peer: [1:7577471370603531497:0] socket: 643 qp: -1
2025-11-27T18:18:56.891321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471369253244569:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:56.891357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:56.891932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471369253244547:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.891955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471369253244547:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.892698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471369253244467:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18314) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471369253244549:2124] held: no
2025-11-27T18:18:56.892813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471369253244467:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:56.892848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:56.892898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471369253244467:2049] [node 2] ICP22 created new session: [4:7577471369253244571:2049]
2025-11-27T18:18:56.892935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471369253244571:2049] [node 2] ICS09 handshake done sender: [4:7577471369253244549:2124] self: [4:7577471369253244565:0] peer: [2:7577471371959389322:0] socket: 620 qp: -1
2025-11-27T18:18:56.892963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471369253244571:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:56.892996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:56.905788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471369253244547:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:56.941445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471369253244538:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.953400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:56.953483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471369253244538:2108] [node 5] ICP22 created new session: [4:7577471369253244575:2108]
2025-11-27T18:18:56.953547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471369253244575:2108] [node 5] ICS09 handshake done sender: [4:7577471369253244547:2122] self: [4:7577471369253244546:0] peer: [5:7577471372239178825:0] socket: 624 qp: -1
2025-11-27T18:18:56.953589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471369253244575:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:56.953646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.964110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471372239178832:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39244
2025-11-27T18:18:56.961502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471369212260213:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18314) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471369212260291:2115] held: no
2025-11-27T18:18:56.961643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471369212260213:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:56.961695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:56.961758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471369212260213:2049] [node 2] ICP22 created new session: [3:7577471369212260306:2049]
2025-11-27T18:18:56.961803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471369212260306:2049] [node 2] ICS09 handshake done sender: [3:7577471369212260291:2115] self: [3:7577471369212260300:0] peer: [2:7577471371959389318:0] socket: 615 qp: -1
2025-11-27T18:18:56.961841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471369212260306:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:56.961889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:56.964428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471370603531502:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.964469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471370603531502:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.976355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471369212260212:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3854) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471369212260293:2117] held: no
2025-11-27T18:18:56.976668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471369212260303:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65331
2025-11-27T18:18:56.976863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471369212260304:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65331
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.978846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471372523095161:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-11-27T18:18:56.979578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471370603531500:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:56.979896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471372239178740:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18314) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471372239178824:2117] held: no
2025-11-27T18:18:56.980035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471372239178823:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:56.980517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471370603531502:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:56.980825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471372523095159:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.981400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471372523095153:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:56.981692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471372523095135:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:56.981748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471372523095068:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:56.981792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:56.981839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471372523095068:2048] [node 1] ICP22 created new session: [6:7577471372523095163:2048]
2025-11-27T18:18:56.981888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471372523095163:2048] [node 1] ICS09 handshake done sender: [6:7577471372523095153:2118] self: [6:7577471372523095160:0] peer: [1:7577471370603531501:0] socket: 664 qp: -1
2025-11-27T18:18:56.981929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471372523095163:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:56.981967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:56.983747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471369253244551:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.983786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471369253244551:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:56.984943Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471369253244551:2125] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:56.985547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471369212260285:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:56.986170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471372239178802:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471372239178823:2116] held: no
2025-11-27T18:18:56.986284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471372239178802:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:56.986333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:56.986380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471372239178802:2108] [node 4] ICP22 created new session: [5:7577471372239178835:2108]
2025-11-27T18:18:56.986425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471372239178835:2108] [node 4] ICS09 handshake done sender: [5:7577471372239178823:2116] self: [5:7577471372239178825:0] peer: [4:7577471369253244546:0] socket: 629 qp: -1
2025-11-27T18:18:56.986461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471372239178835:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:56.986491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:56.987023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471372239178824:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:56.987617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471370603531493:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.987658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:56.987725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471370603531493:2109] [node 5] ICP22 created new session: [1:7577471370603531525:2109]
2025-11-27T18:18:56.987765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471370603531525:2109] [node 5] ICS09 handshake done sender: [1:7577471370603531500:2115] self: [1:7577471370603531499:0] peer: [5:7577471372239178830:0] socket: 622 qp: -1
2025-11-27T18:18:56.987795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471370603531525:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:56.987821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:56.988186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471370603531494:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.988226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:56.988268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471370603531494:2110] [node 6] ICP22 created new session: [1:7577471370603531527:2110]
2025-11-27T18:18:56.988308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471370603531527:2110] [node 6] ICS09 handshake done sender: [1:7577471370603531502:2116] self: [1:7577471370603531501:0] peer: [6:7577471372523095160:0] socket: 662 qp: -1
2025-11-27T18:18:56.988333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471370603531527:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:56.988368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:56.989888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471369212260303:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:56.989959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471369212260276:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.989998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:56.990037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471369212260276:2108] [node 5] ICP22 created new session: [3:7577471369212260310:2108]
2025-11-27T18:18:56.990078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471369212260310:2108] [node 5] ICS09 handshake done sender: [3:7577471369212260285:2113] self: [3:7577471369212260284:0] peer: [5:7577471372239178829:0] socket: 660 qp: -1
2025-11-27T18:18:56.990103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471369212260310:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:56.990134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:56.990597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471369212260293:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:56.993885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471369253244541:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:56.993949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:56.993987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471369253244541:2109] [node 6] ICP22 created new session: [4:7577471369253244578:2109]
2025-11-27T18:18:56.994028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471369253244578:2109] [node 6] ICS09 handshake done sender: [4:7577471369253244551:2125] self: [4:7577471369253244550:0] peer: [6:7577471372523095162:0] socket: 670 qp: -1
2025-11-27T18:18:56.994055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471369253244578:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:56.994092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.995266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471372239178740:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:56.995317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:56.995358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471372239178740:2049] [node 2] ICP22 created new session: [5:7577471372239178837:2049]
2025-11-27T18:18:56.995410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471372239178837:2049] [node 2] ICS09 handshake done sender: [5:7577471372239178824:2117] self: [5:7577471372239178828:0] peer: [2:7577471371959389324:0] socket: 634 qp: -1
2025-11-27T18:18:56.995437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471372239178837:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:56.995463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:56.995894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471372239178833:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39244
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:56.997747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471372523095165:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-11-27T18:18:56.998643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471369212260212:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:56.998685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:56.998730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471369212260212:2048] [node 1] ICP22 created new session: [3:7577471369212260312:2048]
2025-11-27T18:18:56.998772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471369212260312:2048] [node 1] ICS09 handshake done sender: [3:7577471369212260293:2117] self: [3:7577471369212260301:0] peer: [1:7577471370603531485:0] socket: 637 qp: -1
2025-11-27T18:18:56.998798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471369212260312:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:56.998829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:57.001034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471372239178834:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39244
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:57.003934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471372523095166:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:57.007930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471374898498825:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3599
2025-11-27T18:18:57.008368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471372239178741:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13311) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471372239178826:2118] held: no
2025-11-27T18:18:57.008623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471372239178827:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:57.009049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471372523095135:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471372523095159:2120] held: no
2025-11-27T18:18:57.009252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471372523095159:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:57.009991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471372523095166:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:57.010046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471372523095135:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:57.010091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:57.010134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471372523095135:2110] [node 4] ICP22 created new session: [6:7577471376818062463:2110]
2025-11-27T18:18:57.010177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471376818062463:2110] [node 4] ICS09 handshake done sender: [6:7577471372523095159:2120] self: [6:7577471372523095162:0] peer: [4:7577471369253244550:0] socket: 671 qp: -1
2025-11-27T18:18:57.010215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471376818062463:2110] [node 4] ICS10 traffic start
2025-11-27T18:18:57.010252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:57.010794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471372523095070:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:57.011581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471369212260287:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:57.011612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471369212260287:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:57.013025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471372239178826:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:57.013429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471369212260287:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:57.013699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471369212260277:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:57.013761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:57.013818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471369212260277:2109] [node 6] ICP22 created new session: [3:7577471373507227612:2109]
2025-11-27T18:18:57.013865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471373507227612:2109] [node 6] ICS09 handshake done sender: [3:7577471369212260287:2114] self: [3:7577471369212260286:0] peer: [6:7577471376818062465:0] socket: 675 qp: -1
2025-11-27T18:18:57.013901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471373507227612:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:57.013934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:57.015072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471376818062466:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-11-27T18:18:57.015485Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471372523095166:2123] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:57.017021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471372523095070:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:57.017078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:57.017121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471372523095070:2050] [node 3] ICP22 created new session: [6:7577471376818062467:2050]
2025-11-27T18:18:57.017162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471376818062467:2050] [node 3] ICS09 handshake done sender: [6:7577471372523095166:2123] self: [6:7577471376818062465:0] peer: [3:7577471369212260286:0] socket: 676 qp: -1
2025-11-27T18:18:57.017941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471372239178739:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:57.017196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471376818062467:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:57.018008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:57.017227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:57.017445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471372523095070:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:57.018055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471372239178739:2048] [node 1] ICP22 created new session: [5:7577471376534146136:2048]
2025-11-27T18:18:57.018099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471376534146136:2048] [node 1] ICS09 handshake done sender: [5:7577471372239178827:2119] self: [5:7577471372239178830:0] peer: [1:7577471370603531499:0] socket: 663 qp: -1
2025-11-27T18:18:57.018122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471376534146136:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:57.018153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:57.018670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471372239178741:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:57.018712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:57.018754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471372239178741:2050] [node 3] ICP22 created new session: [5:7577471376534146138:2050]
2025-11-27T18:18:57.018790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471376534146138:2050] [node 3] ICS09 handshake done sender: [5:7577471372239178826:2118] self: [5:7577471372239178829:0] peer: [3:7577471369212260284:0] socket: 661 qp: -1
2025-11-27T18:18:57.018825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471376534146138:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:57.018868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:57.045445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:57.045701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471371959389344:2048] [node 1] ICS01 socket: 645 reason# EndOfStream
2025-11-27T18:18:57.045751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471367664421930:2048] [node 1] ICP30 unregister session Session# [2:7577471371959389344:2048] VirtualId# [2:7577471371959389338:0]
2025-11-27T18:18:57.045787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471371959389344:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.045859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:57.045913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:57.049729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:57.049954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471369212260312:2048] [node 1] ICS01 socket: 637 reason# EndOfStream
2025-11-27T18:18:57.050001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471369212260212:2048] [node 1] ICP30 unregister session Session# [3:7577471369212260312:2048] VirtualId# [3:7577471369212260301:0]
2025-11-27T18:18:57.050037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471369212260312:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.050119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:57.050161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:57.057108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:57.057408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471376534146136:2048] [node 1] ICS07 socket disconnect 663 reason# EndOfStream
2025-11-27T18:18:57.057445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471376534146136:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.057568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:57.057630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471376534146136:2048] [node 1] ICS15 start handshake
2025-11-27T18:18:57.058167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:57.058360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471369253244569:2048] [node 1] ICS01 socket: 643 reason# EndOfStream
2025-11-27T18:18:57.058407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471369253244466:2048] [node 1] ICP30 unregister session Session# [4:7577471369253244569:2048] VirtualId# [4:7577471369253244566:0]
2025-11-27T18:18:57.058443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471369253244569:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.058514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:57.058558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:57.059210Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:57.069375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471372239178739:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3854) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:57.069446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471376534146136:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:57.069486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471372239178739:2048] [node 1] ICP30 unregister session Session# [5:7577471376534146136:2048] VirtualId# [5:7577471372239178830:0]
2025-11-27T18:18:57.069536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:57.069571Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471372239178739:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3854) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:57.069589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3854) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:57.068676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471372523095163:2048] [node 1] ICS01 socket: 664 reason# EndOfStream
2025-11-27T18:18:57.068753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471372523095068:2048] [node 1] ICP30 unregister session Session# [6:7577471372523095163:2048] VirtualId# [6:7577471372523095160:0]
2025-11-27T18:18:57.069236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471372523095163:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.069352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:57.069407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:57.076535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471372239178739:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:18:57.092438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:57.092674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471369212260306:2049] [node 2] ICS01 socket: 615 reason# EndOfStream
2025-11-27T18:18:57.092739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471369212260213:2049] [node 2] ICP30 unregister session Session# [3:7577471369212260306:2049] VirtualId# [3:7577471369212260300:0]
2025-11-27T18:18:57.092776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471369212260306:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.092864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:57.092908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:57.101453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:57.101685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471372239178837:2049] [node 2] ICS01 socket: 634 reason# EndOfStream
2025-11-27T18:18:57.101733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471372239178740:2049] [node 2] ICP30 unregister session Session# [5:7577471372239178837:2049] VirtualId# [5:7577471372239178828:0]
2025-11-27T18:18:57.101776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471372239178837:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.101850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:57.101924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:57.105486Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:57.105700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471369253244571:2049] [node 2] ICS01 socket: 620 reason# EndOfStream
2025-11-27T18:18:57.105745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471369253244467:2049] [node 2] ICP30 unregister session Session# [4:7577471369253244571:2049] VirtualId# [4:7577471369253244565:0]
2025-11-27T18:18:57.105781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471369253244571:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.105875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:57.105914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:57.105896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:57.108588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471372523095156:2049] [node 2] ICS01 socket: 659 reason# EndOfStream
2025-11-27T18:18:57.108636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471372523095069:2049] [node 2] ICP30 unregister session Session# [6:7577471372523095156:2049] VirtualId# [6:7577471372523095154:0]
2025-11-27T18:18:57.108674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471372523095156:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.108788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:57.108886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:57.158103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:57.157451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:57.157708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471369253244561:2050] [node 3] ICS01 socket: 619 reason# EndOfStream
2025-11-27T18:18:57.157751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471369253244468:2050] [node 3] ICP30 unregister session Session# [4:7577471369253244561:2050] VirtualId# [4:7577471369253244558:0]
2025-11-27T18:18:57.160105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471376818062467:2050] [node 3] ICS01 socket: 676 reason# EndOfStream
2025-11-27T18:18:57.160062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:57.161583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471376534146138:2050] [node 3] ICS01 socket: 661 reason# EndOfStream
2025-11-27T18:18:57.161622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471372239178741:2050] [node 3] ICP30 unregister session Session# [5:7577471376534146138:2050] VirtualId# [5:7577471372239178829:0]
2025-11-27T18:18:57.161648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471376534146138:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.161716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:57.161752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:57.161172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471372523095070:2050] [node 3] ICP30 unregister session Session# [6:7577471376818062467:2050] VirtualId# [6:7577471376818062465:0]
2025-11-27T18:18:57.161463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471376818062467:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.161558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:57.161613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:57.157789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471369253244561:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.157873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:57.158001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:57.216335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:57.216571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471376818062463:2110] [node 4] ICS01 socket: 671 reason# EndOfStream
2025-11-27T18:18:57.216616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471372523095135:2110] [node 4] ICP30 unregister session Session# [6:7577471376818062463:2110] VirtualId# [6:7577471372523095162:0]
2025-11-27T18:18:57.216658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471376818062463:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.216742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:57.216782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:57.216996Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:57.217171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471372239178835:2108] [node 4] ICS01 socket: 629 reason# EndOfStream
2025-11-27T18:18:57.217208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471372239178802:2108] [node 4] ICP30 unregister session Session# [5:7577471372239178835:2108] VirtualId# [5:7577471372239178825:0]
2025-11-27T18:18:57.217234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471372239178835:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.217322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:57.217359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:57.259978Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:18:57.260280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471372523095149:2105] [node 5] ICS01 socket: 639 reason# EndOfStream
2025-11-27T18:18:57.260351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471372523095129:2105] [node 5] ICP30 unregister session Session# [6:7577471372523095149:2105] VirtualId# [6:7577471372523095147:0]
2025-11-27T18:18:57.260396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471372523095149:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:57.260482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:18:57.260542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:57.394486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471373054907107:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:57.395437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471373054907106:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:57.418286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471375314607768:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:57.418314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471375314607769:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:57.445507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471374216952932:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:57.445556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471374216952933:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:57.471736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471373503844570:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:57.471793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471373503844571:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:57.471823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471373503844572:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:57.494116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471373156081077:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:57.494160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471373156081078:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:57.494187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471373156081079:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:57.524712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471374705921490:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:57.524753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471374705921491:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:57.524788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471374705921492:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:57.810456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471374705921547:2091] [node 4] ICP01 ready to work
2025-11-27T18:18:57.923545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471374705921552:2096] [node 5] ICP01 ready to work
2025-11-27T18:18:57.893805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471375314607831:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:57.901433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471374216952996:2107] [node 4] ICP01 ready to work
2025-11-27T18:18:57.901962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471374216952997:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:57.960170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471375314607832:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:57.973949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471373503844634:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:57.978856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471374216952932:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(177138097ns) {Configure:104(177131961ns) {ProcessConfigured:135(88847ns) {StartInitialHandshake:157(85892ns) {DropHandshakes:470(3166ns) {DropIncomingHandshake:442(704ns) DropOutgoingHandshake:457(524ns)} GenerateSessionVirtualId:988(1258ns) PrepareNewSessionHandshake:477(2527ns) {SwitchToState:179(1038ns)}}}}}
2025-11-27T18:18:57.961805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471375314607833:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:57.998135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471373503844635:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:57.979355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471374216953001:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:58.013281Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471375314607769:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31364483ns) {Configure:104(21001801ns) {ProcessConfigured:135(84166ns) {StartInitialHandshake:157(78616ns) {DropHandshakes:470(3374ns) {DropIncomingHandshake:442(710ns) DropOutgoingHandshake:457(355ns)} GenerateSessionVirtualId:988(1363ns) PrepareNewSessionHandshake:477(2203ns) {SwitchToState:179(925ns)}}}}}
2025-11-27T18:18:58.040536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471373156081077:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(174821779ns) {Configure:104(174815661ns) {ProcessConfigured:135(4920901ns) {StartInitialHandshake:157(4917797ns) {DropHandshakes:470(3119ns) {DropIncomingHandshake:442(946ns) DropOutgoingHandshake:457(502ns)} GenerateSessionVirtualId:988(1036ns) PrepareNewSessionHandshake:477(4242ns) {SwitchToState:179(2356ns)}}}}}
2025-11-27T18:18:58.041121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471377451048438:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:57.986017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471374216952933:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(83735949ns) {Configure:104(83729491ns) {ProcessConfigured:135(11991793ns) {StartInitialHandshake:157(11987471ns) {DropHandshakes:470(3576ns) {DropIncomingHandshake:442(899ns) DropOutgoingHandshake:457(420ns)} GenerateSessionVirtualId:988(1320ns) PrepareNewSessionHandshake:477(5381ns) {SwitchToState:179(2242ns)}}}}}
2025-11-27T18:18:58.025745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471374705921492:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(268702270ns) {Configure:104(268696269ns) {ProcessConfigured:135(86195ns) {StartInitialHandshake:157(82896ns) {DropHandshakes:470(3068ns) {DropIncomingHandshake:442(587ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(1666ns) PrepareNewSessionHandshake:477(1912ns) {SwitchToState:179(455ns)}}}}}
2025-11-27T18:18:58.046707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471374705921491:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20653003ns) {Configure:104(20647636ns) {ProcessConfigured:135(71797ns) {StartInitialHandshake:157(68235ns) {DropHandshakes:470(2375ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(351ns)} GenerateSessionVirtualId:988(1292ns) PrepareNewSessionHandshake:477(1393ns) {SwitchToState:179(425ns)}}}}}
2025-11-27T18:18:58.034758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471373054907106:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(280619153ns) {Configure:104(280613949ns) {ProcessConfigured:135(7409559ns) {StartInitialHandshake:157(7406933ns) {DropHandshakes:470(2874ns) {DropIncomingHandshake:442(809ns) DropOutgoingHandshake:457(484ns)} GenerateSessionVirtualId:988(895ns) PrepareNewSessionHandshake:477(3356ns) {SwitchToState:179(1006ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.035176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471377349874467:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.077451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471377451048440:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:58.093700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471373503844570:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(241735248ns) {Configure:104(241728624ns) {ProcessConfigured:135(9370759ns) {StartInitialHandshake:157(9366453ns) {DropHandshakes:470(3019ns) {DropIncomingHandshake:442(868ns) DropOutgoingHandshake:457(399ns)} GenerateSessionVirtualId:988(1069ns) PrepareNewSessionHandshake:477(3149ns) {SwitchToState:179(1494ns)}}}}}
2025-11-27T18:18:58.073796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471374705921547:2091] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(70690044ns) {Configure:104(70679602ns) {ProcessConfigured:135(75437ns) {StartInitialHandshake:157(69247ns) {DropHandshakes:470(2687ns) {DropIncomingHandshake:442(576ns) DropOutgoingHandshake:457(659ns)} GenerateSessionVirtualId:988(1314ns) PrepareNewSessionHandshake:477(1691ns) {SwitchToState:179(641ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.098351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471373156081078:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(173697879ns) {Configure:104(173686437ns) {ProcessConfigured:135(16231169ns) {StartInitialHandshake:157(16227649ns) {DropHandshakes:470(3539ns) {DropIncomingHandshake:442(998ns) DropOutgoingHandshake:457(594ns)} GenerateSessionVirtualId:988(913ns) PrepareNewSessionHandshake:477(4967ns) {SwitchToState:179(1161ns)}}}}}
2025-11-27T18:18:58.047329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471377349874468:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:58.067607Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471375314607831:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(53923813ns) {Configure:104(53918498ns) {ProcessConfigured:135(57019ns) {StartInitialHandshake:157(54429ns) {DropHandshakes:470(1234ns) {DropIncomingHandshake:442(361ns) DropOutgoingHandshake:457(275ns)} GenerateSessionVirtualId:988(638ns) PrepareNewSessionHandshake:477(4168ns) {SwitchToState:179(384ns)}}}}}
2025-11-27T18:18:58.081094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471379609575138:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29563
2025-11-27T18:18:58.055089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471373054907107:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(289301307ns) {Configure:104(289286640ns) {ProcessConfigured:135(7642101ns) {StartInitialHandshake:157(7639401ns) {DropHandshakes:470(1185ns) {DropIncomingHandshake:442(383ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(771ns) PrepareNewSessionHandshake:477(2578ns) {SwitchToState:179(955ns)}}}}}
2025-11-27T18:18:58.103687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471374216953001:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(109991579ns) {Configure:104(109981660ns) {ProcessConfigured:135(13341819ns) {StartInitialHandshake:157(13339252ns) {DropHandshakes:470(2280ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(487ns)} GenerateSessionVirtualId:988(894ns) PrepareNewSessionHandshake:477(6034ns) {SwitchToState:179(3163ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.058848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471377349874473:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32245
2025-11-27T18:18:58.081163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471379609575139:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29563
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.139186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471378511920308:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58677
2025-11-27T18:18:58.062432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471377349874478:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:58.100948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471375314607768:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(87372757ns) {Configure:104(87365237ns) {ProcessConfigured:135(88392ns) {StartInitialHandshake:157(85531ns) {DropHandshakes:470(3062ns) {DropIncomingHandshake:442(839ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(1775ns) PrepareNewSessionHandshake:477(2440ns) {SwitchToState:179(858ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.141348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471378511920309:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58677
2025-11-27T18:18:58.157439Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471374705921552:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(94589673ns) {Configure:104(94582122ns) {ProcessConfigured:135(59087ns) {StartInitialHandshake:157(56736ns) {DropHandshakes:470(2786ns) {DropIncomingHandshake:442(789ns) DropOutgoingHandshake:457(362ns)} GenerateSessionVirtualId:988(558ns) PrepareNewSessionHandshake:477(3602ns) {SwitchToState:179(137ns)}}}}}
2025-11-27T18:18:58.101661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471379609575145:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29563
2025-11-27T18:18:58.147933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471373503844572:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(114067769ns) {Configure:104(114059466ns) {ProcessConfigured:135(17831250ns) {StartInitialHandshake:157(17828388ns) {DropHandshakes:470(2844ns) {DropIncomingHandshake:442(631ns) DropOutgoingHandshake:457(483ns)} GenerateSessionVirtualId:988(643ns) PrepareNewSessionHandshake:477(6230ns) {SwitchToState:179(2764ns)}}}}}
2025-11-27T18:18:58.146117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471379609575138:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.149230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471379609575139:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.149795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471375314607768:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.143141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471378511920310:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58677
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.153364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471377349874473:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.159210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471377798811936:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-27T18:18:58.157659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471378511920308:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.160920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471377349874466:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.170972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471379609575145:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.157940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471374216952932:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.164603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471377798811937:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-27T18:18:58.160963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471377349874466:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.169147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471377451048438:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(49322189ns) {Configure:104(49314294ns) {ProcessConfigured:135(38168ns) {StartInitialHandshake:157(36472ns) {DropHandshakes:470(474ns) {DropIncomingHandshake:442(205ns) DropOutgoingHandshake:457(170ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:477(365ns) {SwitchToState:179(83ns)}}}}}
2025-11-27T18:18:58.163108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471377349874470:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.173384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471379609575131:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.165024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471378511920310:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.169799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471373503844634:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(41511668ns) {Configure:104(41504952ns) {ProcessConfigured:135(68065ns) {StartInitialHandshake:157(66398ns) {DropHandshakes:470(3245ns) {DropIncomingHandshake:442(1037ns) DropOutgoingHandshake:457(509ns)} GenerateSessionVirtualId:988(614ns) PrepareNewSessionHandshake:477(1772ns) {SwitchToState:179(686ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.170886Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471373156081079:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24125697ns) {Configure:104(24121696ns) {ProcessConfigured:135(2238388ns) {StartInitialHandshake:157(2235180ns) {DropHandshakes:470(1376ns) {DropIncomingHandshake:442(218ns) DropOutgoingHandshake:457(342ns)} GenerateSessionVirtualId:988(511ns) PrepareNewSessionHandshake:477(1580ns) {SwitchToState:179(940ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.173425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471379609575131:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.163146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471377349874470:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.173704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471377798811938:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-27T18:18:58.174977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471379609575131:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:58.165299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471378511920309:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.172448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471377451048442:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33856
2025-11-27T18:18:58.178111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471377349874472:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.178737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471375314607769:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.168659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471374216952933:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.177671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471378511920315:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58677
2025-11-27T18:18:58.177977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471378511920314:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58677
2025-11-27T18:18:58.178138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471377349874472:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.176482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471377798811936:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.178782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:58.181598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471377349874470:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:58.172562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471377451048443:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33856
2025-11-27T18:18:58.176966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471377798811937:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.182833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471377349874466:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.173879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471377451048443:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.187214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471379000888864:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:18:58.178814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471375314607769:2049] [node 3] ICP22 created new session: [2:7577471379609575151:2049]
2025-11-27T18:18:58.179912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471377798811938:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.189553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471374216952933:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31603) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471378511920309:2117] held: no
2025-11-27T18:18:58.178845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471379609575151:2049] [node 3] ICS09 handshake done sender: [2:7577471379609575131:2110] self: [2:7577471379609575130:0] peer: [3:7577471378511920313:0] socket: 621 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.196623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471377349874472:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:58.180145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471373503844572:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.203011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471374216953003:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.173893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471377451048442:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.178870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471379609575151:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:58.201993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471377349874486:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32245
2025-11-27T18:18:58.180445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471373503844571:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.178892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:58.180651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471379609575150:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29563
2025-11-27T18:18:58.208767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471377349874484:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32245
2025-11-27T18:18:58.209324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471379000888864:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.175625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471373156081078:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.203056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471374216953003:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.186694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471379609575135:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.212663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471377798811935:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.209517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471374705921547:2091] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.208884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471373054907107:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.205983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471378511920309:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:58.175687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471373156081077:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.186728Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471379609575135:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.212714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471377798811935:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.208942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:58.209505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471374216952933:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.209541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:58.209586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471374216952933:2049] [node 2] ICP22 created new session: [3:7577471378511920319:2049]
2025-11-27T18:18:58.209625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471378511920319:2049] [node 2] ICS09 handshake done sender: [3:7577471378511920309:2117] self: [3:7577471378511920313:0] peer: [2:7577471379609575130:0] socket: 631 qp: -1
2025-11-27T18:18:58.209659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471378511920319:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:58.209689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:58.210489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471378511920316:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58677
2025-11-27T18:18:58.216941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471374216953003:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:58.196749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471377451048455:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33856
2025-11-27T18:18:58.210304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471379000888865:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:18:58.190391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471379609575154:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29563
2025-11-27T18:18:58.213183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471379000888865:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.213381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471374705921491:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.222080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471379000888868:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:18:58.210051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471373054907107:2049] [node 3] ICP22 created new session: [1:7577471377349874488:2049]
2025-11-27T18:18:58.202299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471377451048456:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33856
2025-11-27T18:18:58.190488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471379609575155:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29563
2025-11-27T18:18:58.210585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471373054907106:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.229900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471378511920315:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.191256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471379609575150:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.229922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471374216952996:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.235970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471373503844571:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31603) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471377798811938:2115] held: no
2025-11-27T18:18:58.237626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471377798811952:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.225389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471377451048442:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:58.191524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471379609575139:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:58.241569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471379000888865:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.210619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:58.229967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:58.233808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471374216952996:2107] [node 4] ICP22 created new session: [3:7577471378511920321:2107]
2025-11-27T18:18:58.211211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471377349874488:2049] [node 3] ICS09 handshake done sender: [1:7577471377349874470:2111] self: [1:7577471377349874469:0] peer: [3:7577471378511920311:0] socket: 618 qp: -1
2025-11-27T18:18:58.192052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471379609575135:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:58.237625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471377451048443:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:58.211258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471373054907106:2048] [node 2] ICP22 created new session: [1:7577471377349874489:2048]
2025-11-27T18:18:58.245060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471377798811938:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:58.202610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471375314607768:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.243843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471379000888869:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:18:58.236649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471378511920321:2107] [node 4] ICS09 handshake done sender: [3:7577471374216953003:2112] self: [3:7577471374216953002:0] peer: [4:7577471377798811948:0] socket: 613 qp: -1
2025-11-27T18:18:58.240871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471373156081077:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.211732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471377349874488:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:58.236692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471378511920308:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:58.237704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471378511920321:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:58.237746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:58.239958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471374216952932:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.239992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:58.240025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471374216952932:2048] [node 1] ICP22 created new session: [3:7577471378511920323:2048]
2025-11-27T18:18:58.240054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471378511920323:2048] [node 1] ICS09 handshake done sender: [3:7577471378511920308:2116] self: [3:7577471378511920311:0] peer: [1:7577471377349874469:0] socket: 630 qp: -1
2025-11-27T18:18:58.240086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471378511920323:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:58.240118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:58.245293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471377798811951:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-27T18:18:58.202668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:58.254188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471377798811935:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:58.240924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:58.211938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.202725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471375314607768:2048] [node 1] ICP22 created new session: [2:7577471379609575158:2048]
2025-11-27T18:18:58.240972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471373156081077:2048] [node 1] ICP22 created new session: [5:7577471377451048458:2048]
2025-11-27T18:18:58.255810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471373503844571:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.254846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471374705921491:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31603) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471379000888865:2116] held: no
2025-11-27T18:18:58.212188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471377349874489:2048] [node 2] ICS09 handshake done sender: [1:7577471377349874466:2110] self: [1:7577471377349874465:0] peer: [2:7577471379609575147:0] socket: 617 qp: -1
2025-11-27T18:18:58.202782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471379609575158:2048] [node 1] ICS09 handshake done sender: [2:7577471379609575139:2115] self: [2:7577471379609575147:0] peer: [1:7577471377349874465:0] socket: 620 qp: -1
2025-11-27T18:18:58.241023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471377451048458:2048] [node 1] ICS09 handshake done sender: [5:7577471377451048442:2112] self: [5:7577471377451048454:0] peer: [1:7577471377349874471:0] socket: 638 qp: -1
2025-11-27T18:18:58.254983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471374705921491:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.202818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471379609575158:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:58.202856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:58.203592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471375314607832:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.203655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.203702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471375314607832:2108] [node 5] ICP22 created new session: [2:7577471379609575160:2108]
2025-11-27T18:18:58.203746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471379609575160:2108] [node 5] ICS09 handshake done sender: [2:7577471379609575135:2112] self: [2:7577471379609575134:0] peer: [5:7577471377451048453:0] socket: 627 qp: -1
2025-11-27T18:18:58.203781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471379609575160:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:58.203821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:58.204225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471379609575133:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.204249Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471379609575133:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.209820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471379609575154:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.211458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471379609575133:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:58.219590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471379609575137:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.219618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471379609575137:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.277746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471374216952996:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(26448811ns) {HandleHandshakeStatus:329(24892748ns) {DropIncomingHandshake:442(586ns) DropOutgoingHandshake:457(736ns) SwitchToState:179(795ns) ProcessPendingSessionEvents:495(5202741ns) {ForwardSessionEventToSession:612(1514665ns) ForwardSessionEventToSession:612(1587828ns) ForwardSessionEventToSession:612(256967ns) ForwardSessionEventToSession:612(5795ns) ForwardSessionEventToSession:612(1786020ns) ForwardSessionEventToSession:612(10037ns) ForwardSessionEventToSession:612(8714ns) ForwardSessionEventToSession:612(3318ns) ForwardSessionEventToSession:612(3099ns) ForwardSessionEventToSession:612(5443ns)}}}
2025-11-27T18:18:58.220152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471375314607831:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.277898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471378511920327:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58677
2025-11-27T18:18:58.220212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:58.220280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471375314607831:2107] [node 4] ICP22 created new session: [2:7577471379609575164:2107]
2025-11-27T18:18:58.220331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471379609575164:2107] [node 4] ICS09 handshake done sender: [2:7577471379609575133:2111] self: [2:7577471379609575132:0] peer: [4:7577471377798811950:0] socket: 626 qp: -1
2025-11-27T18:18:58.220363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471379609575164:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:58.220401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:58.232021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471379609575137:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:58.236746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471375314607833:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.237331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:58.255025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:58.255081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471374705921491:2049] [node 2] ICP22 created new session: [6:7577471379000888870:2049]
2025-11-27T18:18:58.255130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471379000888870:2049] [node 2] ICS09 handshake done sender: [6:7577471379000888865:2116] self: [6:7577471379000888867:0] peer: [2:7577471379609575136:0] socket: 661 qp: -1
2025-11-27T18:18:58.255166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471379000888870:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:58.255203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:58.277832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471379000888864:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:58.257187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:58.212238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471377349874489:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:58.241062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471377451048458:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:58.238394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471375314607833:2109] [node 6] ICP22 created new session: [2:7577471379609575167:2109]
2025-11-27T18:18:58.257408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471373503844571:2049] [node 2] ICP22 created new session: [4:7577471377798811955:2049]
2025-11-27T18:18:58.212528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:58.241087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:58.289763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471374705921547:2091] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.274303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471373503844635:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.238479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471379609575167:2109] [node 6] ICS09 handshake done sender: [2:7577471379609575137:2113] self: [2:7577471379609575136:0] peer: [6:7577471379000888867:0] socket: 658 qp: -1
2025-11-27T18:18:58.244573Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471373156081078:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31603) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471377451048443:2113] held: no
2025-11-27T18:18:58.289831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:58.304930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471374216953006:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.237004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471373054907107:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(24675597ns) {HandleHandshakeStatus:329(24672547ns) {DropIncomingHandshake:442(13415ns) DropOutgoingHandshake:457(977ns) SwitchToState:179(802ns) ProcessPendingSessionEvents:495(9040554ns) {ForwardSessionEventToSession:612(3837992ns) ForwardSessionEventToSession:612(5551ns) ForwardSessionEventToSession:612(3244ns) ForwardSessionEventToSession:612(5148818ns) ForwardSessionEventToSession:612(8207ns) ForwardSessionEventToSession:612(3246ns) ForwardSessionEventToSession:612(6010ns) ForwardSessionEventToSession:612(2661ns) ForwardSessionEventToSession:612(4973ns) ForwardSessionEventToSession:612(3831ns)}}}
2025-11-27T18:18:58.274391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:58.289882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471374705921547:2091] [node 4] ICP22 created new session: [6:7577471379000888876:2091]
2025-11-27T18:18:58.289937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471379000888876:2091] [node 4] ICS09 handshake done sender: [6:7577471379000888864:2115] self: [6:7577471379000888866:0] peer: [4:7577471377798811934:0] socket: 651 qp: -1
2025-11-27T18:18:58.289970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471379000888876:2091] [node 4] ICS10 traffic start
2025-11-27T18:18:58.290002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.318176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471374705921547:2091] [node 4] ICP35 event processing took too much time PendingConnection:279(28184205ns) {HandleHandshakeStatus:329(28180179ns) {DropIncomingHandshake:442(804ns) DropOutgoingHandshake:457(4938ns) SwitchToState:179(917ns) ProcessPendingSessionEvents:495(12630456ns) {ForwardSessionEventToSession:612(11716275ns) ForwardSessionEventToSession:612(6868ns) ForwardSessionEventToSession:612(3415ns) ForwardSessionEventToSession:612(858020ns) ForwardSessionEventToSession:612(3584ns) ForwardSessionEventToSession:612(3423ns) ForwardSessionEventToSession:612(6614ns) ForwardSessionEventToSession:612(5344ns) ForwardSessionEventToSession:612(8662ns) ForwardSessionEventToSession:612(2385ns)}}}
2025-11-27T18:18:58.318304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471379000888872:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:18:58.318498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471379000888873:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:18:58.318634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471379000888874:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:18:58.319452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471379000888874:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.319538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471379000888872:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.321167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471374705921490:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.321246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471374705921552:2096] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.238513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471379609575167:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.245673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471373156081078:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.237069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471377349874468:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.304988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471374216953006:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.277308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471377798811955:2049] [node 2] ICS09 handshake done sender: [4:7577471377798811938:2115] self: [4:7577471377798811950:0] peer: [2:7577471379609575132:0] socket: 634 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.248582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.327947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471379000888881:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:18:58.238549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:58.237871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:58.333784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471374216953006:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:58.250388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471373156081078:2049] [node 2] ICP22 created new session: [5:7577471377451048460:2049]
2025-11-27T18:18:58.252986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471377451048460:2049] [node 2] ICS09 handshake done sender: [5:7577471377451048443:2113] self: [5:7577471377451048453:0] peer: [2:7577471379609575134:0] socket: 639 qp: -1
2025-11-27T18:18:58.256004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471377451048460:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:58.256461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471377451048457:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33856
2025-11-27T18:18:58.257228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:58.273555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471377451048457:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.274337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471373156081078:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17290685ns) {HandleHandshakeStatus:329(17289267ns) {DropIncomingHandshake:442(486ns) DropOutgoingHandshake:457(134ns) SwitchToState:179(824ns) ProcessPendingSessionEvents:495(1056ns)}}
2025-11-27T18:18:58.281831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471373156081079:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.321774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471377451048449:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.321803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471377451048449:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.322587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471377451048465:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33856
2025-11-27T18:18:58.333728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471377451048449:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:58.278080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471373503844635:2110] [node 6] ICP22 created new session: [4:7577471377798811956:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.350162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471378511920307:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.350197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471378511920307:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.350794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471374216952997:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.350848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:58.350897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471374216952997:2108] [node 5] ICP22 created new session: [3:7577471378511920329:2108]
2025-11-27T18:18:58.350942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471378511920329:2108] [node 5] ICS09 handshake done sender: [3:7577471374216953006:2114] self: [3:7577471374216953005:0] peer: [5:7577471377451048464:0] socket: 645 qp: -1
2025-11-27T18:18:58.350978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471378511920329:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:58.351007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:58.334739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471379000888873:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.237929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471377349874468:2108] [node 5] ICP22 created new session: [1:7577471377349874492:2108]
2025-11-27T18:18:58.278135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471377798811956:2110] [node 6] ICS09 handshake done sender: [4:7577471377798811935:2112] self: [4:7577471377798811934:0] peer: [6:7577471379000888866:0] socket: 650 qp: -1
2025-11-27T18:18:58.278464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471377798811956:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:58.278504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:58.281340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471377798811955:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:58.281388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:58.305832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471373503844571:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(45752155ns) {HandleHandshakeStatus:329(45749657ns) {DropIncomingHandshake:442(337ns) DropOutgoingHandshake:457(118ns) SwitchToState:179(258ns) ProcessPendingSessionEvents:495(2439ns)}}
2025-11-27T18:18:58.306132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471373503844635:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(30588339ns) {HandleHandshakeStatus:329(30584334ns) {DropIncomingHandshake:442(18449ns) DropOutgoingHandshake:457(761ns) SwitchToState:179(767ns) ProcessPendingSessionEvents:495(26262549ns) {ForwardSessionEventToSession:612(5086486ns) ForwardSessionEventToSession:612(6817433ns) ForwardSessionEventToSession:612(8436ns) ForwardSessionEventToSession:612(3031ns) ForwardSessionEventToSession:612(2212899ns) ForwardSessionEventToSession:612(4204ns) ForwardSessionEventToSession:612(3472ns) ForwardSessionEventToSession:612(8907888ns) ForwardSessionEventToSession:612(11290ns) ForwardSessionEventToSession:612(3179052ns)}}}
2025-11-27T18:18:58.306221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471377798811936:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:58.324281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471373503844572:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.324330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:58.324385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471373503844572:2050] [node 3] ICP22 created new session: [4:7577471377798811961:2050]
2025-11-27T18:18:58.324428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471377798811961:2050] [node 3] ICS09 handshake done sender: [4:7577471377798811936:2113] self: [4:7577471377798811948:0] peer: [3:7577471374216953002:0] socket: 632 qp: -1
2025-11-27T18:18:58.324460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471377798811961:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:58.324491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:58.343244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471377798811959:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-27T18:18:58.343376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471377798811960:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-27T18:18:58.342026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471374705921492:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.342127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471379000888882:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:18:58.347148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471379000888872:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:58.348057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471377451048440:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.348114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:58.348160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471377451048440:2109] [node 6] ICP22 created new session: [5:7577471377451048469:2109]
2025-11-27T18:18:58.348240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471377451048469:2109] [node 6] ICS09 handshake done sender: [5:7577471377451048449:2116] self: [5:7577471377451048448:0] peer: [6:7577471379000888880:0] socket: 616 qp: -1
2025-11-27T18:18:58.348275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471377451048469:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:58.348308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:58.348749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471377451048465:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.356161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471378511920307:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:58.243986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471373054907106:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(26082163ns) {HandleHandshakeStatus:329(26077686ns) {DropIncomingHandshake:442(208ns) DropOutgoingHandshake:457(210ns) SwitchToState:179(153ns) ProcessPendingSessionEvents:495(12006936ns) {ForwardSessionEventToSession:612(7214884ns) ForwardSessionEventToSession:612(1935331ns) ForwardSessionEventToSession:612(3456ns) ForwardSessionEventToSession:612(3580ns) ForwardSessionEventToSession:612(1727631ns) ForwardSessionEventToSession:612(3290ns) ForwardSessionEventToSession:612(3234ns) ForwardSessionEventToSession:612(877674ns) ForwardSessionEventToSession:612(3491ns) ForwardSessionEventToSession:612(213949ns)}}}
2025-11-27T18:18:58.349181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471377451048467:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33856
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.244043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471377349874492:2108] [node 5] ICS09 handshake done sender: [1:7577471377349874472:2112] self: [1:7577471377349874471:0] peer: [5:7577471377451048454:0] socket: 625 qp: -1
2025-11-27T18:18:58.349545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471377451048468:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33856
2025-11-27T18:18:58.349643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577471377451048440:2109] [node 6] ICP18 handshake (actor [5:7577471377451048465:2120]) is too old
2025-11-27T18:18:58.365730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471374705921552:2096] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.365793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:58.365843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471374705921552:2096] [node 5] ICP22 created new session: [6:7577471379000888884:2096]
2025-11-27T18:18:58.365888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471379000888884:2096] [node 5] ICS09 handshake done sender: [6:7577471379000888872:2119] self: [6:7577471379000888880:0] peer: [5:7577471377451048448:0] socket: 672 qp: -1
2025-11-27T18:18:58.365925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471379000888884:2096] [node 5] ICS10 traffic start
2025-11-27T18:18:58.365960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:58.366196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471374705921552:2096] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:58.367900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471374216953001:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.367971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:58.368024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471374216953001:2109] [node 6] ICP22 created new session: [3:7577471378511920332:2109]
2025-11-27T18:18:58.368067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471378511920332:2109] [node 6] ICS09 handshake done sender: [3:7577471378511920307:2115] self: [3:7577471378511920306:0] peer: [6:7577471379000888883:0] socket: 612 qp: -1
2025-11-27T18:18:58.368105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471378511920332:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:58.368137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:58.370166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471377451048457:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:58.356323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471377798811960:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.244607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471377349874492:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:58.356448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471373503844570:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:58.371414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471373156081079:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13797) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471377451048457:2119] held: no
2025-11-27T18:18:58.244666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:58.248787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471377349874486:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.252771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471377349874484:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.286348Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471377349874468:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(21592172ns) {HandleHandshakeStatus:329(21589668ns) {DropIncomingHandshake:442(518ns) DropOutgoingHandshake:457(305ns) SwitchToState:179(1390ns) ProcessPendingSessionEvents:495(9222330ns) {ForwardSessionEventToSession:612(6466024ns) ForwardSessionEventToSession:612(4052ns) ForwardSessionEventToSession:612(3173ns) ForwardSessionEventToSession:612(1293829ns) ForwardSessionEventToSession:612(2793ns) ForwardSessionEventToSession:612(404305ns) ForwardSessionEventToSession:612(3240ns) ForwardSessionEventToSession:612(1020488ns) ForwardSessionEventToSession:612(6732ns) ForwardSessionEventToSession:612(2531ns)}}}
2025-11-27T18:18:58.286890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471377349874468:2108] [node 5] ICP18 handshake (actor [1:7577471377349874484:2116]) is too old
2025-11-27T18:18:58.330284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471377349874481:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.330327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471377349874481:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.341480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471377349874497:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32245
2025-11-27T18:18:58.342494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471377349874481:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:58.352634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471377349874497:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.352679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471377349874478:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.352725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:58.352769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471377349874478:2109] [node 6] ICP22 created new session: [1:7577471377349874499:2109]
2025-11-27T18:18:58.352808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471377349874499:2109] [node 6] ICS09 handshake done sender: [1:7577471377349874481:2115] self: [1:7577471377349874480:0] peer: [6:7577471379000888879:0] socket: 671 qp: -1
2025-11-27T18:18:58.352836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471377349874499:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:58.352866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:58.353569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471377349874478:2109] [node 6] ICP18 handshake (actor [1:7577471377349874497:2118]) is too old
2025-11-27T18:18:58.363495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471377349874475:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.363526Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471377349874475:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.364760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471377349874475:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:58.369042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471377349874467:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.369090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:58.369131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471377349874467:2107] [node 4] ICP22 created new session: [1:7577471377349874502:2107]
2025-11-27T18:18:58.369176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471377349874502:2107] [node 4] ICS09 handshake done sender: [1:7577471377349874475:2114] self: [1:7577471377349874474:0] peer: [4:7577471377798811964:0] socket: 670 qp: -1
2025-11-27T18:18:58.369200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471377349874502:2107] [node 4] ICS10 traffic start
2025-11-27T18:18:58.369228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:58.371254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471379000888874:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:58.356851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471377798811959:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.379231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471373156081079:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.376890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471374705921490:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471379000888874:2121] held: no
2025-11-27T18:18:58.365106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471377798811966:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13596
2025-11-27T18:18:58.373616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471377798811960:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:58.382215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471373503844570:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471377798811960:2122] held: no
2025-11-27T18:18:58.382314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471373503844570:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.382348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:58.382393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471373503844570:2048] [node 1] ICP22 created new session: [4:7577471377798811967:2048]
2025-11-27T18:18:58.382435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471377798811967:2048] [node 1] ICS09 handshake done sender: [4:7577471377798811960:2122] self: [4:7577471377798811964:0] peer: [1:7577471377349874474:0] socket: 677 qp: -1
2025-11-27T18:18:58.382466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471377798811967:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:58.382493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:58.382967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471377798811944:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.382991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471377798811944:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:58.388764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471377798811944:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:58.397030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471373503844634:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:58.377246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471379000888886:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:18:58.389025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471374705921490:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.389074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:58.389130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471374705921490:2048] [node 1] ICP22 created new session: [6:7577471379000888887:2048]
2025-11-27T18:18:58.389182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471379000888887:2048] [node 1] ICS09 handshake done sender: [6:7577471379000888874:2121] self: [6:7577471379000888879:0] peer: [1:7577471377349874480:0] socket: 674 qp: -1
2025-11-27T18:18:58.389212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471379000888887:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:58.389244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:58.389673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471374705921492:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13797) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471379000888873:2120] held: no
2025-11-27T18:18:58.390013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471379000888873:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:58.402192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471374705921492:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.402232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:58.402286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471374705921492:2050] [node 3] ICP22 created new session: [6:7577471379000888889:2050]
2025-11-27T18:18:58.402334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471379000888889:2050] [node 3] ICS09 handshake done sender: [6:7577471379000888873:2120] self: [6:7577471379000888883:0] peer: [3:7577471378511920306:0] socket: 673 qp: -1
2025-11-27T18:18:58.402376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471379000888889:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:58.402412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:58.379291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:58.380445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471373156081079:2050] [node 3] ICP22 created new session: [5:7577471377451048471:2050]
2025-11-27T18:18:58.380495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471377451048471:2050] [node 3] ICS09 handshake done sender: [5:7577471377451048457:2119] self: [5:7577471377451048464:0] peer: [3:7577471374216953005:0] socket: 668 qp: -1
2025-11-27T18:18:58.380532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471377451048471:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:58.380575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:58.381093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471377451048468:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:58.381725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471377451048438:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:58.405688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471377451048474:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33856
2025-11-27T18:18:58.402078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:58.403494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471373503844634:2108] [node 5] ICP22 created new session: [4:7577471377798811970:2108]
2025-11-27T18:18:58.403549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471377798811970:2108] [node 5] ICS09 handshake done sender: [4:7577471377798811944:2118] self: [4:7577471377798811943:0] peer: [5:7577471377451048473:0] socket: 681 qp: -1
2025-11-27T18:18:58.403589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471377798811970:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:58.403625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:58.407480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471377451048438:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471377451048468:2122] held: no
2025-11-27T18:18:58.407659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471377451048468:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:58.408182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471377451048438:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:58.408230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:58.408280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471377451048438:2108] [node 4] ICP22 created new session: [5:7577471377451048475:2108]
2025-11-27T18:18:58.408328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471377451048475:2108] [node 4] ICS09 handshake done sender: [5:7577471377451048468:2122] self: [5:7577471377451048473:0] peer: [4:7577471377798811943:0] socket: 683 qp: -1
2025-11-27T18:18:58.408359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471377451048475:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:58.408389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:58.429619Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:18:58.429841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471379609575158:2048] [node 1] ICS01 socket: 620 reason# EndOfStream
2025-11-27T18:18:58.429880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471375314607768:2048] [node 1] ICP30 unregister session Session# [2:7577471379609575158:2048] VirtualId# [2:7577471379609575147:0]
2025-11-27T18:18:58.429914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471379609575158:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.429989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:58.430032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:58.431109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:18:58.431303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471378511920323:2048] [node 1] ICS01 socket: 630 reason# EndOfStream
2025-11-27T18:18:58.431344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471374216952932:2048] [node 1] ICP30 unregister session Session# [3:7577471378511920323:2048] VirtualId# [3:7577471378511920311:0]
2025-11-27T18:18:58.431370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471378511920323:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.431424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:58.431459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:58.441726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:18:58.441935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471377451048458:2048] [node 1] ICS01 socket: 638 reason# EndOfStream
2025-11-27T18:18:58.441355Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:18:58.441646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471377798811967:2048] [node 1] ICS01 socket: 677 reason# EndOfStream
2025-11-27T18:18:58.441696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471373503844570:2048] [node 1] ICP30 unregister session Session# [4:7577471377798811967:2048] VirtualId# [4:7577471377798811964:0]
2025-11-27T18:18:58.441989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:18:58.441980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471373156081077:2048] [node 1] ICP30 unregister session Session# [5:7577471377451048458:2048] VirtualId# [5:7577471377451048454:0]
2025-11-27T18:18:58.442019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471377451048458:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.442103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:58.442147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:58.441736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471377798811967:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.445691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471379000888887:2048] [node 1] ICS01 socket: 674 reason# EndOfStream
2025-11-27T18:18:58.445732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471374705921490:2048] [node 1] ICP30 unregister session Session# [6:7577471379000888887:2048] VirtualId# [6:7577471379000888879:0]
2025-11-27T18:18:58.445765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471379000888887:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.445841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:58.445884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:58.441841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:18:58.441885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:18:58.539936Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:18:58.540155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471378511920319:2049] [node 2] ICS01 socket: 631 reason# EndOfStream
2025-11-27T18:18:58.540197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471374216952933:2049] [node 2] ICP30 unregister session Session# [3:7577471378511920319:2049] VirtualId# [3:7577471378511920313:0]
2025-11-27T18:18:58.540235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471378511920319:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.540306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:58.540343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:58.545414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:18:58.545655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471379000888870:2049] [node 2] ICS01 socket: 661 reason# EndOfStream
2025-11-27T18:18:58.545701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471374705921491:2049] [node 2] ICP30 unregister session Session# [6:7577471379000888870:2049] VirtualId# [6:7577471379000888867:0]
2025-11-27T18:18:58.545736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471379000888870:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.545813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:58.545858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:58.545960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:18:58.546136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471377798811955:2049] [node 2] ICS01 socket: 634 reason# EndOfStream
2025-11-27T18:18:58.546172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471373503844571:2049] [node 2] ICP30 unregister session Session# [4:7577471377798811955:2049] VirtualId# [4:7577471377798811950:0]
2025-11-27T18:18:58.546197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471377798811955:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.546250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:58.546285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:58.546378Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:18:58.546555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471377451048460:2049] [node 2] ICS01 socket: 639 reason# EndOfStream
2025-11-27T18:18:58.546586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471373156081078:2049] [node 2] ICP30 unregister session Session# [5:7577471377451048460:2049] VirtualId# [5:7577471377451048453:0]
2025-11-27T18:18:58.546611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471377451048460:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.546676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:18:58.546714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:18:58.618583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:18:58.618824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471377798811961:2050] [node 3] ICS01 socket: 632 reason# EndOfStream
2025-11-27T18:18:58.618868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471373503844572:2050] [node 3] ICP30 unregister session Session# [4:7577471377798811961:2050] VirtualId# [4:7577471377798811948:0]
2025-11-27T18:18:58.618903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471377798811961:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.618979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:58.619020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:58.620685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:18:58.620853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471379000888889:2050] [node 3] ICS01 socket: 673 reason# EndOfStream
2025-11-27T18:18:58.620879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471374705921492:2050] [node 3] ICP30 unregister session Session# [6:7577471379000888889:2050] VirtualId# [6:7577471379000888883:0]
2025-11-27T18:18:58.620897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471379000888889:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.620952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:58.620978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:58.621390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:18:58.621588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471377451048471:2050] [node 3] ICS01 socket: 668 reason# EndOfStream
2025-11-27T18:18:58.621643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471373156081079:2050] [node 3] ICP30 unregister session Session# [5:7577471377451048471:2050] VirtualId# [5:7577471377451048464:0]
2025-11-27T18:18:58.621673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471377451048471:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.621735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:18:58.621811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:18:58.671710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:18:58.671965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471377451048475:2108] [node 4] ICS01 socket: 683 reason# EndOfStream
2025-11-27T18:18:58.672007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471377451048438:2108] [node 4] ICP30 unregister session Session# [5:7577471377451048475:2108] VirtualId# [5:7577471377451048473:0]
2025-11-27T18:18:58.672045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471377451048475:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.672106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:58.672142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:58.676529Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:18:58.676925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471379000888876:2091] [node 4] ICS07 socket disconnect 651 reason# EndOfStream
2025-11-27T18:18:58.676989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471379000888876:2091] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:18:58.677175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:18:58.677289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471379000888876:2091] [node 4] ICS15 start handshake
2025-11-27T18:18:58.680884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471374705921547:2091] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:18:58.680930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471379000888876:2091] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:18:58.680959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471374705921547:2091] [node 4] ICP30 unregister session Session# [6:7577471379000888876:2091] VirtualId# [6:7577471379000888866:0]
2025-11-27T18:18:58.680992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:18:58.681013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471374705921547:2091] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:58.681022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7221) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:18:58.693344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471374705921547:2091] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:18:58.873786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471380969863541:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:58.893484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471380969863540:2048] [node 2] ICP01 ready to work
2025-11-27T18:18:58.899276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471377170595937:2049] [node 3] ICP01 ready to work
2025-11-27T18:18:58.899831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471377170595936:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:58.918336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471381046054206:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:58.918377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471381046054207:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:58.943986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471378357822905:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:58.944027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471378357822906:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:58.945013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471378357822904:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:58.966994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471379048775501:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:58.967044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471379048775502:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:58.967076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471379048775503:2050] [node 3] ICP01 ready to work
2025-11-27T18:18:58.989996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471379715900091:2048] [node 1] ICP01 ready to work
2025-11-27T18:18:58.990057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471379715900092:2049] [node 2] ICP01 ready to work
2025-11-27T18:18:58.990090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471379715900093:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.272708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471381465563292:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30763
2025-11-27T18:18:59.273364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471384010867447:2105] [node 5] ICP01 ready to work
2025-11-27T18:18:59.313453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471381465563297:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:59.302995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471385264830903:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:59.366765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471382652790265:2108] [node 5] ICP01 ready to work
2025-11-27T18:18:59.305458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471385264830902:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:59.390982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471384010867447:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(117033862ns) {Configure:104(117029751ns) {ProcessConfigured:135(97553ns) {StartInitialHandshake:157(95015ns) {DropHandshakes:470(4117ns) {DropIncomingHandshake:442(723ns) DropOutgoingHandshake:457(649ns)} GenerateSessionVirtualId:988(902ns) PrepareNewSessionHandshake:477(2868ns) {SwitchToState:179(1039ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.418046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471384010867453:2110] [node 4] ICP01 ready to work
2025-11-27T18:18:59.363054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471381465563299:2110] [node 5] ICP01 ready to work
2025-11-27T18:18:59.366944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471382652790266:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:59.345835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471385264830904:2109] [node 5] ICP01 ready to work
2025-11-27T18:18:59.410657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471385341021565:2107] [node 5] ICP01 ready to work
2025-11-27T18:18:59.398358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471381465563292:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.425077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471385264830905:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39444
2025-11-27T18:18:59.437405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471385341021566:2109] [node 6] ICP01 ready to work
2025-11-27T18:18:59.453205Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471380969863541:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(149895387ns) {Configure:104(149888085ns) {ProcessConfigured:135(15574075ns) {StartInitialHandshake:157(15570794ns) {DropHandshakes:470(3569ns) {DropIncomingHandshake:442(917ns) DropOutgoingHandshake:457(549ns)} GenerateSessionVirtualId:988(963ns) PrepareNewSessionHandshake:477(6127ns) {SwitchToState:179(3102ns)}}}}}
2025-11-27T18:18:59.471026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471383343742864:2109] [node 4] ICP01 ready to work
2025-11-27T18:18:59.398522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471381465563301:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.445173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471385341021567:2108] [node 4] ICP01 ready to work
2025-11-27T18:18:59.473310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471383343742863:2110] [node 6] ICP01 ready to work
2025-11-27T18:18:59.489851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471382652790266:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(121158669ns) {Configure:104(121155191ns) {ProcessConfigured:135(7070167ns) {StartInitialHandshake:157(7067357ns) {DropHandshakes:470(3172ns) {DropIncomingHandshake:442(701ns) DropOutgoingHandshake:457(443ns)} GenerateSessionVirtualId:988(1150ns) PrepareNewSessionHandshake:477(5266ns) {SwitchToState:179(2230ns)}}}}}
2025-11-27T18:18:59.518482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471385264830903:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(172263471ns) {Configure:104(172258236ns) {ProcessConfigured:135(104181ns) {StartInitialHandshake:157(101957ns) {DropHandshakes:470(8328ns) {DropIncomingHandshake:442(765ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(987ns) PrepareNewSessionHandshake:477(2260ns) {SwitchToState:179(629ns)}}}}}
2025-11-27T18:18:59.533304Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471379048775502:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(107525591ns) {Configure:104(107520423ns) {ProcessConfigured:135(9622891ns) {StartInitialHandshake:157(9619121ns) {DropHandshakes:470(3834ns) {DropIncomingHandshake:442(784ns) DropOutgoingHandshake:457(658ns)} GenerateSessionVirtualId:988(860ns) PrepareNewSessionHandshake:477(3621ns) {SwitchToState:179(954ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.527032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471381046054206:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53385848ns) {Configure:104(53379603ns) {ProcessConfigured:135(61311ns) {StartInitialHandshake:157(58371ns) {DropHandshakes:470(2745ns) {DropIncomingHandshake:442(386ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(773ns) PrepareNewSessionHandshake:477(2830ns) {SwitchToState:179(1108ns)}}}}}
2025-11-27T18:18:59.490728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471382652790272:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57094
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.538109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471378357822904:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(82552079ns) {Configure:104(82546114ns) {ProcessConfigured:135(61122ns) {StartInitialHandshake:157(59092ns) {DropHandshakes:470(6359ns) {DropIncomingHandshake:442(757ns) DropOutgoingHandshake:457(401ns)} GenerateSessionVirtualId:988(882ns) PrepareNewSessionHandshake:477(1760ns) {SwitchToState:179(646ns)}}}}}
2025-11-27T18:18:59.578567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471381465563299:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(107857989ns) {Configure:104(107848766ns) {ProcessConfigured:135(53927ns) {StartInitialHandshake:157(51456ns) {DropHandshakes:470(1761ns) {DropIncomingHandshake:442(431ns) DropOutgoingHandshake:457(448ns)} GenerateSessionVirtualId:988(594ns) PrepareNewSessionHandshake:477(1891ns) {SwitchToState:179(632ns)}}}}}
2025-11-27T18:18:59.578688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471377170595936:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.542852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471383343742867:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34061
2025-11-27T18:18:59.541774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471382652790277:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57094
2025-11-27T18:18:59.549701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471382652790272:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.554347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471378357822904:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.554929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471385264830907:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.599003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471381465563301:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19882570ns) {Configure:104(19877796ns) {ProcessConfigured:135(71093ns) {StartInitialHandshake:157(68212ns) {DropHandshakes:470(2999ns) {DropIncomingHandshake:442(828ns) DropOutgoingHandshake:457(399ns)} GenerateSessionVirtualId:988(816ns) PrepareNewSessionHandshake:477(1837ns) {SwitchToState:179(685ns)}}}}}
2025-11-27T18:18:59.599982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471381465563308:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30763
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.542978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471383343742868:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34061
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.554965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471385264830907:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.600128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471381465563309:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30763
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.604492Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471378357822906:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(244779070ns) {Configure:104(244772961ns) {ProcessConfigured:135(10546167ns) {StartInitialHandshake:157(10542036ns) {DropHandshakes:470(3551ns) {DropIncomingHandshake:442(816ns) DropOutgoingHandshake:457(366ns)} GenerateSessionVirtualId:988(973ns) PrepareNewSessionHandshake:477(4490ns) {SwitchToState:179(1215ns)}}}}}
2025-11-27T18:18:59.543038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471383343742869:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34061
2025-11-27T18:18:59.555822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471385264830916:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39444
2025-11-27T18:18:59.562767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471383343742867:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.610034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471381465563309:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.604639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471382652790268:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.556353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471385264830905:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.610116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471381465563308:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.654026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471385341021565:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(188247704ns) {Configure:104(188239782ns) {ProcessConfigured:135(58972ns) {StartInitialHandshake:157(57292ns) {DropHandshakes:470(5135ns) {DropIncomingHandshake:442(944ns) DropOutgoingHandshake:457(438ns)} GenerateSessionVirtualId:988(734ns) PrepareNewSessionHandshake:477(1776ns) {SwitchToState:179(605ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.562876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471383343742869:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.604663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471382652790268:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:59.569991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471385264830907:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.678260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471379715900093:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(215375598ns) {Configure:104(215370210ns) {ProcessConfigured:135(18744141ns) {StartInitialHandshake:157(18741480ns) {DropHandshakes:470(1499ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(606ns)} GenerateSessionVirtualId:988(336ns) PrepareNewSessionHandshake:477(5198ns) {SwitchToState:179(2532ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.596280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471385264830909:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.596335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471385264830909:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:59.634428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471385264830897:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.563164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471379048775501:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.626560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471381465563297:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(194263351ns) {Configure:104(194251140ns) {ProcessConfigured:135(12179737ns) {StartInitialHandshake:157(12177472ns) {DropHandshakes:470(2282ns) {DropIncomingHandshake:442(422ns) DropOutgoingHandshake:457(441ns)} GenerateSessionVirtualId:988(369ns) PrepareNewSessionHandshake:477(2464ns) {SwitchToState:179(1548ns)}}}}}
2025-11-27T18:18:59.605938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471382652790283:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57094
2025-11-27T18:18:59.634454Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471385264830897:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:18:59.635476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471385264830919:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39444
2025-11-27T18:18:59.637674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471385264830918:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39444
2025-11-27T18:18:59.645830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471385264830909:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:59.646309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471385264830897:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:59.649356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471385264830902:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.649418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:59.649477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471385264830902:2108] [node 4] ICP22 created new session: [1:7577471385264830923:2108]
2025-11-27T18:18:59.649525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471385264830923:2108] [node 4] ICS09 handshake done sender: [1:7577471385264830907:2112] self: [1:7577471385264830906:0] peer: [4:7577471382652790278:0] socket: 624 qp: -1
2025-11-27T18:18:59.649561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471385264830923:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:59.649599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:59.650033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471385264830904:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.650075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:59.650112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471385264830904:2109] [node 5] ICP22 created new session: [1:7577471385264830925:2109]
2025-11-27T18:18:59.650152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471385264830925:2109] [node 5] ICS09 handshake done sender: [1:7577471385264830909:2113] self: [1:7577471385264830908:0] peer: [5:7577471383343742877:0] socket: 625 qp: -1
2025-11-27T18:18:59.650178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471385264830925:2109] [node 5] ICS10 traffic start
2025-11-27T18:18:59.650201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:59.665804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471385264830919:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.681691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471380969863540:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.681758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:59.689785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471380969863540:2048] [node 2] ICP22 created new session: [1:7577471385264830930:2048]
2025-11-27T18:18:59.689847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471385264830930:2048] [node 2] ICS09 handshake done sender: [1:7577471385264830897:2107] self: [1:7577471385264830896:0] peer: [2:7577471381465563305:0] socket: 618 qp: -1
2025-11-27T18:18:59.689891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471385264830930:2048] [node 2] ICS10 traffic start
2025-11-27T18:18:59.689935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:59.713827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471385264830916:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.726334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471385264830918:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.701069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471379715900092:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(217577547ns) {Configure:104(217572161ns) {ProcessConfigured:135(11929852ns) {StartInitialHandshake:157(11926715ns) {DropHandshakes:470(3262ns) {DropIncomingHandshake:442(812ns) DropOutgoingHandshake:457(420ns)} GenerateSessionVirtualId:988(1225ns) PrepareNewSessionHandshake:477(9001ns) {SwitchToState:179(2134ns)}}}}}
2025-11-27T18:18:59.657502Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471377170595937:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(269215705ns) {Configure:104(269208181ns) {ProcessConfigured:135(38439538ns) {StartInitialHandshake:157(38435966ns) {DropHandshakes:470(3773ns) {DropIncomingHandshake:442(1061ns) DropOutgoingHandshake:457(413ns)} GenerateSessionVirtualId:988(875ns) PrepareNewSessionHandshake:477(2821ns) {SwitchToState:179(1416ns)}}}}}
2025-11-27T18:18:59.729442Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471381046054207:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(237644208ns) {Configure:104(237634721ns) {ProcessConfigured:135(23802548ns) {StartInitialHandshake:157(23795961ns) {DropHandshakes:470(2617ns) {DropIncomingHandshake:442(551ns) DropOutgoingHandshake:457(379ns)} GenerateSessionVirtualId:988(1001ns) PrepareNewSessionHandshake:477(4531ns) {SwitchToState:179(2449ns)}}}}}
2025-11-27T18:18:59.563240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471383343742864:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.617483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471382652790268:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:59.701193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471384010867465:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46629
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.677490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471381465563304:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.589569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471383343742863:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(47780114ns) {Configure:104(47775550ns) {ProcessConfigured:135(3340798ns) {StartInitialHandshake:157(3338352ns) {DropHandshakes:470(3964ns) {DropIncomingHandshake:442(746ns) DropOutgoingHandshake:457(492ns)} GenerateSessionVirtualId:988(808ns) PrepareNewSessionHandshake:477(3271ns) {SwitchToState:179(910ns)}}}}}
2025-11-27T18:18:59.625439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471382652790265:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.737118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471385341021571:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.702285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471384010867467:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46629
2025-11-27T18:18:59.613688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471383343742880:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34061
2025-11-27T18:18:59.613751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471383343742881:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34061
2025-11-27T18:18:59.618163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471383343742869:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:59.618333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471383343742864:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.618380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:59.618433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471383343742864:2109] [node 4] ICP22 created new session: [5:7577471383343742882:2109]
2025-11-27T18:18:59.618476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471383343742882:2109] [node 4] ICS09 handshake done sender: [5:7577471383343742869:2114] self: [5:7577471383343742878:0] peer: [4:7577471382652790267:0] socket: 634 qp: -1
2025-11-27T18:18:59.618504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471383343742882:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:59.618536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:59.629190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471379048775503:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(97738216ns) {Configure:104(97732763ns) {ProcessConfigured:135(25648624ns) {StartInitialHandshake:157(25646610ns) {DropHandshakes:470(1098ns) {DropIncomingHandshake:442(157ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(255ns) PrepareNewSessionHandshake:477(3588666ns) {SwitchToState:179(2287ns)}}}}}
2025-11-27T18:18:59.637939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471383343742880:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.661109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471379048775502:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.676807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471379048775501:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471383343742867:2112] held: no
2025-11-27T18:18:59.676949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471383343742885:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34061
2025-11-27T18:18:59.706631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471383343742867:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:59.725894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471379048775501:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.725934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:59.726516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471379048775501:2048] [node 1] ICP22 created new session: [5:7577471383343742889:2048]
2025-11-27T18:18:59.735766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471383343742868:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.757778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471383343742889:2048] [node 1] ICS09 handshake done sender: [5:7577471383343742867:2112] self: [5:7577471383343742877:0] peer: [1:7577471385264830908:0] socket: 632 qp: -1
2025-11-27T18:18:59.757848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471383343742889:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:59.757897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:59.758146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471379048775501:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32153716ns) {HandleHandshakeStatus:329(32151567ns) {DropIncomingHandshake:442(667ns) DropOutgoingHandshake:457(395ns) SwitchToState:179(558ns) ProcessPendingSessionEvents:495(1070ns)}}
2025-11-27T18:18:59.758242Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471379048775502:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11128) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471383343742880:2118] held: no
2025-11-27T18:18:59.758598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471383343742887:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34061
2025-11-27T18:18:59.677518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471381465563304:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.625498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.737154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471385341021571:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:18:59.677974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471381465563317:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30763
2025-11-27T18:18:59.717536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471381465563304:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:59.718005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471381465563299:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.718069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:59.718126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471381465563299:2110] [node 5] ICP22 created new session: [2:7577471381465563319:2110]
2025-11-27T18:18:59.718173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471381465563319:2110] [node 5] ICS09 handshake done sender: [2:7577471381465563304:2112] self: [2:7577471381465563303:0] peer: [5:7577471383343742886:0] socket: 653 qp: -1
2025-11-27T18:18:59.718217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471381465563319:2110] [node 5] ICS10 traffic start
2025-11-27T18:18:59.718253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:59.718743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471381465563292:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:59.719431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471377170595936:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.719476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:59.719598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471377170595936:2048] [node 1] ICP22 created new session: [2:7577471381465563321:2048]
2025-11-27T18:18:59.719640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471381465563321:2048] [node 1] ICS09 handshake done sender: [2:7577471381465563292:2107] self: [2:7577471381465563305:0] peer: [1:7577471385264830896:0] socket: 619 qp: -1
2025-11-27T18:18:59.719673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471381465563321:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:59.719702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:59.758949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471377170595936:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:59.797628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471381465563307:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.797663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471381465563307:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:59.785562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471383343742876:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.785626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471383343742876:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:59.786523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471383343742880:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:59.794742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471379048775502:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.794808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:59.794854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471379048775502:2049] [node 2] ICP22 created new session: [5:7577471383343742893:2049]
2025-11-27T18:18:59.794895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471383343742893:2049] [node 2] ICS09 handshake done sender: [5:7577471383343742880:2118] self: [5:7577471383343742886:0] peer: [2:7577471381465563303:0] socket: 656 qp: -1
2025-11-27T18:18:59.794929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471383343742893:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:59.794962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:59.795887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471383343742892:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34061
2025-11-27T18:18:59.806578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471383343742876:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:59.806920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471383343742863:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.806974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:59.807018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471383343742863:2110] [node 6] ICP22 created new session: [5:7577471383343742896:2110]
2025-11-27T18:18:59.807067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471383343742896:2110] [node 6] ICS09 handshake done sender: [5:7577471383343742876:2116] self: [5:7577471383343742874:0] peer: [6:7577471384010867471:0] socket: 661 qp: -1
2025-11-27T18:18:59.807098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471383343742896:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:59.807133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.810359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471381465563307:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.702605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471384010867468:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46629
2025-11-27T18:18:59.814008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471385264830915:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.814056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471385264830915:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.625552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471382652790265:2108] [node 5] ICP22 created new session: [4:7577471382652790285:2108]
2025-11-27T18:18:59.750031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471385341021571:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.706855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471384010867465:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.786840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471385341021573:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.833340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471381465563301:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.833404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:59.833450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471381465563301:2111] [node 6] ICP22 created new session: [2:7577471381465563326:2111]
2025-11-27T18:18:59.833509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471381465563326:2111] [node 6] ICS09 handshake done sender: [2:7577471381465563307:2113] self: [2:7577471381465563306:0] peer: [6:7577471384010867474:0] socket: 655 qp: -1
2025-11-27T18:18:59.833560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471381465563326:2111] [node 6] ICS10 traffic start
2025-11-27T18:18:59.833591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:59.625604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471382652790285:2108] [node 5] ICS09 handshake done sender: [4:7577471382652790268:2111] self: [4:7577471382652790267:0] peer: [5:7577471383343742878:0] socket: 629 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.839979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471381465563328:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30763
2025-11-27T18:18:59.814485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471385264830932:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39444
2025-11-27T18:18:59.625645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471382652790285:2108] [node 5] ICS10 traffic start
2025-11-27T18:18:59.625689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:59.651266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471382652790279:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57094
2025-11-27T18:18:59.709317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471384010867461:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46629
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.701987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471382652790265:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:18:59.705502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471382652790277:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.705951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471382652790283:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.829871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471385264830902:2108] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.786864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471385341021573:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:59.706935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471378357822906:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.737082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471382652790280:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57094
2025-11-27T18:18:59.753960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471382652790272:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:59.765612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471382652790290:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57094
2025-11-27T18:18:59.777835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471378357822904:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.777888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:59.792188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471378357822904:2048] [node 1] ICP22 created new session: [4:7577471382652790291:2048]
2025-11-27T18:18:59.813634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471382652790271:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.813678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471382652790271:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:18:59.821824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471382652790291:2048] [node 1] ICS09 handshake done sender: [4:7577471382652790272:2113] self: [4:7577471382652790278:0] peer: [1:7577471385264830906:0] socket: 628 qp: -1
2025-11-27T18:18:59.822427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471382652790283:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:59.822770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471378357822906:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.822836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:59.822883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471378357822906:2050] [node 3] ICP22 created new session: [4:7577471382652790294:2050]
2025-11-27T18:18:59.822930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471382652790294:2050] [node 3] ICS09 handshake done sender: [4:7577471382652790283:2119] self: [4:7577471382652790289:0] peer: [3:7577471385341021570:0] socket: 657 qp: -1
2025-11-27T18:18:59.822957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471382652790294:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:59.822992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:59.824058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471382652790292:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57094
2025-11-27T18:18:59.824680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471382652790291:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:59.829401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:59.835375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471382652790271:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:59.853645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471382652790292:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.853875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471378357822905:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.869556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471378357822904:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(87654746ns) {HandleHandshakeStatus:329(87651832ns) {DropIncomingHandshake:442(1109ns) DropOutgoingHandshake:457(4865ns) SwitchToState:179(659ns) ProcessPendingSessionEvents:495(15829910ns) {ForwardSessionEventToSession:612(15755361ns) ForwardSessionEventToSession:612(11353ns) ForwardSessionEventToSession:612(3954ns) ForwardSessionEventToSession:612(8519ns) ForwardSessionEventToSession:612(11225ns) ForwardSessionEventToSession:612(3312ns) ForwardSessionEventToSession:612(10600ns) ForwardSessionEventToSession:612(3235ns) ForwardSessionEventToSession:612(3286ns) ForwardSessionEventToSession:612(6678ns)}}}
2025-11-27T18:18:59.714280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471384010867467:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.858165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471381465563315:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.877339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471382652790266:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.877419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:59.883801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471382652790266:2109] [node 6] ICP22 created new session: [4:7577471382652790299:2109]
2025-11-27T18:18:59.787631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471385341021567:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.870988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471383343742892:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.840511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471385264830915:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:59.858215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471381465563315:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.721673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471384010867468:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.849719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471385264830913:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.894399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471382652790299:2109] [node 6] ICS09 handshake done sender: [4:7577471382652790271:2112] self: [4:7577471382652790270:0] peer: [6:7577471384010867473:0] socket: 630 qp: -1
2025-11-27T18:18:59.849764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471385264830913:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:18:59.874398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471385264830903:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.874483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:59.894292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471385264830903:2110] [node 6] ICP22 created new session: [1:7577471385264830936:2110]
2025-11-27T18:18:59.906660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471385264830913:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:59.907188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471385264830936:2110] [node 6] ICS09 handshake done sender: [1:7577471385264830915:2115] self: [1:7577471385264830914:0] peer: [6:7577471384010867472:0] socket: 667 qp: -1
2025-11-27T18:18:59.917691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471385264830936:2110] [node 6] ICS10 traffic start
2025-11-27T18:18:59.882909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471379048775503:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.730715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471384010867466:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46629
2025-11-27T18:18:59.738710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471379715900093:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.741347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471384010867447:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.754207Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471379715900093:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17022781ns) {IncomingHandshake:261(17020564ns) {DropIncomingHandshake:442(328ns) PrepareNewSessionHandshake:477(1563ns) {SwitchToState:179(646ns)} GenerateSessionVirtualId:988(810ns) IssueIncomingHandshakeReply:193(7954400ns) {SwitchToState:179(2471ns)}}}
2025-11-27T18:18:59.754373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471384010867447:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(19581597ns) {IncomingHandshake:261(19579874ns) {DropIncomingHandshake:442(1003ns) PrepareNewSessionHandshake:477(4404756ns) {SwitchToState:179(1866ns)} GenerateSessionVirtualId:988(1074ns) IssueIncomingHandshakeReply:193(10455785ns) {SwitchToState:179(1984ns)}}}
2025-11-27T18:18:59.759006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471379715900091:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.761452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471384010867461:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.762134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471384010867466:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.769636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471384010867453:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.769720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471379715900092:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.809118Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471384010867447:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3461) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471384010867467:2117] held: no
2025-11-27T18:18:59.809793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471384010867475:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46629
2025-11-27T18:18:59.810557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471384010867476:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46629
2025-11-27T18:18:59.824530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471384010867465:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:59.835052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471384010867477:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46629
2025-11-27T18:18:59.840118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471384010867467:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:59.853689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471379715900093:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.853753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:59.859199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471379715900093:2050] [node 3] ICP22 created new session: [6:7577471384010867480:2050]
2025-11-27T18:18:59.870511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471384010867478:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46629
2025-11-27T18:18:59.883199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471384010867480:2050] [node 3] ICS09 handshake done sender: [6:7577471384010867465:2115] self: [6:7577471384010867470:0] peer: [3:7577471385341021572:0] socket: 623 qp: -1
2025-11-27T18:18:59.889808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471384010867480:2050] [node 3] ICS10 traffic start
2025-11-27T18:18:59.889850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:59.894526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471384010867447:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.905464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:59.917825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471384010867447:2105] [node 5] ICP22 created new session: [6:7577471384010867483:2105]
2025-11-27T18:18:59.925847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471384010867483:2105] [node 5] ICS09 handshake done sender: [6:7577471384010867467:2117] self: [6:7577471384010867471:0] peer: [5:7577471383343742874:0] socket: 669 qp: -1
2025-11-27T18:18:59.787683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:59.933533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471384010867466:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:59.906184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471382652790300:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57094
2025-11-27T18:18:59.907233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471382652790299:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:59.914636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:59.931647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471382652790292:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:59.933025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471378357822905:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11128) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471382652790292:2121] held: no
2025-11-27T18:18:59.933132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471378357822905:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.933166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:59.933212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471378357822905:2049] [node 2] ICP22 created new session: [4:7577471382652790302:2049]
2025-11-27T18:18:59.933273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471382652790302:2049] [node 2] ICS09 handshake done sender: [4:7577471382652790292:2121] self: [4:7577471382652790298:0] peer: [2:7577471381465563313:0] socket: 676 qp: -1
2025-11-27T18:18:59.937494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471382652790302:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:59.925678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:59.865701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471381465563315:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:18:59.787739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471385341021567:2108] [node 4] ICP22 created new session: [3:7577471385341021585:2108]
2025-11-27T18:18:59.938660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471384010867483:2105] [node 5] ICS10 traffic start
2025-11-27T18:18:59.871074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471381465563328:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.937546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:59.787798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471385341021585:2108] [node 4] ICS09 handshake done sender: [3:7577471385341021571:2111] self: [3:7577471385341021570:0] peer: [4:7577471382652790289:0] socket: 650 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.931968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471380969863541:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.960100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471379048775503:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29425) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471383343742892:2122] held: no
2025-11-27T18:18:59.945342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:59.787838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471385341021585:2108] [node 4] ICS10 traffic start
2025-11-27T18:18:59.787874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:59.788854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471385341021584:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61810
2025-11-27T18:18:59.797859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471385341021573:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:18:59.824969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471385341021581:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61810
2025-11-27T18:18:59.837917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471385341021584:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.838288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471385341021566:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.838326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:18:59.838373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471385341021566:2109] [node 6] ICP22 created new session: [3:7577471385341021589:2109]
2025-11-27T18:18:59.838417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471385341021589:2109] [node 6] ICS09 handshake done sender: [3:7577471385341021573:2112] self: [3:7577471385341021572:0] peer: [6:7577471384010867470:0] socket: 654 qp: -1
2025-11-27T18:18:59.838453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471385341021589:2109] [node 6] ICS10 traffic start
2025-11-27T18:18:59.838494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:18:59.838917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471385341021581:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.839319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471381046054206:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.877388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471385341021583:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61810
2025-11-27T18:18:59.885968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471385341021583:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.886085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471381046054207:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:18:59.893407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471385341021578:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.893444Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471385341021578:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:18:59.905705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471385341021578:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:18:59.913989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471385341021597:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61810
2025-11-27T18:18:59.914099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471385341021598:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61810
2025-11-27T18:18:59.943048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471385341021565:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.943103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:18:59.949452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471385341021565:2107] [node 5] ICP22 created new session: [3:7577471385341021599:2107]
2025-11-27T18:18:59.949519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471385341021599:2107] [node 5] ICS09 handshake done sender: [3:7577471385341021578:2113] self: [3:7577471385341021577:0] peer: [5:7577471383343742898:0] socket: 679 qp: -1
2025-11-27T18:18:59.949567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471385341021599:2107] [node 5] ICS10 traffic start
2025-11-27T18:18:59.955421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471381046054206:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471385341021581:2115] held: no
2025-11-27T18:18:59.956999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471385341021581:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:18:59.957710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:18:59.960666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471385341021583:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:18:59.961983Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471385341021565:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(18697688ns) {HandleHandshakeStatus:329(18694730ns) {DropIncomingHandshake:442(14641ns) DropOutgoingHandshake:457(226ns) SwitchToState:179(295ns) ProcessPendingSessionEvents:495(111933ns) {ForwardSessionEventToSession:612(59265ns) ForwardSessionEventToSession:612(6775ns) ForwardSessionEventToSession:612(2792ns) ForwardSessionEventToSession:612(8647ns) ForwardSessionEventToSession:612(2418ns) ForwardSessionEventToSession:612(3277ns) ForwardSessionEventToSession:612(8342ns) ForwardSessionEventToSession:612(3228ns) ForwardSessionEventToSession:612(2616ns) ForwardSessionEventToSession:612(7827ns)}}}
2025-11-27T18:18:59.962060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471381046054207:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11128) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471385341021583:2116] held: no
2025-11-27T18:18:59.965947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471385341021595:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61810
2025-11-27T18:18:59.968206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471381046054207:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.969886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:59.886334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471381465563314:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:18:59.932036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:59.989500Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471384010867447:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(88953612ns) {HandleHandshakeStatus:329(88949672ns) {DropIncomingHandshake:442(6310ns) DropOutgoingHandshake:457(481ns) SwitchToState:179(925ns) ProcessPendingSessionEvents:495(1239ns)}}
2025-11-27T18:18:59.973393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471378357822905:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(24370580ns) {HandleHandshakeStatus:329(24369072ns) {DropIncomingHandshake:442(425ns) DropOutgoingHandshake:457(142ns) SwitchToState:179(392ns) ProcessPendingSessionEvents:495(1469ns)}}
2025-11-27T18:18:59.977706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471381046054207:2049] [node 2] ICP22 created new session: [3:7577471385341021601:2049]
2025-11-27T18:18:59.886361Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471381465563314:2116] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:18:59.932089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471380969863541:2049] [node 3] ICP22 created new session: [1:7577471385264830939:2049]
2025-11-27T18:19:00.001334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471382652790266:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(83468224ns) {HandleHandshakeStatus:329(83465011ns) {DropIncomingHandshake:442(20676ns) DropOutgoingHandshake:457(459ns) SwitchToState:179(507ns) ProcessPendingSessionEvents:495(7239315ns) {ForwardSessionEventToSession:612(7167464ns) ForwardSessionEventToSession:612(9592ns) ForwardSessionEventToSession:612(3887ns) ForwardSessionEventToSession:612(3839ns) ForwardSessionEventToSession:612(5829ns) ForwardSessionEventToSession:612(3825ns) ForwardSessionEventToSession:612(13566ns) ForwardSessionEventToSession:612(9441ns) ForwardSessionEventToSession:612(2998ns) ForwardSessionEventToSession:612(5103ns)}}}
2025-11-27T18:18:59.932142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471385264830939:2049] [node 3] ICS09 handshake done sender: [1:7577471385264830913:2114] self: [1:7577471385264830912:0] peer: [3:7577471385341021593:0] socket: 666 qp: -1
2025-11-27T18:19:00.005736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471383343742899:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34061
2025-11-27T18:18:59.977781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471385341021601:2049] [node 2] ICS09 handshake done sender: [3:7577471385341021583:2116] self: [3:7577471385341021594:0] peer: [2:7577471381465563302:0] socket: 673 qp: -1
2025-11-27T18:18:59.932183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471385264830939:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:59.977822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471385341021601:2049] [node 2] ICS10 traffic start
2025-11-27T18:18:59.932221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:18:59.977868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:18:59.985815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471385264830903:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(111188158ns) {HandleHandshakeStatus:329(111183025ns) {DropIncomingHandshake:442(24683ns) DropOutgoingHandshake:457(528ns) SwitchToState:179(906ns) ProcessPendingSessionEvents:495(39793421ns) {ForwardSessionEventToSession:612(16559457ns) ForwardSessionEventToSession:612(7782ns) ForwardSessionEventToSession:612(3949021ns) ForwardSessionEventToSession:612(3714ns) ForwardSessionEventToSession:612(3193868ns) ForwardSessionEventToSession:612(4030ns) ForwardSessionEventToSession:612(3739ns) ForwardSessionEventToSession:612(5285866ns) ForwardSessionEventToSession:612(10747857ns) ForwardSessionEventToSession:612(11227ns)}}}
2025-11-27T18:18:59.978017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471381046054206:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.978055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:18:59.978091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471381046054206:2048] [node 1] ICP22 created new session: [3:7577471385341021603:2048]
2025-11-27T18:18:59.978152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471385341021603:2048] [node 1] ICS09 handshake done sender: [3:7577471385341021581:2115] self: [3:7577471385341021593:0] peer: [1:7577471385264830912:0] socket: 641 qp: -1
2025-11-27T18:18:59.978185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471385341021603:2048] [node 1] ICS10 traffic start
2025-11-27T18:18:59.978212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:18:59.993930Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471379715900093:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(107287498ns) {HandleHandshakeStatus:329(107284146ns) {DropIncomingHandshake:442(1205ns) DropOutgoingHandshake:457(4543ns) SwitchToState:179(688ns) ProcessPendingSessionEvents:495(35550196ns) {ForwardSessionEventToSession:612(8162155ns) ForwardSessionEventToSession:612(4993790ns) ForwardSessionEventToSession:612(4021ns) ForwardSessionEventToSession:612(7249539ns) ForwardSessionEventToSession:612(6471ns) ForwardSessionEventToSession:612(2901ns) ForwardSessionEventToSession:612(11743923ns) ForwardSessionEventToSession:612(4710ns) ForwardSessionEventToSession:612(3026ns) ForwardSessionEventToSession:612(3354814ns)}}}
2025-11-27T18:18:59.998427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471379715900092:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:18:59.998513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:18:59.889392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471381465563297:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.889450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:18:59.901462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471381465563297:2109] [node 4] ICP22 created new session: [2:7577471381465563332:2109]
2025-11-27T18:18:59.901538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471381465563332:2109] [node 4] ICS09 handshake done sender: [2:7577471381465563315:2117] self: [2:7577471381465563313:0] peer: [4:7577471382652790298:0] socket: 647 qp: -1
2025-11-27T18:18:59.901580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471381465563332:2109] [node 4] ICS10 traffic start
2025-11-27T18:18:59.901623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:18:59.914486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471381465563314:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:18:59.942161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471377170595937:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:18:59.942226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:18:59.942289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471377170595937:2049] [node 3] ICP22 created new session: [2:7577471381465563334:2049]
2025-11-27T18:18:59.942351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471381465563334:2049] [node 3] ICS09 handshake done sender: [2:7577471381465563314:2116] self: [2:7577471381465563302:0] peer: [3:7577471385341021594:0] socket: 652 qp: -1
2025-11-27T18:18:59.942386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471381465563334:2049] [node 3] ICS10 traffic start
2025-11-27T18:18:59.942421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:00.014417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471379715900092:2049] [node 2] ICP22 created new session: [6:7577471384010867485:2049]
2025-11-27T18:19:00.014501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471384010867485:2049] [node 2] ICS09 handshake done sender: [6:7577471384010867466:2116] self: [6:7577471384010867474:0] peer: [2:7577471381465563306:0] socket: 668 qp: -1
2025-11-27T18:19:00.014548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471384010867485:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:00.014600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:00.018013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471384010867481:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46629
2025-11-27T18:19:00.018834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471384010867461:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:00.019014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471379715900091:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471384010867468:2118] held: no
2025-11-27T18:19:00.019441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471384010867453:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1759) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471384010867461:2113] held: no
2025-11-27T18:19:00.019532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471384010867453:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:00.019567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:00.019606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471384010867453:2110] [node 4] ICP22 created new session: [6:7577471388305834783:2110]
2025-11-27T18:19:00.019649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471388305834783:2110] [node 4] ICS09 handshake done sender: [6:7577471384010867461:2113] self: [6:7577471384010867473:0] peer: [4:7577471382652790270:0] socket: 642 qp: -1
2025-11-27T18:19:00.019683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471388305834783:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:00.019718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:00.020384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471384010867468:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:00.033498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471379715900091:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:00.037697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:00.042000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471379715900091:2048] [node 1] ICP22 created new session: [6:7577471388305834786:2048]
2025-11-27T18:19:00.033593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471383343742892:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:00.042070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471388305834786:2048] [node 1] ICS09 handshake done sender: [6:7577471384010867468:2118] self: [6:7577471384010867472:0] peer: [1:7577471385264830914:0] socket: 670 qp: -1
2025-11-27T18:19:00.049510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471388305834786:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:00.053535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471379715900092:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(51480973ns) {HandleHandshakeStatus:329(51477187ns) {DropIncomingHandshake:442(2875ns) DropOutgoingHandshake:457(6644ns) SwitchToState:179(746ns) ProcessPendingSessionEvents:495(30740670ns) {ForwardSessionEventToSession:612(14734133ns) ForwardSessionEventToSession:612(4099905ns) ForwardSessionEventToSession:612(4202ns) ForwardSessionEventToSession:612(486846ns) ForwardSessionEventToSession:612(3098ns) ForwardSessionEventToSession:612(13357ns) ForwardSessionEventToSession:612(3687962ns) ForwardSessionEventToSession:612(3207ns) ForwardSessionEventToSession:612(7605527ns) ForwardSessionEventToSession:612(64523ns)}}}
2025-11-27T18:19:00.057421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:00.057691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471379715900091:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24098110ns) {HandleHandshakeStatus:329(24094712ns) {DropIncomingHandshake:442(974ns) DropOutgoingHandshake:457(434ns) SwitchToState:179(538ns) ProcessPendingSessionEvents:495(1015ns)}}
2025-11-27T18:19:00.045464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471379048775503:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:00.049585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:00.061479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471379048775503:2050] [node 3] ICP22 created new session: [5:7577471387638710196:2050]
2025-11-27T18:19:00.062859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471387638710196:2050] [node 3] ICS09 handshake done sender: [5:7577471383343742892:2122] self: [5:7577471383343742898:0] peer: [3:7577471385341021577:0] socket: 680 qp: -1
2025-11-27T18:19:00.062926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471387638710196:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:00.062981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:00.063219Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471379048775503:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(21083720ns) {HandleHandshakeStatus:329(21079659ns) {DropIncomingHandshake:442(2499ns) DropOutgoingHandshake:457(479ns) SwitchToState:179(715ns) ProcessPendingSessionEvents:495(1006ns)}}
2025-11-27T18:19:00.128224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:00.128465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471381465563321:2048] [node 1] ICS01 socket: 619 reason# EndOfStream
2025-11-27T18:19:00.128511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471377170595936:2048] [node 1] ICP30 unregister session Session# [2:7577471381465563321:2048] VirtualId# [2:7577471381465563305:0]
2025-11-27T18:19:00.128548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471381465563321:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.128629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:00.128680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:00.130639Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:00.130874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471385341021603:2048] [node 1] ICS01 socket: 641 reason# EndOfStream
2025-11-27T18:19:00.130911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471381046054206:2048] [node 1] ICP30 unregister session Session# [3:7577471385341021603:2048] VirtualId# [3:7577471385341021593:0]
2025-11-27T18:19:00.130953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471385341021603:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.131034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:00.131072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:00.137489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:00.137718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471383343742889:2048] [node 1] ICS01 socket: 632 reason# EndOfStream
2025-11-27T18:19:00.137417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:00.137751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471379048775501:2048] [node 1] ICP30 unregister session Session# [5:7577471383343742889:2048] VirtualId# [5:7577471383343742877:0]
2025-11-27T18:19:00.137787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471383343742889:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.137869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:00.137918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:00.137660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471388305834786:2048] [node 1] ICS01 socket: 670 reason# EndOfStream
2025-11-27T18:19:00.137704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471379715900091:2048] [node 1] ICP30 unregister session Session# [6:7577471388305834786:2048] VirtualId# [6:7577471384010867472:0]
2025-11-27T18:19:00.137738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471388305834786:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.137819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:00.137865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:00.145415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:00.145678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471382652790291:2048] [node 1] ICS01 socket: 628 reason# EndOfStream
2025-11-27T18:19:00.145728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471378357822904:2048] [node 1] ICP30 unregister session Session# [4:7577471382652790291:2048] VirtualId# [4:7577471382652790278:0]
2025-11-27T18:19:00.145773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471382652790291:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.145870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:00.145918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:00.206881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:00.207133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471385341021601:2049] [node 2] ICS01 socket: 673 reason# EndOfStream
2025-11-27T18:19:00.207181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471381046054207:2049] [node 2] ICP30 unregister session Session# [3:7577471385341021601:2049] VirtualId# [3:7577471385341021594:0]
2025-11-27T18:19:00.207220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471385341021601:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.207312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:00.207367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:00.213475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:00.213723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471383343742893:2049] [node 2] ICS01 socket: 656 reason# EndOfStream
2025-11-27T18:19:00.213775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471379048775502:2049] [node 2] ICP30 unregister session Session# [5:7577471383343742893:2049] VirtualId# [5:7577471383343742886:0]
2025-11-27T18:19:00.213810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471383343742893:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.213895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:00.213952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:00.213476Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:00.213722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471382652790302:2049] [node 2] ICS01 socket: 676 reason# EndOfStream
2025-11-27T18:19:00.213774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471378357822905:2049] [node 2] ICP30 unregister session Session# [4:7577471382652790302:2049] VirtualId# [4:7577471382652790298:0]
2025-11-27T18:19:00.213810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471382652790302:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.213897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:00.213940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:00.214034Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:00.214226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471384010867485:2049] [node 2] ICS07 socket disconnect 668 reason# EndOfStream
2025-11-27T18:19:00.214255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471384010867485:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.214330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:00.214370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471384010867485:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:00.215904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471379715900092:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11128) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:00.215968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471384010867485:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:00.216002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471379715900092:2049] [node 2] ICP30 unregister session Session# [6:7577471384010867485:2049] VirtualId# [6:7577471384010867474:0]
2025-11-27T18:19:00.216050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:00.216088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471379715900092:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:00.216108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:00.229398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471379715900092:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:00.277593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:00.277773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471387638710196:2050] [node 3] ICS01 socket: 680 reason# EndOfStream
2025-11-27T18:19:00.277804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471379048775503:2050] [node 3] ICP30 unregister session Session# [5:7577471387638710196:2050] VirtualId# [5:7577471383343742898:0]
2025-11-27T18:19:00.277832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471387638710196:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.277894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:00.277925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:00.278024Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:00.278461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:00.278639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471384010867480:2050] [node 3] ICS07 socket disconnect 623 reason# EndOfStream
2025-11-27T18:19:00.278669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471384010867480:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.278739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:00.278774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471384010867480:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:00.293513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471379715900093:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29425) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:00.297038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471384010867480:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:00.297090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471379715900093:2050] [node 3] ICP30 unregister session Session# [6:7577471384010867480:2050] VirtualId# [6:7577471384010867470:0]
2025-11-27T18:19:00.297155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:00.297201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471379715900093:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29425) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:00.297218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29425) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:00.307316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471379715900093:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:00.340577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:00.340798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471388305834783:2110] [node 4] ICS01 socket: 642 reason# EndOfStream
2025-11-27T18:19:00.340830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471384010867453:2110] [node 4] ICP30 unregister session Session# [6:7577471388305834783:2110] VirtualId# [6:7577471384010867473:0]
2025-11-27T18:19:00.340857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471388305834783:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.340949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:00.340998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:00.341499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:00.341708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471383343742882:2109] [node 4] ICS01 socket: 634 reason# EndOfStream
2025-11-27T18:19:00.341741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471383343742864:2109] [node 4] ICP30 unregister session Session# [5:7577471383343742882:2109] VirtualId# [5:7577471383343742878:0]
2025-11-27T18:19:00.341771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471383343742882:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.341856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:00.341899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:00.393446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:00.393700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471384010867483:2105] [node 5] ICS01 socket: 669 reason# EndOfStream
2025-11-27T18:19:00.393756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471384010867447:2105] [node 5] ICP30 unregister session Session# [6:7577471384010867483:2105] VirtualId# [6:7577471384010867471:0]
2025-11-27T18:19:00.393787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471384010867483:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:00.393871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:00.393919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:19:00.485044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471388852542874:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:00.485097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471388852542875:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:00.514723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471388490714871:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:00.514769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471388490714872:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:00.538822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471386074973352:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:00.538861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471386074973353:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:00.561180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471387987726841:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:00.561225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471387987726842:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:00.561483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471387987726843:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:00.583292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471386982054106:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:00.583327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471386982054107:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:00.583355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471386982054108:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:00.605278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471389034177702:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:00.605318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471389034177703:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:00.605348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471389034177704:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:00.763289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471389034177764:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:00.774297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471388852542938:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:00.795178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471388490714934:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:00.861108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471388852542941:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:00.930895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471389034177769:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:00.910245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471388490714936:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:00.910642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471388490714937:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:00.942120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471388490714933:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-11-27T18:19:00.950234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471386982054170:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:00.950332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471386982054171:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:00.951479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471386074973415:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:00.957580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471386982054167:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56109
2025-11-27T18:19:00.937371Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471388852542938:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(47863219ns) {Configure:104(47857926ns) {ProcessConfigured:135(77782ns) {StartInitialHandshake:157(74794ns) {DropHandshakes:470(5593ns) {DropIncomingHandshake:442(639ns) DropOutgoingHandshake:457(276ns)} GenerateSessionVirtualId:988(681ns) PrepareNewSessionHandshake:477(1248ns) {SwitchToState:179(328ns)}}}}}
2025-11-27T18:19:00.951235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471388852542947:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:00.967692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471389034177704:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(90111037ns) {Configure:104(90107624ns) {ProcessConfigured:135(72346ns) {StartInitialHandshake:157(70315ns) {DropHandshakes:470(2666ns) {DropIncomingHandshake:442(599ns) DropOutgoingHandshake:457(614ns)} GenerateSessionVirtualId:988(1140ns) PrepareNewSessionHandshake:477(1664ns) {SwitchToState:179(394ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:00.987776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471386074973353:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36002041ns) {Configure:104(35998362ns) {ProcessConfigured:135(6199868ns) {StartInitialHandshake:157(6196127ns) {DropHandshakes:470(3874ns) {DropIncomingHandshake:442(1120ns) DropOutgoingHandshake:457(590ns)} GenerateSessionVirtualId:988(983ns) PrepareNewSessionHandshake:477(2375ns) {SwitchToState:179(185ns)}}}}}
2025-11-27T18:19:00.990471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471386074973420:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-27T18:19:00.990541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471386074973421:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-27T18:19:00.987103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471387987726843:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(123663947ns) {Configure:104(101369496ns) {ProcessConfigured:135(75686ns) {StartInitialHandshake:157(73209ns) {DropHandshakes:470(3522ns) {DropIncomingHandshake:442(743ns) DropOutgoingHandshake:457(403ns)} GenerateSessionVirtualId:988(872ns) PrepareNewSessionHandshake:477(2146ns) {SwitchToState:179(864ns)}}}}}
2025-11-27T18:19:00.959187Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471388490714936:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(46537277ns) {Configure:104(46533348ns) {ProcessConfigured:135(69860ns) {StartInitialHandshake:157(67733ns) {DropHandshakes:470(1939ns) {DropIncomingHandshake:442(338ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(692ns) PrepareNewSessionHandshake:477(1977ns) {SwitchToState:179(854ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.000116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471387987726842:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(138314833ns) {Configure:104(138308807ns) {ProcessConfigured:135(79691ns) {StartInitialHandshake:157(77101ns) {DropHandshakes:470(2634ns) {DropIncomingHandshake:442(729ns) DropOutgoingHandshake:457(359ns)} GenerateSessionVirtualId:988(931ns) PrepareNewSessionHandshake:477(2165ns) {SwitchToState:179(807ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.000955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471387987726909:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42317
2025-11-27T18:19:00.960497Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471388490714934:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(164926317ns) {Configure:104(164921168ns) {ProcessConfigured:135(6946461ns) {StartInitialHandshake:157(6944039ns) {DropHandshakes:470(1276ns) {DropIncomingHandshake:442(670ns) DropOutgoingHandshake:457(94ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:477(2472ns) {SwitchToState:179(1091ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:00.983357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471388852542875:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(203231861ns) {Configure:104(203225594ns) {ProcessConfigured:135(58485ns) {StartInitialHandshake:157(56518ns) {DropHandshakes:470(2669ns) {DropIncomingHandshake:442(619ns) DropOutgoingHandshake:457(670ns)} GenerateSessionVirtualId:988(732ns) PrepareNewSessionHandshake:477(766ns) {SwitchToState:179(306ns)}}}}}
2025-11-27T18:19:00.982739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471388490714871:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(189472733ns) {Configure:104(189466967ns) {ProcessConfigured:135(8629143ns) {StartInitialHandshake:157(8626053ns) {DropHandshakes:470(1222ns) {DropIncomingHandshake:442(300ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(413ns) PrepareNewSessionHandshake:477(4121ns) {SwitchToState:179(1607ns)}}}}}
2025-11-27T18:19:01.016612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471388490714933:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.017550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471392785682244:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-11-27T18:19:01.001137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471387987726910:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42317
2025-11-27T18:19:01.001646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471387987726909:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.015932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471387987726906:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:01.017736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471387987726910:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.008721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471386074973424:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.029473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471388490714871:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.008894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471386074973425:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.061473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471393147510251:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42018
2025-11-27T18:19:01.042423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471392282694209:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:01.042351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471386982054107:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(87158528ns) {Configure:104(87153895ns) {ProcessConfigured:135(9045406ns) {StartInitialHandshake:157(9041816ns) {DropHandshakes:470(2847ns) {DropIncomingHandshake:442(579ns) DropOutgoingHandshake:457(619ns)} GenerateSessionVirtualId:988(1072ns) PrepareNewSessionHandshake:477(3237ns) {SwitchToState:179(1609ns)}}}}}
2025-11-27T18:19:01.041340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471392785682245:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-11-27T18:19:01.045457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471386074973352:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(226038188ns) {Configure:104(220111182ns) {ProcessConfigured:135(52655ns) {StartInitialHandshake:157(47402ns) {DropHandshakes:470(2085ns) {DropIncomingHandshake:442(533ns) DropOutgoingHandshake:457(363ns)} GenerateSessionVirtualId:988(838ns) PrepareNewSessionHandshake:477(917ns) {SwitchToState:179(382ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.064679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471392785682248:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-11-27T18:19:01.073319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471388852542935:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.053515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471386982054106:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(83788999ns) {Configure:104(83783935ns) {ProcessConfigured:135(15791036ns) {StartInitialHandshake:157(15787176ns) {DropHandshakes:470(2291ns) {DropIncomingHandshake:442(483ns) DropOutgoingHandshake:457(345ns)} GenerateSessionVirtualId:988(807ns) PrepareNewSessionHandshake:477(3604ns) {SwitchToState:179(541ns)}}}}}
2025-11-27T18:19:01.045627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471386074973421:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.073378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471388852542935:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.098924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471388852542935:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:01.099624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471388852542874:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.099671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:01.099716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471388852542874:2048] [node 2] ICP22 created new session: [1:7577471393147510253:2048]
2025-11-27T18:19:01.099760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471393147510253:2048] [node 2] ICS09 handshake done sender: [1:7577471388852542935:2107] self: [1:7577471388852542934:0] peer: [2:7577471392785682247:0] socket: 625 qp: -1
2025-11-27T18:19:01.099799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471393147510253:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:01.099830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.053863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471386982054167:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.111860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471393147510249:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42018
2025-11-27T18:19:01.046032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471390369940722:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-27T18:19:01.077535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471392785682245:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.111127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471393329145075:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25187
2025-11-27T18:19:01.054592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471391277021472:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56109
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.071420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471390369940722:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.085316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471392785682244:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.127925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471388852542945:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.128198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471393329145075:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.123880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471387987726906:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(57516746ns) {Configure:104(57515175ns) {ProcessConfigured:135(88949ns) {StartInitialHandshake:157(45901ns) {DropHandshakes:470(916ns) {DropIncomingHandshake:442(250ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(4092ns) PrepareNewSessionHandshake:477(651ns) {SwitchToState:179(356ns)}} IncomingHandshake:261(19432ns) {DropIncomingHandshake:442(159ns) PrepareNewSessionHandshake:477(250ns) {SwitchToState:179(96ns)} GenerateSessionVirtualId:988(109ns) IssueIncomingHandshakeReply:193(778ns)}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.054653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471391277021473:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56109
2025-11-27T18:19:01.135883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471392282694218:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42317
2025-11-27T18:19:01.142323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471392282694218:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.157739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471387987726841:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(140958585ns) {Configure:104(140953129ns) {ProcessConfigured:135(6301493ns) {StartInitialHandshake:157(6298258ns) {DropHandshakes:470(1319ns) {DropIncomingHandshake:442(326ns) DropOutgoingHandshake:457(382ns)} GenerateSessionVirtualId:988(1727ns) PrepareNewSessionHandshake:477(2164ns) {SwitchToState:179(812ns)}}}}}
2025-11-27T18:19:01.157781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471387987726843:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.097827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471388490714937:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(144231560ns) {Configure:104(144222487ns) {ProcessConfigured:135(59327ns) {StartInitialHandshake:157(56130ns) {DropHandshakes:470(1867ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(626ns) PrepareNewSessionHandshake:477(1096ns) {SwitchToState:179(340ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.164254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471387987726841:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.141525Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471389034177702:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(170844302ns) {Configure:104(160741611ns) {ProcessConfigured:135(6823184ns) {StartInitialHandshake:157(6819707ns) {DropHandshakes:470(2037ns) {DropIncomingHandshake:442(422ns) DropOutgoingHandshake:457(563ns)} GenerateSessionVirtualId:988(415ns) PrepareNewSessionHandshake:477(2139ns) {SwitchToState:179(1015ns)}}}}}
2025-11-27T18:19:01.127977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471388852542945:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.121656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471386074973425:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(25599841ns) {IncomingHandshake:261(9193385ns) {DropIncomingHandshake:442(678ns) PrepareNewSessionHandshake:477(2249661ns) {SwitchToState:179(4603ns)} GenerateSessionVirtualId:988(628ns) IssueIncomingHandshakeReply:193(8483ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.054813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471391277021474:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56109
2025-11-27T18:19:01.098387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471392785682252:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-11-27T18:19:01.130880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471390369940729:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.167286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471392282694211:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.141668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471389034177702:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.137627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471393147510250:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42018
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.171771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471388490714939:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.089322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471391277021473:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.167326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471392282694211:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.118834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471386982054108:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(67055179ns) {Configure:104(67044778ns) {ProcessConfigured:135(58102ns) {StartInitialHandshake:157(55763ns) {DropHandshakes:470(2045ns) {DropIncomingHandshake:442(665ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(971ns) PrepareNewSessionHandshake:477(1432ns) {SwitchToState:179(431ns)}}}}}
2025-11-27T18:19:01.119408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471391277021477:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56109
2025-11-27T18:19:01.119530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471386982054106:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.131554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471391277021474:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.132910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471386982054108:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.143792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471391277021477:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.133330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471386074973419:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.166882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471393329145079:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25187
2025-11-27T18:19:01.145500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471388852542949:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.179238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471392282694222:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42317
2025-11-27T18:19:01.171795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471388490714939:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.144472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471391277021487:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56109
2025-11-27T18:19:01.133353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471386074973419:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.145523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471388852542949:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.177607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471393329145078:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25187
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.179462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471392282694211:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:01.172560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471388490714942:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.148999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471386982054108:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25701) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471391277021474:2115] held: no
2025-11-27T18:19:01.134299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471386074973419:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.148652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471388852542949:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:01.180260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471393329145075:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:01.172584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471388490714942:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.179826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471392282694222:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.149247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471391277021474:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:01.201085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471389034177702:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.134619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471386074973415:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.211174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:01.218391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471389034177702:2048] [node 1] ICP22 created new session: [6:7577471393329145080:2048]
2025-11-27T18:19:01.221527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471393329145080:2048] [node 1] ICS09 handshake done sender: [6:7577471393329145075:2114] self: [6:7577471393329145077:0] peer: [1:7577471388852542948:0] socket: 667 qp: -1
2025-11-27T18:19:01.225174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471393329145079:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.228705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471393329145081:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25187
2025-11-27T18:19:01.231231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471393329145080:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:01.233775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471389034177764:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.236650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471393329145081:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.238082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:01.238300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471389034177769:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.242961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471393329145086:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25187
2025-11-27T18:19:01.243026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471393329145087:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25187
2025-11-27T18:19:01.257116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471393329145087:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.257288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471389034177704:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.273640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471393329145086:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.278819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471389034177703:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.279932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471389034177702:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(78603538ns) {HandleHandshakeStatus:329(78601175ns) {DropIncomingHandshake:442(848ns) DropOutgoingHandshake:457(21649ns) SwitchToState:179(439ns) ProcessPendingSessionEvents:495(7181960ns) {ForwardSessionEventToSession:612(7125254ns) ForwardSessionEventToSession:612(5037ns) ForwardSessionEventToSession:612(6823ns) ForwardSessionEventToSession:612(3642ns) ForwardSessionEventToSession:612(2636ns) ForwardSessionEventToSession:612(10467ns) ForwardSessionEventToSession:612(2448ns) ForwardSessionEventToSession:612(5231ns) ForwardSessionEventToSession:612(2276ns) ForwardSessionEventToSession:612(2567ns)}}}
2025-11-27T18:19:01.280642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471393329145091:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25187
2025-11-27T18:19:01.150397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471388852542947:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.173411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471392785682248:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.179879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471392282694209:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.293970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471393329145087:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.294270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471393329145092:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25187
2025-11-27T18:19:01.294608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471389034177704:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.294658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:01.294697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471389034177704:2050] [node 3] ICP22 created new session: [6:7577471393329145093:2050]
2025-11-27T18:19:01.294736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471393329145093:2050] [node 3] ICS09 handshake done sender: [6:7577471393329145087:2120] self: [6:7577471393329145088:0] peer: [3:7577471390369940735:0] socket: 694 qp: -1
2025-11-27T18:19:01.294770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471393329145093:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:01.294800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:01.296373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471393329145086:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:01.134659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:01.149560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471386982054108:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.150442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:01.296664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471389034177703:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.134708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471386074973415:2108] [node 5] ICP22 created new session: [3:7577471390369940731:2108]
2025-11-27T18:19:01.150481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471388852542947:2110] [node 6] ICP22 created new session: [1:7577471393147510258:2110]
2025-11-27T18:19:01.179930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:01.176308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471388490714933:2109] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.149605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:01.206661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471388490714942:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:01.209626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471388490714871:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.209684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:01.209738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471388490714871:2048] [node 1] ICP22 created new session: [2:7577471392785682257:2048]
2025-11-27T18:19:01.209802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471392785682257:2048] [node 1] ICS09 handshake done sender: [2:7577471388490714933:2109] self: [2:7577471392785682247:0] peer: [1:7577471388852542934:0] socket: 627 qp: -1
2025-11-27T18:19:01.209840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471392785682257:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:01.209879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:01.210138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471388490714871:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:01.213402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471388490714939:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:01.221032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471388490714936:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.221091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:01.221136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471388490714936:2107] [node 5] ICP22 created new session: [2:7577471392785682259:2107]
2025-11-27T18:19:01.221177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471392785682259:2107] [node 5] ICS09 handshake done sender: [2:7577471388490714942:2112] self: [2:7577471388490714941:0] peer: [5:7577471391277021497:0] socket: 633 qp: -1
2025-11-27T18:19:01.221208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471392785682259:2107] [node 5] ICS10 traffic start
2025-11-27T18:19:01.221237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:01.234341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471388490714872:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.234397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:01.234444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471388490714872:2049] [node 3] ICP22 created new session: [2:7577471392785682261:2049]
2025-11-27T18:19:01.234493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471392785682261:2049] [node 3] ICS09 handshake done sender: [2:7577471388490714939:2111] self: [2:7577471388490714938:0] peer: [3:7577471390369940737:0] socket: 632 qp: -1
2025-11-27T18:19:01.234524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471392785682261:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:01.234550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:01.245773Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471388490714936:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(24524336ns) {HandleHandshakeStatus:329(24521204ns) {DropIncomingHandshake:442(15112ns) DropOutgoingHandshake:457(328ns) SwitchToState:179(367ns) ProcessPendingSessionEvents:495(12111413ns) {ForwardSessionEventToSession:612(12044332ns) ForwardSessionEventToSession:612(4599ns) ForwardSessionEventToSession:612(6479ns) ForwardSessionEventToSession:612(2939ns) ForwardSessionEventToSession:612(11853ns) ForwardSessionEventToSession:612(14845ns) ForwardSessionEventToSession:612(2890ns) ForwardSessionEventToSession:612(2263ns) ForwardSessionEventToSession:612(9872ns) ForwardSessionEventToSession:612(1933ns)}}}
2025-11-27T18:19:01.249410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471392785682265:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-11-27T18:19:01.283042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471392785682251:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.283059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471392785682251:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.289412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471392785682251:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:01.305984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471388490714945:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.306029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471388490714945:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.330322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471388490714945:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:01.296725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:01.150526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471393147510258:2110] [node 6] ICS09 handshake done sender: [1:7577471388852542949:2113] self: [1:7577471388852542948:0] peer: [6:7577471393329145077:0] socket: 666 qp: -1
2025-11-27T18:19:01.179989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471392282694209:2108] [node 5] ICP22 created new session: [4:7577471392282694223:2108]
2025-11-27T18:19:01.134752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471390369940731:2108] [node 5] ICS09 handshake done sender: [3:7577471386074973419:2111] self: [3:7577471386074973418:0] peer: [5:7577471391277021485:0] socket: 639 qp: -1
2025-11-27T18:19:01.149650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471386982054108:2050] [node 3] ICP22 created new session: [5:7577471391277021489:2050]
2025-11-27T18:19:01.134785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471390369940731:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:01.149690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471391277021489:2050] [node 3] ICS09 handshake done sender: [5:7577471391277021474:2115] self: [5:7577471391277021485:0] peer: [3:7577471386074973418:0] socket: 656 qp: -1
2025-11-27T18:19:01.149724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471391277021489:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:01.149758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:01.155605Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471386982054171:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(166915085ns) {Configure:104(160912223ns) {ProcessConfigured:135(19782149ns) {StartInitialHandshake:157(19778858ns) {DropHandshakes:470(614575ns) {DropIncomingHandshake:442(222ns) DropOutgoingHandshake:457(875ns)} GenerateSessionVirtualId:988(738ns) PrepareNewSessionHandshake:477(2531ns) {SwitchToState:179(1110ns)}}}}}
2025-11-27T18:19:01.155713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471391277021488:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56109
2025-11-27T18:19:01.337907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471388490714937:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.194793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471392282694223:2108] [node 5] ICS09 handshake done sender: [4:7577471392282694211:2114] self: [4:7577471392282694210:0] peer: [5:7577471391277021492:0] socket: 659 qp: -1
2025-11-27T18:19:01.296766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471389034177703:2049] [node 2] ICP22 created new session: [6:7577471393329145095:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.150554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471393147510258:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:01.134826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:01.156673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471386982054171:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.150584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:01.337999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:01.200569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471392282694223:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:01.296817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471393329145095:2049] [node 2] ICS09 handshake done sender: [6:7577471393329145086:2119] self: [6:7577471393329145089:0] peer: [2:7577471392785682250:0] socket: 693 qp: -1
2025-11-27T18:19:01.157018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471391277021473:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:01.146798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471390369940729:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.296846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471393329145095:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:01.155174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471388852542945:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:01.338079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471388490714937:2110] [node 6] ICP22 created new session: [2:7577471392785682268:2110]
2025-11-27T18:19:01.200614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:01.221530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471392282694226:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42317
2025-11-27T18:19:01.237679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471387987726910:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:01.265001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471392282694215:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.265047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471392282694215:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.269623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471387987726843:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25701) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471392282694218:2117] held: no
2025-11-27T18:19:01.269726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471387987726841:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.269780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:01.269830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471387987726841:2048] [node 1] ICP22 created new session: [4:7577471392282694229:2048]
2025-11-27T18:19:01.269879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471392282694229:2048] [node 1] ICS09 handshake done sender: [4:7577471387987726910:2113] self: [4:7577471392282694220:0] peer: [1:7577471388852542942:0] socket: 643 qp: -1
2025-11-27T18:19:01.269920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471392282694229:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:01.269954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:01.271259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471392282694228:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42317
2025-11-27T18:19:01.276654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471392282694228:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.281407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471387987726842:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.285770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471392282694215:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:01.311624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471392282694227:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42317
2025-11-27T18:19:01.316907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471387987726906:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.316967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:01.317019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471387987726906:2109] [node 6] ICP22 created new session: [4:7577471392282694233:2109]
2025-11-27T18:19:01.317070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471392282694233:2109] [node 6] ICS09 handshake done sender: [4:7577471392282694215:2115] self: [4:7577471392282694214:0] peer: [6:7577471393329145085:0] socket: 683 qp: -1
2025-11-27T18:19:01.317100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471392282694233:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:01.317137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:01.318092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471392282694209:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(95713849ns) {HandleHandshakeStatus:329(95710973ns) {DropIncomingHandshake:442(781ns) DropOutgoingHandshake:457(860ns) SwitchToState:179(626ns) ProcessPendingSessionEvents:495(54801190ns) {ForwardSessionEventToSession:612(15186607ns) ForwardSessionEventToSession:612(15273880ns) ForwardSessionEventToSession:612(9088ns) ForwardSessionEventToSession:612(4560934ns) ForwardSessionEventToSession:612(4716ns) ForwardSessionEventToSession:612(3085ns) ForwardSessionEventToSession:612(6483577ns) ForwardSessionEventToSession:612(4606ns) ForwardSessionEventToSession:612(2773ns) ForwardSessionEventToSession:612(13243466ns)}}}
2025-11-27T18:19:01.326922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471392282694235:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42317
2025-11-27T18:19:01.337311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471392282694209:2108] [node 5] ICP18 handshake (actor [4:7577471392282694222:2118]) is too old
2025-11-27T18:19:01.339916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471392282694218:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:01.342281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471392282694228:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:01.348841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471387987726842:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.348889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:01.348932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471387987726842:2049] [node 2] ICP22 created new session: [4:7577471392282694236:2049]
2025-11-27T18:19:01.348979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471392282694236:2049] [node 2] ICS09 handshake done sender: [4:7577471392282694228:2121] self: [4:7577471392282694232:0] peer: [2:7577471388490714940:0] socket: 644 qp: -1
2025-11-27T18:19:01.349014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471392282694236:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:01.349047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:01.350811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471387987726843:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.350848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:01.350894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471387987726843:2050] [node 3] ICP22 created new session: [4:7577471392282694238:2050]
2025-11-27T18:19:01.350937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471392282694238:2050] [node 3] ICS09 handshake done sender: [4:7577471392282694218:2117] self: [4:7577471392282694219:0] peer: [3:7577471390369940726:0] socket: 672 qp: -1
2025-11-27T18:19:01.350961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471392282694238:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:01.350992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:01.165604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471393147510249:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.165701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471393147510250:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.169382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471388852542874:2048] [node 2] ICP18 handshake (actor [1:7577471393147510249:2115]) is too old
2025-11-27T18:19:01.173528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471388852542941:2109] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:01.191584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471388852542943:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.191609Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471388852542943:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.194445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471388852542943:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:01.196207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471388852542941:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.206891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:01.217718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471388852542941:2109] [node 5] ICP22 created new session: [1:7577471393147510263:2109]
2025-11-27T18:19:01.219399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471388852542938:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.219444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:01.219498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471388852542938:2108] [node 4] ICP22 created new session: [1:7577471393147510264:2108]
2025-11-27T18:19:01.219551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471393147510264:2108] [node 4] ICS09 handshake done sender: [1:7577471388852542943:2111] self: [1:7577471388852542942:0] peer: [4:7577471392282694220:0] socket: 631 qp: -1
2025-11-27T18:19:01.219586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471393147510264:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:01.219621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:01.220856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471388852542952:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.220890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471388852542952:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.230152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471388852542952:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:01.232790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471393147510263:2109] [node 5] ICS09 handshake done sender: [1:7577471388852542945:2112] self: [1:7577471388852542944:0] peer: [5:7577471391277021481:0] socket: 635 qp: -1
2025-11-27T18:19:01.232843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471393147510263:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:01.233637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:01.237964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471388852542875:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.243853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:01.251706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471388852542875:2049] [node 3] ICP22 created new session: [1:7577471393147510268:2049]
2025-11-27T18:19:01.275644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471393147510268:2049] [node 3] ICS09 handshake done sender: [1:7577471388852542952:2114] self: [1:7577471388852542951:0] peer: [3:7577471390369940740:0] socket: 676 qp: -1
2025-11-27T18:19:01.278736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471393147510268:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:01.281661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:01.310372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471388852542875:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(72202355ns) {HandleHandshakeStatus:329(72200387ns) {DropIncomingHandshake:442(11461ns) DropOutgoingHandshake:457(1532ns) SwitchToState:179(303ns) ProcessPendingSessionEvents:495(19408054ns) {ForwardSessionEventToSession:612(19344238ns) ForwardSessionEventToSession:612(8654ns) ForwardSessionEventToSession:612(8540ns) ForwardSessionEventToSession:612(5185ns) ForwardSessionEventToSession:612(2662ns) ForwardSessionEventToSession:612(8054ns) ForwardSessionEventToSession:612(2943ns) ForwardSessionEventToSession:612(5352ns) ForwardSessionEventToSession:612(3104ns) ForwardSessionEventToSession:612(2884ns)}}}
2025-11-27T18:19:01.348340Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471388852542941:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(94543686ns) {HandleHandshakeStatus:329(94539558ns) {DropIncomingHandshake:442(400ns) DropOutgoingHandshake:457(414ns) SwitchToState:179(447ns) ProcessPendingSessionEvents:495(28339598ns) {ForwardSessionEventToSession:612(20525245ns) ForwardSessionEventToSession:612(4120ns) ForwardSessionEventToSession:612(2097ns) ForwardSessionEventToSession:612(1619035ns) ForwardSessionEventToSession:612(2223ns) ForwardSessionEventToSession:612(4912154ns) ForwardSessionEventToSession:612(772551ns) ForwardSessionEventToSession:612(2949ns) ForwardSessionEventToSession:612(478875ns) ForwardSessionEventToSession:612(3645ns)}}}
2025-11-27T18:19:01.146911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471386074973415:2108] [node 5] ICP18 handshake (actor [3:7577471390369940729:2117]) is too old
2025-11-27T18:19:01.151673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471390369940734:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-27T18:19:01.164254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471386074973420:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.166257Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471386074973424:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(144443929ns) {Configure:104(144437236ns) {ProcessConfigured:135(12737596ns) {StartInitialHandshake:157(12734877ns) {DropHandshakes:470(1475ns) {DropIncomingHandshake:442(396ns) DropOutgoingHandshake:457(201ns)} GenerateSessionVirtualId:988(546ns) PrepareNewSessionHandshake:477(2974ns) {SwitchToState:179(1902ns)}}}}}
2025-11-27T18:19:01.166325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471386074973353:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.167228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471390369940734:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.191211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471390369940727:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.191270Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471390369940727:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.197387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471386074973352:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.197538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471386074973352:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17048155ns) {IncomingHandshake:261(17046047ns) {DropIncomingHandshake:442(1994ns) PrepareNewSessionHandshake:477(4263ns) {SwitchToState:179(3573ns)} GenerateSessionVirtualId:988(695ns) IssueIncomingHandshakeReply:193(48619ns) {SwitchToState:179(118ns)}}}
2025-11-27T18:19:01.197916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471390369940739:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-27T18:19:01.211367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471390369940727:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:01.231155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471386074973425:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.236032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:01.238582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471386074973420:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:01.240760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471386074973353:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.240801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:01.240843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471386074973353:2049] [node 2] ICP22 created new session: [3:7577471390369940744:2049]
2025-11-27T18:19:01.240888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471390369940744:2049] [node 2] ICS09 handshake done sender: [3:7577471386074973420:2112] self: [3:7577471390369940737:0] peer: [2:7577471388490714938:0] socket: 636 qp: -1
2025-11-27T18:19:01.240918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471390369940744:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:01.240948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:01.243760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471386074973425:2107] [node 4] ICP22 created new session: [3:7577471390369940742:2107]
2025-11-27T18:19:01.255272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471390369940746:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25956
2025-11-27T18:19:01.256727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471386074973352:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8868) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471390369940734:2118] held: no
2025-11-27T18:19:01.272307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471390369940742:2107] [node 4] ICS09 handshake done sender: [3:7577471390369940727:2116] self: [3:7577471390369940726:0] peer: [4:7577471392282694219:0] socket: 671 qp: -1
2025-11-27T18:19:01.272355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471390369940742:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:01.275362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:01.277533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471390369940736:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.277555Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471390369940736:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.278343Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471390369940734:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:01.289881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471390369940736:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:01.298527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471386074973352:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.298576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:01.298615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471386074973352:2048] [node 1] ICP22 created new session: [3:7577471390369940749:2048]
2025-11-27T18:19:01.298669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471390369940749:2048] [node 1] ICS09 handshake done sender: [3:7577471390369940734:2118] self: [3:7577471390369940740:0] peer: [1:7577471388852542951:0] socket: 677 qp: -1
2025-11-27T18:19:01.298697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471390369940749:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:01.298727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:01.299013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471386074973424:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.299057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:01.299093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471386074973424:2109] [node 6] ICP22 created new session: [3:7577471390369940751:2109]
2025-11-27T18:19:01.299130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471390369940751:2109] [node 6] ICS09 handshake done sender: [3:7577471390369940736:2119] self: [3:7577471390369940735:0] peer: [6:7577471393329145088:0] socket: 662 qp: -1
2025-11-27T18:19:01.299157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471390369940751:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:01.299188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:01.364603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471386074973425:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(106137116ns) {HandleHandshakeStatus:329(106133587ns) {DropIncomingHandshake:442(16889ns) DropOutgoingHandshake:457(465ns) SwitchToState:179(2700ns) ProcessPendingSessionEvents:495(51091695ns) {ForwardSessionEventToSession:612(24779317ns) ForwardSessionEventToSession:612(7069725ns) ForwardSessionEventToSession:612(7491234ns) ForwardSessionEventToSession:612(4623ns) ForwardSessionEventToSession:612(2785ns) ForwardSessionEventToSession:612(1794480ns) ForwardSessionEventToSession:612(3944ns) ForwardSessionEventToSession:612(2599ns) ForwardSessionEventToSession:612(3191714ns) ForwardSessionEventToSession:612(6718763ns)}}}
2025-11-27T18:19:01.161857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471386982054106:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.161904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:01.161941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471386982054106:2048] [node 1] ICP22 created new session: [5:7577471391277021493:2048]
2025-11-27T18:19:01.161971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471391277021493:2048] [node 1] ICS09 handshake done sender: [5:7577471391277021473:2114] self: [5:7577471391277021481:0] peer: [1:7577471388852542944:0] socket: 655 qp: -1
2025-11-27T18:19:01.161992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471391277021493:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:01.162013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:01.169177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471386982054170:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(168868606ns) {Configure:104(168864675ns) {ProcessConfigured:135(9236393ns) {StartInitialHandshake:157(9233753ns) {DropHandshakes:470(2025ns) {DropIncomingHandshake:442(381ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(559ns) PrepareNewSessionHandshake:477(1432ns) {SwitchToState:179(432ns)}}}}}
2025-11-27T18:19:01.170818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471391277021472:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.171413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471386982054107:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:01.179618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471391277021498:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56109
2025-11-27T18:19:01.199249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471391277021477:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:01.221565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471391277021499:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56109
2025-11-27T18:19:01.230321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471386982054171:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.230396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:01.233310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471386982054171:2109] [node 4] ICP22 created new session: [5:7577471391277021500:2109]
2025-11-27T18:19:01.243693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471391277021500:2109] [node 4] ICS09 handshake done sender: [5:7577471391277021477:2116] self: [5:7577471391277021492:0] peer: [4:7577471392282694210:0] socket: 661 qp: -1
2025-11-27T18:19:01.252358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471391277021500:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:01.253533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471391277021484:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:01.253570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471391277021484:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:01.260284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471391277021472:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:01.261780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471386982054107:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471391277021472:2113] held: no
2025-11-27T18:19:01.262583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:01.271802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471391277021484:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:01.273941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471386982054107:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.273988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:01.274034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471386982054107:2049] [node 2] ICP22 created new session: [5:7577471391277021503:2049]
2025-11-27T18:19:01.274075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471391277021503:2049] [node 2] ICS09 handshake done sender: [5:7577471391277021472:2113] self: [5:7577471391277021497:0] peer: [2:7577471388490714941:0] socket: 654 qp: -1
2025-11-27T18:19:01.274107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471391277021503:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:01.274135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:01.289744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471386982054170:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.289812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:01.308832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471386982054170:2110] [node 6] ICP22 created new session: [5:7577471391277021505:2110]
2025-11-27T18:19:01.325556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471391277021505:2110] [node 6] ICS09 handshake done sender: [5:7577471391277021484:2118] self: [5:7577471391277021478:0] peer: [6:7577471393329145082:0] socket: 678 qp: -1
2025-11-27T18:19:01.327389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471391277021505:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:01.327434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:01.339526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471386982054171:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(68497281ns) {HandleHandshakeStatus:329(68493825ns) {DropIncomingHandshake:442(4332ns) DropOutgoingHandshake:457(16095ns) SwitchToState:179(416ns) ProcessPendingSessionEvents:495(5621447ns) {ForwardSessionEventToSession:612(5568597ns) ForwardSessionEventToSession:612(11722ns) ForwardSessionEventToSession:612(2883ns) ForwardSessionEventToSession:612(2787ns) ForwardSessionEventToSession:612(8404ns) ForwardSessionEventToSession:612(3323ns) ForwardSessionEventToSession:612(2506ns) ForwardSessionEventToSession:612(5207ns) ForwardSessionEventToSession:612(2570ns) ForwardSessionEventToSession:612(5626ns)}}}
2025-11-27T18:19:01.296871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:01.346952Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471389034177764:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11739) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471393329145079:2117] held: no
2025-11-27T18:19:01.348090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471393329145090:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25187
2025-11-27T18:19:01.348288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471389034177769:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19877) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471393329145081:2118] held: no
2025-11-27T18:19:01.356963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471393329145079:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:01.357485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471389034177764:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.361882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:01.361946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471389034177764:2106] [node 5] ICP22 created new session: [6:7577471393329145098:2106]
2025-11-27T18:19:01.364155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471393329145098:2106] [node 5] ICS09 handshake done sender: [6:7577471393329145079:2117] self: [6:7577471393329145082:0] peer: [5:7577471391277021478:0] socket: 679 qp: -1
2025-11-27T18:19:01.364195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471393329145098:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:01.364234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:01.370461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471393329145097:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25187
2025-11-27T18:19:01.338140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471392785682268:2110] [node 6] ICS09 handshake done sender: [2:7577471392785682251:2118] self: [2:7577471392785682250:0] peer: [6:7577471393329145089:0] socket: 650 qp: -1
2025-11-27T18:19:01.338182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471392785682268:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:01.338237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:01.353632Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471388490714937:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(23855181ns) {HandleHandshakeStatus:329(23852102ns) {DropIncomingHandshake:442(1731ns) DropOutgoingHandshake:457(7036ns) SwitchToState:179(738ns) ProcessPendingSessionEvents:495(5971306ns) {ForwardSessionEventToSession:612(5905109ns) ForwardSessionEventToSession:612(5444ns) ForwardSessionEventToSession:612(5976ns) ForwardSessionEventToSession:612(2397ns) ForwardSessionEventToSession:612(5608ns) ForwardSessionEventToSession:612(3239ns) ForwardSessionEventToSession:612(3002ns) ForwardSessionEventToSession:612(18994ns) ForwardSessionEventToSession:612(5094ns) ForwardSessionEventToSession:612(2461ns)}}}
2025-11-27T18:19:01.353688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471388490714934:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:01.353736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:01.353777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471388490714934:2108] [node 4] ICP22 created new session: [2:7577471392785682270:2108]
2025-11-27T18:19:01.353822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471392785682270:2108] [node 4] ICS09 handshake done sender: [2:7577471388490714945:2113] self: [2:7577471388490714940:0] peer: [4:7577471392282694232:0] socket: 648 qp: -1
2025-11-27T18:19:01.353866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471392785682270:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:01.353898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:01.354594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471388490714937:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:01.375758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471393329145081:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:01.378821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471389034177769:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:01.378877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:01.378925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471389034177769:2110] [node 4] ICP22 created new session: [6:7577471393329145100:2110]
2025-11-27T18:19:01.378974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471393329145100:2110] [node 4] ICS09 handshake done sender: [6:7577471393329145081:2118] self: [6:7577471393329145085:0] peer: [4:7577471392282694214:0] socket: 684 qp: -1
2025-11-27T18:19:01.379009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471393329145100:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:01.379047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:01.372252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471386982054170:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(75784980ns) {HandleHandshakeStatus:329(75780244ns) {DropIncomingHandshake:442(22562ns) DropOutgoingHandshake:457(576ns) SwitchToState:179(336ns) ProcessPendingSessionEvents:495(18109479ns) {ForwardSessionEventToSession:612(9127808ns) ForwardSessionEventToSession:612(763760ns) ForwardSessionEventToSession:612(3365ns) ForwardSessionEventToSession:612(4587778ns) ForwardSessionEventToSession:612(6458ns) ForwardSessionEventToSession:612(3252ns) ForwardSessionEventToSession:612(503673ns) ForwardSessionEventToSession:612(4058ns) ForwardSessionEventToSession:612(2474580ns) ForwardSessionEventToSession:612(617153ns)}}}
2025-11-27T18:19:01.457644Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:01.457897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471392785682257:2048] [node 1] ICS07 socket disconnect 627 reason# EndOfStream
2025-11-27T18:19:01.457960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471392785682257:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.458062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:01.458105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471392785682257:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:01.460482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471388490714871:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8868) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:01.461623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:01.460508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471392785682257:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:01.461048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471388490714871:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8868) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:01.461082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471392785682257:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:01.461104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471388490714871:2048] [node 1] ICP30 unregister session Session# [2:7577471392785682257:2048] VirtualId# [2:7577471392785682247:0]
2025-11-27T18:19:01.461150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:01.461177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471388490714871:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8868) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:01.461194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8868) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:01.461864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471390369940749:2048] [node 1] ICS01 socket: 677 reason# EndOfStream
2025-11-27T18:19:01.461936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471386074973352:2048] [node 1] ICP30 unregister session Session# [3:7577471390369940749:2048] VirtualId# [3:7577471390369940740:0]
2025-11-27T18:19:01.461959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471390369940749:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.462025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:01.462072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:01.465416Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:01.465639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471393329145080:2048] [node 1] ICS07 socket disconnect 667 reason# EndOfStream
2025-11-27T18:19:01.465667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471393329145080:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.465737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:01.465778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471393329145080:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:01.466865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471389034177702:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8868) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:01.466917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471393329145080:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:01.466949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471389034177702:2048] [node 1] ICP30 unregister session Session# [6:7577471393329145080:2048] VirtualId# [6:7577471393329145077:0]
2025-11-27T18:19:01.466990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:01.467021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471389034177702:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8868) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:01.467034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8868) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:01.470446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471388490714871:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:01.472221Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:01.472430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471391277021493:2048] [node 1] ICS01 socket: 655 reason# EndOfStream
2025-11-27T18:19:01.472481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471386982054106:2048] [node 1] ICP30 unregister session Session# [5:7577471391277021493:2048] VirtualId# [5:7577471391277021481:0]
2025-11-27T18:19:01.472520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471391277021493:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.472616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:01.472665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:01.472766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:01.472941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471392282694229:2048] [node 1] ICS01 socket: 643 reason# EndOfStream
2025-11-27T18:19:01.472973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471387987726841:2048] [node 1] ICP30 unregister session Session# [4:7577471392282694229:2048] VirtualId# [4:7577471392282694220:0]
2025-11-27T18:19:01.472998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471392282694229:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.473053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:01.473088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:01.476640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471389034177702:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:01.547963Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:01.548189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471390369940744:2049] [node 2] ICS01 socket: 636 reason# EndOfStream
2025-11-27T18:19:01.548231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471386074973353:2049] [node 2] ICP30 unregister session Session# [3:7577471390369940744:2049] VirtualId# [3:7577471390369940737:0]
2025-11-27T18:19:01.548262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471390369940744:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.548350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:01.548395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:01.557446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:01.557707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:01.557924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471392282694236:2049] [node 2] ICS01 socket: 644 reason# EndOfStream
2025-11-27T18:19:01.557967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471387987726842:2049] [node 2] ICP30 unregister session Session# [4:7577471392282694236:2049] VirtualId# [4:7577471392282694232:0]
2025-11-27T18:19:01.557995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471392282694236:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.558064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:01.558108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:01.558195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:01.558369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471393329145095:2049] [node 2] ICS01 socket: 693 reason# EndOfStream
2025-11-27T18:19:01.558406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471389034177703:2049] [node 2] ICP30 unregister session Session# [6:7577471393329145095:2049] VirtualId# [6:7577471393329145089:0]
2025-11-27T18:19:01.558431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471393329145095:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.558492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:01.558524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:01.557697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471391277021503:2049] [node 2] ICS01 socket: 654 reason# EndOfStream
2025-11-27T18:19:01.557761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471386982054107:2049] [node 2] ICP30 unregister session Session# [5:7577471391277021503:2049] VirtualId# [5:7577471391277021497:0]
2025-11-27T18:19:01.557813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471391277021503:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.557900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:01.557945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:01.639492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:01.639739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471392282694238:2050] [node 3] ICS01 socket: 672 reason# EndOfStream
2025-11-27T18:19:01.639794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471387987726843:2050] [node 3] ICP30 unregister session Session# [4:7577471392282694238:2050] VirtualId# [4:7577471392282694219:0]
2025-11-27T18:19:01.639854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471392282694238:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.639925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:01.639965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:01.641140Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:01.641367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471391277021489:2050] [node 3] ICS01 socket: 656 reason# EndOfStream
2025-11-27T18:19:01.641408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471386982054108:2050] [node 3] ICP30 unregister session Session# [5:7577471391277021489:2050] VirtualId# [5:7577471391277021485:0]
2025-11-27T18:19:01.641444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471391277021489:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.641618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:01.641774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471393329145093:2050] [node 3] ICS01 socket: 694 reason# EndOfStream
2025-11-27T18:19:01.641822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471389034177704:2050] [node 3] ICP30 unregister session Session# [6:7577471393329145093:2050] VirtualId# [6:7577471393329145088:0]
2025-11-27T18:19:01.641846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471393329145093:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.641894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:01.641920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:01.641530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:01.641586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:01.773568Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:01.773804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471393329145100:2110] [node 4] ICS01 socket: 684 reason# EndOfStream
2025-11-27T18:19:01.773846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471389034177769:2110] [node 4] ICP30 unregister session Session# [6:7577471393329145100:2110] VirtualId# [6:7577471393329145085:0]
2025-11-27T18:19:01.773887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471393329145100:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.773982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:01.774029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:01.773492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:01.773751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471391277021500:2109] [node 4] ICS07 socket disconnect 661 reason# EndOfStream
2025-11-27T18:19:01.773786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471391277021500:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.773869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:01.773912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471391277021500:2109] [node 4] ICS15 start handshake
2025-11-27T18:19:01.777713Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471386982054171:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19877) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:01.777791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471391277021500:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:01.777839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471386982054171:2109] [node 4] ICP30 unregister session Session# [5:7577471391277021500:2109] VirtualId# [5:7577471391277021492:0]
2025-11-27T18:19:01.777892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:01.777926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471386982054171:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19877) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:01.777946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19877) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:01.865438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:01.865697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471393329145098:2106] [node 5] ICS01 socket: 679 reason# EndOfStream
2025-11-27T18:19:01.865738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471389034177764:2106] [node 5] ICP30 unregister session Session# [6:7577471393329145098:2106] VirtualId# [6:7577471393329145082:0]
2025-11-27T18:19:01.865775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471393329145098:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:01.865875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:01.865922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:19:02.074558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471397673384909:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:02.074645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471397673384910:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:02.126343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471395969532052:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:02.126392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471395969532053:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:02.149342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471394745713209:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:02.150297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471394745713210:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:02.175486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471394415003763:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:02.175523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471394415003764:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:02.176474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471394415003762:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:02.221405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471396232601937:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:02.221475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471396232601938:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:02.221507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471396232601939:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:02.235566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471396873364830:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:02.235605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471396873364831:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:02.235633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471396873364832:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:02.524499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471396873364892:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:02.562079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471397673384973:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:02.561585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471396873364893:2102] [node 5] ICP01 ready to work
2025-11-27T18:19:02.716506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471396873364892:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(191716719ns) {Configure:104(191713373ns) {ProcessConfigured:135(17960621ns) {StartInitialHandshake:157(17958220ns) {DropHandshakes:470(2271ns) {DropIncomingHandshake:442(467ns) DropOutgoingHandshake:457(289ns)} GenerateSessionVirtualId:988(1029ns) PrepareNewSessionHandshake:477(2914ns) {SwitchToState:179(1432ns)}}}}}
2025-11-27T18:19:02.657305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471397673384977:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:02.710954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471395969532116:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:02.711512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471394745713278:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.745477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471394745713280:2114] [node 5] ICP01 ready to work
2025-11-27T18:19:02.727553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471396232602001:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:02.713301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471397673384978:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:02.715475Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471395969532053:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(64079460ns) {Configure:104(64069515ns) {ProcessConfigured:135(73384ns) {StartInitialHandshake:157(70666ns) {DropHandshakes:470(3079ns) {DropIncomingHandshake:442(458ns) DropOutgoingHandshake:457(792ns)} GenerateSessionVirtualId:988(1071ns) PrepareNewSessionHandshake:477(1692ns) {SwitchToState:179(745ns)}}}}}
2025-11-27T18:19:02.756305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471394745713279:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:02.742572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471397673384909:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(161924904ns) {Configure:104(161921114ns) {ProcessConfigured:135(108698ns) {StartInitialHandshake:157(105118ns) {DropHandshakes:470(2644ns) {DropIncomingHandshake:442(979ns) DropOutgoingHandshake:457(305ns)} GenerateSessionVirtualId:988(1039ns) PrepareNewSessionHandshake:477(2411ns) {SwitchToState:179(1181ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.792495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471396232602000:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44552
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.794384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471397673384982:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:549
2025-11-27T18:19:02.761867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471394415003825:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:02.728008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471395969532120:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.812634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471397673384982:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.745774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471395969532121:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:02.782301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471394415003762:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(109469378ns) {Configure:104(81551954ns) {ProcessConfigured:135(63431ns) {StartInitialHandshake:157(61560ns) {DropHandshakes:470(2581ns) {DropIncomingHandshake:442(368ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(867ns) PrepareNewSessionHandshake:477(1428ns) {SwitchToState:179(722ns)}}}}}
2025-11-27T18:19:02.817488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471396232602002:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.831303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471396873364898:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51499
2025-11-27T18:19:02.782684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471394415003826:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1309
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.844561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471396873364898:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.845034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471396873364899:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51499
2025-11-27T18:19:02.828579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471397673384977:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(110687769ns) {Configure:104(110681697ns) {ProcessConfigured:135(83483ns) {StartInitialHandshake:157(80665ns) {DropHandshakes:470(2857ns) {DropIncomingHandshake:442(1011ns) DropOutgoingHandshake:457(288ns)} GenerateSessionVirtualId:988(937ns) PrepareNewSessionHandshake:477(1410ns) {SwitchToState:179(448ns)}}}}}
2025-11-27T18:19:02.747830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471395969532122:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38444
2025-11-27T18:19:02.850299Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471396873364893:2102] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(196734648ns) {Configure:104(193475828ns) {ProcessConfigured:135(4432539ns) {StartInitialHandshake:157(4430268ns) {DropHandshakes:470(2408ns) {DropIncomingHandshake:442(553ns) DropOutgoingHandshake:457(471ns)} GenerateSessionVirtualId:988(1980ns) PrepareNewSessionHandshake:477(1660ns) {SwitchToState:179(497ns)}}}}}
2025-11-27T18:19:02.796261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471395969532122:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.782724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471394415003827:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1309
2025-11-27T18:19:02.846600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471396232602000:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.872587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471397673384978:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(139247397ns) {Configure:104(139241596ns) {ProcessConfigured:135(77303ns) {StartInitialHandshake:157(75387ns) {DropHandshakes:470(2189ns) {DropIncomingHandshake:442(388ns) DropOutgoingHandshake:457(320ns)} GenerateSessionVirtualId:988(964ns) PrepareNewSessionHandshake:477(1676ns) {SwitchToState:179(392ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.880408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471397673384984:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.869921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471396873364899:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.879571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471396873364830:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(151689248ns) {Configure:104(151680348ns) {ProcessConfigured:135(70296ns) {StartInitialHandshake:157(68121ns) {DropHandshakes:470(2490ns) {DropIncomingHandshake:442(533ns) DropOutgoingHandshake:457(377ns)} GenerateSessionVirtualId:988(1153ns) PrepareNewSessionHandshake:477(1991ns) {SwitchToState:179(902ns)}}}}}
2025-11-27T18:19:02.879888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471396873364830:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:02.880222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471396873364893:2102] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:02.890336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471394745713210:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(296814965ns) {Configure:104(296800109ns) {ProcessConfigured:135(62621ns) {StartInitialHandshake:157(59772ns) {DropHandshakes:470(2350ns) {DropIncomingHandshake:442(572ns) DropOutgoingHandshake:457(311ns)} GenerateSessionVirtualId:988(825ns) PrepareNewSessionHandshake:477(1617ns) {SwitchToState:179(503ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.802960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471394415003831:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:02.874094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471395969532126:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38444
2025-11-27T18:19:02.880442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471397673384984:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:02.811547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471394415003832:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1309
2025-11-27T18:19:02.900478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471396232602007:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.900527Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471396232602007:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:02.902349Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471396232601937:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(183815922ns) {Configure:104(183807654ns) {ProcessConfigured:135(10041440ns) {StartInitialHandshake:157(10039625ns) {DropHandshakes:470(1296ns) {DropIncomingHandshake:442(569ns) DropOutgoingHandshake:457(157ns)} GenerateSessionVirtualId:988(311ns) PrepareNewSessionHandshake:477(2039ns) {SwitchToState:179(1316ns)}}}}}
2025-11-27T18:19:02.902496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471396232601937:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:02.891754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471396873364831:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(141415369ns) {Configure:104(141409813ns) {ProcessConfigured:135(12820200ns) {StartInitialHandshake:157(12816566ns) {DropHandshakes:470(1538ns) {DropIncomingHandshake:442(175ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(1411ns) PrepareNewSessionHandshake:477(2964ns) {SwitchToState:179(924ns)}}}}}
2025-11-27T18:19:02.898134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471395969532126:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.881045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471397673384910:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(290609629ns) {Configure:104(290602167ns) {ProcessConfigured:135(10080105ns) {StartInitialHandshake:157(10077822ns) {DropHandshakes:470(2955ns) {DropIncomingHandshake:442(508ns) DropOutgoingHandshake:457(421ns)} GenerateSessionVirtualId:988(919ns) PrepareNewSessionHandshake:477(2104ns) {SwitchToState:179(372ns)}}}}}
2025-11-27T18:19:02.812631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471394415003827:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.890756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471394745713281:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31752
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.898355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471397673384984:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:02.901571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471395969532116:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(160507214ns) {Configure:104(160501947ns) {ProcessConfigured:135(53423ns) {StartInitialHandshake:157(51294ns) {DropHandshakes:470(3033ns) {DropIncomingHandshake:442(793ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(1037ns) PrepareNewSessionHandshake:477(1706ns) {SwitchToState:179(742ns)}}}}}
2025-11-27T18:19:02.900515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471396873364832:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(250259663ns) {Configure:104(250256317ns) {ProcessConfigured:135(12061462ns) {StartInitialHandshake:157(12059666ns) {DropHandshakes:470(1117ns) {DropIncomingHandshake:442(327ns) DropOutgoingHandshake:457(164ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1430ns) {SwitchToState:179(207ns)}}}}}
2025-11-27T18:19:02.929709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471396232602007:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.914594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471395969532118:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.914620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471395969532118:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:02.915269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471395969532124:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.915294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471395969532124:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:02.919852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471395969532118:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:02.922856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471395969532135:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38444
2025-11-27T18:19:02.929835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471395969532124:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:02.934386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471395969532052:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(247105541ns) {Configure:104(247097937ns) {ProcessConfigured:135(11430347ns) {StartInitialHandshake:157(11427772ns) {DropHandshakes:470(2159ns) {DropIncomingHandshake:442(466ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(583ns) PrepareNewSessionHandshake:477(2599ns) {SwitchToState:179(1011ns)}}}}}
2025-11-27T18:19:02.846075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471394415003826:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.939708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471397673384975:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.891221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471394745713284:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31752
2025-11-27T18:19:02.938272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471395969532052:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:02.939752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471397673384975:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:02.940349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471397673384977:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:02.940396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:02.940450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471397673384977:2109] [node 6] ICP22 created new session: [1:7577471397673384992:2109]
2025-11-27T18:19:02.940517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471397673384992:2109] [node 6] ICS09 handshake done sender: [1:7577471397673384984:2114] self: [1:7577471397673384983:0] peer: [6:7577471396873364909:0] socket: 649 qp: -1
2025-11-27T18:19:02.940558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471397673384992:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:02.940598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:02.923866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471396873364914:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51499
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.848915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471394415003764:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(147290512ns) {Configure:104(147281014ns) {ProcessConfigured:135(61053ns) {StartInitialHandshake:157(58685ns) {DropHandshakes:470(1450ns) {DropIncomingHandshake:442(523ns) DropOutgoingHandshake:457(170ns)} GenerateSessionVirtualId:988(818ns) PrepareNewSessionHandshake:477(14834ns) {SwitchToState:179(13584ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.891369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471394745713285:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31752
2025-11-27T18:19:02.941885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471396232602002:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:02.954518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471397673384980:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.931701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471396873364913:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51499
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.896999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471394745713284:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.941933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:02.853646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471394415003832:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.932044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471396873364912:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51499
2025-11-27T18:19:02.977610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471395969532120:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(182140111ns) {Configure:104(182130412ns) {ProcessConfigured:135(14152259ns) {StartInitialHandshake:157(14149376ns) {DropHandshakes:470(1973ns) {DropIncomingHandshake:442(517ns) DropOutgoingHandshake:457(329ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(2316ns) {SwitchToState:179(748ns)}}}}}
2025-11-27T18:19:02.954558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471397673384980:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:02.898184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471394415003763:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(284557767ns) {Configure:104(284554198ns) {ProcessConfigured:135(10635425ns) {StartInitialHandshake:157(10632824ns) {DropHandshakes:470(1065ns) {DropIncomingHandshake:442(285ns) DropOutgoingHandshake:457(178ns)} GenerateSessionVirtualId:988(490ns) PrepareNewSessionHandshake:477(3041ns) {SwitchToState:179(1290ns)}}}}}
2025-11-27T18:19:02.977961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471395969532139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38444
2025-11-27T18:19:02.982633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471395969532135:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.982801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471395969532053:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:02.982842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:02.994362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471395969532122:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:02.995008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471395969532121:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:02.999879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471395969532053:2049] [node 3] ICP22 created new session: [2:7577471395969532140:2049]
2025-11-27T18:19:03.011832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:03.011902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471395969532121:2107] [node 4] ICP22 created new session: [2:7577471395969532141:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:03.012880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471395969532052:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.013162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471395969532129:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.906911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471394745713281:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.993744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471396873364915:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51499
2025-11-27T18:19:02.941972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471396232602002:2109] [node 6] ICP22 created new session: [5:7577471396232602019:2109]
2025-11-27T18:19:03.013188Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471395969532129:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:02.956489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471397673384980:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.899734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471394415003831:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(72682049ns) {Configure:104(72674911ns) {ProcessConfigured:135(73529ns) {StartInitialHandshake:157(71112ns) {DropHandshakes:470(1415ns) {DropIncomingHandshake:442(400ns) DropOutgoingHandshake:457(272ns)} GenerateSessionVirtualId:988(573ns) PrepareNewSessionHandshake:477(1110ns) {SwitchToState:179(413ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.942020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471396232602019:2109] [node 6] ICS09 handshake done sender: [5:7577471396232602007:2112] self: [5:7577471396232602006:0] peer: [6:7577471396873364910:0] socket: 651 qp: -1
2025-11-27T18:19:02.907141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471394745713285:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.007868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471396873364913:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.015040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:02.957145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471397673384909:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.906259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471394415003825:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(119542184ns) {Configure:104(119535680ns) {ProcessConfigured:135(15872683ns) {StartInitialHandshake:157(15869690ns) {DropHandshakes:470(5450189ns) {DropIncomingHandshake:442(1540ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(1482ns) PrepareNewSessionHandshake:477(3027ns) {SwitchToState:179(1381ns)}}}}}
2025-11-27T18:19:02.942075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471396232602019:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:03.009604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471396873364831:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.907332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471394745713210:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:02.957190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:02.957244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471397673384909:2048] [node 2] ICP22 created new session: [1:7577471397673384999:2048]
2025-11-27T18:19:02.962935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471397673384975:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:02.972113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471397673384993:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:549
2025-11-27T18:19:02.981268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471397673384999:2048] [node 2] ICS09 handshake done sender: [1:7577471397673384980:2112] self: [1:7577471397673384979:0] peer: [2:7577471395969532137:0] socket: 636 qp: -1
2025-11-27T18:19:02.985400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471397673384999:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:02.985487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:02.985977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471397673384909:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(28632917ns) {HandleHandshakeStatus:329(28629321ns) {DropIncomingHandshake:442(452ns) DropOutgoingHandshake:457(498ns) SwitchToState:179(713ns) ProcessPendingSessionEvents:495(128909ns) {ForwardSessionEventToSession:612(58290ns) ForwardSessionEventToSession:612(7442ns) ForwardSessionEventToSession:612(2870ns) ForwardSessionEventToSession:612(2634ns) ForwardSessionEventToSession:612(5579ns) ForwardSessionEventToSession:612(2509ns) ForwardSessionEventToSession:612(8144ns) ForwardSessionEventToSession:612(2638ns) ForwardSessionEventToSession:612(2524ns) ForwardSessionEventToSession:612(26485ns)}}}
2025-11-27T18:19:02.986410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471397673384993:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.994710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471397673384973:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:02.994777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:03.004082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471397673384973:2108] [node 5] ICP22 created new session: [1:7577471397673385003:2108]
2025-11-27T18:19:03.021391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471401968352300:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:549
2025-11-27T18:19:03.022788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471397673385003:2108] [node 5] ICS09 handshake done sender: [1:7577471397673384975:2111] self: [1:7577471397673384974:0] peer: [5:7577471396232602012:0] socket: 630 qp: -1
2025-11-27T18:19:03.025648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471397673384990:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.025683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471397673384990:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:03.026955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471397673384990:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:03.027680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471397673385003:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:03.027728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:03.039126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471397673384994:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:549
2025-11-27T18:19:03.055700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471397673384994:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.056618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471397673384910:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:03.056691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:03.056736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471397673384910:2049] [node 3] ICP22 created new session: [1:7577471401968352303:2049]
2025-11-27T18:19:03.056780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471401968352303:2049] [node 3] ICS09 handshake done sender: [1:7577471397673384990:2116] self: [1:7577471397673384986:0] peer: [3:7577471399040680603:0] socket: 688 qp: -1
2025-11-27T18:19:03.056815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471401968352303:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:03.056852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:03.059781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471397673384973:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(71504697ns) {HandleHandshakeStatus:329(64839453ns) {DropIncomingHandshake:442(16440ns) DropOutgoingHandshake:457(506ns) SwitchToState:179(292ns) ProcessPendingSessionEvents:495(5489982ns) {ForwardSessionEventToSession:612(5440362ns) ForwardSessionEventToSession:612(6265ns) ForwardSessionEventToSession:612(2660ns) ForwardSessionEventToSession:612(5661ns) ForwardSessionEventToSession:612(2860ns) ForwardSessionEventToSession:612(2223ns) ForwardSessionEventToSession:612(5613ns) ForwardSessionEventToSession:612(3062ns) ForwardSessionEventToSession:612(5547ns) ForwardSessionEventToSession:612(6027ns)}}}
2025-11-27T18:19:03.060256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471397673384910:2049] [node 3] ICP18 handshake (actor [1:7577471397673384994:2118]) is too old
2025-11-27T18:19:03.066632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471397673384988:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.066666Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471397673384988:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:03.018076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471395969532140:2049] [node 3] ICS09 handshake done sender: [2:7577471395969532118:2111] self: [2:7577471395969532117:0] peer: [3:7577471394745713287:0] socket: 632 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.916846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471394415003845:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1309
2025-11-27T18:19:02.925622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471394745713278:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(193157594ns) {Configure:104(193151706ns) {ProcessConfigured:135(59199ns) {StartInitialHandshake:157(55973ns) {DropHandshakes:470(1761ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(627ns)} GenerateSessionVirtualId:988(339ns) PrepareNewSessionHandshake:477(1699ns) {SwitchToState:179(558ns)}}}}}
2025-11-27T18:19:03.010201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471396873364898:2111] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:02.942104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.926174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471394745713291:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31752
2025-11-27T18:19:02.926600Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471394745713209:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(269816817ns) {Configure:104(269812180ns) {ProcessConfigured:135(6838852ns) {StartInitialHandshake:157(6836146ns) {DropHandshakes:470(1201ns) {DropIncomingHandshake:442(325ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(337ns) PrepareNewSessionHandshake:477(1470ns) {SwitchToState:179(337ns)}}}}}
2025-11-27T18:19:02.927619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471394745713280:2114] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(108406462ns) {Configure:104(108400726ns) {ProcessConfigured:135(48266ns) {StartInitialHandshake:157(45803ns) {DropHandshakes:470(2208ns) {DropIncomingHandshake:442(789ns) DropOutgoingHandshake:457(288ns)} GenerateSessionVirtualId:988(179ns) PrepareNewSessionHandshake:477(1522ns) {SwitchToState:179(407ns)}}}}}
2025-11-27T18:19:02.988393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471394745713281:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:02.990742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471394745713300:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31752
2025-11-27T18:19:02.990929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471394745713301:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31752
2025-11-27T18:19:02.993447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471394745713300:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:02.994895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471394745713279:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(183849324ns) {Configure:104(183837923ns) {ProcessConfigured:135(24777688ns) {StartInitialHandshake:157(24774914ns) {DropHandshakes:470(1733ns) {DropIncomingHandshake:442(340ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(387ns) PrepareNewSessionHandshake:477(3553ns) {SwitchToState:179(1925ns)}}}}}
2025-11-27T18:19:03.010364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471394745713210:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.018237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:03.018294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471394745713210:2049] [node 2] ICP22 created new session: [3:7577471399040680600:2049]
2025-11-27T18:19:03.018336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471399040680600:2049] [node 2] ICS09 handshake done sender: [3:7577471394745713281:2116] self: [3:7577471394745713287:0] peer: [2:7577471395969532117:0] socket: 637 qp: -1
2025-11-27T18:19:03.018359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471399040680600:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:03.018398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:03.021625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471394745713301:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.021749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471394745713209:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:03.043598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471399040680604:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31752
2025-11-27T18:19:03.054172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471394745713301:2126] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:03.066305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471394745713209:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.066357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:03.072151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471394745713209:2048] [node 1] ICP22 created new session: [3:7577471399040680605:2048]
2025-11-27T18:19:03.083795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471394745713293:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.083827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471394745713293:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:03.084196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471394745713289:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.084216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471394745713289:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:03.085915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471399040680605:2048] [node 1] ICS09 handshake done sender: [3:7577471394745713301:2126] self: [3:7577471399040680603:0] peer: [1:7577471397673384986:0] socket: 656 qp: -1
2025-11-27T18:19:03.085960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471399040680605:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:03.087975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:03.018123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471395969532140:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:02.917763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471394415003763:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:02.942999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471396232602021:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44552
2025-11-27T18:19:03.010250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471396873364899:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.937454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471394415003827:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:03.090771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471394745713289:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:03.025427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:02.950009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471396232601938:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(185137701ns) {Configure:104(185131477ns) {ProcessConfigured:135(23404070ns) {StartInitialHandshake:157(23402221ns) {DropHandshakes:470(1796ns) {DropIncomingHandshake:442(362ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(302ns) PrepareNewSessionHandshake:477(2183ns) {SwitchToState:179(429ns)}}}}}
2025-11-27T18:19:03.080574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471397673384988:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:03.014207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471401168332213:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51499
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.969547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471394415003763:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.027920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471395969532129:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:03.102400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471394745713293:2122] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.980852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471396232601939:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(197084820ns) {Configure:104(197080460ns) {ProcessConfigured:135(11465882ns) {StartInitialHandshake:157(11463418ns) {DropHandshakes:470(2272ns) {DropIncomingHandshake:442(469ns) DropOutgoingHandshake:457(294ns)} GenerateSessionVirtualId:988(571ns) PrepareNewSessionHandshake:477(2199ns) {SwitchToState:179(1031ns)}}}}}
2025-11-27T18:19:03.090442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471397673384978:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:03.034513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471396873364830:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:02.974452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:03.029349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471395969532052:2048] [node 1] ICP22 created new session: [2:7577471400264499441:2048]
2025-11-27T18:19:02.985407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471396232602009:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44552
2025-11-27T18:19:03.091640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:03.112433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471394745713280:2114] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:02.974512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471394415003763:2049] [node 2] ICP22 created new session: [4:7577471394415003847:2049]
2025-11-27T18:19:03.112510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:03.034573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:03.029411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471400264499441:2048] [node 1] ICS09 handshake done sender: [2:7577471395969532122:2112] self: [2:7577471395969532137:0] peer: [1:7577471397673384979:0] socket: 638 qp: -1
2025-11-27T18:19:02.988237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471396232602000:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:03.004773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471396232601937:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.004832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:03.022838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471396232601937:2048] [node 1] ICP22 created new session: [5:7577471400527569319:2048]
2025-11-27T18:19:03.033592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471400527569319:2048] [node 1] ICS09 handshake done sender: [5:7577471396232602000:2110] self: [5:7577471396232602012:0] peer: [1:7577471397673384974:0] socket: 631 qp: -1
2025-11-27T18:19:03.037861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471396232602009:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.038621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471400527569320:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44552
2025-11-27T18:19:03.038735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577471396232602002:2109] [node 6] ICP18 handshake (actor [5:7577471396232602009:2113]) is too old
2025-11-27T18:19:03.043319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471400527569319:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:03.049970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471400527569322:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44552
2025-11-27T18:19:03.050638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:03.062758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471400527569320:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.066453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471400527569322:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.069430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471396232601938:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:03.079431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471396232601937:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(74469855ns) {HandleHandshakeStatus:329(74466888ns) {DropIncomingHandshake:442(802ns) DropOutgoingHandshake:457(12930ns) SwitchToState:179(471ns) ProcessPendingSessionEvents:495(9979699ns) {ForwardSessionEventToSession:612(9928876ns) ForwardSessionEventToSession:612(10221ns) ForwardSessionEventToSession:612(2825ns) ForwardSessionEventToSession:612(2616ns) ForwardSessionEventToSession:612(5448ns) ForwardSessionEventToSession:612(2423ns) ForwardSessionEventToSession:612(2501ns) ForwardSessionEventToSession:612(7851ns) ForwardSessionEventToSession:612(2647ns) ForwardSessionEventToSession:612(5079ns)}}}
2025-11-27T18:19:03.029440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471400264499441:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:03.082354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471396232601939:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:03.029491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:03.104059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471400527569326:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44552
2025-11-27T18:19:03.121718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471396232601939:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2172) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471400527569322:2119] held: no
2025-11-27T18:19:03.029948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471395969532052:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16853361ns) {HandleHandshakeStatus:329(16850922ns) {DropIncomingHandshake:442(2742ns) DropOutgoingHandshake:457(9004ns) SwitchToState:179(544ns) ProcessPendingSessionEvents:495(100470ns) {ForwardSessionEventToSession:612(53777ns) ForwardSessionEventToSession:612(2903ns) ForwardSessionEventToSession:612(8674ns) ForwardSessionEventToSession:612(2271ns) ForwardSessionEventToSession:612(3186ns) ForwardSessionEventToSession:612(4707ns) ForwardSessionEventToSession:612(6315ns) ForwardSessionEventToSession:612(4786ns) ForwardSessionEventToSession:612(2849ns) ForwardSessionEventToSession:612(2507ns)}}}
2025-11-27T18:19:03.121893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471400527569322:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:03.037594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471395969532141:2107] [node 4] ICS09 handshake done sender: [2:7577471395969532124:2113] self: [2:7577471395969532123:0] peer: [4:7577471394415003844:0] socket: 639 qp: -1
2025-11-27T18:19:03.129531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471396232601939:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.043842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471395969532116:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:03.043880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:03.129577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:03.043926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471395969532116:2109] [node 6] ICP22 created new session: [2:7577471400264499444:2109]
2025-11-27T18:19:03.129646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471396232601939:2050] [node 3] ICP22 created new session: [5:7577471400527569329:2050]
2025-11-27T18:19:03.043972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471400264499444:2109] [node 6] ICS09 handshake done sender: [2:7577471395969532129:2115] self: [2:7577471395969532128:0] peer: [6:7577471401168332212:0] socket: 665 qp: -1
2025-11-27T18:19:03.129687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471400527569329:2050] [node 3] ICS09 handshake done sender: [5:7577471400527569322:2119] self: [5:7577471400527569325:0] peer: [3:7577471394745713288:0] socket: 689 qp: -1
2025-11-27T18:19:03.129719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471400527569329:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:03.043999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471400264499444:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:03.129754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:03.044034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:03.130405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471400527569328:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44552
2025-11-27T18:19:03.050552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471395969532141:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:03.133173Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471396232601938:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11414) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471400527569320:2118] held: no
2025-11-27T18:19:03.050608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:03.151157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471400527569328:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.061683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471400264499447:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38444
2025-11-27T18:19:03.156879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471396232602001:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:03.061822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471400264499448:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38444
2025-11-27T18:19:03.160092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471400527569327:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44552
2025-11-27T18:19:03.080695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471400264499447:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.173563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471400527569320:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:03.174199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471400527569332:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44552
2025-11-27T18:19:03.080738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471395969532138:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.080780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471395969532138:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:03.089686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471395969532121:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(106756758ns) {HandleHandshakeStatus:329(101138543ns) {DropIncomingHandshake:442(11823ns) DropOutgoingHandshake:457(252ns) SwitchToState:179(179ns) ProcessPendingSessionEvents:495(6008038ns) {ForwardSessionEventToSession:612(5949877ns) ForwardSessionEventToSession:612(5635ns) ForwardSessionEventToSession:612(2831ns) ForwardSessionEventToSession:612(6602ns) ForwardSessionEventToSession:612(12064ns) ForwardSessionEventToSession:612(2883ns) ForwardSessionEventToSession:612(6028ns) ForwardSessionEventToSession:612(2439ns) ForwardSessionEventToSession:612(5857ns) ForwardSessionEventToSession:612(2945ns)}}}
2025-11-27T18:19:03.103751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471395969532138:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:03.112058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471395969532053:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(102976533ns) {HandleHandshakeStatus:329(102974698ns) {DropIncomingHandshake:442(530ns) DropOutgoingHandshake:457(416ns) SwitchToState:179(756ns) ProcessPendingSessionEvents:495(42673252ns) {ForwardSessionEventToSession:612(16702422ns) ForwardSessionEventToSession:612(2319422ns) ForwardSessionEventToSession:612(3255ns) ForwardSessionEventToSession:612(9018250ns) ForwardSessionEventToSession:612(3419ns) ForwardSessionEventToSession:612(2961ns) ForwardSessionEventToSession:612(1378949ns) ForwardSessionEventToSession:612(3108ns) ForwardSessionEventToSession:612(1607158ns) ForwardSessionEventToSession:612(2203233ns)}}}
2025-11-27T18:19:03.120565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471395969532053:2049] [node 3] ICP18 handshake (actor [2:7577471395969532135:2117]) is too old
2025-11-27T18:19:03.120957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471395969532120:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:03.121001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:03.135668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471395969532120:2108] [node 5] ICP22 created new session: [2:7577471400264499451:2108]
2025-11-27T18:19:03.141357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471400264499451:2108] [node 5] ICS09 handshake done sender: [2:7577471395969532138:2118] self: [2:7577471395969532136:0] peer: [5:7577471400527569324:0] socket: 678 qp: -1
2025-11-27T18:19:03.146264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471400264499451:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:03.150637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:03.169032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471395969532120:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(47886232ns) {HandleHandshakeStatus:329(47883589ns) {DropIncomingHandshake:442(11114ns) DropOutgoingHandshake:457(410ns) SwitchToState:179(326ns) ProcessPendingSessionEvents:495(5783953ns) {ForwardSessionEventToSession:612(5723470ns) ForwardSessionEventToSession:612(11222ns) ForwardSessionEventToSession:612(3457ns) ForwardSessionEventToSession:612(2979ns) ForwardSessionEventToSession:612(6164ns) ForwardSessionEventToSession:612(3160ns) ForwardSessionEventToSession:612(3400ns) ForwardSessionEventToSession:612(7086ns) ForwardSessionEventToSession:612(7659ns) ForwardSessionEventToSession:612(6323ns)}}}
2025-11-27T18:19:03.034632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471396873364830:2048] [node 1] ICP22 created new session: [6:7577471401168332214:2048]
2025-11-27T18:19:03.034674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471401168332214:2048] [node 1] ICS09 handshake done sender: [6:7577471396873364898:2111] self: [6:7577471396873364909:0] peer: [1:7577471397673384983:0] socket: 650 qp: -1
2025-11-27T18:19:03.034703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471401168332214:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:03.034735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:03.035357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471396873364893:2102] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.035403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:03.035441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471396873364893:2102] [node 5] ICP22 created new session: [6:7577471401168332216:2102]
2025-11-27T18:19:03.035487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471401168332216:2102] [node 5] ICS09 handshake done sender: [6:7577471396873364899:2112] self: [6:7577471396873364910:0] peer: [5:7577471396232602006:0] socket: 652 qp: -1
2025-11-27T18:19:03.035514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471401168332216:2102] [node 5] ICS10 traffic start
2025-11-27T18:19:03.035544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:03.043142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471396873364915:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.053387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471396873364913:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:03.057997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471396873364832:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:03.058072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471396873364831:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.058116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:03.058168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471396873364831:2049] [node 2] ICP22 created new session: [6:7577471401168332221:2049]
2025-11-27T18:19:03.058211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471401168332221:2049] [node 2] ICS09 handshake done sender: [6:7577471396873364913:2118] self: [6:7577471401168332212:0] peer: [2:7577471395969532128:0] socket: 667 qp: -1
2025-11-27T18:19:03.058240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471401168332221:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:03.058270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:03.081826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471401168332218:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51499
2025-11-27T18:19:03.102108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471401168332218:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.102226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471401168332223:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51499
2025-11-27T18:19:03.119010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471396873364915:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:03.121694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471396873364892:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:03.136186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471396873364832:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.136240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:03.136279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471396873364832:2050] [node 3] ICP22 created new session: [6:7577471401168332226:2050]
2025-11-27T18:19:03.136321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471401168332226:2050] [node 3] ICS09 handshake done sender: [6:7577471396873364915:2120] self: [6:7577471401168332220:0] peer: [3:7577471394745713292:0] socket: 677 qp: -1
2025-11-27T18:19:03.136355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471401168332226:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:03.136391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:03.147520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471401168332225:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51499
2025-11-27T18:19:03.151383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471396873364892:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7429) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471401168332218:2122] held: no
2025-11-27T18:19:03.169711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471401168332218:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:03.102327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471397673384978:2107] [node 4] ICP22 created new session: [1:7577471401968352306:2107]
2025-11-27T18:19:02.988246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471394415003847:2049] [node 2] ICS09 handshake done sender: [4:7577471394415003827:2111] self: [4:7577471394415003844:0] peer: [2:7577471395969532123:0] socket: 641 qp: -1
2025-11-27T18:19:03.002664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471394415003848:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1309
2025-11-27T18:19:03.005713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471394415003847:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:03.005794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:03.021227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471394415003848:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.027241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471394415003762:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:03.041606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471394415003763:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(78142603ns) {HandleHandshakeStatus:329(78140101ns) {DropIncomingHandshake:442(1289ns) DropOutgoingHandshake:457(17974ns) SwitchToState:179(962ns) ProcessPendingSessionEvents:495(14218394ns) {ForwardSessionEventToSession:612(7762143ns) ForwardSessionEventToSession:612(12956ns) ForwardSessionEventToSession:612(3075ns) ForwardSessionEventToSession:612(2687ns) ForwardSessionEventToSession:612(6583ns) ForwardSessionEventToSession:612(5404ns) ForwardSessionEventToSession:612(5546ns) ForwardSessionEventToSession:612(2682ns) ForwardSessionEventToSession:612(2170ns) ForwardSessionEventToSession:612(4930ns)}}}
2025-11-27T18:19:03.086195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471398709971148:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1309
2025-11-27T18:19:03.093555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471394415003762:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471394415003848:2119] held: no
2025-11-27T18:19:03.103324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471394415003848:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:03.103701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471394415003762:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.103771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:03.103822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471394415003762:2048] [node 1] ICP22 created new session: [4:7577471398709971150:2048]
2025-11-27T18:19:03.105085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471398709971150:2048] [node 1] ICS09 handshake done sender: [4:7577471394415003848:2119] self: [4:7577471398709971147:0] peer: [1:7577471397673384987:0] socket: 685 qp: -1
2025-11-27T18:19:03.105138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471398709971150:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:03.105185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:03.122058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471394415003842:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.122083Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471394415003842:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:03.122826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471394415003842:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:03.138149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471394415003831:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:03.138214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:03.138262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471394415003831:2109] [node 6] ICP22 created new session: [4:7577471398709971154:2109]
2025-11-27T18:19:03.138307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471398709971154:2109] [node 6] ICS09 handshake done sender: [4:7577471394415003842:2117] self: [4:7577471394415003841:0] peer: [6:7577471401168332224:0] socket: 694 qp: -1
2025-11-27T18:19:03.138352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471398709971154:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:03.138386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:03.161453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471394415003840:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.161481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471394415003840:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:03.168516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471394415003840:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:03.173369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471398709971149:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1309
2025-11-27T18:19:03.177436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471394415003825:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:03.112568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471394745713280:2114] [node 5] ICP22 created new session: [3:7577471399040680610:2114]
2025-11-27T18:19:03.116947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471401968352306:2107] [node 4] ICS09 handshake done sender: [1:7577471397673384988:2115] self: [1:7577471397673384987:0] peer: [4:7577471398709971147:0] socket: 684 qp: -1
2025-11-27T18:19:03.177864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471396232601938:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.112624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471399040680610:2114] [node 5] ICS09 handshake done sender: [3:7577471394745713289:2120] self: [3:7577471394745713288:0] peer: [5:7577471400527569325:0] socket: 693 qp: -1
2025-11-27T18:19:03.177789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471396873364892:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.178358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:03.180252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471396232601938:2049] [node 2] ICP22 created new session: [5:7577471400527569333:2049]
2025-11-27T18:19:03.184235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471396232602001:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7429) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471400527569328:2122] held: no
2025-11-27T18:19:03.185577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471400527569333:2049] [node 2] ICS09 handshake done sender: [5:7577471400527569320:2118] self: [5:7577471400527569324:0] peer: [2:7577471395969532136:0] socket: 690 qp: -1
2025-11-27T18:19:03.185634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471400527569333:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:03.185680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:03.180285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:03.116999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471401968352306:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:03.190052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471400527569328:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:03.178319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:03.112668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471399040680610:2114] [node 5] ICS10 traffic start
2025-11-27T18:19:03.117059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:03.182105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471394415003825:2108] [node 5] ICP22 created new session: [4:7577471398709971157:2108]
2025-11-27T18:19:03.195130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471396232602001:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.195195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:03.195251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471396232602001:2108] [node 4] ICP22 created new session: [5:7577471400527569335:2108]
2025-11-27T18:19:03.195309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471400527569335:2108] [node 4] ICS09 handshake done sender: [5:7577471400527569328:2122] self: [5:7577471400527569331:0] peer: [4:7577471394415003839:0] socket: 676 qp: -1
2025-11-27T18:19:03.195354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471400527569335:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:03.195405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:03.184364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471398709971149:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.184397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471398709971157:2108] [node 5] ICS09 handshake done sender: [4:7577471394415003840:2116] self: [4:7577471394415003839:0] peer: [5:7577471400527569331:0] socket: 702 qp: -1
2025-11-27T18:19:03.112721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:03.184442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471398709971157:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:03.119308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471394745713278:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:03.186447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471394415003764:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:03.182008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471396873364892:2107] [node 4] ICP22 created new session: [6:7577471401168332228:2107]
2025-11-27T18:19:03.170074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471397673384978:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(78191766ns) {HandleHandshakeStatus:329(78189360ns) {DropIncomingHandshake:442(26307ns) DropOutgoingHandshake:457(3018ns) SwitchToState:179(488ns) ProcessPendingSessionEvents:495(22262216ns) {ForwardSessionEventToSession:612(10100518ns) ForwardSessionEventToSession:612(5684ns) ForwardSessionEventToSession:612(3398525ns) ForwardSessionEventToSession:612(1051824ns) ForwardSessionEventToSession:612(6052ns) ForwardSessionEventToSession:612(7663693ns) ForwardSessionEventToSession:612(3870ns) ForwardSessionEventToSession:612(6964ns) ForwardSessionEventToSession:612(3426ns) ForwardSessionEventToSession:612(2695ns)}}}
2025-11-27T18:19:03.119373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:03.119423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471394745713278:2108] [node 6] ICP22 created new session: [3:7577471399040680612:2108]
2025-11-27T18:19:03.119469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471399040680612:2108] [node 6] ICS09 handshake done sender: [3:7577471394745713293:2122] self: [3:7577471394745713292:0] peer: [6:7577471401168332220:0] socket: 675 qp: -1
2025-11-27T18:19:03.119509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471399040680612:2108] [node 6] ICS10 traffic start
2025-11-27T18:19:03.119541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:03.186218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471401168332228:2107] [node 4] ICS09 handshake done sender: [6:7577471401168332218:2122] self: [6:7577471401168332224:0] peer: [4:7577471394415003841:0] socket: 695 qp: -1
2025-11-27T18:19:03.193399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:03.149682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471394745713209:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(72850988ns) {HandleHandshakeStatus:329(72848905ns) {DropIncomingHandshake:442(785ns) DropOutgoingHandshake:457(5342ns) SwitchToState:179(466ns) ProcessPendingSessionEvents:495(15215310ns) {ForwardSessionEventToSession:612(15156067ns) ForwardSessionEventToSession:612(12693ns) ForwardSessionEventToSession:612(2399ns) ForwardSessionEventToSession:612(2850ns) ForwardSessionEventToSession:612(11207ns) ForwardSessionEventToSession:612(2349ns) ForwardSessionEventToSession:612(6019ns) ForwardSessionEventToSession:612(5149ns) ForwardSessionEventToSession:612(2785ns) ForwardSessionEventToSession:612(4870ns)}}}
2025-11-27T18:19:03.196459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471394745713299:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:03.196496Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471394745713299:2124] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:03.186274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471401168332228:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:03.190459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:03.196888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471394415003825:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17757231ns) {HandleHandshakeStatus:329(17754904ns) {DropIncomingHandshake:442(15798ns) DropOutgoingHandshake:457(351ns) SwitchToState:179(472ns) ProcessPendingSessionEvents:495(121481ns) {ForwardSessionEventToSession:612(67464ns) ForwardSessionEventToSession:612(2973ns) ForwardSessionEventToSession:612(3501ns) ForwardSessionEventToSession:612(9571ns) ForwardSessionEventToSession:612(2492ns) ForwardSessionEventToSession:612(11897ns) ForwardSessionEventToSession:612(2784ns) ForwardSessionEventToSession:612(2731ns) ForwardSessionEventToSession:612(5942ns) ForwardSessionEventToSession:612(3156ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:03.200361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471398709971160:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1309
2025-11-27T18:19:03.200353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471394745713299:2124] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:03.200814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471394745713279:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:03.200894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:03.200949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471394745713279:2107] [node 4] ICP22 created new session: [3:7577471399040680615:2107]
2025-11-27T18:19:03.200997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471399040680615:2107] [node 4] ICS09 handshake done sender: [3:7577471394745713299:2124] self: [3:7577471394745713298:0] peer: [4:7577471398709971159:0] socket: 698 qp: -1
2025-11-27T18:19:03.201032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471399040680615:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:03.201075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:03.201813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471398709971149:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:03.202712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471394415003764:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:03.202796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:03.202848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471394415003764:2050] [node 3] ICP22 created new session: [4:7577471398709971161:2050]
2025-11-27T18:19:03.202889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471398709971161:2050] [node 3] ICS09 handshake done sender: [4:7577471398709971149:2121] self: [4:7577471398709971159:0] peer: [3:7577471394745713298:0] socket: 697 qp: -1
2025-11-27T18:19:03.202922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471398709971161:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:03.202961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:03.273428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:03.273735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471400264499441:2048] [node 1] ICS01 socket: 638 reason# EndOfStream
2025-11-27T18:19:03.273800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471395969532052:2048] [node 1] ICP30 unregister session Session# [2:7577471400264499441:2048] VirtualId# [2:7577471395969532137:0]
2025-11-27T18:19:03.273836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471400264499441:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.273919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:03.273966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:03.276837Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:03.277064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471399040680605:2048] [node 1] ICS07 socket disconnect 656 reason# EndOfStream
2025-11-27T18:19:03.277093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471399040680605:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.277171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:03.277217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471399040680605:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:03.280383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471394745713209:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:03.280432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471399040680605:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:03.281166Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471394745713209:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:03.281381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471399040680605:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:03.281555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471394745713209:2048] [node 1] ICP30 unregister session Session# [3:7577471399040680605:2048] VirtualId# [3:7577471399040680603:0]
2025-11-27T18:19:03.281617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:03.281657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471394745713209:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:03.281673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:03.293009Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:03.293178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471400527569319:2048] [node 1] ICS01 socket: 631 reason# EndOfStream
2025-11-27T18:19:03.293203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471396232601937:2048] [node 1] ICP30 unregister session Session# [5:7577471400527569319:2048] VirtualId# [5:7577471396232602012:0]
2025-11-27T18:19:03.293227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471400527569319:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.293317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:03.293355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:03.293680Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:03.293931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471398709971150:2048] [node 1] ICS01 socket: 685 reason# EndOfStream
2025-11-27T18:19:03.293981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471394415003762:2048] [node 1] ICP30 unregister session Session# [4:7577471398709971150:2048] VirtualId# [4:7577471398709971147:0]
2025-11-27T18:19:03.294016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471398709971150:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.294098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:03.294142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:03.294244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:03.294433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471401168332214:2048] [node 1] ICS07 socket disconnect 650 reason# EndOfStream
2025-11-27T18:19:03.294461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471401168332214:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.294519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:03.294555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471401168332214:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:03.295286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471396873364830:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:03.295387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471401168332214:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:03.295428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471396873364830:2048] [node 1] ICP30 unregister session Session# [6:7577471401168332214:2048] VirtualId# [6:7577471396873364909:0]
2025-11-27T18:19:03.295483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:03.295526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471396873364830:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:03.295563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:03.298645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471394745713209:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:03.308494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471396873364830:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:03.352765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:03.353065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471399040680600:2049] [node 2] ICS01 socket: 637 reason# EndOfStream
2025-11-27T18:19:03.353122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471394745713210:2049] [node 2] ICP30 unregister session Session# [3:7577471399040680600:2049] VirtualId# [3:7577471394745713287:0]
2025-11-27T18:19:03.353181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471399040680600:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.353291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:03.353368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:03.356040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:03.356260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471394415003847:2049] [node 2] ICS01 socket: 641 reason# EndOfStream
2025-11-27T18:19:03.356309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471394415003763:2049] [node 2] ICP30 unregister session Session# [4:7577471394415003847:2049] VirtualId# [4:7577471394415003844:0]
2025-11-27T18:19:03.356342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471394415003847:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.356419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:03.356459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:03.357461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:03.357700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471401168332221:2049] [node 2] ICS01 socket: 667 reason# EndOfStream
2025-11-27T18:19:03.357744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471396873364831:2049] [node 2] ICP30 unregister session Session# [6:7577471401168332221:2049] VirtualId# [6:7577471401168332212:0]
2025-11-27T18:19:03.358462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:03.358656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471400527569333:2049] [node 2] ICS01 socket: 690 reason# EndOfStream
2025-11-27T18:19:03.358686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471396232601938:2049] [node 2] ICP30 unregister session Session# [5:7577471400527569333:2049] VirtualId# [5:7577471400527569324:0]
2025-11-27T18:19:03.358713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471400527569333:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.358783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:03.358820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:03.357782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471401168332221:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.357893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:03.357955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:03.474136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:03.473576Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:03.473819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471398709971161:2050] [node 3] ICS01 socket: 697 reason# EndOfStream
2025-11-27T18:19:03.473862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471394415003764:2050] [node 3] ICP30 unregister session Session# [4:7577471398709971161:2050] VirtualId# [4:7577471398709971159:0]
2025-11-27T18:19:03.473902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471398709971161:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.473986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:03.474036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:03.474562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:03.474737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471400527569329:2050] [node 3] ICS01 socket: 689 reason# EndOfStream
2025-11-27T18:19:03.474774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471396232601939:2050] [node 3] ICP30 unregister session Session# [5:7577471400527569329:2050] VirtualId# [5:7577471400527569325:0]
2025-11-27T18:19:03.474799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471400527569329:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.474855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:03.474892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:03.474315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471401168332226:2050] [node 3] ICS01 socket: 677 reason# EndOfStream
2025-11-27T18:19:03.474351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471396873364832:2050] [node 3] ICP30 unregister session Session# [6:7577471401168332226:2050] VirtualId# [6:7577471401168332220:0]
2025-11-27T18:19:03.474390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471401168332226:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.474445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:03.474482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:03.544611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:03.544911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471400527569335:2108] [node 4] ICS01 socket: 676 reason# EndOfStream
2025-11-27T18:19:03.544960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471396232602001:2108] [node 4] ICP30 unregister session Session# [5:7577471400527569335:2108] VirtualId# [5:7577471400527569331:0]
2025-11-27T18:19:03.544998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471400527569335:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.545075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:03.545140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:03.549475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:03.549713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471401168332228:2107] [node 4] ICS01 socket: 695 reason# EndOfStream
2025-11-27T18:19:03.549754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471396873364892:2107] [node 4] ICP30 unregister session Session# [6:7577471401168332228:2107] VirtualId# [6:7577471401168332224:0]
2025-11-27T18:19:03.549790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471401168332228:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.549871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:03.549914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:03.679982Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:03.680293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471401168332216:2102] [node 5] ICS01 socket: 652 reason# EndOfStream
2025-11-27T18:19:03.680341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471396873364893:2102] [node 5] ICP30 unregister session Session# [6:7577471401168332216:2102] VirtualId# [6:7577471396873364910:0]
2025-11-27T18:19:03.680381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471401168332216:2102] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:03.680458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:03.680526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-27T18:19:03.902190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471399641025687:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:03.902264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471399641025688:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:03.923077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471399445836905:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:03.923137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471399445836906:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:03.955402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471399423503589:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:03.955438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471399423503590:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:03.981182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471401070206956:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:03.981240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471401070206957:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:03.981299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471401070206958:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:04.004086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471406450775256:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:04.004133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471406450775257:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:04.004161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471406450775258:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:04.025092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471405406529521:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:04.025120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471405406529522:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:04.025138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471405406529523:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:04.298887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471403740804264:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:04.385964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471406450775319:2106] [node 4] ICP01 ready to work
2025-11-27T18:19:04.364028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471403935993047:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:04.386320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471406450775320:2107] [node 6] ICP01 ready to work
2025-11-27T18:19:04.368776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471405406529564:2079] [node 4] ICP01 ready to work
2025-11-27T18:19:04.413708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471403935993048:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:04.417891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471403718470958:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:04.418464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471403718470959:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:04.438512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471403718470964:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:04.440075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471405365174316:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:04.457423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471403935993049:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:04.461793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471403740804265:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:04.475376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471405406529585:2083] [node 5] ICP01 ready to work
2025-11-27T18:19:04.485550Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471406450775256:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(74927085ns) {Configure:104(74920923ns) {ProcessConfigured:135(35393366ns) {StartInitialHandshake:157(35389600ns) {DropHandshakes:470(7540023ns) {DropIncomingHandshake:442(721ns) DropOutgoingHandshake:457(1718ns)} GenerateSessionVirtualId:988(2794ns) PrepareNewSessionHandshake:477(4416ns) {SwitchToState:179(2617ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.521588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471403718470974:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18191
2025-11-27T18:19:04.523586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471403740804266:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:04.506728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471405365174317:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.530453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471403935993063:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20316
2025-11-27T18:19:04.530307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471405406529521:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38274742ns) {Configure:104(38270410ns) {ProcessConfigured:135(16700065ns) {StartInitialHandshake:157(16697147ns) {DropHandshakes:470(3071ns) {DropIncomingHandshake:442(686ns) DropOutgoingHandshake:457(477ns)} GenerateSessionVirtualId:988(841ns) PrepareNewSessionHandshake:477(4814ns) {SwitchToState:179(2852ns)}}}}}
2025-11-27T18:19:04.548182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471401070206957:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27441297ns) {Configure:104(27436030ns) {ProcessConfigured:135(74559ns) {StartInitialHandshake:157(70775ns) {DropHandshakes:470(2719ns) {DropIncomingHandshake:442(689ns) DropOutgoingHandshake:457(305ns)} GenerateSessionVirtualId:988(947ns) PrepareNewSessionHandshake:477(1514ns) {SwitchToState:179(571ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:19:04.563212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471405365174326:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27401
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.563631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471405365174327:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27401
2025-11-27T18:19:04.563727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471405365174328:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27401
2025-11-27T18:19:04.568759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471403718470974:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.573674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471403740804265:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25711902ns) {Configure:104(25707770ns) {ProcessConfigured:135(7960554ns) {StartInitialHandshake:157(7956540ns) {DropHandshakes:470(2412ns) {DropIncomingHandshake:442(697ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(1889ns) PrepareNewSessionHandshake:477(4657ns) {SwitchToState:179(2026ns)}}}}}
2025-11-27T18:19:04.569764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471403718470975:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18191
2025-11-27T18:19:04.582400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471403935993053:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.582444Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471403935993053:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.559742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471406450775257:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(33915953ns) {Configure:104(33911796ns) {ProcessConfigured:135(67695ns) {StartInitialHandshake:157(65411ns) {DropHandshakes:470(788ns) {DropIncomingHandshake:442(181ns) DropOutgoingHandshake:457(104ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(1302ns) {SwitchToState:179(477ns)}}}}}
2025-11-27T18:19:04.569872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471399423503589:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:04.584409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471403935993064:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20316
2025-11-27T18:19:04.589766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471403935993053:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:04.569945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471403718470976:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18191
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.590866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471403718470975:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.591328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471403718470978:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18191
2025-11-27T18:19:04.591767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471405365174326:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.592196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471405365174327:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.592344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471401070206956:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:04.592410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471401070206958:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.593439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471403740804275:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
2025-11-27T18:19:04.593625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471403740804276:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
2025-11-27T18:19:04.593739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471403740804277:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.595692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471406450775331:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62319
2025-11-27T18:19:04.595825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471406450775332:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62319
2025-11-27T18:19:04.596969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471403718470976:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.597107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471403718470966:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.597128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471403718470966:2122] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.597764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471405365174328:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.598428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471405365174336:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27401
2025-11-27T18:19:04.598488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471405365174337:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27401
2025-11-27T18:19:04.599051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471403935993063:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.599143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471403935993064:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.599233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471403935993055:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.599251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471403935993055:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.599951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471399641025688:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.599997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:04.600045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471399641025688:2049] [node 3] ICP22 created new session: [1:7577471403935993067:2049]
2025-11-27T18:19:04.600091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471403935993067:2049] [node 3] ICS09 handshake done sender: [1:7577471403935993053:2112] self: [1:7577471403935993052:0] peer: [3:7577471403718470977:0] socket: 37 qp: -1
2025-11-27T18:19:04.600124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471403935993067:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:04.600186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:04.601923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471399641025688:2049] [node 3] ICP18 handshake (actor [1:7577471403935993063:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.597698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471406450775331:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.602230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471403935993066:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20316
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.602952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471403740804277:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.607109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471403935993055:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:04.610083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471405365174339:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27401
2025-11-27T18:19:04.615672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471403718470974:2125] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.616376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471399423503589:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23631) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471403718470974:2125] held: no
2025-11-27T18:19:04.620029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471403935993047:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.620096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:04.617024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471399423503589:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.620167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471403935993047:2107] [node 4] ICP22 created new session: [1:7577471403935993072:2107]
2025-11-27T18:19:04.617075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:04.617133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471399423503589:2048] [node 1] ICP22 created new session: [3:7577471403718470983:2048]
2025-11-27T18:19:04.617191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471403718470983:2048] [node 1] ICS09 handshake done sender: [3:7577471403718470974:2125] self: [3:7577471403718470977:0] peer: [1:7577471403935993052:0] socket: 44 qp: -1
2025-11-27T18:19:04.617230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471403718470983:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:04.617284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:04.619003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471403718470980:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18191
2025-11-27T18:19:04.619077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471403718470981:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18191
2025-11-27T18:19:04.621532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471403718470966:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:04.621934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471403718470958:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.621986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:04.622032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471403718470958:2107] [node 4] ICP22 created new session: [3:7577471403718470987:2107]
2025-11-27T18:19:04.622074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471403718470987:2107] [node 4] ICS09 handshake done sender: [3:7577471403718470966:2122] self: [3:7577471403718470965:0] peer: [4:7577471405365174335:0] socket: 41 qp: -1
2025-11-27T18:19:04.622104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471403718470987:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:04.626940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:04.622450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471403740804275:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.620217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471403935993072:2107] [node 4] ICS09 handshake done sender: [1:7577471403935993055:2113] self: [1:7577471403935993054:0] peer: [4:7577471405365174334:0] socket: 38 qp: -1
2025-11-27T18:19:04.663456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471403718470980:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.663629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471399423503590:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.665274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471405365174326:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:04.620254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471403935993072:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:04.643882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471403740804276:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.644159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471399445836905:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:04.664308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471403740804270:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.664335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471403740804270:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.664972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471403740804286:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.666114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471405406529599:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40060
2025-11-27T18:19:04.666277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471405406529600:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40060
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.666985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471406450775336:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62319
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.667305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471406450775338:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62319
2025-11-27T18:19:04.620291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:04.653448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471403935993051:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.653481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471403935993051:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.654413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471403935993051:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:04.657316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471399641025687:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.657377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:04.657447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471399641025687:2048] [node 2] ICP22 created new session: [1:7577471403935993076:2048]
2025-11-27T18:19:04.657495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471403935993076:2048] [node 2] ICS09 handshake done sender: [1:7577471403935993051:2111] self: [1:7577471403935993050:0] peer: [2:7577471403740804284:0] socket: 36 qp: -1
2025-11-27T18:19:04.657539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471403935993076:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:04.657578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:04.668059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471401070206956:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23631) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471405365174326:2115] held: no
2025-11-27T18:19:04.669232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471401070206956:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.669396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:04.669449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471401070206956:2048] [node 1] ICP22 created new session: [4:7577471405365174341:2048]
2025-11-27T18:19:04.667993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471403935993066:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.669487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471406450775336:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.669624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471406450775256:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:04.671600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471405365174341:2048] [node 1] ICS09 handshake done sender: [4:7577471405365174326:2115] self: [4:7577471405365174334:0] peer: [1:7577471403935993054:0] socket: 51 qp: -1
2025-11-27T18:19:04.671659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471405365174341:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:04.671707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:04.675641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471403935993057:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.675684Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471403935993057:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.677283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471403740804275:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:04.679042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471403740804270:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:04.679348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471399445836905:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.679412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:04.679476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471399445836905:2048] [node 1] ICP22 created new session: [2:7577471403740804288:2048]
2025-11-27T18:19:04.679523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471403740804288:2048] [node 1] ICS09 handshake done sender: [2:7577471403740804275:2114] self: [2:7577471403740804284:0] peer: [1:7577471403935993050:0] socket: 60 qp: -1
2025-11-27T18:19:04.679563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471403740804288:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:04.679604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:04.681885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471403935993057:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:04.669389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471405406529600:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.677747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471406450775332:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.688644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471403718470981:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.677889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471405406529521:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:04.693082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471399445836906:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.694242Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471401070206958:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3911) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471405365174327:2116] held: no
2025-11-27T18:19:04.689170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471403718470991:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18191
2025-11-27T18:19:04.685449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471405406529599:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.701448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471403935993048:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.701520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:04.701593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471403935993048:2108] [node 5] ICP22 created new session: [1:7577471403935993080:2108]
2025-11-27T18:19:04.701640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471403935993080:2108] [node 5] ICS09 handshake done sender: [1:7577471403935993057:2114] self: [1:7577471403935993056:0] peer: [5:7577471406450775339:0] socket: 42 qp: -1
2025-11-27T18:19:04.701675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471403935993080:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:04.701707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:04.702722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471403935993059:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.702750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471403935993059:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.697042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:04.700365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471399445836906:2049] [node 3] ICP22 created new session: [2:7577471403740804290:2049]
2025-11-27T18:19:04.700423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471403740804290:2049] [node 3] ICS09 handshake done sender: [2:7577471403740804270:2111] self: [2:7577471403740804269:0] peer: [3:7577471403718470990:0] socket: 66 qp: -1
2025-11-27T18:19:04.700462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471403740804290:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:04.704240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:04.704222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471406450775338:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.704667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471406450775340:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62319
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.705462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471405365174340:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27401
2025-11-27T18:19:04.704706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471403718470968:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.704739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471403718470968:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.693215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471405406529523:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:04.709652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471405365174337:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.710067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471405365174336:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.710373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471405365174327:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:04.711071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471403718470980:2129] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:04.710705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471406450775257:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.711641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471406450775258:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:04.711704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471406450775336:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:04.711659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471405406529603:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40060
2025-11-27T18:19:04.711717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471405406529604:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40060
2025-11-27T18:19:04.712298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471406450775256:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.712357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:04.712419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471406450775256:2048] [node 1] ICP22 created new session: [5:7577471406450775344:2048]
2025-11-27T18:19:04.712469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471406450775344:2048] [node 1] ICS09 handshake done sender: [5:7577471406450775336:2117] self: [5:7577471406450775339:0] peer: [1:7577471403935993056:0] socket: 79 qp: -1
2025-11-27T18:19:04.712507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471406450775344:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:04.712541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:04.721628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471401070206958:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.721692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:04.721741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471401070206958:2050] [node 3] ICP22 created new session: [4:7577471405365174346:2050]
2025-11-27T18:19:04.721789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471405365174346:2050] [node 3] ICS09 handshake done sender: [4:7577471405365174327:2116] self: [4:7577471405365174335:0] peer: [3:7577471403718470965:0] socket: 52 qp: -1
2025-11-27T18:19:04.721822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471405365174346:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:04.721859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:04.722871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471401070206957:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.724227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471403740804272:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.724264Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471403740804272:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.725184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471403740804268:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.725208Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471403740804268:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.725877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471403935993059:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:04.724109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471405406529605:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40060
2025-11-27T18:19:04.735654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471403718470970:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.735696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471403718470970:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.736266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471399423503590:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.736328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:04.736380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471399423503590:2049] [node 2] ICP22 created new session: [3:7577471403718470994:2049]
2025-11-27T18:19:04.736438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471403718470994:2049] [node 2] ICS09 handshake done sender: [3:7577471403718470980:2129] self: [3:7577471403718470990:0] peer: [2:7577471403740804269:0] socket: 80 qp: -1
2025-11-27T18:19:04.736480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471403718470994:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:04.736515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:04.737252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471399423503590:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:04.728881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471403935993049:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.728932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:04.728970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471403935993049:2109] [node 6] ICP22 created new session: [1:7577471403935993083:2109]
2025-11-27T18:19:04.729012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471403935993083:2109] [node 6] ICS09 handshake done sender: [1:7577471403935993059:2115] self: [1:7577471403935993058:0] peer: [6:7577471405406529601:0] socket: 59 qp: -1
2025-11-27T18:19:04.729051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471403935993083:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:04.729078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:04.742388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471403740804272:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:04.742839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471403740804268:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:04.744022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471403740804265:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.744068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:04.744124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471403740804265:2108] [node 5] ICP22 created new session: [2:7577471403740804295:2108]
2025-11-27T18:19:04.744196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471403740804295:2108] [node 5] ICS09 handshake done sender: [2:7577471403740804272:2112] self: [2:7577471403740804271:0] peer: [5:7577471406450775341:0] socket: 67 qp: -1
2025-11-27T18:19:04.744240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471403740804295:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:04.744296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:04.745316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471403740804264:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.745364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:04.745406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471403740804264:2107] [node 4] ICP22 created new session: [2:7577471403740804297:2107]
2025-11-27T18:19:04.745454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471403740804297:2107] [node 4] ICS09 handshake done sender: [2:7577471403740804268:2110] self: [2:7577471403740804267:0] peer: [4:7577471405365174348:0] socket: 65 qp: -1
2025-11-27T18:19:04.745482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471403740804297:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:04.745510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.747695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471405365174349:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27401
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.748271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471406450775342:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62319
2025-11-27T18:19:04.749442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471406450775346:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62319
2025-11-27T18:19:04.750118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471405365174336:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.750697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471406450775347:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62319
2025-11-27T18:19:04.753029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471405406529600:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:04.755531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471406450775342:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.757384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471401070206957:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.757464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:04.757515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471401070206957:2049] [node 2] ICP22 created new session: [4:7577471405365174350:2049]
2025-11-27T18:19:04.757562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471405365174350:2049] [node 2] ICS09 handshake done sender: [4:7577471405365174336:2119] self: [4:7577471405365174348:0] peer: [2:7577471403740804267:0] socket: 74 qp: -1
2025-11-27T18:19:04.757595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471405365174350:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:04.757627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:04.760596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471403718470968:2123] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:04.753291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471405406529521:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.755690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471406450775332:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:04.753341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:04.753406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471405406529521:2048] [node 1] ICP22 created new session: [6:7577471405406529608:2048]
2025-11-27T18:19:04.753453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471405406529608:2048] [node 1] ICS09 handshake done sender: [6:7577471405406529600:2116] self: [6:7577471405406529601:0] peer: [1:7577471403935993058:0] socket: 85 qp: -1
2025-11-27T18:19:04.753487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471405406529608:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:04.761619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471406450775257:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.762784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471403718470970:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:04.753524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:04.756468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471405406529606:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40060
2025-11-27T18:19:04.756584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471405406529607:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40060
2025-11-27T18:19:04.761138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471405406529604:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.772946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471405406529603:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.773025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471405406529599:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:04.761672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:04.761719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471406450775257:2049] [node 2] ICP22 created new session: [5:7577471406450775349:2049]
2025-11-27T18:19:04.761763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471406450775349:2049] [node 2] ICS09 handshake done sender: [5:7577471406450775332:2116] self: [5:7577471406450775341:0] peer: [2:7577471403740804271:0] socket: 69 qp: -1
2025-11-27T18:19:04.761794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471406450775349:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:04.761821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:04.765615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471406450775319:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:04.773654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471403718470964:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.774087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:04.778229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471406450775338:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:04.781798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471403718470964:2109] [node 6] ICP22 created new session: [3:7577471403718470997:2109]
2025-11-27T18:19:04.781882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471403718470997:2109] [node 6] ICS09 handshake done sender: [3:7577471403718470968:2123] self: [3:7577471403718470967:0] peer: [6:7577471405406529602:0] socket: 83 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:19:04.781919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471403718470997:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:04.781963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:04.778526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471405406529564:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:04.778635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471405406529523:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.778679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:04.778729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471405406529523:2050] [node 3] ICP22 created new session: [6:7577471405406529612:2050]
2025-11-27T18:19:04.778803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471405406529612:2050] [node 3] ICS09 handshake done sender: [6:7577471405406529599:2115] self: [6:7577471405406529602:0] peer: [3:7577471403718470967:0] socket: 84 qp: -1
2025-11-27T18:19:04.778847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471405406529612:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:04.778883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:04.783692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471403718470959:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.783761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:04.783804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471403718470959:2108] [node 5] ICP22 created new session: [3:7577471403718470999:2108]
2025-11-27T18:19:04.784131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471405406529522:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:04.782008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471403740804299:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
2025-11-27T18:19:04.785434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471405365174325:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.785471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471405365174325:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.786327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471405365174330:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.786348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471405365174330:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.784311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471405406529607:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.791314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471405406529585:2083] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:04.791643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471403718470999:2108] [node 5] ICS09 handshake done sender: [3:7577471403718470970:2124] self: [3:7577471403718470969:0] peer: [5:7577471406450775343:0] socket: 54 qp: -1
2025-11-27T18:19:04.791986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471403718470999:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:04.799984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471405365174325:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:04.801746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471405365174330:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:04.791839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471403740804279:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.793432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471406450775258:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.791882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471403740804279:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.802059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471403740804279:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:04.792037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:04.797565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:04.801812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471406450775258:2050] [node 3] ICP22 created new session: [5:7577471406450775352:2050]
2025-11-27T18:19:04.801877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471406450775352:2050] [node 3] ICS09 handshake done sender: [5:7577471406450775338:2118] self: [5:7577471406450775343:0] peer: [3:7577471403718470969:0] socket: 86 qp: -1
2025-11-27T18:19:04.801916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471406450775352:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:04.801959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.831436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471405406529616:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40060
2025-11-27T18:19:04.831527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471406450775258:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(37411555ns) {HandleHandshakeStatus:329(37409036ns) {DropIncomingHandshake:442(797ns) DropOutgoingHandshake:457(6370ns) SwitchToState:179(562ns) ProcessPendingSessionEvents:495(13017238ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12964675ns max# 12560161ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.832982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471405406529617:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40060
2025-11-27T18:19:04.832679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471406450775328:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.832711Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471406450775328:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:04.825515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471405365174316:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.825613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:04.825670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471405365174316:2108] [node 5] ICP22 created new session: [4:7577471405365174354:2108]
2025-11-27T18:19:04.825723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471405365174354:2108] [node 5] ICS09 handshake done sender: [4:7577471405365174325:2114] self: [4:7577471405365174324:0] peer: [5:7577471406450775351:0] socket: 93 qp: -1
2025-11-27T18:19:04.825762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471405365174354:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:04.825805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:04.826981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471405365174317:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.827034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:04.827090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471405365174317:2109] [node 6] ICP22 created new session: [4:7577471405365174356:2109]
2025-11-27T18:19:04.827129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471405365174356:2109] [node 6] ICS09 handshake done sender: [4:7577471405365174330:2118] self: [4:7577471405365174329:0] peer: [6:7577471405406529611:0] socket: 94 qp: -1
2025-11-27T18:19:04.827157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471405365174356:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:04.827185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:04.828706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471403740804299:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:04.828766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471403740804266:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.828799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:04.828845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471403740804266:2109] [node 6] ICP22 created new session: [2:7577471403740804301:2109]
2025-11-27T18:19:04.828888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471403740804301:2109] [node 6] ICS09 handshake done sender: [2:7577471403740804279:2117] self: [2:7577471403740804278:0] peer: [6:7577471405406529614:0] socket: 61 qp: -1
2025-11-27T18:19:04.828917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471403740804301:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:04.828947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:04.830001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471403740804266:2109] [node 6] ICP18 handshake (actor [2:7577471403740804299:2119]) is too old
2025-11-27T18:19:04.836912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471405406529564:2079] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2411) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471405406529604:2118] held: no
2025-11-27T18:19:04.846112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471405406529522:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28923) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471405406529603:2117] held: no
2025-11-27T18:19:04.846789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471405406529604:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:04.850441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471405406529603:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:19:04.852387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471406450775319:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2411) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471406450775342:2120] held: no
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:04.861469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471406450775354:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62319
2025-11-27T18:19:04.855143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471405406529564:2079] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.862099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471406450775342:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:04.855227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:04.855283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471405406529564:2079] [node 4] ICP22 created new session: [6:7577471405406529618:2079]
2025-11-27T18:19:04.855336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471405406529618:2079] [node 4] ICS09 handshake done sender: [6:7577471405406529604:2118] self: [6:7577471405406529611:0] peer: [4:7577471405365174329:0] socket: 95 qp: -1
2025-11-27T18:19:04.855441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471405406529618:2079] [node 4] ICS10 traffic start
2025-11-27T18:19:04.855484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:04.865876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471405406529522:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.865930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:04.865983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471405406529522:2049] [node 2] ICP22 created new session: [6:7577471405406529621:2049]
2025-11-27T18:19:04.866036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471405406529621:2049] [node 2] ICS09 handshake done sender: [6:7577471405406529603:2117] self: [6:7577471405406529614:0] peer: [2:7577471403740804278:0] socket: 92 qp: -1
2025-11-27T18:19:04.866075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471405406529621:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:04.866108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:04.867756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471405406529620:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40060
2025-11-27T18:19:04.868646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471406450775328:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:04.869022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471406450775319:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.869083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:04.869140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471406450775319:2106] [node 4] ICP22 created new session: [5:7577471406450775356:2106]
2025-11-27T18:19:04.869186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471406450775356:2106] [node 4] ICS09 handshake done sender: [5:7577471406450775342:2120] self: [5:7577471406450775351:0] peer: [4:7577471405365174324:0] socket: 96 qp: -1
2025-11-27T18:19:04.869225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471406450775356:2106] [node 4] ICS10 traffic start
2025-11-27T18:19:04.869273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:04.870119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471406450775320:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:04.870174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:04.870217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471406450775320:2107] [node 6] ICP22 created new session: [5:7577471406450775358:2107]
2025-11-27T18:19:04.870259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471406450775358:2107] [node 6] ICS09 handshake done sender: [5:7577471406450775328:2113] self: [5:7577471406450775327:0] peer: [6:7577471405406529615:0] socket: 103 qp: -1
2025-11-27T18:19:04.870291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471406450775358:2107] [node 6] ICS10 traffic start
2025-11-27T18:19:04.870321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:04.877675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471405406529585:2083] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28659) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471405406529607:2121] held: no
2025-11-27T18:19:04.878315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471405406529607:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:04.878880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471405406529585:2083] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:04.878932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:04.878982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471405406529585:2083] [node 5] ICP22 created new session: [6:7577471405406529623:2083]
2025-11-27T18:19:04.879040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471405406529623:2083] [node 5] ICS09 handshake done sender: [6:7577471405406529607:2121] self: [6:7577471405406529615:0] peer: [5:7577471406450775327:0] socket: 105 qp: -1
2025-11-27T18:19:04.879077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471405406529623:2083] [node 5] ICS10 traffic start
2025-11-27T18:19:04.879110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:04.905407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:04.905407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:04.905636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471403718470983:2048] [node 1] ICS01 socket: 44 reason# EndOfStream
2025-11-27T18:19:04.905682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471399423503589:2048] [node 1] ICP30 unregister session Session# [3:7577471403718470983:2048] VirtualId# [3:7577471403718470977:0]
2025-11-27T18:19:04.905721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471403718470983:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:04.905798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:04.905842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:04.906256Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:04.905687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471403740804288:2048] [node 1] ICS07 socket disconnect 60 reason# EndOfStream
2025-11-27T18:19:04.905718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471403740804288:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:04.905799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:04.905847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471403740804288:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:04.913500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:04.913746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471405406529608:2048] [node 1] ICS07 socket disconnect 85 reason# EndOfStream
2025-11-27T18:19:04.913777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471405406529608:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:04.913852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:04.913892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471405406529608:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:04.914700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:04.914859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471405406529521:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23631) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:04.914925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471405406529608:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:04.914958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471405406529521:2048] [node 1] ICP30 unregister session Session# [6:7577471405406529608:2048] VirtualId# [6:7577471405406529601:0]
2025-11-27T18:19:04.915002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:04.915048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471405406529521:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23631) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:04.915064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23631) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:04.917944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471406450775344:2048] [node 1] ICS01 socket: 79 reason# EndOfStream
2025-11-27T18:19:04.917988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471406450775256:2048] [node 1] ICP30 unregister session Session# [5:7577471406450775344:2048] VirtualId# [5:7577471406450775339:0]
2025-11-27T18:19:04.918015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471406450775344:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:04.918081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:04.906652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471405365174341:2048] [node 1] ICS01 socket: 51 reason# EndOfStream
2025-11-27T18:19:04.918257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:04.918423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471399445836905:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23631) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:04.918485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471403740804288:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:04.918518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471399445836905:2048] [node 1] ICP30 unregister session Session# [2:7577471403740804288:2048] VirtualId# [2:7577471403740804284:0]
2025-11-27T18:19:04.918555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:04.918585Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471399445836905:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23631) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:04.918601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23631) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:04.917347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471401070206956:2048] [node 1] ICP30 unregister session Session# [4:7577471405365174341:2048] VirtualId# [4:7577471405365174334:0]
2025-11-27T18:19:04.917398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471405365174341:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:04.917480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:04.917522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:04.929493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471399445836905:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:04.929382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471405406529521:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:05.009422Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:05.009656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471403718470994:2049] [node 2] ICS07 socket disconnect 80 reason# EndOfStream
2025-11-27T18:19:05.009687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471403718470994:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:05.009776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:05.009812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471403718470994:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:05.021605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471399423503590:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28923) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:05.021658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471403718470994:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:05.022198Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:05.022433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471406450775349:2049] [node 2] ICS01 socket: 69 reason# EndOfStream
2025-11-27T18:19:05.022500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471406450775257:2049] [node 2] ICP30 unregister session Session# [5:7577471406450775349:2049] VirtualId# [5:7577471406450775341:0]
2025-11-27T18:19:05.022538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471406450775349:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:05.022609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:05.022646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:05.022739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:05.022896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471405406529621:2049] [node 2] ICS01 socket: 92 reason# EndOfStream
2025-11-27T18:19:05.022930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471405406529522:2049] [node 2] ICP30 unregister session Session# [6:7577471405406529621:2049] VirtualId# [6:7577471405406529614:0]
2025-11-27T18:19:05.022959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471405406529621:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:05.023013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:05.023060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:05.021491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:05.021735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471405365174350:2049] [node 2] ICS01 socket: 74 reason# EndOfStream
2025-11-27T18:19:05.021800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471401070206957:2049] [node 2] ICP30 unregister session Session# [4:7577471405365174350:2049] VirtualId# [4:7577471405365174348:0]
2025-11-27T18:19:05.021861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471405365174350:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:05.021988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:05.022063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:05.109050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:05.109585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:05.109789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471405365174346:2050] [node 3] ICS01 socket: 52 reason# EndOfStream
2025-11-27T18:19:05.109834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471401070206958:2050] [node 3] ICP30 unregister session Session# [4:7577471405365174346:2050] VirtualId# [4:7577471405365174335:0]
2025-11-27T18:19:05.109868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471405365174346:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:05.109939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:05.109982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:05.109563Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:05.109769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471405406529612:2050] [node 3] ICS07 socket disconnect 84 reason# EndOfStream
2025-11-27T18:19:05.109795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471405406529612:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:05.109859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:05.109893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471405406529612:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:05.110898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471405406529523:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3911) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:05.110955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471405406529612:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:05.110991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471405406529523:2050] [node 3] ICP30 unregister session Session# [6:7577471405406529612:2050] VirtualId# [6:7577471405406529602:0]
2025-11-27T18:19:05.111035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:05.111072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471405406529523:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3911) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:05.111087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3911) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:05.109344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471406450775352:2050] [node 3] ICS07 socket disconnect 86 reason# EndOfStream
2025-11-27T18:19:05.109382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471406450775352:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:05.109467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:05.109510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471406450775352:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:05.112240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471406450775258:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3911) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:05.112283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471406450775352:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:05.112314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471406450775258:2050] [node 3] ICP30 unregister session Session# [5:7577471406450775352:2050] VirtualId# [5:7577471406450775343:0]
2025-11-27T18:19:05.112436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:05.112510Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471406450775258:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3911) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:05.112542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3911) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:05.121446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471405406529523:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:05.125488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471406450775258:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:05.187131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:05.187410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471406450775356:2106] [node 4] ICS01 socket: 96 reason# EndOfStream
2025-11-27T18:19:05.187462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471406450775319:2106] [node 4] ICP30 unregister session Session# [5:7577471406450775356:2106] VirtualId# [5:7577471406450775351:0]
2025-11-27T18:19:05.187502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471406450775356:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:05.187609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:05.187672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:05.189367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:05.189591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471405406529618:2079] [node 4] ICS01 socket: 95 reason# EndOfStream
2025-11-27T18:19:05.189640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471405406529564:2079] [node 4] ICP30 unregister session Session# [6:7577471405406529618:2079] VirtualId# [6:7577471405406529611:0]
2025-11-27T18:19:05.189672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471405406529618:2079] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:05.189745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:05.189785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:05.246506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:05.246780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471405406529623:2083] [node 5] ICS01 socket: 105 reason# EndOfStream
2025-11-27T18:19:05.246841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471405406529585:2083] [node 5] ICP30 unregister session Session# [6:7577471405406529623:2083] VirtualId# [6:7577471405406529615:0]
2025-11-27T18:19:05.246882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471405406529623:2083] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:05.246972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:05.247030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:05.448395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471409556718038:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:05.448440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471409556718039:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:05.477477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471408133898699:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:05.477534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471408133898700:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:05.510716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471408037456423:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:05.510770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471408037456424:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:05.528526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471409804874310:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:05.528560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471409804874311:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:05.528590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471409804874312:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:05.553828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471407659644088:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:05.553893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471407659644089:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:05.553955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471407659644090:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:05.580168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471409750992202:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:05.580211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471409750992203:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:05.580240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471409750992204:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:05.790348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471409804874370:2106] [node 6] ICP01 ready to work
2025-11-27T18:19:05.901848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471409750992204:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(94148292ns) {Configure:104(94142176ns) {ProcessConfigured:135(26567785ns) {StartInitialHandshake:157(26564264ns) {DropHandshakes:470(3504ns) {DropIncomingHandshake:442(902ns) DropOutgoingHandshake:457(438ns)} GenerateSessionVirtualId:988(1127ns) PrepareNewSessionHandshake:477(6105ns) {SwitchToState:179(2197ns)}}}}}
2025-11-27T18:19:05.889821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471408133898762:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:05.912694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471409750992221:2059] [node 4] ICP01 ready to work
2025-11-27T18:19:05.909477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471409556718102:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:05.922508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471409804874375:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:05.921917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471408037456495:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:05.921402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471407659644136:2077] [node 6] ICP01 ready to work
2025-11-27T18:19:05.926552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471408037456496:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:05.929359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471408037456499:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:05.921953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471409556718101:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:05.917084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471408133898763:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:05.976366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471408133898764:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:05.994148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471407659644141:2079] [node 4] ICP01 ready to work
2025-11-27T18:19:05.937791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471409556718103:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:05.998475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471409750992269:2066] [node 5] ICP01 ready to work
2025-11-27T18:19:06.077560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471408133898700:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33373536ns) {Configure:104(33369740ns) {ProcessConfigured:135(18610224ns) {StartInitialHandshake:157(18606495ns) {DropHandshakes:470(3348ns) {DropIncomingHandshake:442(821ns) DropOutgoingHandshake:457(384ns)} GenerateSessionVirtualId:988(1065ns) PrepareNewSessionHandshake:477(8312ns) {SwitchToState:179(2176ns)}}}}}
2025-11-27T18:19:06.075520Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471408037456499:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(109079107ns) {Configure:104(109071685ns) {ProcessConfigured:135(103918ns) {StartInitialHandshake:157(101080ns) {DropHandshakes:470(3358ns) {DropIncomingHandshake:442(806ns) DropOutgoingHandshake:457(351ns)} GenerateSessionVirtualId:988(1241ns) PrepareNewSessionHandshake:477(2428ns) {SwitchToState:179(853ns)}}}}}
2025-11-27T18:19:06.097304Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471408037456424:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(123506667ns) {Configure:104(123502298ns) {ProcessConfigured:135(49184654ns) {StartInitialHandshake:157(42526700ns) {DropHandshakes:470(4311ns) {DropIncomingHandshake:442(935ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(3456ns) PrepareNewSessionHandshake:477(4349ns) {SwitchToState:179(2268ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.128151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471407659644089:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55410499ns) {Configure:104(55398311ns) {ProcessConfigured:135(56950ns) {StartInitialHandshake:157(55011ns) {DropHandshakes:470(2790ns) {DropIncomingHandshake:442(776ns) DropOutgoingHandshake:457(487ns)} GenerateSessionVirtualId:988(861ns) PrepareNewSessionHandshake:477(2104ns) {SwitchToState:179(891ns)}}}}}
2025-11-27T18:19:06.121083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471409804874312:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28705975ns) {Configure:104(28702042ns) {ProcessConfigured:135(74291ns) {StartInitialHandshake:157(71518ns) {DropHandshakes:470(3692ns) {DropIncomingHandshake:442(1052ns) DropOutgoingHandshake:457(642ns)} GenerateSessionVirtualId:988(963ns) PrepareNewSessionHandshake:477(2352ns) {SwitchToState:179(789ns)}}}}}
2025-11-27T18:19:06.137413Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471408037456495:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(53579479ns) {Configure:104(53573143ns) {ProcessConfigured:135(11645716ns) {StartInitialHandshake:157(11643112ns) {DropHandshakes:470(3446ns) {DropIncomingHandshake:442(766ns) DropOutgoingHandshake:457(516ns)} GenerateSessionVirtualId:988(1016ns) PrepareNewSessionHandshake:477(2505ns) {SwitchToState:179(1182ns)}}}}}
2025-11-27T18:19:06.140634Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471409750992202:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(60605557ns) {Configure:104(60600618ns) {ProcessConfigured:135(8297609ns) {StartInitialHandshake:157(8295427ns) {DropHandshakes:470(2752ns) {DropIncomingHandshake:442(819ns) DropOutgoingHandshake:457(595ns)} GenerateSessionVirtualId:988(652ns) PrepareNewSessionHandshake:477(7681ns) {SwitchToState:179(1514ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.150892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471414045959576:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24667
2025-11-27T18:19:06.123615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471414099841672:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:328
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.165956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471412428866069:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60694
2025-11-27T18:19:06.166040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471412428866070:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60694
2025-11-27T18:19:06.166106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471412428866071:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60694
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.166743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471414045959577:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24667
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.167469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471412332423808:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23850
2025-11-27T18:19:06.173847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471409556718038:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21714214ns) {Configure:104(21704888ns) {ProcessConfigured:135(74584ns) {StartInitialHandshake:157(72347ns) {DropHandshakes:470(4204ns) {DropIncomingHandshake:442(1391ns) DropOutgoingHandshake:457(704ns)} GenerateSessionVirtualId:988(1871ns) PrepareNewSessionHandshake:477(2299ns) {SwitchToState:179(577ns)}}}}}
2025-11-27T18:19:06.181891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471407659644088:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53417274ns) {Configure:104(53409782ns) {ProcessConfigured:135(74424ns) {StartInitialHandshake:157(71924ns) {DropHandshakes:470(2415ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(828ns) PrepareNewSessionHandshake:477(2133ns) {SwitchToState:179(812ns)}}}}}
2025-11-27T18:19:06.164533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471414099841672:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.201158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471409556718039:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25080239ns) {Configure:104(25074932ns) {ProcessConfigured:135(93121ns) {StartInitialHandshake:157(88326ns) {DropHandshakes:470(1003ns) {DropIncomingHandshake:442(99ns) DropOutgoingHandshake:457(314ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:477(1986ns) {SwitchToState:179(671ns)}}}}}
2025-11-27T18:19:06.194031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471412428866069:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.201635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471409804874370:2106] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(291618749ns) {Configure:104(291610296ns) {ProcessConfigured:135(18855883ns) {StartInitialHandshake:157(18852251ns) {DropHandshakes:470(3094ns) {DropIncomingHandshake:442(933ns) DropOutgoingHandshake:457(572ns)} GenerateSessionVirtualId:988(702ns) PrepareNewSessionHandshake:477(6970306ns) {SwitchToState:179(5060ns)}}}}}
2025-11-27T18:19:06.198198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471412332423809:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23850
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.206069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471408133898762:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(93409640ns) {Configure:104(93402021ns) {ProcessConfigured:135(9668690ns) {StartInitialHandshake:157(5512130ns) {DropHandshakes:470(3006ns) {DropIncomingHandshake:442(550ns) DropOutgoingHandshake:457(436ns)} GenerateSessionVirtualId:988(2087ns) PrepareNewSessionHandshake:477(4431ns) {SwitchToState:179(1749ns)}}}}}
2025-11-27T18:19:06.209872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471414045959577:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.209494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471412332423809:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.206272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471412428866071:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.214705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471412428866066:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.214734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471412428866066:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.218245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471412428866066:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:06.226117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471412428866064:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.226158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471412428866064:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.239638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471413851685406:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2823
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.210012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471409750992203:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.246547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471407659644136:2077] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(41962849ns) {Configure:104(41956674ns) {ProcessConfigured:135(17043911ns) {StartInitialHandshake:157(17040976ns) {DropHandshakes:470(4462453ns) {DropIncomingHandshake:442(1658ns) DropOutgoingHandshake:457(434ns)} GenerateSessionVirtualId:988(1779ns) PrepareNewSessionHandshake:477(4266ns) {SwitchToState:179(2017ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.246225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471413851685407:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2823
2025-11-27T18:19:06.209633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471408037456424:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.214569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471414045959576:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.266934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471412428866064:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.218002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471412332423811:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23850
2025-11-27T18:19:06.262231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471413851685408:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2823
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.284677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471412428866070:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.284917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471408133898700:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.284978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:06.285026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471408133898700:2049] [node 3] ICP22 created new session: [2:7577471412428866083:2049]
2025-11-27T18:19:06.285074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471412428866083:2049] [node 3] ICS09 handshake done sender: [2:7577471412428866066:2112] self: [2:7577471412428866065:0] peer: [3:7577471412332423810:0] socket: 62 qp: -1
2025-11-27T18:19:06.285109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471412428866083:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:06.285144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:06.286965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471408133898763:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.287018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:06.287068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471408133898763:2109] [node 6] ICP22 created new session: [2:7577471412428866085:2109]
2025-11-27T18:19:06.287116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471412428866085:2109] [node 6] ICS09 handshake done sender: [2:7577471412428866064:2111] self: [2:7577471412428866063:0] peer: [6:7577471414045959578:0] socket: 61 qp: -1
2025-11-27T18:19:06.287150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471412428866085:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:06.287181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:06.246427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471412332423799:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.219231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471409750992204:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.245832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471414045959580:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24667
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.266783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471413851685406:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.319696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471414099841687:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:328
2025-11-27T18:19:06.246466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471412332423799:2121] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.271669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471414045959577:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:06.306638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471413851685407:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.329249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471411954611464:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59924
2025-11-27T18:19:06.329579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471411954611465:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59924
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.369781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471412428866081:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60694
2025-11-27T18:19:06.369948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471412428866082:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60694
2025-11-27T18:19:06.344266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471414099841687:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.262303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471412332423808:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.377831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471411954611464:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.271845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471409750992203:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.317442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471413851685408:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.344447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471409804874312:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.267575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471412332423809:2126] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.392343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471411954611463:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59924
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.345669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471413851685416:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2823
2025-11-27T18:19:06.394401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471407659644090:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.271905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:06.396742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471411954611465:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.272696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471412332423807:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23850
2025-11-27T18:19:06.377269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471414099841689:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:328
2025-11-27T18:19:06.397637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471411954611463:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.397753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471407659644088:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.399699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471411954611469:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59924
2025-11-27T18:19:06.395205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471412428866081:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.386667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471414099841687:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:06.273222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471412332423812:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23850
2025-11-27T18:19:06.350527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471413851685420:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2823
2025-11-27T18:19:06.271975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471409750992203:2049] [node 2] ICP22 created new session: [6:7577471414045959581:2049]
2025-11-27T18:19:06.403929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471411954611455:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.403962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471411954611455:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.405314Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471411954611455:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.410478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471407659644136:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.410540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:06.410582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471407659644136:2077] [node 6] ICP22 created new session: [5:7577471411954611472:2077]
2025-11-27T18:19:06.410647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471411954611472:2077] [node 6] ICS09 handshake done sender: [5:7577471411954611455:2113] self: [5:7577471411954611454:0] peer: [6:7577471414045959589:0] socket: 79 qp: -1
2025-11-27T18:19:06.410681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471411954611472:2077] [node 6] ICS10 traffic start
2025-11-27T18:19:06.410712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.422683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471411954611471:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59924
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.436304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471407659644088:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1803) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471411954611463:2116] held: no
2025-11-27T18:19:06.439406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471411954611463:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:06.398465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471412428866082:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.365660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471413851685405:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.272021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471414045959581:2049] [node 2] ICS09 handshake done sender: [6:7577471414045959577:2116] self: [6:7577471414045959578:0] peer: [2:7577471412428866063:0] socket: 63 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.282161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471412332423799:2121] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.401945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471409804874312:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.446624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471411954611474:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59924
2025-11-27T18:19:06.446796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471411954611469:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.446835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471407659644088:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.446876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:06.446937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471407659644088:2048] [node 1] ICP22 created new session: [5:7577471411954611477:2048]
2025-11-27T18:19:06.446979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471411954611477:2048] [node 1] ICS09 handshake done sender: [5:7577471411954611463:2116] self: [5:7577471411954611468:0] peer: [1:7577471413851685402:0] socket: 85 qp: -1
2025-11-27T18:19:06.447016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471411954611477:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:06.447052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:06.447945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471411954611476:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59924
2025-11-27T18:19:06.272058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471414045959581:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:06.399096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471408133898699:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.452470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471407659644089:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.453011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471411954611464:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.365697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471413851685405:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.282357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471408037456424:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.402021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.378646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471413851685405:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:06.423336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471412428866092:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60694
2025-11-27T18:19:06.282415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:06.272099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.381780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471413851685420:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.385691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471413851685416:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.385741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471409556718039:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.385808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:06.385868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471409556718039:2049] [node 3] ICP22 created new session: [1:7577471413851685422:2049]
2025-11-27T18:19:06.385926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471413851685422:2049] [node 3] ICS09 handshake done sender: [1:7577471413851685405:2112] self: [1:7577471413851685404:0] peer: [3:7577471412332423822:0] socket: 74 qp: -1
2025-11-27T18:19:06.385962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471413851685422:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:06.385998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:06.407750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471413851685403:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.407790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471413851685403:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.408629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471413851685401:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.408649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471413851685401:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.416205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471413851685403:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:06.423699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471413851685401:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:06.424535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471409556718101:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.424595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:06.424639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471409556718101:2108] [node 5] ICP22 created new session: [1:7577471413851685429:2108]
2025-11-27T18:19:06.424700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471413851685429:2108] [node 5] ICS09 handshake done sender: [1:7577471413851685403:2111] self: [1:7577471413851685402:0] peer: [5:7577471411954611468:0] socket: 73 qp: -1
2025-11-27T18:19:06.424729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471413851685429:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:06.424763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:06.425698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471409556718038:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.425745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:06.425787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471409556718038:2048] [node 2] ICP22 created new session: [1:7577471413851685431:2048]
2025-11-27T18:19:06.425828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471413851685431:2048] [node 2] ICS09 handshake done sender: [1:7577471413851685401:2110] self: [1:7577471413851685400:0] peer: [2:7577471412428866090:0] socket: 72 qp: -1
2025-11-27T18:19:06.425854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471413851685431:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:06.425910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:06.428795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471413851685410:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.428817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471413851685410:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.442325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471413851685410:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:06.444868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471409556718103:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.444917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:06.444959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471409556718103:2109] [node 6] ICP22 created new session: [1:7577471413851685435:2109]
2025-11-27T18:19:06.445029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471413851685435:2109] [node 6] ICS09 handshake done sender: [1:7577471413851685410:2116] self: [1:7577471413851685409:0] peer: [6:7577471414045959594:0] socket: 100 qp: -1
2025-11-27T18:19:06.445063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471413851685435:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:06.445090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:06.474608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471413851685412:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.474631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471413851685412:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.424109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471412428866081:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:06.457796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471411954611481:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59924
2025-11-27T18:19:06.282469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471408037456424:2049] [node 2] ICP22 created new session: [3:7577471412332423815:2049]
2025-11-27T18:19:06.282525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471412332423815:2049] [node 2] ICS09 handshake done sender: [3:7577471412332423809:2126] self: [3:7577471412332423810:0] peer: [2:7577471412428866065:0] socket: 67 qp: -1
2025-11-27T18:19:06.282568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471412332423815:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:06.460051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471411954611474:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.282607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:06.292323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471412332423807:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.460087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471407659644090:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.292372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471408037456499:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.460130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:06.292413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:06.460178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471407659644090:2050] [node 3] ICP22 created new session: [5:7577471411954611482:2050]
2025-11-27T18:19:06.292469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471408037456499:2109] [node 6] ICP22 created new session: [3:7577471412332423817:2109]
2025-11-27T18:19:06.292516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471412332423817:2109] [node 6] ICS09 handshake done sender: [3:7577471412332423799:2121] self: [3:7577471412332423798:0] peer: [6:7577471414045959579:0] socket: 53 qp: -1
2025-11-27T18:19:06.460238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471411954611482:2050] [node 3] ICS09 handshake done sender: [5:7577471411954611464:2117] self: [5:7577471411954611466:0] peer: [3:7577471412332423800:0] socket: 91 qp: -1
2025-11-27T18:19:06.292550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471412332423817:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:06.460275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471411954611482:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:06.292588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:06.460303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:06.295298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471408037456499:2109] [node 6] ICP18 handshake (actor [3:7577471412332423807:2124]) is too old
2025-11-27T18:19:06.469991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471407659644089:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5869) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471411954611469:2119] held: no
2025-11-27T18:19:06.350740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471412332423803:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.470661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471407659644141:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.350768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471412332423803:2123] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.470787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471411954611469:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:06.351395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471412332423812:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.351740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471408037456423:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.377017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471412332423803:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:06.389948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471408037456496:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.390005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:06.390075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471408037456496:2107] [node 4] ICP22 created new session: [3:7577471412332423824:2107]
2025-11-27T18:19:06.390126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471412332423824:2107] [node 4] ICS09 handshake done sender: [3:7577471412332423803:2123] self: [3:7577471412332423802:0] peer: [4:7577471414099841688:0] socket: 90 qp: -1
2025-11-27T18:19:06.429532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471408133898699:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.290138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471414045959583:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24667
2025-11-27T18:19:06.402075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471409804874312:2050] [node 3] ICP22 created new session: [4:7577471414099841690:2050]
2025-11-27T18:19:06.429573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:06.390161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471412332423824:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:06.390195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:06.392690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471412332423823:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23850
2025-11-27T18:19:06.395607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471408037456423:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1803) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471412332423812:2128] held: no
2025-11-27T18:19:06.396190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471412332423812:2128] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:06.413343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471412332423801:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.413367Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471412332423801:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.414899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471408037456423:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.414944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:06.414991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471408037456423:2048] [node 1] ICP22 created new session: [3:7577471412332423826:2048]
2025-11-27T18:19:06.415033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471412332423826:2048] [node 1] ICS09 handshake done sender: [3:7577471412332423812:2128] self: [3:7577471412332423822:0] peer: [1:7577471413851685404:0] socket: 84 qp: -1
2025-11-27T18:19:06.415077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471412332423826:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:06.415109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:06.439832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471412332423801:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:06.441126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471412332423829:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23850
2025-11-27T18:19:06.450494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471408037456495:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.450534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:06.450581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471408037456495:2108] [node 5] ICP22 created new session: [3:7577471412332423830:2108]
2025-11-27T18:19:06.450627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471412332423830:2108] [node 5] ICS09 handshake done sender: [3:7577471412332423801:2122] self: [3:7577471412332423800:0] peer: [5:7577471411954611466:0] socket: 89 qp: -1
2025-11-27T18:19:06.450662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471412332423830:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:06.450689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:06.453295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471412332423829:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.290202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471414045959584:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24667
2025-11-27T18:19:06.429614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471408133898699:2048] [node 1] ICP22 created new session: [2:7577471412428866093:2048]
2025-11-27T18:19:06.402121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471414099841690:2050] [node 3] ICS09 handshake done sender: [4:7577471414099841687:2117] self: [4:7577471414099841688:0] peer: [3:7577471412332423802:0] socket: 47 qp: -1
2025-11-27T18:19:06.478553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471407659644089:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.310736Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471409750992204:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471414045959576:2115] held: no
2025-11-27T18:19:06.478611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:06.478665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471407659644089:2049] [node 2] ICP22 created new session: [5:7577471411954611485:2049]
2025-11-27T18:19:06.478718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471411954611485:2049] [node 2] ICS09 handshake done sender: [5:7577471411954611469:2119] self: [5:7577471411954611480:0] peer: [2:7577471412428866067:0] socket: 96 qp: -1
2025-11-27T18:19:06.478755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471411954611485:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:06.478785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:06.402154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471414099841690:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:06.484981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471408037456495:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(28272619ns) {HandleHandshakeStatus:329(28270661ns) {DropIncomingHandshake:442(250ns) DropOutgoingHandshake:457(663ns) SwitchToState:179(616ns) ProcessPendingSessionEvents:495(27971212ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27890707ns max# 5913392ns}}}
2025-11-27T18:19:06.485945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471408037456495:2108] [node 5] ICP18 handshake (actor [3:7577471412332423829:2130]) is too old
2025-11-27T18:19:06.429666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471412428866093:2048] [node 1] ICS09 handshake done sender: [2:7577471412428866081:2118] self: [2:7577471412428866090:0] peer: [1:7577471413851685400:0] socket: 81 qp: -1
2025-11-27T18:19:06.483484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471413851685412:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:06.311567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471414045959576:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:06.338352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471409750992204:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.402200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:06.406295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471409804874312:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:06.440334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471414099841693:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:328
2025-11-27T18:19:06.445796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471414099841693:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.465696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471409804874311:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.465787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471414099841694:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:328
2025-11-27T18:19:06.473966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471414099841696:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:328
2025-11-27T18:19:06.474177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471414099841694:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.474301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471409804874310:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.475579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471414099841698:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:328
2025-11-27T18:19:06.481334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471414099841680:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.481360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471414099841680:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.481645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471414099841696:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.487317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471414099841694:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:06.489287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471409804874310:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.489337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:06.489383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471409804874310:2048] [node 1] ICP22 created new session: [4:7577471414099841701:2048]
2025-11-27T18:19:06.489431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471414099841701:2048] [node 1] ICS09 handshake done sender: [4:7577471414099841694:2120] self: [4:7577471414099841697:0] peer: [1:7577471413851685411:0] socket: 52 qp: -1
2025-11-27T18:19:06.489463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471414099841701:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:06.489492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:06.429700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471412428866093:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:06.429725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:06.453892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471412428866068:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.453917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471412428866068:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.455802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471412428866068:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:06.456009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471408133898764:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.456062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:06.456109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471408133898764:2108] [node 5] ICP22 created new session: [2:7577471412428866096:2108]
2025-11-27T18:19:06.456167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471412428866096:2108] [node 5] ICS09 handshake done sender: [2:7577471412428866068:2113] self: [2:7577471412428866067:0] peer: [5:7577471411954611480:0] socket: 54 qp: -1
2025-11-27T18:19:06.338411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:06.456203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471412428866096:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:06.456232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:06.338463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471409750992204:2050] [node 3] ICP22 created new session: [6:7577471414045959586:2050]
2025-11-27T18:19:06.479189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471412428866077:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.338519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471414045959586:2050] [node 3] ICS09 handshake done sender: [6:7577471414045959576:2115] self: [6:7577471414045959579:0] peer: [3:7577471412332423798:0] socket: 55 qp: -1
2025-11-27T18:19:06.479270Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471412428866077:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.338566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471414045959586:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:06.338601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:06.483720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471412428866077:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:06.366444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471414045959583:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.398144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471409750992269:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.405089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471414045959590:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24667
2025-11-27T18:19:06.410146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471414045959591:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24667
2025-11-27T18:19:06.412700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471414045959583:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:06.418639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471409750992269:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.418682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:06.418726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471409750992269:2066] [node 5] ICP22 created new session: [6:7577471414045959592:2066]
2025-11-27T18:19:06.418770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471414045959592:2066] [node 5] ICS09 handshake done sender: [6:7577471414045959583:2118] self: [6:7577471414045959589:0] peer: [5:7577471411954611454:0] socket: 80 qp: -1
2025-11-27T18:19:06.418803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471414045959592:2066] [node 5] ICS10 traffic start
2025-11-27T18:19:06.418833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:06.428357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471414045959591:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.428483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471409750992202:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.434468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471414045959595:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24667
2025-11-27T18:19:06.442767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471414045959591:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:06.442917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471409750992202:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.442975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:06.443017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471409750992202:2048] [node 1] ICP22 created new session: [6:7577471414045959596:2048]
2025-11-27T18:19:06.443072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471414045959596:2048] [node 1] ICS09 handshake done sender: [6:7577471414045959591:2121] self: [6:7577471414045959594:0] peer: [1:7577471413851685409:0] socket: 103 qp: -1
2025-11-27T18:19:06.443114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471414045959596:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:06.443145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:06.496456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471408133898762:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.496295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471409556718102:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.496521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:06.496572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471408133898762:2107] [node 4] ICP22 created new session: [2:7577471412428866099:2107]
2025-11-27T18:19:06.496611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471412428866099:2107] [node 4] ICS09 handshake done sender: [2:7577471412428866077:2117] self: [2:7577471412428866076:0] peer: [4:7577471414099841695:0] socket: 68 qp: -1
2025-11-27T18:19:06.496643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471412428866099:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:06.496703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:06.506870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471409804874310:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16818213ns) {HandleHandshakeStatus:329(16816111ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(4920ns) SwitchToState:179(1030ns) ProcessPendingSessionEvents:495(16491106ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16383012ns max# 5480825ns}}}
2025-11-27T18:19:06.507022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471414099841699:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:328
2025-11-27T18:19:06.496380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:06.496449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471409556718102:2107] [node 4] ICP22 created new session: [1:7577471413851685438:2107]
2025-11-27T18:19:06.496498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471413851685438:2107] [node 4] ICS09 handshake done sender: [1:7577471413851685412:2117] self: [1:7577471413851685411:0] peer: [4:7577471414099841697:0] socket: 42 qp: -1
2025-11-27T18:19:06.497540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471409556718103:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(50136978ns) {HandleHandshakeStatus:329(50134676ns) {DropIncomingHandshake:442(12684ns) DropOutgoingHandshake:457(522ns) SwitchToState:179(402ns) ProcessPendingSessionEvents:495(49811290ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 49722020ns max# 8252661ns}}}
2025-11-27T18:19:06.499288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471413851685438:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:06.499354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.514754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471411954611487:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59924
2025-11-27T18:19:06.516380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471409804874311:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5869) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471414099841693:2119] held: no
2025-11-27T18:19:06.517757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471414099841693:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:06.518743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471414099841680:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:06.519083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471411954611474:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.523573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471414045959598:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24667
2025-11-27T18:19:06.523675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471407659644141:2079] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.523733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:06.523773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471407659644141:2079] [node 4] ICP22 created new session: [5:7577471411954611488:2079]
2025-11-27T18:19:06.523816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471411954611488:2079] [node 4] ICS09 handshake done sender: [5:7577471411954611474:2121] self: [5:7577471411954611484:0] peer: [4:7577471414099841679:0] socket: 106 qp: -1
2025-11-27T18:19:06.523839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471411954611488:2079] [node 4] ICS10 traffic start
2025-11-27T18:19:06.523860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:06.519694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471409804874375:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.527729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471414045959598:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.527866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471409750992221:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:06.519764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:06.521614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471409804874375:2107] [node 5] ICP22 created new session: [4:7577471414099841705:2107]
2025-11-27T18:19:06.521661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471414099841705:2107] [node 5] ICS09 handshake done sender: [4:7577471414099841680:2115] self: [4:7577471414099841679:0] peer: [5:7577471411954611484:0] socket: 50 qp: -1
2025-11-27T18:19:06.521695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471414099841705:2107] [node 5] ICS10 traffic start
2025-11-27T18:19:06.521735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:06.524152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471409804874311:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.524191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:06.524237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471409804874311:2049] [node 2] ICP22 created new session: [4:7577471414099841707:2049]
2025-11-27T18:19:06.524276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471414099841707:2049] [node 2] ICS09 handshake done sender: [4:7577471414099841693:2119] self: [4:7577471414099841695:0] peer: [2:7577471412428866076:0] socket: 99 qp: -1
2025-11-27T18:19:06.524307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471414099841707:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:06.524334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:06.529235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471414099841685:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:06.529292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471414099841685:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:06.531008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471407659644141:2079] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:06.532323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471414045959600:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24667
2025-11-27T18:19:06.533186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471414099841685:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:06.533697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471414045959598:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:06.533922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471409750992221:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:06.533967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:06.534018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471409750992221:2059] [node 4] ICP22 created new session: [6:7577471414045959601:2059]
2025-11-27T18:19:06.534067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471414045959601:2059] [node 4] ICS09 handshake done sender: [6:7577471414045959598:2123] self: [6:7577471414045959599:0] peer: [4:7577471414099841684:0] socket: 64 qp: -1
2025-11-27T18:19:06.534099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471414045959601:2059] [node 4] ICS10 traffic start
2025-11-27T18:19:06.534141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:06.537296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471409804874370:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:06.537357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:06.537397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471409804874370:2106] [node 6] ICP22 created new session: [4:7577471414099841710:2106]
2025-11-27T18:19:06.537439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471414099841710:2106] [node 6] ICS09 handshake done sender: [4:7577471414099841685:2116] self: [4:7577471414099841684:0] peer: [6:7577471414045959599:0] socket: 49 qp: -1
2025-11-27T18:19:06.537485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471414099841710:2106] [node 6] ICS10 traffic start
2025-11-27T18:19:06.537512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:06.592612Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:06.592811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471412428866093:2048] [node 1] ICS01 socket: 81 reason# EndOfStream
2025-11-27T18:19:06.592866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471408133898699:2048] [node 1] ICP30 unregister session Session# [2:7577471412428866093:2048] VirtualId# [2:7577471412428866090:0]
2025-11-27T18:19:06.592908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471412428866093:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.592981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:06.593023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:06.593672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:06.593875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471412332423826:2048] [node 1] ICS01 socket: 84 reason# EndOfStream
2025-11-27T18:19:06.593912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471408037456423:2048] [node 1] ICP30 unregister session Session# [3:7577471412332423826:2048] VirtualId# [3:7577471412332423822:0]
2025-11-27T18:19:06.593943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471412332423826:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.593997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:06.594030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:06.599261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:06.599447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471414045959596:2048] [node 1] ICS01 socket: 103 reason# EndOfStream
2025-11-27T18:19:06.599480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471409750992202:2048] [node 1] ICP30 unregister session Session# [6:7577471414045959596:2048] VirtualId# [6:7577471414045959594:0]
2025-11-27T18:19:06.599511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471414045959596:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.599589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:06.599630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:06.599630Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:06.599808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471411954611477:2048] [node 1] ICS01 socket: 85 reason# EndOfStream
2025-11-27T18:19:06.599839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471407659644088:2048] [node 1] ICP30 unregister session Session# [5:7577471411954611477:2048] VirtualId# [5:7577471411954611468:0]
2025-11-27T18:19:06.599862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471411954611477:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.599918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:06.599948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:06.597490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:06.597681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471414099841701:2048] [node 1] ICS07 socket disconnect 52 reason# EndOfStream
2025-11-27T18:19:06.597713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471414099841701:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.597801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:06.597858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471414099841701:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:06.602772Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471409804874310:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1803) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:06.603340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471414099841701:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:06.603367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471409804874310:2048] [node 1] ICP30 unregister session Session# [4:7577471414099841701:2048] VirtualId# [4:7577471414099841697:0]
2025-11-27T18:19:06.607115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:06.607168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471409804874310:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1803) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:06.607200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1803) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:06.617141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471409804874310:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:06.673700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:06.673932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471412332423815:2049] [node 2] ICS07 socket disconnect 67 reason# EndOfStream
2025-11-27T18:19:06.673963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471412332423815:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.674045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:06.674079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471412332423815:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:06.677401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471408037456424:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5869) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:06.677457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471412332423815:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:06.678052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471408037456424:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5869) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:06.678121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471412332423815:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:06.678162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471408037456424:2049] [node 2] ICP30 unregister session Session# [3:7577471412332423815:2049] VirtualId# [3:7577471412332423810:0]
2025-11-27T18:19:06.678215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:06.678260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471408037456424:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5869) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:06.678275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5869) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:06.679242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:06.679459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471414099841707:2049] [node 2] ICS01 socket: 99 reason# EndOfStream
2025-11-27T18:19:06.679518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471409804874311:2049] [node 2] ICP30 unregister session Session# [4:7577471414099841707:2049] VirtualId# [4:7577471414099841695:0]
2025-11-27T18:19:06.679560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471414099841707:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.679629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:06.679673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:06.681373Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:06.681610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471414045959581:2049] [node 2] ICS07 socket disconnect 63 reason# EndOfStream
2025-11-27T18:19:06.681642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471414045959581:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.681744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:06.681786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471414045959581:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:06.682791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471409750992203:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5869) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:06.682837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471414045959581:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:06.682875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471409750992203:2049] [node 2] ICP30 unregister session Session# [6:7577471414045959581:2049] VirtualId# [6:7577471414045959578:0]
2025-11-27T18:19:06.682912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:06.682950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471409750992203:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5869) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:06.682965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5869) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:06.683469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:06.683706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471411954611485:2049] [node 2] ICS01 socket: 96 reason# EndOfStream
2025-11-27T18:19:06.683775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471407659644089:2049] [node 2] ICP30 unregister session Session# [5:7577471411954611485:2049] VirtualId# [5:7577471411954611480:0]
2025-11-27T18:19:06.683830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471411954611485:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.683961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:06.684026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:06.693625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471409750992203:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:06.732496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:06.732744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471414099841690:2050] [node 3] ICS07 socket disconnect 47 reason# EndOfStream
2025-11-27T18:19:06.732777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471414099841690:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.732841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:06.732876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471414099841690:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:06.733835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471409804874312:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10845) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:06.733895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471414099841690:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:06.733928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471409804874312:2050] [node 3] ICP30 unregister session Session# [4:7577471414099841690:2050] VirtualId# [4:7577471414099841688:0]
2025-11-27T18:19:06.733969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:06.733998Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471409804874312:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:06.734013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:06.737379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:06.737603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:06.737594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471414045959586:2050] [node 3] ICS01 socket: 55 reason# EndOfStream
2025-11-27T18:19:06.737770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471411954611482:2050] [node 3] ICS01 socket: 91 reason# EndOfStream
2025-11-27T18:19:06.737637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471409750992204:2050] [node 3] ICP30 unregister session Session# [6:7577471414045959586:2050] VirtualId# [6:7577471414045959579:0]
2025-11-27T18:19:06.737810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471407659644090:2050] [node 3] ICP30 unregister session Session# [5:7577471411954611482:2050] VirtualId# [5:7577471411954611466:0]
2025-11-27T18:19:06.737673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471414045959586:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.737841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471411954611482:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.737747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:06.737900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:06.737810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:06.737933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:06.745447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471409804874312:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:06.810229Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:06.810449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471414045959601:2059] [node 4] ICS01 socket: 64 reason# EndOfStream
2025-11-27T18:19:06.810494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471409750992221:2059] [node 4] ICP30 unregister session Session# [6:7577471414045959601:2059] VirtualId# [6:7577471414045959599:0]
2025-11-27T18:19:06.810535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471414045959601:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.810611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:06.810656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:06.807801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:06.808053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471411954611488:2079] [node 4] ICS07 socket disconnect 106 reason# EndOfStream
2025-11-27T18:19:06.808080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471411954611488:2079] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.808155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:06.808190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471411954611488:2079] [node 4] ICS15 start handshake
2025-11-27T18:19:06.812894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471407659644141:2079] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18433) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:06.812959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471411954611488:2079] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:06.813004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471407659644141:2079] [node 4] ICP30 unregister session Session# [5:7577471411954611488:2079] VirtualId# [5:7577471411954611484:0]
2025-11-27T18:19:06.813101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:06.813154Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471407659644141:2079] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18433) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:06.813174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18433) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:06.825706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471407659644141:2079] [node 4] ICP33 wake up from error state
2025-11-27T18:19:06.919632Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:06.919846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471414045959592:2066] [node 5] ICS01 socket: 80 reason# EndOfStream
2025-11-27T18:19:06.919876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471409750992269:2066] [node 5] ICP30 unregister session Session# [6:7577471414045959592:2066] VirtualId# [6:7577471414045959589:0]
2025-11-27T18:19:06.919906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471414045959592:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:06.919973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:06.920008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:07.072771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471418759186577:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:07.072823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471418759186578:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:07.102572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471417986727408:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:07.102640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471417986727409:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:07.127330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471417046260191:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:07.127395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471417046260192:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:07.162018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471416602297259:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:07.162064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471416602297260:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:07.162092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471416602297261:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:07.189181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471418195040478:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:07.189228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471418195040480:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:07.189272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471418195040479:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:07.221577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471418720772388:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:07.221620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471418720772389:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:07.221649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471418720772390:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:07.684821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471418720772421:2080] [node 5] ICP01 ready to work
2025-11-27T18:19:07.713924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471416602297322:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:07.714258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471416602297323:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:07.781961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471417046260254:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:07.780834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471418195040541:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:07.748711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471418759186641:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:07.780635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471417986727472:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:07.805907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471418195040542:2108] [node 6] ICP01 ready to work
2025-11-27T18:19:07.791054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471418759186643:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:07.813563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471417046260255:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.781644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471418759186642:2107] [node 6] ICP01 ready to work
2025-11-27T18:19:07.842300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471417046260258:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.879887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471416602297329:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60975
2025-11-27T18:19:07.879990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471416602297330:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60975
2025-11-27T18:19:07.849818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471417986727473:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:07.889528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471418720772450:2095] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.912090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471417046260260:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26733
2025-11-27T18:19:07.912242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471417046260261:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26733
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.918316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471417046260260:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:07.945595Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471418759186577:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(69818291ns) {Configure:104(69812126ns) {ProcessConfigured:135(55141ns) {StartInitialHandshake:157(54142ns) {DropHandshakes:470(2601ns) {DropIncomingHandshake:442(952ns) DropOutgoingHandshake:457(479ns)} GenerateSessionVirtualId:988(596ns) PrepareNewSessionHandshake:477(2159ns) {SwitchToState:179(954ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.941636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471418720772421:2080] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(256421086ns) {Configure:104(256415749ns) {ProcessConfigured:135(27806679ns) {StartInitialHandshake:157(27804044ns) {DropHandshakes:470(3110ns) {DropIncomingHandshake:442(685ns) DropOutgoingHandshake:457(447ns)} GenerateSessionVirtualId:988(916ns) PrepareNewSessionHandshake:477(5337ns) {SwitchToState:179(2796ns)}}}}}
2025-11-27T18:19:07.860057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471417986727476:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17251
2025-11-27T18:19:07.917909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471418195040556:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16692
2025-11-27T18:19:07.926217Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471417046260191:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(101627687ns) {Configure:104(101623749ns) {ProcessConfigured:135(28157076ns) {StartInitialHandshake:157(28154104ns) {DropHandshakes:470(2214ns) {DropIncomingHandshake:442(482ns) DropOutgoingHandshake:457(583ns)} GenerateSessionVirtualId:988(510ns) PrepareNewSessionHandshake:477(4025ns) {SwitchToState:179(1463ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.953813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471418720772388:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(155628855ns) {Configure:104(155621194ns) {ProcessConfigured:135(52603ns) {StartInitialHandshake:157(50582ns) {DropHandshakes:470(1896ns) {DropIncomingHandshake:442(558ns) DropOutgoingHandshake:457(388ns)} GenerateSessionVirtualId:988(370ns) PrepareNewSessionHandshake:477(1888ns) {SwitchToState:179(764ns)}}}}}
2025-11-27T18:19:07.956702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471418720772457:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3671
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.958129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471417986727484:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17251
2025-11-27T18:19:07.958215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471417986727486:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17251
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.956998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471418720772460:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3671
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.949695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471418759186642:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20584834ns) {Configure:104(20578596ns) {ProcessConfigured:135(37363ns) {StartInitialHandshake:157(32723ns) {DropHandshakes:470(842ns) {DropIncomingHandshake:442(250ns) DropOutgoingHandshake:457(104ns)} GenerateSessionVirtualId:988(348ns) PrepareNewSessionHandshake:477(480ns) {SwitchToState:179(138ns)}}}}}
2025-11-27T18:19:07.961708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471417046260271:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26733
2025-11-27T18:19:07.966133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471418720772457:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:07.958961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471416602297322:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(211153296ns) {Configure:104(211146630ns) {ProcessConfigured:135(35012835ns) {StartInitialHandshake:157(35009612ns) {DropHandshakes:470(1209ns) {DropIncomingHandshake:442(287ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(624ns) PrepareNewSessionHandshake:477(4849ns) {SwitchToState:179(2469ns)}}}}}
2025-11-27T18:19:07.950105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471418759186644:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8038
2025-11-27T18:19:07.950262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471418759186645:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8038
2025-11-27T18:19:07.962294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471418759186644:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.966694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471417986727486:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.974320Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471416602297323:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(166789514ns) {Configure:104(163461378ns) {ProcessConfigured:135(66156ns) {StartInitialHandshake:157(63608ns) {DropHandshakes:470(2489ns) {DropIncomingHandshake:442(633ns) DropOutgoingHandshake:457(434ns)} GenerateSessionVirtualId:988(943ns) PrepareNewSessionHandshake:477(2337ns) {SwitchToState:179(1148ns)}}}}}
2025-11-27T18:19:07.977341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471417986727485:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:07.970409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471417046260263:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:07.989614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471417986727476:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:07.997924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471417986727409:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(89112613ns) {Configure:104(89103440ns) {ProcessConfigured:135(64385ns) {StartInitialHandshake:157(62284ns) {DropHandshakes:470(2512ns) {DropIncomingHandshake:442(275ns) DropOutgoingHandshake:457(437ns)} GenerateSessionVirtualId:988(618ns) PrepareNewSessionHandshake:477(875ns) {SwitchToState:179(255ns)}}}}}
2025-11-27T18:19:07.998166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471417986727480:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:07.998191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471417986727480:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:08.005366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471417986727480:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:08.009343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471417986727478:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.009403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471417986727478:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:07.966435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471418720772390:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:07.992041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471418195040556:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:07.992179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471418195040479:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:07.994024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471418195040557:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16692
2025-11-27T18:19:07.978518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471416602297329:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.969993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471418720772389:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26002171ns) {Configure:104(25995662ns) {ProcessConfigured:135(59341ns) {StartInitialHandshake:157(57587ns) {DropHandshakes:470(2817ns) {DropIncomingHandshake:442(709ns) DropOutgoingHandshake:457(529ns)} GenerateSessionVirtualId:988(801ns) PrepareNewSessionHandshake:477(1846ns) {SwitchToState:179(693ns)}}}}}
2025-11-27T18:19:07.982320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471416602297330:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:07.970437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471417046260263:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:08.018055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471417986727478:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:08.032793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471422490007855:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16692
2025-11-27T18:19:08.033664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471418195040479:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25411) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471418195040556:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.991487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471416602297338:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60975
2025-11-27T18:19:08.046229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471418195040557:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.046361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471418195040478:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.971813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471418720772469:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3671
2025-11-27T18:19:07.978790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471417046260263:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:08.040788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471423054153953:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8038
2025-11-27T18:19:08.019456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471417986727472:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.051308Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471418759186578:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(76443035ns) {Configure:104(76435387ns) {ProcessConfigured:135(51532ns) {StartInitialHandshake:157(49776ns) {DropHandshakes:470(2076ns) {DropIncomingHandshake:442(299ns) DropOutgoingHandshake:457(536ns)} GenerateSessionVirtualId:988(396ns) PrepareNewSessionHandshake:477(1116ns) {SwitchToState:179(585ns)}}}}}
2025-11-27T18:19:08.051458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471418759186645:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.051733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471423054153955:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8038
2025-11-27T18:19:08.052230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471423054153958:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8038
2025-11-27T18:19:07.993651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471417046260254:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(115959682ns) {Configure:104(115953629ns) {ProcessConfigured:135(32797301ns) {StartInitialHandshake:157(32794746ns) {DropHandshakes:470(2777ns) {DropIncomingHandshake:442(599ns) DropOutgoingHandshake:457(509ns)} GenerateSessionVirtualId:988(654ns) PrepareNewSessionHandshake:477(1850ns) {SwitchToState:179(574ns)}}}}}
2025-11-27T18:19:08.019508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:08.023044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471418720772457:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:08.058742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471423054153953:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.059187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471418759186651:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.059219Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471418759186651:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:07.991626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471416602297260:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.060411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471423054153955:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.019561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471417986727472:2107] [node 4] ICP22 created new session: [2:7577471422281694789:2107]
2025-11-27T18:19:08.032874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471417046260255:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:07.993812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471416602297343:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60975
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.073008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471422490007858:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16692
2025-11-27T18:19:08.074378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471418195040479:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(43835620ns) {HandleHandshakeStatus:395(43830846ns) {LogHandshakeFail:472(16442ns) {UpdateErrorStateLog:545(5939ns)} DropOutgoingHandshake:457(1579ns) ProcessPendingSessionEvents:495(39911245ns) {unordered DropSessionEvent:512 num# 100 sum# 9998859ns max# 3288462ns}}}
2025-11-27T18:19:08.074598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471418195040556:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.037756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471418720772390:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.078560Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471418195040557:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:08.063785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471418759186653:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.019607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471422281694789:2107] [node 4] ICS09 handshake done sender: [2:7577471417986727480:2114] self: [2:7577471417986727479:0] peer: [4:7577471416602297342:0] socket: 54 qp: -1
2025-11-27T18:19:08.042177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:07.999589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471416602297344:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60975
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.037818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:08.043348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471417046260271:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.019645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471422281694789:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.086335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471418195040479:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.063811Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471418759186653:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:07.999643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471416602297345:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60975
2025-11-27T18:19:08.037868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471418720772390:2050] [node 3] ICP22 created new session: [6:7577471423015739766:2050]
2025-11-27T18:19:08.019676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:08.065778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471418759186649:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.087462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471418195040478:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.048875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471416602297330:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:08.046766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471417046260255:2108] [node 6] ICP22 created new session: [3:7577471421341227572:2108]
2025-11-27T18:19:08.037917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471423015739766:2050] [node 3] ICS09 handshake done sender: [6:7577471418720772457:2112] self: [6:7577471418720772466:0] peer: [3:7577471417046260262:0] socket: 72 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.020712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471417986727473:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.087531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:08.065806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471418759186649:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.047225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471417046260261:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.037960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471423015739766:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:08.053054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471416602297343:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.020757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:08.053163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471416602297259:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.053727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471416602297338:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.053837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471416602297260:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.053888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:08.059635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471416602297260:2049] [node 2] ICP22 created new session: [4:7577471420897264644:2049]
2025-11-27T18:19:08.059688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471420897264644:2049] [node 2] ICS09 handshake done sender: [4:7577471416602297330:2114] self: [4:7577471416602297342:0] peer: [2:7577471417986727479:0] socket: 57 qp: -1
2025-11-27T18:19:08.059723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471420897264644:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:08.059762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:08.060636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471416602297261:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.068746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471416602297344:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.072115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471420897264647:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60975
2025-11-27T18:19:08.080099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471416602297260:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(28711806ns) {HandleHandshakeStatus:329(28710061ns) {DropIncomingHandshake:442(1069ns) DropOutgoingHandshake:457(5829ns) SwitchToState:179(78ns) ProcessPendingSessionEvents:495(3398492ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3332167ns max# 2900953ns}}}
2025-11-27T18:19:08.095902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471420897264648:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60975
2025-11-27T18:19:08.119670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471416602297343:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:08.125984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471416602297259:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.126077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:08.126136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471416602297259:2048] [node 1] ICP22 created new session: [4:7577471420897264650:2048]
2025-11-27T18:19:08.126185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471420897264650:2048] [node 1] ICS09 handshake done sender: [4:7577471416602297343:2119] self: [4:7577471420897264642:0] peer: [1:7577471418759186648:0] socket: 84 qp: -1
2025-11-27T18:19:08.126231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471420897264650:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:08.126287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:08.127228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471416602297338:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:08.133273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471416602297261:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.133359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:08.133409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471416602297261:2050] [node 3] ICP22 created new session: [4:7577471420897264652:2050]
2025-11-27T18:19:08.133457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471420897264652:2050] [node 3] ICS09 handshake done sender: [4:7577471416602297338:2117] self: [4:7577471420897264643:0] peer: [3:7577471417046260264:0] socket: 76 qp: -1
2025-11-27T18:19:08.133492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471420897264652:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:08.133525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:08.020793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471417986727473:2108] [node 5] ICP22 created new session: [2:7577471422281694791:2108]
2025-11-27T18:19:08.020833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471422281694791:2108] [node 5] ICS09 handshake done sender: [2:7577471417986727478:2113] self: [2:7577471417986727477:0] peer: [5:7577471418195040558:0] socket: 53 qp: -1
2025-11-27T18:19:08.020854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471422281694791:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:08.020875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:08.021733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471417986727484:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.058912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471417986727472:2107] [node 4] ICP18 handshake (actor [2:7577471417986727484:2115]) is too old
2025-11-27T18:19:08.061334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471422281694797:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17251
2025-11-27T18:19:08.077861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471422281694797:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.085505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471422281694795:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.069519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471418759186653:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:08.037994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:08.085551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471422281694795:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:08.087473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471422281694795:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:08.089610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471417986727488:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.089654Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471417986727488:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:08.095539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471417986727485:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.098440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471422281694801:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17251
2025-11-27T18:19:08.099102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471417986727488:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:08.099319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471417986727409:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.099375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:08.099427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471417986727409:2049] [node 3] ICP22 created new session: [2:7577471422281694803:2049]
2025-11-27T18:19:08.099479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471422281694803:2049] [node 3] ICS09 handshake done sender: [2:7577471417986727488:2117] self: [2:7577471417986727487:0] peer: [3:7577471421341227577:0] socket: 78 qp: -1
2025-11-27T18:19:08.099524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471422281694803:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:08.099556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:08.102262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:08.104449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471417986727485:2110] [node 6] ICP22 created new session: [2:7577471422281694800:2110]
2025-11-27T18:19:08.108442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471422281694801:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.108584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471417986727408:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.125437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471422281694800:2110] [node 6] ICS09 handshake done sender: [2:7577471422281694795:2118] self: [2:7577471422281694794:0] peer: [6:7577471423015739772:0] socket: 100 qp: -1
2025-11-27T18:19:08.140217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471422281694800:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:08.153751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:08.047467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471421341227573:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26733
2025-11-27T18:19:08.087594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471418195040478:2048] [node 1] ICP22 created new session: [5:7577471422490007860:2048]
2025-11-27T18:19:08.168842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471417986727485:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(72454008ns) {HandleHandshakeStatus:329(72451162ns) {DropIncomingHandshake:442(15068ns) DropOutgoingHandshake:457(377ns) SwitchToState:179(505ns) ProcessPendingSessionEvents:495(4802429ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4745691ns max# 4306484ns}}}
2025-11-27T18:19:08.170846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471416602297340:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.170910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471416602297340:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:08.180367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471416602297340:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:08.180789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471416602297323:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.180862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:08.180905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471416602297323:2109] [node 6] ICP22 created new session: [4:7577471420897264656:2109]
2025-11-27T18:19:08.180952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471420897264656:2109] [node 6] ICS09 handshake done sender: [4:7577471416602297340:2118] self: [4:7577471416602297339:0] peer: [6:7577471423015739779:0] socket: 116 qp: -1
2025-11-27T18:19:08.087779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471422490007860:2048] [node 1] ICS09 handshake done sender: [5:7577471418195040557:2116] self: [5:7577471422490007856:0] peer: [1:7577471418759186650:0] socket: 81 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.182059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471420897264656:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:08.182105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.188078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471422281694806:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17251
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.069893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471418759186651:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:08.046816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471418720772460:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.053557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471421341227572:2108] [node 6] ICS09 handshake done sender: [3:7577471417046260263:2113] self: [3:7577471417046260262:0] peer: [6:7577471418720772466:0] socket: 68 qp: -1
2025-11-27T18:19:08.087875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:08.060541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471421341227572:2108] [node 6] ICS10 traffic start
2025-11-27T18:19:08.059304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471418720772388:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.070090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471418759186642:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.197338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471416602297337:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.197617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471417986727408:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26143) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471422281694801:2120] held: no
2025-11-27T18:19:08.089692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471418195040479:2049] [node 2] ICP22 created new session: [5:7577471422490007859:2049]
2025-11-27T18:19:08.074952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471423015739770:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3671
2025-11-27T18:19:08.077161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471423015739770:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.077457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471423015739771:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3671
2025-11-27T18:19:08.083396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471418720772389:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.086961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471418720772460:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:08.092746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471418720772388:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26143) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471418720772460:2114] held: no
2025-11-27T18:19:08.094077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471423015739773:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3671
2025-11-27T18:19:08.103888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471418720772388:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.103944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:08.103995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471418720772388:2048] [node 1] ICP22 created new session: [6:7577471423015739774:2048]
2025-11-27T18:19:08.104045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471423015739774:2048] [node 1] ICS09 handshake done sender: [6:7577471418720772460:2114] self: [6:7577471423015739769:0] peer: [1:7577471418759186652:0] socket: 77 qp: -1
2025-11-27T18:19:08.104087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471423015739774:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:08.104123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:08.135845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471418720772389:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25411) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471423015739770:2118] held: no
2025-11-27T18:19:08.137716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471423015739776:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3671
2025-11-27T18:19:08.142497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471423015739778:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3671
2025-11-27T18:19:08.147379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471423015739770:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:08.163619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471423015739778:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.165996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471418720772450:2095] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.167086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471418720772389:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.167175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:08.167222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471418720772389:2049] [node 2] ICP22 created new session: [6:7577471423015739780:2049]
2025-11-27T18:19:08.167281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471423015739780:2049] [node 2] ICS09 handshake done sender: [6:7577471423015739770:2118] self: [6:7577471423015739772:0] peer: [2:7577471422281694794:0] socket: 102 qp: -1
2025-11-27T18:19:08.167314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471423015739780:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:08.167377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:08.186548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471423015739776:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.199440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471423015739782:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3671
2025-11-27T18:19:08.230968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471418720772450:2095] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12270) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471423015739778:2122] held: no
2025-11-27T18:19:08.061742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:08.197408Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471416602297337:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:08.070148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:08.231798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471418720772421:2080] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.090203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471422490007860:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.075888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471421341227573:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.210323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471417986727408:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(31679264ns) {HandleHandshakeStatus:395(31675911ns) {LogHandshakeFail:472(12458ns) {UpdateErrorStateLog:545(5159ns)} DropOutgoingHandshake:457(1783ns) ProcessPendingSessionEvents:495(11861891ns) {unordered DropSessionEvent:512 num# 100 sum# 1500847ns max# 1017508ns}}}
2025-11-27T18:19:08.070201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471418759186642:2107] [node 6] ICP22 created new session: [1:7577471423054153965:2107]
2025-11-27T18:19:08.227678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471416602297337:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:08.234079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471423015739778:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:08.076052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471417046260265:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.090245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.070276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471423054153965:2107] [node 6] ICS09 handshake done sender: [1:7577471418759186653:2116] self: [1:7577471418759186652:0] peer: [6:7577471423015739769:0] socket: 75 qp: -1
2025-11-27T18:19:08.210661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471422281694801:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:08.241785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471416602297322:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.095422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471422490007861:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16692
2025-11-27T18:19:08.070323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471423054153965:2107] [node 6] ICS10 traffic start
2025-11-27T18:19:08.213169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471417986727408:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.108899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471422490007859:2049] [node 2] ICS09 handshake done sender: [5:7577471418195040556:2115] self: [5:7577471418195040558:0] peer: [2:7577471417986727477:0] socket: 63 qp: -1
2025-11-27T18:19:08.115480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471422490007859:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:08.123577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:08.135001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471422490007861:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.163062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471418195040541:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.171670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471418195040479:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(75197505ns) {HandleHandshakeStatus:329(75185701ns) {DropIncomingHandshake:442(774ns) DropOutgoingHandshake:457(433ns) SwitchToState:179(515ns) ProcessPendingSessionEvents:495(1900ns)}}
2025-11-27T18:19:08.177021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471418195040541:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(22847140ns) {IncomingHandshake:261(17875857ns) {DropIncomingHandshake:442(720ns) PrepareNewSessionHandshake:477(1932ns) {SwitchToState:179(755ns)} GenerateSessionVirtualId:988(858ns) IssueIncomingHandshakeReply:193(7678620ns) {SwitchToState:179(1848ns)}}}
2025-11-27T18:19:08.183607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471422490007866:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16692
2025-11-27T18:19:08.192841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471422490007867:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16692
2025-11-27T18:19:08.207158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471422490007866:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.245325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471422490007867:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.245467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471418195040555:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.245491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471418195040555:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:08.261524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471418195040480:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.265976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471418195040555:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:08.273509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471418195040542:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.273591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:08.273638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471418195040542:2108] [node 6] ICP22 created new session: [5:7577471422490007872:2108]
2025-11-27T18:19:08.273686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471422490007872:2108] [node 6] ICS09 handshake done sender: [5:7577471418195040555:2114] self: [5:7577471418195040554:0] peer: [6:7577471423015739783:0] socket: 49 qp: -1
2025-11-27T18:19:08.273722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471422490007872:2108] [node 6] ICS10 traffic start
2025-11-27T18:19:08.273754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:08.285872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471422490007869:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16692
2025-11-27T18:19:08.253436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471418720772450:2095] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.076083Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471417046260265:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:08.083946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471417046260192:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.087805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471417046260265:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:08.096310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471417046260255:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(60609211ns) {HandleHandshakeStatus:329(60607072ns) {DropIncomingHandshake:442(1861ns) DropOutgoingHandshake:457(875ns) SwitchToState:179(563ns) ProcessPendingSessionEvents:495(1760922ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 1681582ns max# 1184264ns}}}
2025-11-27T18:19:08.101859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471417046260255:2108] [node 6] ICP18 handshake (actor [3:7577471417046260271:2116]) is too old
2025-11-27T18:19:08.115650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471417046260258:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.121205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:08.131433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471417046260258:2107] [node 4] ICP22 created new session: [3:7577471421341227580:2107]
2025-11-27T18:19:08.166519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471421341227580:2107] [node 4] ICS09 handshake done sender: [3:7577471417046260265:2114] self: [3:7577471417046260264:0] peer: [4:7577471420897264643:0] socket: 69 qp: -1
2025-11-27T18:19:08.170008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471417046260192:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25411) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471421341227573:2118] held: no
2025-11-27T18:19:08.172693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471421341227580:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:08.172737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:08.184287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471421341227578:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26733
2025-11-27T18:19:08.184426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471421341227579:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26733
2025-11-27T18:19:08.185723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471421341227573:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:08.185948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471417046260192:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.186006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:08.186054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471417046260192:2049] [node 2] ICP22 created new session: [3:7577471421341227583:2049]
2025-11-27T18:19:08.186091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471421341227583:2049] [node 2] ICS09 handshake done sender: [3:7577471421341227573:2118] self: [3:7577471421341227577:0] peer: [2:7577471417986727487:0] socket: 97 qp: -1
2025-11-27T18:19:08.186120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471421341227583:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:08.186158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:08.205649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471421341227578:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.216577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471417046260191:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:08.247674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471421341227586:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26733
2025-11-27T18:19:08.274294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471417046260272:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.274332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471417046260272:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:08.273342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:08.300372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471418195040541:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12270) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471422490007861:2119] held: no
2025-11-27T18:19:08.301064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471422490007861:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:08.307724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471417046260191:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26143) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471421341227578:2119] held: no
2025-11-27T18:19:08.315047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471418195040541:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.315135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:08.315199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471418195040541:2109] [node 4] ICP22 created new session: [5:7577471422490007874:2109]
2025-11-27T18:19:08.315250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471422490007874:2109] [node 4] ICS09 handshake done sender: [5:7577471422490007861:2119] self: [5:7577471422490007865:0] peer: [4:7577471416602297336:0] socket: 111 qp: -1
2025-11-27T18:19:08.315291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471422490007874:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:08.315336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:08.253489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:08.257390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471418720772450:2095] [node 4] ICP22 created new session: [6:7577471423015739784:2095]
2025-11-27T18:19:08.321598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471421341227578:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:08.213225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:08.257457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471423015739784:2095] [node 4] ICS09 handshake done sender: [6:7577471423015739778:2122] self: [6:7577471423015739779:0] peer: [4:7577471416602297339:0] socket: 59 qp: -1
2025-11-27T18:19:08.257503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471423015739784:2095] [node 4] ICS10 traffic start
2025-11-27T18:19:08.257559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:08.322431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471418720772421:2080] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13377) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471423015739776:2121] held: no
2025-11-27T18:19:08.070366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:08.281455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471416602297322:2108] [node 5] ICP22 created new session: [4:7577471420897264659:2108]
2025-11-27T18:19:08.281535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471420897264659:2108] [node 5] ICS09 handshake done sender: [4:7577471416602297337:2116] self: [4:7577471416602297336:0] peer: [5:7577471422490007865:0] socket: 110 qp: -1
2025-11-27T18:19:08.281585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471420897264659:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:08.281655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:08.283288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471416602297322:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(40601281ns) {HandleHandshakeStatus:329(40597273ns) {DropIncomingHandshake:442(22274ns) DropOutgoingHandshake:457(698ns) SwitchToState:179(642ns) ProcessPendingSessionEvents:495(575048ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 488942ns max# 45370ns}}}
2025-11-27T18:19:08.339260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471418720772421:2080] [node 5] ICP35 event processing took too much time PendingConnection:279(36661124ns) {HandleHandshakeStatus:395(30913763ns) {LogHandshakeFail:472(14812925ns) {UpdateErrorStateLog:545(8956ns)} DropOutgoingHandshake:457(696ns) ProcessPendingSessionEvents:495(16026292ns) {unordered DropSessionEvent:512 num# 100 sum# 15995566ns max# 13123610ns}}}
2025-11-27T18:19:08.090291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471423054153958:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.213295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471417986727408:2048] [node 1] ICP22 created new session: [2:7577471422281694808:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.346685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471417046260272:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:08.213372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471422281694808:2048] [node 1] ICS09 handshake done sender: [2:7577471422281694801:2120] self: [2:7577471422281694805:0] peer: [1:7577471418759186646:0] socket: 114 qp: -1
2025-11-27T18:19:08.350968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471422490007876:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16692
2025-11-27T18:19:08.346416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471423015739786:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3671
2025-11-27T18:19:08.090351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471418759186641:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.363030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471423015739776:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:08.213410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471422281694808:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:08.090389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:08.090446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471418759186641:2108] [node 5] ICP22 created new session: [1:7577471423054153967:2108]
2025-11-27T18:19:08.090494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471423054153967:2108] [node 5] ICS09 handshake done sender: [1:7577471418759186651:2115] self: [1:7577471418759186650:0] peer: [5:7577471422490007856:0] socket: 74 qp: -1
2025-11-27T18:19:08.090534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471423054153967:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:08.090576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:08.091774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471418759186649:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:08.092294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471418759186641:2108] [node 5] ICP18 handshake (actor [1:7577471423054153958:2120]) is too old
2025-11-27T18:19:08.109117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471418759186643:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.131733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471418759186647:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.131766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471418759186647:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:08.140166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:08.145100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471418759186643:2109] [node 4] ICP22 created new session: [1:7577471423054153971:2109]
2025-11-27T18:19:08.147757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471418759186647:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:08.154545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471423054153971:2109] [node 4] ICS09 handshake done sender: [1:7577471418759186649:2114] self: [1:7577471418759186648:0] peer: [4:7577471420897264642:0] socket: 73 qp: -1
2025-11-27T18:19:08.154570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471418759186577:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.161683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471423054153971:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:08.163444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:08.171206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:08.172604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471418759186577:2048] [node 2] ICP22 created new session: [1:7577471423054153974:2048]
2025-11-27T18:19:08.186393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471423054153974:2048] [node 2] ICS09 handshake done sender: [1:7577471418759186647:2113] self: [1:7577471418759186646:0] peer: [2:7577471422281694805:0] socket: 112 qp: -1
2025-11-27T18:19:08.192949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471423054153974:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:08.193004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:08.238748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471423054153957:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:08.238798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471423054153957:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:08.247286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471423054153957:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:08.262339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471418759186578:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.262398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:08.262457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471418759186578:2049] [node 3] ICP22 created new session: [1:7577471423054153977:2049]
2025-11-27T18:19:08.262515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471423054153977:2049] [node 3] ICS09 handshake done sender: [1:7577471423054153957:2119] self: [1:7577471423054153956:0] peer: [3:7577471421341227585:0] socket: 99 qp: -1
2025-11-27T18:19:08.262564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471423054153977:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:08.262599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:08.333932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471418759186577:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(167291087ns) {HandleHandshakeStatus:329(167286862ns) {DropIncomingHandshake:442(777138ns) DropOutgoingHandshake:457(1091ns) SwitchToState:179(789ns) ProcessPendingSessionEvents:495(128696448ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 128575766ns max# 13991824ns}}}
2025-11-27T18:19:08.348675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471418759186643:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(237674611ns) {HandleHandshakeStatus:329(237671649ns) {DropIncomingHandshake:442(13051ns) DropOutgoingHandshake:457(607ns) SwitchToState:179(105ns) ProcessPendingSessionEvents:495(173366921ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 166811929ns max# 15144817ns}}}
2025-11-27T18:19:08.358361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471417046260258:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(241806539ns) {HandleHandshakeStatus:329(241803256ns) {DropIncomingHandshake:442(5487246ns) DropOutgoingHandshake:457(1690ns) SwitchToState:179(1032ns) ProcessPendingSessionEvents:495(158881273ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 158750658ns max# 22357448ns}}}
2025-11-27T18:19:08.213448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:08.362795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471422490007867:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:08.367776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471418195040478:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(183316399ns) {HandleHandshakeStatus:329(183313760ns) {DropIncomingHandshake:442(873ns) DropOutgoingHandshake:457(5539ns) SwitchToState:179(747ns) ProcessPendingSessionEvents:495(139244620ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 139105875ns max# 18302257ns}}}
2025-11-27T18:19:08.368707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471418195040542:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(94262863ns) {HandleHandshakeStatus:329(94260240ns) {DropIncomingHandshake:442(14946ns) DropOutgoingHandshake:457(553ns) SwitchToState:179(611ns) ProcessPendingSessionEvents:495(93888768ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 89277417ns max# 22466698ns}}}
2025-11-27T18:19:08.385578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471418195040480:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.358465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471417046260254:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:08.386063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471418720772421:2080] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.358537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:08.358597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471417046260254:2109] [node 5] ICP22 created new session: [3:7577471421341227588:2109]
2025-11-27T18:19:08.358647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471421341227588:2109] [node 5] ICS09 handshake done sender: [3:7577471417046260272:2117] self: [3:7577471417046260270:0] peer: [5:7577471422490007871:0] socket: 67 qp: -1
2025-11-27T18:19:08.358688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471421341227588:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:08.358723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:08.405266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:08.405364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471418195040480:2050] [node 3] ICP22 created new session: [5:7577471422490007877:2050]
2025-11-27T18:19:08.405404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471422490007877:2050] [node 3] ICS09 handshake done sender: [5:7577471422490007867:2121] self: [5:7577471422490007871:0] peer: [3:7577471417046260270:0] socket: 71 qp: -1
2025-11-27T18:19:08.405436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471422490007877:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:08.405498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:08.406769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471418195040480:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(20627214ns) {HandleHandshakeStatus:329(20624232ns) {DropIncomingHandshake:442(1743ns) DropOutgoingHandshake:457(6541ns) SwitchToState:179(678ns) ProcessPendingSessionEvents:495(528837ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 479039ns max# 68499ns}}}
2025-11-27T18:19:08.406837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471418195040480:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:08.407462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:08.409065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471418720772421:2080] [node 5] ICP22 created new session: [6:7577471423015739787:2080]
2025-11-27T18:19:08.409113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471423015739787:2080] [node 5] ICS09 handshake done sender: [6:7577471423015739776:2121] self: [6:7577471423015739783:0] peer: [5:7577471418195040554:0] socket: 80 qp: -1
2025-11-27T18:19:08.409154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471423015739787:2080] [node 5] ICS10 traffic start
2025-11-27T18:19:08.409206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:08.409440Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471418720772421:2080] [node 5] ICP35 event processing took too much time PendingConnection:279(23310178ns) {HandleHandshakeStatus:329(23306637ns) {DropIncomingHandshake:442(913ns) DropOutgoingHandshake:457(505ns) SwitchToState:179(1189ns) ProcessPendingSessionEvents:495(1187ns)}}
2025-11-27T18:19:08.411565Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471417046260191:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(107036995ns) {HandleHandshakeStatus:395(106913394ns) {LogHandshakeFail:472(6077863ns) {UpdateErrorStateLog:545(6050ns)} DropOutgoingHandshake:457(564ns) ProcessPendingSessionEvents:495(100766799ns) {unordered DropSessionEvent:512 num# 100 sum# 66836524ns max# 39133587ns}}}
2025-11-27T18:19:08.411647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471417046260191:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:08.411707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:08.411756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471417046260191:2048] [node 1] ICP22 created new session: [3:7577471421341227590:2048]
2025-11-27T18:19:08.411786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471421341227590:2048] [node 1] ICS09 handshake done sender: [3:7577471421341227578:2119] self: [3:7577471421341227585:0] peer: [1:7577471423054153956:0] socket: 101 qp: -1
2025-11-27T18:19:08.411809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471421341227590:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:08.411845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:08.466945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:08.467176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471422281694808:2048] [node 1] ICS01 socket: 114 reason# EndOfStream
2025-11-27T18:19:08.467223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471417986727408:2048] [node 1] ICP30 unregister session Session# [2:7577471422281694808:2048] VirtualId# [2:7577471422281694805:0]
2025-11-27T18:19:08.467260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471422281694808:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.467327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:08.467368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:08.469054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:08.469252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471421341227590:2048] [node 1] ICS01 socket: 101 reason# EndOfStream
2025-11-27T18:19:08.469303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471417046260191:2048] [node 1] ICP30 unregister session Session# [3:7577471421341227590:2048] VirtualId# [3:7577471421341227585:0]
2025-11-27T18:19:08.469331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471421341227590:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.469387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:08.469423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:08.477487Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:08.477836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:08.478005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:08.478193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471420897264650:2048] [node 1] ICS01 socket: 84 reason# EndOfStream
2025-11-27T18:19:08.478614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471416602297259:2048] [node 1] ICP30 unregister session Session# [4:7577471420897264650:2048] VirtualId# [4:7577471420897264642:0]
2025-11-27T18:19:08.477734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471423015739774:2048] [node 1] ICS01 socket: 77 reason# EndOfStream
2025-11-27T18:19:08.477776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471418720772388:2048] [node 1] ICP30 unregister session Session# [6:7577471423015739774:2048] VirtualId# [6:7577471423015739769:0]
2025-11-27T18:19:08.477806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471423015739774:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.477873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:08.477919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:08.478040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471422490007860:2048] [node 1] ICS07 socket disconnect 81 reason# EndOfStream
2025-11-27T18:19:08.479944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471420897264650:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.480003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:08.480048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:08.478067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471422490007860:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.478146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:08.478192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471422490007860:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:08.483882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471418195040478:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26143) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:08.484281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471422490007860:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:08.484331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471418195040478:2048] [node 1] ICP30 unregister session Session# [5:7577471422490007860:2048] VirtualId# [5:7577471422490007856:0]
2025-11-27T18:19:08.484444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:08.484485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471418195040478:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26143) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:08.484502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26143) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:08.497509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471418195040478:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:08.555400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:08.555648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471421341227583:2049] [node 2] ICS01 socket: 97 reason# EndOfStream
2025-11-27T18:19:08.555690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471417046260192:2049] [node 2] ICP30 unregister session Session# [3:7577471421341227583:2049] VirtualId# [3:7577471421341227577:0]
2025-11-27T18:19:08.555725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471421341227583:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.555807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:08.555850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:08.560352Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:08.560572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471423015739780:2049] [node 2] ICS01 socket: 102 reason# EndOfStream
2025-11-27T18:19:08.560608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471418720772389:2049] [node 2] ICP30 unregister session Session# [6:7577471423015739780:2049] VirtualId# [6:7577471423015739772:0]
2025-11-27T18:19:08.560657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471423015739780:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.560718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:08.560752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:08.560845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:08.561031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471420897264644:2049] [node 2] ICS07 socket disconnect 57 reason# EndOfStream
2025-11-27T18:19:08.561056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471420897264644:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.561107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:08.561141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471420897264644:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:08.561659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:08.561831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471422490007859:2049] [node 2] ICS01 socket: 63 reason# EndOfStream
2025-11-27T18:19:08.561864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471418195040479:2049] [node 2] ICP30 unregister session Session# [5:7577471422490007859:2049] VirtualId# [5:7577471418195040558:0]
2025-11-27T18:19:08.561889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471422490007859:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.561960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:08.561993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:08.564695Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471416602297260:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25411) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:08.564747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471420897264644:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:08.564782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471416602297260:2049] [node 2] ICP30 unregister session Session# [4:7577471420897264644:2049] VirtualId# [4:7577471416602297342:0]
2025-11-27T18:19:08.564892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:08.564923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471416602297260:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25411) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:08.564945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25411) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:08.576950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471416602297260:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:08.654070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:08.654258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471422490007877:2050] [node 3] ICS07 socket disconnect 71 reason# EndOfStream
2025-11-27T18:19:08.654289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471422490007877:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.654347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:08.654388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471422490007877:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:08.653476Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:08.653730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471423015739766:2050] [node 3] ICS01 socket: 72 reason# EndOfStream
2025-11-27T18:19:08.653779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471418720772390:2050] [node 3] ICP30 unregister session Session# [6:7577471423015739766:2050] VirtualId# [6:7577471418720772466:0]
2025-11-27T18:19:08.653817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471423015739766:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.653900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:08.653955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:08.655334Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471418195040480:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28008) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:08.655395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471422490007877:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:08.655434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471418195040480:2050] [node 3] ICP30 unregister session Session# [5:7577471422490007877:2050] VirtualId# [5:7577471422490007871:0]
2025-11-27T18:19:08.655520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:08.655552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471418195040480:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28008) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:08.655566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28008) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:08.656137Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:08.656338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471420897264652:2050] [node 3] ICS01 socket: 76 reason# EndOfStream
2025-11-27T18:19:08.656379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471416602297261:2050] [node 3] ICP30 unregister session Session# [4:7577471420897264652:2050] VirtualId# [4:7577471420897264643:0]
2025-11-27T18:19:08.656410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471420897264652:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.656484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:08.656525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:08.680652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471418195040480:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:08.773581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:08.773835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471423015739784:2095] [node 4] ICS01 socket: 59 reason# EndOfStream
2025-11-27T18:19:08.773884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471418720772450:2095] [node 4] ICP30 unregister session Session# [6:7577471423015739784:2095] VirtualId# [6:7577471423015739779:0]
2025-11-27T18:19:08.773921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471423015739784:2095] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.774001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:08.774052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:08.777302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:08.777624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471422490007874:2109] [node 4] ICS01 socket: 111 reason# EndOfStream
2025-11-27T18:19:08.777687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471418195040541:2109] [node 4] ICP30 unregister session Session# [5:7577471422490007874:2109] VirtualId# [5:7577471422490007865:0]
2025-11-27T18:19:08.777723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471422490007874:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.777832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:08.777902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:08.829702Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:08.829924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471423015739787:2080] [node 5] ICS01 socket: 80 reason# EndOfStream
2025-11-27T18:19:08.829984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471418720772421:2080] [node 5] ICP30 unregister session Session# [6:7577471423015739787:2080] VirtualId# [6:7577471423015739783:0]
2025-11-27T18:19:08.830033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471423015739787:2080] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:08.830126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:08.830177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:08.981124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471421012659049:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:08.981618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471421012659048:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:09.007611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471424655673151:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:09.007660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471424655673152:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:09.038070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471426566287603:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:09.038121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471426566287604:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:09.091536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471427696138988:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:09.091574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471427696138989:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:09.091602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471427696138990:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:09.116681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471426900766314:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:09.116726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471426900766315:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:09.116754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471426900766316:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:09.146029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471427586489839:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:09.146068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471427586489840:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:09.146094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471427586489841:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:09.825897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471425307626405:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:09.941802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471426900766395:2125] [node 6] ICP01 ready to work
2025-11-27T18:19:09.940239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471425307626408:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:09.945928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471427586489894:2091] [node 4] ICP01 ready to work
2025-11-27T18:19:09.950740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471426566287668:2108] [node 6] ICP01 ready to work
2025-11-27T18:19:09.950693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471424655673223:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:09.949038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471425307626409:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:09.961419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471427696139051:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:09.989362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471427586489899:2100] [node 5] ICP01 ready to work
2025-11-27T18:19:09.950418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471426566287667:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:09.955460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471424655673224:2107] [node 6] ICP01 ready to work
2025-11-27T18:19:09.993766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471427696139052:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:09.983181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471426900766396:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:09.955260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471426566287666:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:09.998624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471426900766314:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(56345782ns) {Configure:104(56340784ns) {ProcessConfigured:135(65601ns) {StartInitialHandshake:157(62852ns) {DropHandshakes:470(5456ns) {DropIncomingHandshake:442(584ns) DropOutgoingHandshake:457(316ns)} GenerateSessionVirtualId:988(1014ns) PrepareNewSessionHandshake:477(1522ns) {SwitchToState:179(544ns)}}}}}
2025-11-27T18:19:09.970092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471424655673225:2112] [node 5] ICP01 ready to work
2025-11-27T18:19:10.012401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471427586489899:2100] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20333568ns) {Configure:104(20329568ns) {ProcessConfigured:135(2718795ns) {StartInitialHandshake:157(2714534ns) {DropHandshakes:470(2330ns) {DropIncomingHandshake:442(495ns) DropOutgoingHandshake:457(357ns)} GenerateSessionVirtualId:988(1833ns) PrepareNewSessionHandshake:477(3064ns) {SwitchToState:179(1521ns)}}}}}
2025-11-27T18:19:10.032129Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471427696138989:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(189064997ns) {Configure:104(189058646ns) {ProcessConfigured:135(14103966ns) {StartInitialHandshake:157(14100684ns) {DropHandshakes:470(2276ns) {DropIncomingHandshake:442(717ns) DropOutgoingHandshake:457(334ns)} GenerateSessionVirtualId:988(1128ns) PrepareNewSessionHandshake:477(4298ns) {SwitchToState:179(2444ns)}}}}}
2025-11-27T18:19:10.019434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471426900766315:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44791581ns) {Configure:104(44784271ns) {ProcessConfigured:135(63965ns) {StartInitialHandshake:157(62251ns) {DropHandshakes:470(1211ns) {DropIncomingHandshake:442(399ns) DropOutgoingHandshake:457(295ns)} GenerateSessionVirtualId:988(546ns) PrepareNewSessionHandshake:477(2417ns) {SwitchToState:179(782ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.037963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471427586489841:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(193133268ns) {Configure:104(193123621ns) {ProcessConfigured:135(58926ns) {StartInitialHandshake:157(56381ns) {DropHandshakes:470(2219ns) {DropIncomingHandshake:442(663ns) DropOutgoingHandshake:457(348ns)} GenerateSessionVirtualId:988(910ns) PrepareNewSessionHandshake:477(1752ns) {SwitchToState:179(607ns)}}}}}
2025-11-27T18:19:10.054044Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471421012659048:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24351521ns) {Configure:104(24342159ns) {ProcessConfigured:135(62075ns) {StartInitialHandshake:157(60318ns) {DropHandshakes:470(1784ns) {DropIncomingHandshake:442(533ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(613ns) PrepareNewSessionHandshake:477(1836ns) {SwitchToState:179(716ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.086770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471426900766316:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40570568ns) {Configure:104(39518524ns) {ProcessConfigured:135(57710ns) {StartInitialHandshake:157(55837ns) {DropHandshakes:470(2438ns) {DropIncomingHandshake:442(526ns) DropOutgoingHandshake:457(257ns)} GenerateSessionVirtualId:988(869ns) PrepareNewSessionHandshake:477(1471ns) {SwitchToState:179(545ns)}}}}}
2025-11-27T18:19:10.088850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471431195733706:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1121
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.108980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471428950640522:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:819
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.109127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471428950640523:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:819
2025-11-27T18:19:10.109196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471428950640524:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:819
2025-11-27T18:19:10.081683Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471427586489840:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(184843363ns) {Configure:104(184835823ns) {ProcessConfigured:135(67335ns) {StartInitialHandshake:157(65491ns) {DropHandshakes:470(1912ns) {DropIncomingHandshake:442(424ns) DropOutgoingHandshake:457(496ns)} GenerateSessionVirtualId:988(525ns) PrepareNewSessionHandshake:477(1514ns) {SwitchToState:179(440ns)}}}}}
2025-11-27T18:19:10.098434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471427586489839:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(85891554ns) {Configure:104(85887805ns) {ProcessConfigured:135(47049ns) {StartInitialHandshake:157(45383ns) {DropHandshakes:470(4271ns) {DropIncomingHandshake:442(473ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(562ns) PrepareNewSessionHandshake:477(1643ns) {SwitchToState:179(493ns)}}}}}
2025-11-27T18:19:10.099120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471431881457213:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40516
2025-11-27T18:19:10.099183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471431881457214:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40516
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.121566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471426566287666:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(56980023ns) {Configure:104(56976000ns) {ProcessConfigured:135(21425777ns) {StartInitialHandshake:157(21420942ns) {DropHandshakes:470(1875ns) {DropIncomingHandshake:442(659ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(2247ns) PrepareNewSessionHandshake:477(3063ns) {SwitchToState:179(1562ns)}}}}}
2025-11-27T18:19:10.124872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471424655673152:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(115616845ns) {Configure:104(115607189ns) {ProcessConfigured:135(49620ns) {StartInitialHandshake:157(46300ns) {DropHandshakes:470(2100ns) {DropIncomingHandshake:442(459ns) DropOutgoingHandshake:457(331ns)} GenerateSessionVirtualId:988(704ns) PrepareNewSessionHandshake:477(1255ns) {SwitchToState:179(348ns)}}}}}
2025-11-27T18:19:10.056519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471429602593710:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58134
2025-11-27T18:19:10.132091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471431195733706:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.132258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471426900766314:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.134796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471428950640524:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.135255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471424655673151:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.138141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471428950640522:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.112004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471427696138990:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(37701843ns) {Configure:104(37697716ns) {ProcessConfigured:135(30519700ns) {StartInitialHandshake:157(30510334ns) {DropHandshakes:470(2413ns) {DropIncomingHandshake:442(972ns) DropOutgoingHandshake:457(257ns)} GenerateSessionVirtualId:988(573ns) PrepareNewSessionHandshake:477(4038ns) {SwitchToState:179(2434ns)}}}}}
2025-11-27T18:19:10.096707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471429602593716:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58134
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.148450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471431195733708:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1121
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.149462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471431881457214:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.137927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471430861254977:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-27T18:19:10.141351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471429602593710:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.157348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471427586489841:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.117214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471431991106358:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23854
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.165358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471431881457213:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.172069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471427586489839:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.122057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471431991106360:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23854
2025-11-27T18:19:10.150648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471430861254965:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-27T18:19:10.149619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471429602593707:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.171541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471431195733708:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.152169Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471426566287604:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(78315041ns) {Configure:104(78309138ns) {ProcessConfigured:135(75975ns) {StartInitialHandshake:157(73475ns) {DropHandshakes:470(1278ns) {DropIncomingHandshake:442(376ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(561ns) PrepareNewSessionHandshake:477(1284ns) {SwitchToState:179(596ns)}}}}}
2025-11-27T18:19:10.149658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471429602593707:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.122119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471431991106361:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23854
2025-11-27T18:19:10.171435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471428950640538:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:819
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.153158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471429602593709:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.153182Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471429602593709:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.153922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471429602593716:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.154664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471429602593717:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58134
2025-11-27T18:19:10.154795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471429602593719:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58134
2025-11-27T18:19:10.154921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471429602593707:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:10.178070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471429602593709:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:10.198274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471421012659049:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(170617041ns) {Configure:104(162619931ns) {ProcessConfigured:135(20469628ns) {StartInitialHandshake:157(20466804ns) {DropHandshakes:470(1945ns) {DropIncomingHandshake:442(475ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(641ns) PrepareNewSessionHandshake:477(7352110ns) {SwitchToState:179(3745ns)}}}}}
2025-11-27T18:19:10.181894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471431195733710:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1121
2025-11-27T18:19:10.137198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471431991106365:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23854
2025-11-27T18:19:10.162141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471430861254967:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.215452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471431881457215:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40516
2025-11-27T18:19:10.201622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471424655673224:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(169048585ns) {Configure:104(169042062ns) {ProcessConfigured:135(26002682ns) {StartInitialHandshake:157(26000356ns) {DropHandshakes:470(1116ns) {DropIncomingHandshake:442(203ns) DropOutgoingHandshake:457(180ns)} GenerateSessionVirtualId:988(525ns) PrepareNewSessionHandshake:477(3686ns) {SwitchToState:179(2159ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.206215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471425307626405:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(157708582ns) {Configure:104(157700899ns) {ProcessConfigured:135(8664389ns) {StartInitialHandshake:157(8660072ns) {DropHandshakes:470(2256ns) {DropIncomingHandshake:442(612ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(1346ns) PrepareNewSessionHandshake:477(1907ns) {SwitchToState:179(671ns)}}}}}
2025-11-27T18:19:10.212792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471428950640523:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.226507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471428950640532:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.226543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471428950640532:2125] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.230706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471428950640524:2121] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.235714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471428950640543:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:819
2025-11-27T18:19:10.162167Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471430861254967:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.221591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471431881457218:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40516
2025-11-27T18:19:10.185352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471426900766315:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.206266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471421012659048:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.138750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471431991106361:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.229797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471427586489841:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22017) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471431881457214:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.163652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471430861254967:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:10.238153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471428950640532:2125] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.217598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471431195733710:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.206305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:10.143640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471431991106358:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.246844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471424655673151:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.163895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471426566287667:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.219678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471426900766316:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.158080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471427696138990:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.261459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:10.164455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471430861254965:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.274633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471431881457215:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.279571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471427586489894:2091] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.280762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471431881457220:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40516
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.221603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471426900766314:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471431195733706:2133] held: no
2025-11-27T18:19:10.206357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471421012659048:2048] [node 2] ICP22 created new session: [1:7577471429602593726:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.165311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471431991106360:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.259273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471431195733713:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1121
2025-11-27T18:19:10.300715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471431881457213:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:10.265891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471424655673151:2048] [node 1] ICP22 created new session: [2:7577471428950640544:2048]
2025-11-27T18:19:10.206399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471429602593726:2048] [node 2] ICS09 handshake done sender: [1:7577471429602593707:2110] self: [1:7577471429602593706:0] peer: [2:7577471428950640535:0] socket: 64 qp: -1
2025-11-27T18:19:10.167869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.269434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471428950640526:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.269462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471428950640526:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.270214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471428950640543:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.270270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471424655673225:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.270372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:10.270427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471424655673225:2112] [node 5] ICP22 created new session: [2:7577471428950640545:2112]
2025-11-27T18:19:10.270482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471428950640545:2112] [node 5] ICS09 handshake done sender: [2:7577471428950640532:2125] self: [2:7577471428950640531:0] peer: [5:7577471431195733711:0] socket: 88 qp: -1
2025-11-27T18:19:10.270520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471428950640545:2112] [node 5] ICS10 traffic start
2025-11-27T18:19:10.270566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:10.272597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471428950640544:2048] [node 1] ICS09 handshake done sender: [2:7577471428950640524:2121] self: [2:7577471428950640535:0] peer: [1:7577471429602593706:0] socket: 80 qp: -1
2025-11-27T18:19:10.274428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471428950640544:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:10.274472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:10.286224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471428950640526:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:10.317846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471424655673223:2109] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.206435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471429602593726:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:10.173804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471431991106368:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23854
2025-11-27T18:19:10.302642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471431881457222:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40516
2025-11-27T18:19:10.167928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471426566287667:2107] [node 4] ICP22 created new session: [3:7577471430861254982:2107]
2025-11-27T18:19:10.272524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471431195733714:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1121
2025-11-27T18:19:10.303881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471431881457223:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40516
2025-11-27T18:19:10.345437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:10.230913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471431991106361:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:10.206467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:10.167971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471430861254982:2107] [node 4] ICS09 handshake done sender: [3:7577471430861254967:2111] self: [3:7577471430861254966:0] peer: [4:7577471431991106366:0] socket: 72 qp: -1
2025-11-27T18:19:10.168011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471430861254982:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:10.311312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471426900766315:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13059) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471431195733708:2134] held: no
2025-11-27T18:19:10.168049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:10.178330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471430861254971:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.178365Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471430861254971:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.207689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471425307626409:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.202816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471430861254971:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:10.207741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:10.217483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471426566287668:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.207782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471425307626409:2108] [node 5] ICP22 created new session: [1:7577471429602593728:2108]
2025-11-27T18:19:10.207822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471429602593728:2108] [node 5] ICS09 handshake done sender: [1:7577471429602593709:2111] self: [1:7577471429602593708:0] peer: [5:7577471431195733707:0] socket: 65 qp: -1
2025-11-27T18:19:10.207853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471429602593728:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:10.207881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:10.212708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471429602593717:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.215238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471429602593714:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:19:10.215283Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471429602593714:2113] [node 6] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-27T18:19:10.311972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471431195733709:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1121
2025-11-27T18:19:10.312317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471431195733708:2134] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:10.225811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471429602593719:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.313779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471431195733710:2136] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:10.234324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471429602593714:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:10.319501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471426900766315:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.241564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471421012659048:2048] [node 2] ICP18 handshake (actor [1:7577471429602593719:2116]) is too old
2025-11-27T18:19:10.319555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:10.217564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:10.313022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471431881457215:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:10.231254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471431991106365:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.353529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471428950640530:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.313300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471427586489839:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.217616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471426566287668:2108] [node 6] ICP22 created new session: [3:7577471430861254986:2108]
2025-11-27T18:19:10.246062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471427696138990:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.249767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471425307626408:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.319600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471426900766315:2049] [node 2] ICP22 created new session: [5:7577471431195733716:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.313359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:10.353568Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471428950640530:2124] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.217685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471430861254986:2108] [node 6] ICS09 handshake done sender: [3:7577471430861254971:2113] self: [3:7577471430861254970:0] peer: [6:7577471431881457216:0] socket: 73 qp: -1
2025-11-27T18:19:10.319652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471431195733716:2049] [node 2] ICS09 handshake done sender: [5:7577471431195733708:2134] self: [5:7577471431195733711:0] peer: [2:7577471428950640531:0] socket: 91 qp: -1
2025-11-27T18:19:10.246918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471427696138989:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.249836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:10.355075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471424655673223:2109] [node 4] ICP22 created new session: [2:7577471428950640552:2109]
2025-11-27T18:19:10.313414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471427586489839:2048] [node 1] ICP22 created new session: [6:7577471431881457224:2048]
2025-11-27T18:19:10.319678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471431195733716:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:10.254291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:10.217727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471430861254986:2108] [node 6] ICS10 traffic start
2025-11-27T18:19:10.361642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471428950640552:2109] [node 4] ICS09 handshake done sender: [2:7577471428950640526:2122] self: [2:7577471428950640525:0] peer: [4:7577471431991106371:0] socket: 86 qp: -1
2025-11-27T18:19:10.363445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471428950640552:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:10.366566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:10.373717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471428950640530:2124] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:10.385999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471424655673152:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.422329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:10.319707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:10.249886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471425307626408:2109] [node 6] ICP22 created new session: [1:7577471429602593735:2109]
2025-11-27T18:19:10.318866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471431881457224:2048] [node 1] ICS09 handshake done sender: [6:7577471431881457213:2115] self: [6:7577471431881457217:0] peer: [1:7577471429602593713:0] socket: 74 qp: -1
2025-11-27T18:19:10.217760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:10.265935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471427696138990:2050] [node 3] ICP22 created new session: [4:7577471431991106372:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.249939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471429602593735:2109] [node 6] ICS09 handshake done sender: [1:7577471429602593714:2113] self: [1:7577471429602593713:0] peer: [6:7577471431881457217:0] socket: 70 qp: -1
2025-11-27T18:19:10.318920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471431881457224:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:10.320792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471426900766316:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.434432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471424655673152:2049] [node 3] ICP22 created new session: [2:7577471428950640556:2049]
2025-11-27T18:19:10.230012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471430861254976:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.265965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471431991106370:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23854
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.320861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:10.319018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:10.249975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471429602593735:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:10.437454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471428950640553:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:819
2025-11-27T18:19:10.230028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471430861254976:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.281010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471431991106352:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.320901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471426900766316:2050] [node 3] ICP22 created new session: [5:7577471431195733718:2050]
2025-11-27T18:19:10.250007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:10.440088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471428950640556:2049] [node 3] ICS09 handshake done sender: [2:7577471428950640530:2124] self: [2:7577471428950640529:0] peer: [3:7577471430861254994:0] socket: 83 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.339220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471427586489894:2091] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.281034Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471431991106352:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.320942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471431195733718:2050] [node 3] ICS09 handshake done sender: [5:7577471431195733710:2136] self: [5:7577471431195733712:0] peer: [3:7577471430861254975:0] socket: 103 qp: -1
2025-11-27T18:19:10.340501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471431881457227:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40516
2025-11-27T18:19:10.443516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471428950640556:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:10.231063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471430861254977:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.365903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471429602593725:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.320972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471431195733718:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:10.282063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471431991106372:2050] [node 3] ICS09 handshake done sender: [4:7577471431991106361:2117] self: [4:7577471431991106366:0] peer: [3:7577471430861254966:0] socket: 78 qp: -1
2025-11-27T18:19:10.348544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:10.443559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:10.365927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471429602593725:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.373452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471429602593723:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.444078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471428950640540:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.444099Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471428950640540:2127] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.447242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471424655673152:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(60471404ns) {HandleHandshakeStatus:329(60468958ns) {DropIncomingHandshake:442(13375ns) DropOutgoingHandshake:457(732ns) SwitchToState:179(840ns) ProcessPendingSessionEvents:495(564282ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 510731ns max# 55684ns}}}
2025-11-27T18:19:10.447433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471428950640540:2127] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:10.463184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471424655673224:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.463265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:10.463313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471424655673224:2107] [node 6] ICP22 created new session: [2:7577471428950640559:2107]
2025-11-27T18:19:10.463370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471428950640559:2107] [node 6] ICS09 handshake done sender: [2:7577471428950640540:2127] self: [2:7577471428950640537:0] peer: [6:7577471431881457232:0] socket: 54 qp: -1
2025-11-27T18:19:10.463403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471428950640559:2107] [node 6] ICS10 traffic start
2025-11-27T18:19:10.463438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:10.373489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471429602593725:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:10.373495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471429602593723:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.402100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471425307626405:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.402200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:10.286440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471431991106352:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:10.409589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471425307626405:2107] [node 4] ICP22 created new session: [1:7577471429602593738:2107]
2025-11-27T18:19:10.417475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471429602593738:2107] [node 4] ICS09 handshake done sender: [1:7577471429602593725:2118] self: [1:7577471429602593724:0] peer: [4:7577471431991106379:0] socket: 63 qp: -1
2025-11-27T18:19:10.286703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471431991106372:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:10.417522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471429602593738:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:10.300284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:10.417573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:10.419061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471425307626405:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(32083215ns) {HandleHandshakeStatus:329(32080680ns) {DropIncomingHandshake:442(30526ns) DropOutgoingHandshake:457(412ns) SwitchToState:179(412ns) ProcessPendingSessionEvents:495(532089ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 471475ns max# 56441ns}}}
2025-11-27T18:19:10.309770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471431991106370:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.419251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471429602593723:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:10.313191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471427696139051:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.420263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471421012659049:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.313276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:10.420346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:10.420406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471421012659049:2049] [node 3] ICP22 created new session: [1:7577471429602593741:2049]
2025-11-27T18:19:10.321013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:10.420462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471429602593741:2049] [node 3] ICS09 handshake done sender: [1:7577471429602593723:2117] self: [1:7577471429602593718:0] peer: [3:7577471430861254995:0] socket: 60 qp: -1
2025-11-27T18:19:10.421963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471429602593741:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:10.429429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:10.441797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471421012659049:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(20731182ns) {HandleHandshakeStatus:329(20727972ns) {DropIncomingHandshake:442(20168ns) DropOutgoingHandshake:457(524ns) SwitchToState:179(514ns) ProcessPendingSessionEvents:495(5864283ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5795508ns max# 5356080ns}}}
2025-11-27T18:19:10.349556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471431195733715:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1121
2025-11-27T18:19:10.366984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471431195733715:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.254042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471430861254976:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:10.313326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471427696139051:2109] [node 6] ICP22 created new session: [4:7577471431991106377:2109]
2025-11-27T18:19:10.345818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471431991106377:2109] [node 6] ICS09 handshake done sender: [4:7577471431991106352:2111] self: [4:7577471431991106351:0] peer: [6:7577471431881457219:0] socket: 82 qp: -1
2025-11-27T18:19:10.345864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471431991106377:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:10.348599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471427586489894:2091] [node 4] ICP22 created new session: [6:7577471431881457226:2091]
2025-11-27T18:19:10.254061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471426566287667:2107] [node 4] ICP18 handshake (actor [3:7577471430861254977:2115]) is too old
2025-11-27T18:19:10.345894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:10.365610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471427696138988:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.368364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471431991106375:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23854
2025-11-27T18:19:10.377972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471431991106365:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:10.385644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471431991106381:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23854
2025-11-27T18:19:10.390128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471427696138989:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.390239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:10.390296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471427696138989:2049] [node 2] ICP22 created new session: [4:7577471431991106382:2049]
2025-11-27T18:19:10.390363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471431991106382:2049] [node 2] ICS09 handshake done sender: [4:7577471431991106365:2118] self: [4:7577471431991106371:0] peer: [2:7577471428950640525:0] socket: 90 qp: -1
2025-11-27T18:19:10.390399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471431991106382:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:10.390447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:10.429279Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471427696139051:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(115266368ns) {HandleHandshakeStatus:329(115264074ns) {DropIncomingHandshake:442(15198ns) DropOutgoingHandshake:457(283ns) SwitchToState:179(500ns) ProcessPendingSessionEvents:495(59966362ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 59882612ns max# 18385008ns}}}
2025-11-27T18:19:10.445291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471431991106370:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:10.450885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471427696138988:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.450931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:10.450974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471427696138988:2048] [node 1] ICP22 created new session: [4:7577471431991106384:2048]
2025-11-27T18:19:10.451019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471431991106384:2048] [node 1] ICS09 handshake done sender: [4:7577471431991106370:2120] self: [4:7577471431991106379:0] peer: [1:7577471429602593724:0] socket: 110 qp: -1
2025-11-27T18:19:10.451047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471431991106384:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:10.451084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:10.451690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471427696138988:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:10.472779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471431991106354:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.472800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471431991106354:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.481157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471431991106354:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:10.481575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471427696139052:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.481636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:10.481685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471427696139052:2108] [node 5] ICP22 created new session: [4:7577471431991106387:2108]
2025-11-27T18:19:10.481731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471431991106387:2108] [node 5] ICS09 handshake done sender: [4:7577471431991106354:2112] self: [4:7577471431991106353:0] peer: [5:7577471431195733729:0] socket: 66 qp: -1
2025-11-27T18:19:10.481762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471431991106387:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:10.481796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:10.494829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471427696138989:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(108250275ns) {HandleHandshakeStatus:329(108247536ns) {DropIncomingHandshake:442(2523ns) DropOutgoingHandshake:457(9581ns) SwitchToState:179(434ns) ProcessPendingSessionEvents:495(94800434ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 94696413ns max# 16094317ns}}}
2025-11-27T18:19:10.515038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471427696138990:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(267898600ns) {HandleHandshakeStatus:329(267896083ns) {DropIncomingHandshake:442(437ns) DropOutgoingHandshake:457(5003ns) SwitchToState:179(4241ns) ProcessPendingSessionEvents:495(188311823ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 188174722ns max# 28766807ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:10.391372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471426900766314:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(73684894ns) {HandleHandshakeStatus:395(73681149ns) {LogHandshakeFail:472(15756ns) {UpdateErrorStateLog:545(4156ns)} DropOutgoingHandshake:457(402ns) ProcessPendingSessionEvents:495(73610576ns) {unordered DropSessionEvent:512 num# 100 sum# 15025836ns max# 4223588ns}}}
2025-11-27T18:19:10.348654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471431881457226:2091] [node 4] ICS09 handshake done sender: [6:7577471431881457215:2117] self: [6:7577471431881457219:0] peer: [4:7577471431991106351:0] socket: 85 qp: -1
2025-11-27T18:19:10.514744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471424655673151:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(268296539ns) {HandleHandshakeStatus:329(268293871ns) {DropIncomingHandshake:442(4796ns) DropOutgoingHandshake:457(18304ns) SwitchToState:179(396ns) ProcessPendingSessionEvents:495(199896825ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 198209716ns max# 16674638ns}}}
2025-11-27T18:19:10.277963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471426566287666:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.348694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471431881457226:2091] [node 4] ICS10 traffic start
2025-11-27T18:19:10.402372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471431195733706:2133] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:10.433774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471426900766314:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.433822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:10.433863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471426900766314:2048] [node 1] ICP22 created new session: [5:7577471431195733724:2048]
2025-11-27T18:19:10.436832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471431195733705:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.436873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471431195733705:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:10.439955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471431195733724:2048] [node 1] ICS09 handshake done sender: [5:7577471431195733706:2133] self: [5:7577471431195733707:0] peer: [1:7577471429602593708:0] socket: 69 qp: -1
2025-11-27T18:19:10.445728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471431195733724:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:10.450329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:10.453498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471431195733705:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:10.457426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471426900766395:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:10.461451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:10.461511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471426900766395:2125] [node 6] ICP22 created new session: [5:7577471431195733727:2125]
2025-11-27T18:19:10.461559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471431195733727:2125] [node 6] ICS09 handshake done sender: [5:7577471431195733705:2132] self: [5:7577471431195733704:0] peer: [6:7577471431881457231:0] socket: 119 qp: -1
2025-11-27T18:19:10.461604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471431195733727:2125] [node 6] ICS10 traffic start
2025-11-27T18:19:10.461666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:10.464892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471431195733723:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1121
2025-11-27T18:19:10.465635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471426900766314:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(36119952ns) {HandleHandshakeStatus:329(31760326ns) {DropIncomingHandshake:442(488ns) DropOutgoingHandshake:457(668ns) SwitchToState:179(434ns) ProcessPendingSessionEvents:495(881ns)}}
2025-11-27T18:19:10.472320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471431195733723:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.472461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471426900766396:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.480882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471431195733730:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1121
2025-11-27T18:19:10.483599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471431195733723:2140] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:10.484042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471426900766396:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.484101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:10.484143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471426900766396:2108] [node 4] ICP22 created new session: [5:7577471431195733731:2108]
2025-11-27T18:19:10.484191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471431195733731:2108] [node 4] ICS09 handshake done sender: [5:7577471431195733723:2140] self: [5:7577471431195733729:0] peer: [4:7577471431991106353:0] socket: 121 qp: -1
2025-11-27T18:19:10.484230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471431195733731:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:10.515520Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471424655673223:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(200564837ns) {HandleHandshakeStatus:329(200563031ns) {DropIncomingHandshake:442(2330ns) DropOutgoingHandshake:457(818ns) SwitchToState:179(420ns) ProcessPendingSessionEvents:495(139448998ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 128163364ns max# 24432287ns}}}
2025-11-27T18:19:10.278035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:10.484266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:10.516431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471424655673223:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:10.348751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:10.358079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471427586489841:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(80741030ns) {HandleHandshakeStatus:395(80738983ns) {LogHandshakeFail:472(10529ns) {UpdateErrorStateLog:545(3138ns)} DropOutgoingHandshake:457(426ns) ProcessPendingSessionEvents:495(80686641ns) {unordered DropSessionEvent:512 num# 100 sum# 25311194ns max# 8582323ns}}}
2025-11-27T18:19:10.361743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471431881457214:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:10.369321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471431881457227:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.369367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471427586489841:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.372285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:10.373331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471431881457223:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.375026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471427586489841:2050] [node 3] ICP22 created new session: [6:7577471431881457229:2050]
2025-11-27T18:19:10.381408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471431881457229:2050] [node 3] ICS09 handshake done sender: [6:7577471431881457214:2116] self: [6:7577471431881457216:0] peer: [3:7577471430861254970:0] socket: 77 qp: -1
2025-11-27T18:19:10.391009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471431881457229:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:10.391047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:10.405602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471427586489899:2100] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.433589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471427586489840:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.443265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471427586489841:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(63482097ns) {HandleHandshakeStatus:329(63479289ns) {DropIncomingHandshake:442(348ns) DropOutgoingHandshake:457(318ns) SwitchToState:179(179ns) ProcessPendingSessionEvents:495(1740ns)}}
2025-11-27T18:19:10.450438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471431881457233:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40516
2025-11-27T18:19:10.493735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471431881457223:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:10.517132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471427586489840:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.517209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:10.517254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471427586489840:2049] [node 2] ICP22 created new session: [6:7577471431881457235:2049]
2025-11-27T18:19:10.517318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471431881457235:2049] [node 2] ICS09 handshake done sender: [6:7577471431881457223:2121] self: [6:7577471431881457232:0] peer: [2:7577471428950640537:0] socket: 87 qp: -1
2025-11-27T18:19:10.517350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471431881457235:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:10.517386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:10.527480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471427586489899:2100] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24836) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471431881457227:2122] held: no
2025-11-27T18:19:10.528562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471431881457234:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40516
2025-11-27T18:19:10.534893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471427586489840:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16875036ns) {HandleHandshakeStatus:329(16872797ns) {DropIncomingHandshake:442(681ns) DropOutgoingHandshake:457(4042ns) SwitchToState:179(522ns) ProcessPendingSessionEvents:495(16498193ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16437318ns max# 5666886ns}}}
2025-11-27T18:19:10.534956Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471427586489840:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:10.565452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471431881457227:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:10.278102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471426566287666:2109] [node 5] ICP22 created new session: [3:7577471430861254991:2109]
2025-11-27T18:19:10.278153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471430861254991:2109] [node 5] ICS09 handshake done sender: [3:7577471430861254976:2114] self: [3:7577471430861254975:0] peer: [5:7577471431195733712:0] socket: 101 qp: -1
2025-11-27T18:19:10.278191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471430861254991:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:10.278225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:10.294803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471430861254993:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-27T18:19:10.312740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471430861254993:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.314917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471426566287604:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.341793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471430861254990:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-27T18:19:10.362561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471430861254990:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:10.367080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471426566287603:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:10.374727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471430861254996:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-27T18:19:10.386041Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471430861254993:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:10.411189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471426566287604:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.411246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:10.411300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471426566287604:2049] [node 2] ICP22 created new session: [3:7577471430861254997:2049]
2025-11-27T18:19:10.411352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471430861254997:2049] [node 2] ICS09 handshake done sender: [3:7577471430861254993:2118] self: [3:7577471430861254994:0] peer: [2:7577471428950640529:0] socket: 92 qp: -1
2025-11-27T18:19:10.411393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471430861254997:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:10.411432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:10.412185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471426566287604:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:10.434990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471426566287603:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471430861254990:2117] held: no
2025-11-27T18:19:10.440310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471430861254999:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:342
2025-11-27T18:19:10.471459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471426566287603:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(36159797ns) {HandleHandshakeStatus:395(36155959ns) {LogHandshakeFail:472(394427ns) {UpdateErrorStateLog:545(4706ns)} DropOutgoingHandshake:457(330ns) ProcessPendingSessionEvents:495(35711463ns) {unordered DropSessionEvent:512 num# 100 sum# 35669616ns max# 11717731ns}}}
2025-11-27T18:19:10.471719Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471430861254990:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:10.477715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471426566287603:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.477765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:10.497559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471426566287603:2048] [node 1] ICP22 created new session: [3:7577471430861255000:2048]
2025-11-27T18:19:10.501432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471430861255000:2048] [node 1] ICS09 handshake done sender: [3:7577471430861254990:2117] self: [3:7577471430861254995:0] peer: [1:7577471429602593718:0] socket: 108 qp: -1
2025-11-27T18:19:10.501482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471430861255000:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:10.501558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:10.501783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471426566287603:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23988486ns) {HandleHandshakeStatus:329(23984720ns) {DropIncomingHandshake:442(612ns) DropOutgoingHandshake:457(515ns) SwitchToState:179(515ns) ProcessPendingSessionEvents:495(1008ns)}}
2025-11-27T18:19:10.568601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471427586489899:2100] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:10.568666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:10.568714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471427586489899:2100] [node 5] ICP22 created new session: [6:7577471431881457237:2100]
2025-11-27T18:19:10.568764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471431881457237:2100] [node 5] ICS09 handshake done sender: [6:7577471431881457227:2122] self: [6:7577471431881457231:0] peer: [5:7577471431195733704:0] socket: 120 qp: -1
2025-11-27T18:19:10.568811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471431881457237:2100] [node 5] ICS10 traffic start
2025-11-27T18:19:10.568855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:10.570748Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471427586489839:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(240346565ns) {HandleHandshakeStatus:329(240343524ns) {DropIncomingHandshake:442(586ns) DropOutgoingHandshake:457(4169ns) SwitchToState:179(376ns) ProcessPendingSessionEvents:495(234408904ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 234259150ns max# 27251950ns}}}
2025-11-27T18:19:10.612053Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:10.612265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471428950640544:2048] [node 1] ICS07 socket disconnect 80 reason# EndOfStream
2025-11-27T18:19:10.612290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471428950640544:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.612371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:10.612425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471428950640544:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:10.613558Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:10.613788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471430861255000:2048] [node 1] ICS01 socket: 108 reason# EndOfStream
2025-11-27T18:19:10.613836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471426566287603:2048] [node 1] ICP30 unregister session Session# [3:7577471430861255000:2048] VirtualId# [3:7577471430861254995:0]
2025-11-27T18:19:10.613868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471430861255000:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.613933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:10.613972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:10.616796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471424655673151:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:10.616834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471428950640544:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:10.617620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471424655673151:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:10.617692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471428950640544:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:10.617744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471424655673151:2048] [node 1] ICP30 unregister session Session# [2:7577471428950640544:2048] VirtualId# [2:7577471428950640535:0]
2025-11-27T18:19:10.617884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:10.617913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471424655673151:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.617926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.624147Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:10.622923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:10.623147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471431991106384:2048] [node 1] ICS07 socket disconnect 110 reason# EndOfStream
2025-11-27T18:19:10.623173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471431991106384:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.623254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:10.623298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471431991106384:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:10.624330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471431195733724:2048] [node 1] ICS01 socket: 69 reason# EndOfStream
2025-11-27T18:19:10.624368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471426900766314:2048] [node 1] ICP30 unregister session Session# [5:7577471431195733724:2048] VirtualId# [5:7577471431195733707:0]
2025-11-27T18:19:10.624402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471431195733724:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.624473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:10.624512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:10.625530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471427696138988:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:10.625604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471431991106384:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:10.625641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471427696138988:2048] [node 1] ICP30 unregister session Session# [4:7577471431991106384:2048] VirtualId# [4:7577471431991106379:0]
2025-11-27T18:19:10.625683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:10.627310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471427696138988:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.627583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.624777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:10.624979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471431881457224:2048] [node 1] ICS07 socket disconnect 74 reason# EndOfStream
2025-11-27T18:19:10.625003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471431881457224:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.625072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:10.625118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471431881457224:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:10.631826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471427586489839:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:10.637727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471431881457224:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:10.637736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471427696138988:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:10.628211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471424655673151:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:10.641656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471427586489839:2048] [node 1] ICP30 unregister session Session# [6:7577471431881457224:2048] VirtualId# [6:7577471431881457217:0]
2025-11-27T18:19:10.642602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:10.642644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471427586489839:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.645864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.646180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [6:7577471427586489839:2048] [node 1] ICP35 event processing took too much time StateWork:294(18124832ns) {HandleHandshakeStatus:395(18122430ns) {LogHandshakeFail:472(14437ns) {UpdateErrorStateLog:545(1607ns)} DropOutgoingHandshake:457(2641ns) ProcessPendingSessionEvents:495(308ns) UnregisterSession:538(3943857ns) {ProcessPendingSessionEvents:495(1325ns) SwitchToInitialState:202(1777ns) {SwitchToState:179(1146ns)}} TransitToErrorState:752(3304461ns) {SwitchToState:179(34647ns) ProcessPendingSessionEvents:495(471ns) DropHandshakes:470(1868ns) {DropIncomingHandshake:442(497ns) DropOutgoingHandshake:457(298ns)}}}}
2025-11-27T18:19:10.656833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471427586489839:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:10.682074Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:10.682307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471430861254997:2049] [node 2] ICS01 socket: 92 reason# EndOfStream
2025-11-27T18:19:10.682369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471426566287604:2049] [node 2] ICP30 unregister session Session# [3:7577471430861254997:2049] VirtualId# [3:7577471430861254994:0]
2025-11-27T18:19:10.682406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471430861254997:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.682469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:10.682522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:10.684467Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:10.684688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471431881457235:2049] [node 2] ICS07 socket disconnect 87 reason# EndOfStream
2025-11-27T18:19:10.684729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471431881457235:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.684808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:10.684842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471431881457235:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:10.685700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:10.685903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471431991106382:2049] [node 2] ICS01 socket: 90 reason# EndOfStream
2025-11-27T18:19:10.685940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471427696138989:2049] [node 2] ICP30 unregister session Session# [4:7577471431991106382:2049] VirtualId# [4:7577471431991106371:0]
2025-11-27T18:19:10.685968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471431991106382:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.686036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:10.686090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:10.686682Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471427586489840:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13059) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:10.687254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471431881457235:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:10.687294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471427586489840:2049] [node 2] ICP30 unregister session Session# [6:7577471431881457235:2049] VirtualId# [6:7577471431881457232:0]
2025-11-27T18:19:10.689385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:10.689645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471431195733716:2049] [node 2] ICS01 socket: 91 reason# EndOfStream
2025-11-27T18:19:10.689680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471426900766315:2049] [node 2] ICP30 unregister session Session# [5:7577471431195733716:2049] VirtualId# [5:7577471431195733711:0]
2025-11-27T18:19:10.702567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:10.689711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471431195733716:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.702669Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471427586489840:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13059) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.689792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:10.702693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13059) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.689832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:10.702964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [6:7577471427586489840:2049] [node 2] ICP35 event processing took too much time StateWork:294(16453225ns) {HandleHandshakeStatus:395(16107921ns) {LogHandshakeFail:472(19293ns) {UpdateErrorStateLog:545(2456ns)} DropOutgoingHandshake:457(955ns) ProcessPendingSessionEvents:495(273ns) UnregisterSession:538(35674ns) {ProcessPendingSessionEvents:495(147ns) SwitchToInitialState:202(1253ns) {SwitchToState:179(580ns)}} TransitToErrorState:752(92428ns) {SwitchToState:179(7930ns) ProcessPendingSessionEvents:495(461ns) DropHandshakes:470(3065ns) {DropIncomingHandshake:442(596ns) DropOutgoingHandshake:457(214ns)}}}}
2025-11-27T18:19:10.713459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471427586489840:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:10.808599Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:10.808874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471431991106372:2050] [node 3] ICS07 socket disconnect 78 reason# EndOfStream
2025-11-27T18:19:10.808911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471431991106372:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.808991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:10.809054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471431991106372:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:10.810157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471427696138990:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22017) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:10.810238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471431991106372:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:10.810276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471427696138990:2050] [node 3] ICP30 unregister session Session# [4:7577471431991106372:2050] VirtualId# [4:7577471431991106366:0]
2025-11-27T18:19:10.810384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:10.810425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471427696138990:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.810440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.811605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:10.811835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471431195733718:2050] [node 3] ICS07 socket disconnect 103 reason# EndOfStream
2025-11-27T18:19:10.811883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471431195733718:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.811989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:10.812031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471431195733718:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:10.813110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471426900766316:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22017) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:10.813173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471431195733718:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:10.813210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471426900766316:2050] [node 3] ICP30 unregister session Session# [5:7577471431195733718:2050] VirtualId# [5:7577471431195733712:0]
2025-11-27T18:19:10.813323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:10.813354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471426900766316:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.813380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.819279Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:10.819720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471431881457229:2050] [node 3] ICS01 socket: 77 reason# EndOfStream
2025-11-27T18:19:10.819816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471427586489841:2050] [node 3] ICP30 unregister session Session# [6:7577471431881457229:2050] VirtualId# [6:7577471431881457216:0]
2025-11-27T18:19:10.819884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471431881457229:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.820035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:10.820158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:10.821685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471427696138990:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:10.824098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471426900766316:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:10.877449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:10.877746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471431195733731:2108] [node 4] ICS07 socket disconnect 121 reason# EndOfStream
2025-11-27T18:19:10.877782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471431195733731:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.877876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:10.877933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471431195733731:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:10.879008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471426900766396:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11869) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:10.879098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471431195733731:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:10.879150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471426900766396:2108] [node 4] ICP30 unregister session Session# [5:7577471431195733731:2108] VirtualId# [5:7577471431195733729:0]
2025-11-27T18:19:10.879270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:10.879312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471426900766396:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11869) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.879328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11869) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:10.884111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:10.884523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471431881457226:2091] [node 4] ICS01 socket: 85 reason# EndOfStream
2025-11-27T18:19:10.884619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471427586489894:2091] [node 4] ICP30 unregister session Session# [6:7577471431881457226:2091] VirtualId# [6:7577471431881457219:0]
2025-11-27T18:19:10.884747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471431881457226:2091] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:10.884917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:10.884982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:10.889302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471426900766396:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:19:11.047678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:11.047953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471431881457237:2100] [node 5] ICS01 socket: 120 reason# EndOfStream
2025-11-27T18:19:11.048010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471427586489899:2100] [node 5] ICP30 unregister session Session# [6:7577471431881457237:2100] VirtualId# [6:7577471431881457231:0]
2025-11-27T18:19:11.048051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471431881457237:2100] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:11.048155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:11.048215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:11.249357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471436811373656:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:11.249404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471436811373657:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:11.283893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471433396637871:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:11.283943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471433396637872:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:11.300613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471434462960792:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:11.300635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471434462960793:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:11.325683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471435640568640:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:11.325729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471435640568641:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:11.325762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471435640568642:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:11.352769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471435091935906:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:11.352816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471435091935907:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:11.352848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471435091935908:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:11.381640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471436489211094:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:11.381682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471436489211095:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:11.381725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471436489211096:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:11.849864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471436489211111:2064] [node 5] ICP01 ready to work
2025-11-27T18:19:11.858982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471436489211118:2071] [node 4] ICP01 ready to work
2025-11-27T18:19:11.873214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471435091935951:2095] [node 6] ICP01 ready to work
2025-11-27T18:19:11.870304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471433396637934:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:11.915033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471433396637938:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:12.033547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471438757928154:2111] [node 6] ICP01 ready to work
2025-11-27T18:19:12.019674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471435091935964:2103] [node 4] ICP01 ready to work
2025-11-27T18:19:12.039860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471441106341016:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:12.040156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471439935535999:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:12.052556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471439935536000:2108] [node 6] ICP01 ready to work
2025-11-27T18:19:12.058074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471435640568641:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(136040473ns) {Configure:104(136034652ns) {ProcessConfigured:135(71833ns) {StartInitialHandshake:157(68128ns) {DropHandshakes:470(3098ns) {DropIncomingHandshake:442(699ns) DropOutgoingHandshake:457(584ns)} GenerateSessionVirtualId:988(1075ns) PrepareNewSessionHandshake:477(1512ns) {SwitchToState:179(408ns)}}}}}
2025-11-27T18:19:12.040135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471434462960857:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:12.077311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471441106341018:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:12.080436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471437691605238:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:12.084077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471438757928155:2110] [node 5] ICP01 ready to work
2025-11-27T18:19:12.083112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471441106341017:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:12.116333Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471436489211118:2071] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(232244945ns) {Configure:104(232240216ns) {ProcessConfigured:135(2879628ns) {StartInitialHandshake:157(2876934ns) {DropHandshakes:470(3048ns) {DropIncomingHandshake:442(577ns) DropOutgoingHandshake:457(713ns)} GenerateSessionVirtualId:988(919ns) PrepareNewSessionHandshake:477(3143ns) {SwitchToState:179(1213ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.138253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471440784178459:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.090964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471436811373656:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(226105587ns) {Configure:104(226099360ns) {ProcessConfigured:135(56554942ns) {StartInitialHandshake:157(56551346ns) {DropHandshakes:470(2653ns) {DropIncomingHandshake:442(706ns) DropOutgoingHandshake:457(418ns)} GenerateSessionVirtualId:988(1114ns) PrepareNewSessionHandshake:477(3965ns) {SwitchToState:179(2022ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.130336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471438757928154:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(46712257ns) {Configure:104(46706515ns) {ProcessConfigured:135(87305ns) {StartInitialHandshake:157(85123ns) {DropHandshakes:470(2166ns) {DropIncomingHandshake:442(564ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(1025ns) PrepareNewSessionHandshake:477(4205ns) {SwitchToState:179(556ns)}}}}}
2025-11-27T18:19:12.131465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471438757928156:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28690
2025-11-27T18:19:12.143332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471438757928162:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28690
2025-11-27T18:19:12.159529Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471434462960793:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(141917786ns) {Configure:104(141912307ns) {ProcessConfigured:135(95173ns) {StartInitialHandshake:157(90214ns) {DropHandshakes:470(2183ns) {DropIncomingHandshake:442(562ns) DropOutgoingHandshake:457(365ns)} GenerateSessionVirtualId:988(972ns) PrepareNewSessionHandshake:477(1226ns) {SwitchToState:179(290ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.176999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471441106341016:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31275547ns) {Configure:104(31266996ns) {ProcessConfigured:135(58831ns) {StartInitialHandshake:157(55663ns) {DropHandshakes:470(2393ns) {DropIncomingHandshake:442(639ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(1005ns) PrepareNewSessionHandshake:477(1891ns) {SwitchToState:179(771ns)}}}}}
2025-11-27T18:19:12.166078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471436489211096:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(237883884ns) {Configure:104(237878949ns) {ProcessConfigured:135(14923393ns) {StartInitialHandshake:157(14920145ns) {DropHandshakes:470(2872ns) {DropIncomingHandshake:442(672ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(1625ns) PrepareNewSessionHandshake:477(2119ns) {SwitchToState:179(103ns)}}}}}
2025-11-27T18:19:12.170837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471437691605239:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33811
2025-11-27T18:19:12.170460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471435091935906:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(48489173ns) {Configure:104(48484875ns) {ProcessConfigured:135(30433792ns) {StartInitialHandshake:157(30430600ns) {DropHandshakes:470(1688ns) {DropIncomingHandshake:442(435ns) DropOutgoingHandshake:457(363ns)} GenerateSessionVirtualId:988(617ns) PrepareNewSessionHandshake:477(2896ns) {SwitchToState:179(1613ns)}}}}}
2025-11-27T18:19:12.171087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471438757928156:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.194347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471441106341020:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3633
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.218599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471441106341030:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3633
2025-11-27T18:19:12.174008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471439386903266:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-27T18:19:12.194526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471436489211111:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(307380846ns) {Configure:104(307374074ns) {ProcessConfigured:135(32832577ns) {StartInitialHandshake:157(32827934ns) {DropHandshakes:470(2433ns) {DropIncomingHandshake:442(673ns) DropOutgoingHandshake:457(434ns)} GenerateSessionVirtualId:988(1509ns) PrepareNewSessionHandshake:477(8348595ns) {SwitchToState:179(7972ns)}}}}}
2025-11-27T18:19:12.216326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471433396637940:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.216384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471433396637940:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.189913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471434462960793:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.209948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471440784178459:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.202600Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471434462960793:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(18717760ns) {IncomingHandshake:261(8185737ns) {DropIncomingHandshake:442(909ns) PrepareNewSessionHandshake:477(1302ns) {SwitchToState:179(372ns)} GenerateSessionVirtualId:988(445ns) IssueIncomingHandshakeReply:193(88239ns) {SwitchToState:179(705ns)}}}
2025-11-27T18:19:12.209566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471435091935951:2095] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(149329979ns) {Configure:104(149322269ns) {ProcessConfigured:135(69529ns) {StartInitialHandshake:157(66695ns) {DropHandshakes:470(3106ns) {DropIncomingHandshake:442(828ns) DropOutgoingHandshake:457(605ns)} GenerateSessionVirtualId:988(1263ns) PrepareNewSessionHandshake:477(4126ns) {SwitchToState:179(404ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.235243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471441106341020:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.226925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471439935536005:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50766
2025-11-27T18:19:12.250161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471433396637940:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:12.209819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471439386903266:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.270535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471438757928162:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.274284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471434462960857:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(40624146ns) {Configure:104(40617982ns) {ProcessConfigured:135(63774ns) {StartInitialHandshake:157(61088ns) {DropHandshakes:470(4975ns) {DropIncomingHandshake:442(550ns) DropOutgoingHandshake:457(349ns)} GenerateSessionVirtualId:988(971ns) PrepareNewSessionHandshake:477(1876ns) {SwitchToState:179(795ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.226995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471439935536006:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50766
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.265574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471439386903277:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-27T18:19:12.287349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471438757928172:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28690
2025-11-27T18:19:12.278360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471436489211096:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.258365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471437691605239:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.269885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471439935536004:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50766
2025-11-27T18:19:12.297572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471439386903277:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.306790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471439386903282:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-27T18:19:12.306967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471435091935964:2103] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:12.291759Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471434462960792:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(254045361ns) {Configure:104(254036055ns) {ProcessConfigured:135(33701281ns) {StartInitialHandshake:157(33697117ns) {DropHandshakes:470(2826ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(721ns)} GenerateSessionVirtualId:988(770ns) PrepareNewSessionHandshake:477(5429362ns) {SwitchToState:179(8774ns)}}}}}
2025-11-27T18:19:12.291607Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471436811373656:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(25710237ns) {IncomingHandshake:261(13894578ns) {DropIncomingHandshake:442(1957ns) PrepareNewSessionHandshake:477(3094ns) {SwitchToState:179(385ns)} GenerateSessionVirtualId:988(328ns) IssueIncomingHandshakeReply:193(6639ns)}}
2025-11-27T18:19:12.295706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471436489211094:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(184911774ns) {Configure:104(184899871ns) {ProcessConfigured:135(3835792ns) {StartInitialHandshake:157(3831836ns) {DropHandshakes:470(2864ns) {DropIncomingHandshake:442(678ns) DropOutgoingHandshake:457(477ns)} GenerateSessionVirtualId:988(1138ns) PrepareNewSessionHandshake:477(5189ns) {SwitchToState:179(2422ns)}}}}}
2025-11-27T18:19:12.305396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471435091935907:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(151723015ns) {Configure:104(151718425ns) {ProcessConfigured:135(21376591ns) {StartInitialHandshake:157(21374705ns) {DropHandshakes:470(2101ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(510ns)} GenerateSessionVirtualId:988(454ns) PrepareNewSessionHandshake:477(2128ns) {SwitchToState:179(746ns)}}}}}
2025-11-27T18:19:12.298461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471440784178464:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-27T18:19:12.317449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471440784178464:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.329863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471435091935907:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:12.345749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471439386903282:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.292206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471438757928158:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.270440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471439935536005:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.295806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471441106341030:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.286866Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471433396637938:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(87436483ns) {Configure:104(87428671ns) {ProcessConfigured:135(89858ns) {StartInitialHandshake:157(87095ns) {DropHandshakes:470(3357ns) {DropIncomingHandshake:442(729ns) DropOutgoingHandshake:457(426ns)} GenerateSessionVirtualId:988(934ns) PrepareNewSessionHandshake:477(1919ns) {SwitchToState:179(837ns)}}}}}
2025-11-27T18:19:12.349644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471439386903272:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.370382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471439386903272:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.370710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471435091935908:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:12.297724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471437691605243:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33811
2025-11-27T18:19:12.283659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471439935536004:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.341435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471436489211095:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(208790814ns) {Configure:104(208776190ns) {ProcessConfigured:135(12777507ns) {StartInitialHandshake:157(12773163ns) {DropHandshakes:470(2259ns) {DropIncomingHandshake:442(537ns) DropOutgoingHandshake:457(263ns)} GenerateSessionVirtualId:988(962ns) PrepareNewSessionHandshake:477(4009ns) {SwitchToState:179(1866ns)}}}}}
2025-11-27T18:19:12.292241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471438757928158:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.301429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471437691605244:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33811
2025-11-27T18:19:12.409542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471436489211094:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.365455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471438757928158:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:12.381694Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471436811373657:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(236167759ns) {Configure:104(236160871ns) {ProcessConfigured:135(24297215ns) {StartInitialHandshake:157(24293588ns) {DropHandshakes:470(3000ns) {DropIncomingHandshake:442(659ns) DropOutgoingHandshake:457(627ns)} GenerateSessionVirtualId:988(936ns) PrepareNewSessionHandshake:477(4400ns) {SwitchToState:179(2310ns)}}}}}
2025-11-27T18:19:12.309638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471439935536006:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.329488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471437691605244:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.425909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471439386903289:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-27T18:19:12.393924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471441106341025:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.426421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471439386903290:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-27T18:19:12.426721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471439386903288:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-27T18:19:12.427728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471439386903266:2109] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:12.427952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471435091935907:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.428017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:12.428086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471435091935907:2049] [node 2] ICP22 created new session: [5:7577471439386903291:2049]
2025-11-27T18:19:12.428130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471439386903291:2049] [node 2] ICS09 handshake done sender: [5:7577471439386903266:2109] self: [5:7577471439386903285:0] peer: [2:7577471433396637935:0] socket: 63 qp: -1
2025-11-27T18:19:12.428159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471439386903291:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:12.428209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:12.444672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471439386903277:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:12.457959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471435091935964:2103] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.458729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471439386903288:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.458999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471439386903293:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-27T18:19:12.459478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.374905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471439935536008:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.370116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471438757928175:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28690
2025-11-27T18:19:12.446790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471436489211096:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4720) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471440784178459:2112] held: no
2025-11-27T18:19:12.333385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471433396637872:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.393949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471441106341025:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.465400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471435091935906:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.374951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471439935536008:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.447479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471440784178473:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-27T18:19:12.373522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471438757928156:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:12.395535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471441106341025:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:12.364203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:12.466151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471435091935964:2103] [node 4] ICP22 created new session: [5:7577471439386903294:2103]
2025-11-27T18:19:12.365415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471433396637871:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:12.365660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471433396637936:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.365687Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471433396637936:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.381373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471433396637872:2049] [node 3] ICP22 created new session: [2:7577471437691605252:2049]
2025-11-27T18:19:12.385638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471437691605243:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.385644Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471433396637871:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(35809825ns) {IncomingHandshake:261(12098790ns) {DropIncomingHandshake:442(1209ns) PrepareNewSessionHandshake:477(5523ns) {SwitchToState:179(4499ns)} GenerateSessionVirtualId:988(1146ns) IssueIncomingHandshakeReply:193(79605ns) {SwitchToState:179(106ns)}}}
2025-11-27T18:19:12.393539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471437691605252:2049] [node 3] ICS09 handshake done sender: [2:7577471433396637940:2111] self: [2:7577471433396637939:0] peer: [3:7577471438757928167:0] socket: 62 qp: -1
2025-11-27T18:19:12.393594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471437691605252:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:12.393677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:12.413063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471433396637936:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:12.447868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471433396637934:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.447923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:12.454502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471433396637934:2108] [node 5] ICP22 created new session: [2:7577471437691605257:2108]
2025-11-27T18:19:12.469440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471437691605257:2108] [node 5] ICS09 handshake done sender: [2:7577471433396637936:2110] self: [2:7577471433396637935:0] peer: [5:7577471439386903285:0] socket: 60 qp: -1
2025-11-27T18:19:12.482551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471437691605246:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.482575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471437691605246:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.484955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471437691605257:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:12.489870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471437691605260:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33811
2025-11-27T18:19:12.491149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471437691605246:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:12.504187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:12.385993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471434462960793:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.395834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471441106341018:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.395896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:12.395946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471441106341018:2107] [node 4] ICP22 created new session: [1:7577471441106341036:2107]
2025-11-27T18:19:12.395999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471441106341036:2107] [node 4] ICS09 handshake done sender: [1:7577471441106341025:2113] self: [1:7577471441106341024:0] peer: [4:7577471439935536019:0] socket: 85 qp: -1
2025-11-27T18:19:12.396029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471441106341036:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:12.396059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:12.402006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471441106341019:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.402054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471441106341019:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.410673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471441106341023:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.410703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471441106341023:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.433822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471441106341023:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:12.435892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471441106341016:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.435968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:12.436034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471441106341016:2109] [node 6] ICP22 created new session: [1:7577471441106341040:2109]
2025-11-27T18:19:12.436098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471441106341040:2109] [node 6] ICS09 handshake done sender: [1:7577471441106341023:2112] self: [1:7577471441106341022:0] peer: [6:7577471440784178474:0] socket: 84 qp: -1
2025-11-27T18:19:12.436143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471441106341040:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:12.436192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:12.458184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471441106341019:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:12.463339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471441106341043:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3633
2025-11-27T18:19:12.491152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471441106341027:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.491194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471441106341027:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.491334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471436811373656:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.512103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.465664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471440784178473:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.385489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471435640568640:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(222546693ns) {Configure:104(222537835ns) {ProcessConfigured:135(40642671ns) {StartInitialHandshake:157(40639173ns) {DropHandshakes:470(3140ns) {DropIncomingHandshake:442(862ns) DropOutgoingHandshake:457(402ns)} GenerateSessionVirtualId:988(1037ns) PrepareNewSessionHandshake:477(4570ns) {SwitchToState:179(2093ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.480831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471436489211095:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:12.480926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471440784178472:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-27T18:19:12.482388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471440784178459:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:12.491295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471436489211096:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.501003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:12.501065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471436489211096:2050] [node 3] ICP22 created new session: [6:7577471440784178478:2050]
2025-11-27T18:19:12.501120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471440784178478:2050] [node 3] ICS09 handshake done sender: [6:7577471440784178459:2112] self: [6:7577471440784178469:0] peer: [3:7577471438757928157:0] socket: 71 qp: -1
2025-11-27T18:19:12.501155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471440784178478:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:12.501215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:12.504182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471440784178475:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-27T18:19:12.504251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471440784178471:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-27T18:19:12.517700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471440784178481:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-27T18:19:12.518524Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471436489211096:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(19596386ns) {HandleHandshakeStatus:329(19594073ns) {DropIncomingHandshake:442(1879ns) DropOutgoingHandshake:457(397ns) SwitchToState:179(666ns) ProcessPendingSessionEvents:495(1740ns)}}
2025-11-27T18:19:12.518456Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471441106341027:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:12.473678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471439386903294:2103] [node 4] ICS09 handshake done sender: [5:7577471439386903277:2115] self: [5:7577471439386903284:0] peer: [4:7577471439935536007:0] socket: 94 qp: -1
2025-11-27T18:19:12.537772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471437691605260:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.393495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471435640568640:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:12.386074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:12.546709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471440784178473:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:12.546949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471440784178464:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:12.473806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471439386903272:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:12.386128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471434462960793:2049] [node 2] ICP22 created new session: [3:7577471438757928176:2049]
2025-11-27T18:19:12.386171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471438757928176:2049] [node 2] ICS09 handshake done sender: [3:7577471438757928156:2112] self: [3:7577471438757928167:0] peer: [2:7577471433396637939:0] socket: 67 qp: -1
2025-11-27T18:19:12.386210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471438757928176:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:12.386241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:12.393358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471438757928175:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.417551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471438757928154:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.417653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:12.417718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471438757928154:2111] [node 6] ICP22 created new session: [3:7577471438757928178:2111]
2025-11-27T18:19:12.417772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471438757928178:2111] [node 6] ICS09 handshake done sender: [3:7577471438757928158:2113] self: [3:7577471438757928157:0] peer: [6:7577471440784178469:0] socket: 69 qp: -1
2025-11-27T18:19:12.417818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471438757928178:2111] [node 6] ICS10 traffic start
2025-11-27T18:19:12.417859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:12.429580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471438757928161:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.429631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471438757928161:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.449486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471438757928161:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:12.464532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471438757928155:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.471078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:12.473805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471438757928155:2110] [node 5] ICP22 created new session: [3:7577471438757928184:2110]
2025-11-27T18:19:12.485375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471438757928184:2110] [node 5] ICS09 handshake done sender: [3:7577471438757928161:2114] self: [3:7577471438757928160:0] peer: [5:7577471439386903287:0] socket: 76 qp: -1
2025-11-27T18:19:12.490862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471438757928184:2110] [node 5] ICS10 traffic start
2025-11-27T18:19:12.490916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:12.525844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471438757928187:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28690
2025-11-27T18:19:12.541693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471438757928186:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28690
2025-11-27T18:19:12.551023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471440784178471:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.551084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471436489211095:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.521538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471436811373656:2048] [node 2] ICP22 created new session: [1:7577471441106341046:2048]
2025-11-27T18:19:12.553822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471437691605238:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.553876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:12.553927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471437691605238:2109] [node 6] ICP22 created new session: [2:7577471437691605262:2109]
2025-11-27T18:19:12.553981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471437691605262:2109] [node 6] ICS09 handshake done sender: [2:7577471437691605246:2116] self: [2:7577471437691605245:0] peer: [6:7577471440784178477:0] socket: 105 qp: -1
2025-11-27T18:19:12.554013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471437691605262:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:12.554042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:12.555190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471437691605249:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.555214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471437691605249:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.556527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471433396637871:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471437691605244:2115] held: no
2025-11-27T18:19:12.524468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471441106341043:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.405629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471435640568642:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(221323795ns) {Configure:104(221315279ns) {ProcessConfigured:135(32268610ns) {StartInitialHandshake:157(32264881ns) {DropHandshakes:470(3520ns) {DropIncomingHandshake:442(799ns) DropOutgoingHandshake:457(620ns)} GenerateSessionVirtualId:988(760ns) PrepareNewSessionHandshake:477(3774ns) {SwitchToState:179(916ns)}}}}}
2025-11-27T18:19:12.557450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471438757928171:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.526578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471441106341046:2048] [node 2] ICS09 handshake done sender: [1:7577471441106341019:2110] self: [1:7577471436811373719:0] peer: [2:7577471437691605253:0] socket: 80 qp: -1
2025-11-27T18:19:12.478288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471435091935908:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4720) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471439386903272:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.551136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:12.563805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471437691605238:2109] [node 6] ICP18 handshake (actor [2:7577471437691605260:2119]) is too old
2025-11-27T18:19:12.405879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471439935536008:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.537698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471441106341046:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:12.557497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471438757928171:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.485450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471439386903294:2103] [node 4] ICS10 traffic start
2025-11-27T18:19:12.450132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471439935536021:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50766
2025-11-27T18:19:12.567492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471437691605249:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:12.551343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471436489211094:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.538862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:12.490966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.564519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471438757928186:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.479678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471439935536022:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50766
2025-11-27T18:19:12.551942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471436489211095:2049] [node 2] ICP22 created new session: [6:7577471440784178482:2049]
2025-11-27T18:19:12.593230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471433396637938:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.504243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471435091935908:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.548890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471441106341017:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.564522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:12.567204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471438757928187:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.593321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:12.496343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471439935535999:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.526307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:12.565600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471440784178482:2049] [node 2] ICS09 handshake done sender: [6:7577471440784178473:2119] self: [6:7577471440784178477:0] peer: [2:7577471437691605245:0] socket: 106 qp: -1
2025-11-27T18:19:12.496429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:12.552226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:12.568789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471434462960792:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:12.593391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471433396637938:2107] [node 4] ICP22 created new session: [2:7577471437691605265:2107]
2025-11-27T18:19:12.536704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471435091935908:2050] [node 3] ICP22 created new session: [5:7577471439386903298:2050]
2025-11-27T18:19:12.569181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471438757928171:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:12.552287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471441106341017:2108] [node 5] ICP22 created new session: [1:7577471441106341049:2108]
2025-11-27T18:19:12.568340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471440784178482:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:12.593458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471437691605265:2107] [node 4] ICS09 handshake done sender: [2:7577471437691605249:2117] self: [2:7577471437691605248:0] peer: [4:7577471439935536028:0] socket: 116 qp: -1
2025-11-27T18:19:12.496530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471439935536005:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:12.601473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471434462960857:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.549436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471439386903298:2050] [node 3] ICS09 handshake done sender: [5:7577471439386903272:2113] self: [5:7577471439386903287:0] peer: [3:7577471438757928160:0] socket: 81 qp: -1
2025-11-27T18:19:12.552339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471441106341049:2108] [node 5] ICS09 handshake done sender: [1:7577471441106341027:2114] self: [1:7577471441106341026:0] peer: [5:7577471439386903295:0] socket: 109 qp: -1
2025-11-27T18:19:12.570023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471436489211094:2048] [node 1] ICP22 created new session: [6:7577471440784178483:2048]
2025-11-27T18:19:12.503883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471439935536023:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50766
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:12.593503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471437691605265:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:12.551852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471439386903298:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:12.576900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471440784178480:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-27T18:19:12.563366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471439386903299:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32523
2025-11-27T18:19:12.564078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471439386903288:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:12.565992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:12.588207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471435091935906:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.606492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:12.615270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471435091935906:2048] [node 1] ICP22 created new session: [5:7577471439386903301:2048]
2025-11-27T18:19:12.630454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471435091935908:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(105285055ns) {HandleHandshakeStatus:329(105281623ns) {DropIncomingHandshake:442(913ns) DropOutgoingHandshake:457(335ns) SwitchToState:179(1199ns) ProcessPendingSessionEvents:495(1306ns)}}
2025-11-27T18:19:12.633788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471439386903301:2048] [node 1] ICS09 handshake done sender: [5:7577471439386903288:2119] self: [5:7577471439386903295:0] peer: [1:7577471441106341026:0] socket: 110 qp: -1
2025-11-27T18:19:12.633835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471439386903301:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:12.638741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471439386903276:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.648118Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471439386903276:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.661205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:12.668764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471439386903276:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:12.510635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471439935535999:2107] [node 5] ICP22 created new session: [4:7577471439935536024:2107]
2025-11-27T18:19:12.601541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:12.552374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471441106341049:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:12.593549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:12.678763Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471435091935906:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(100104508ns) {HandleHandshakeStatus:329(100100760ns) {DropIncomingHandshake:442(3160ns) DropOutgoingHandshake:457(7729ns) SwitchToState:179(837ns) ProcessPendingSessionEvents:495(5937240ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5868925ns max# 5007238ns}}}
2025-11-27T18:19:12.552418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:12.601593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471434462960857:2107] [node 4] ICP22 created new session: [3:7577471438757928191:2107]
2025-11-27T18:19:12.577104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471436489211118:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:12.531314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471439935536022:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.601645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471438757928191:2107] [node 4] ICS09 handshake done sender: [3:7577471438757928171:2118] self: [3:7577471438757928170:0] peer: [4:7577471439935536029:0] socket: 118 qp: -1
2025-11-27T18:19:12.601686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471438757928191:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:12.601715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:12.617659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471434462960792:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471438757928186:2121] held: no
2025-11-27T18:19:12.621771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471438757928193:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28690
2025-11-27T18:19:12.638223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471438757928186:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:12.638879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471434462960792:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.666768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:12.670960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471434462960792:2048] [node 1] ICP22 created new session: [3:7577471438757928194:2048]
2025-11-27T18:19:12.679551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471438757928194:2048] [node 1] ICS09 handshake done sender: [3:7577471438757928186:2121] self: [3:7577471438757928189:0] peer: [1:7577471441106341033:0] socket: 107 qp: -1
2025-11-27T18:19:12.681692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471438757928194:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:12.687756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:12.687995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471434462960792:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(49010348ns) {HandleHandshakeStatus:329(49007044ns) {DropIncomingHandshake:442(1092ns) DropOutgoingHandshake:457(437ns) SwitchToState:179(720ns) ProcessPendingSessionEvents:495(1069ns)}}
2025-11-27T18:19:12.597460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471437691605258:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33811
2025-11-27T18:19:12.634044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471437691605244:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:12.655452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471433396637871:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.655508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:12.655563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471433396637871:2048] [node 1] ICP22 created new session: [2:7577471437691605267:2048]
2025-11-27T18:19:12.655614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471437691605267:2048] [node 1] ICS09 handshake done sender: [2:7577471437691605244:2115] self: [2:7577471437691605253:0] peer: [1:7577471436811373719:0] socket: 82 qp: -1
2025-11-27T18:19:12.655654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471437691605267:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:12.655692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:12.575335Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471436811373656:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(82921727ns) {HandleHandshakeStatus:329(82916618ns) {DropIncomingHandshake:442(35048ns) DropOutgoingHandshake:457(472ns) SwitchToState:179(502ns) ProcessPendingSessionEvents:495(11405100ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11308981ns max# 5896377ns}}}
2025-11-27T18:19:12.579753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471441106341034:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.579787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471441106341034:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.581079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471441106341034:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:12.606352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471436811373657:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.614939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:12.621978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471436811373657:2049] [node 3] ICP22 created new session: [1:7577471441106341052:2049]
2025-11-27T18:19:12.638778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471441106341052:2049] [node 3] ICS09 handshake done sender: [1:7577471441106341034:2116] self: [1:7577471441106341033:0] peer: [3:7577471438757928189:0] socket: 124 qp: -1
2025-11-27T18:19:12.640857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471441106341052:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:12.665496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:12.674298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471436811373657:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(73914730ns) {HandleHandshakeStatus:329(73911375ns) {DropIncomingHandshake:442(18527ns) DropOutgoingHandshake:457(746ns) SwitchToState:179(976ns) ProcessPendingSessionEvents:495(724220ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 679805ns max# 209161ns}}}
2025-11-27T18:19:12.577794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471440784178483:2048] [node 1] ICS09 handshake done sender: [6:7577471440784178464:2114] self: [6:7577471440784178474:0] peer: [1:7577471441106341022:0] socket: 86 qp: -1
2025-11-27T18:19:12.577823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:12.597529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471440784178483:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:12.597588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:12.613390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471440784178480:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.622114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471436489211111:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:12.626950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471436489211095:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(75284331ns) {HandleHandshakeStatus:329(75281987ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(4213ns) SwitchToState:179(157ns) ProcessPendingSessionEvents:495(11308843ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11249657ns max# 10863923ns}}}
2025-11-27T18:19:12.639783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471436489211094:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(58248775ns) {HandleHandshakeStatus:329(58247687ns) {DropIncomingHandshake:442(229ns) DropOutgoingHandshake:457(3256ns) SwitchToState:179(324ns) ProcessPendingSessionEvents:495(11880106ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11812754ns max# 11284572ns}}}
2025-11-27T18:19:12.661502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471436489211118:2071] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20166) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471440784178471:2117] held: no
2025-11-27T18:19:12.691281Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471436489211118:2071] [node 4] ICP35 event processing took too much time PendingConnection:279(34121364ns) {HandleHandshakeStatus:395(34118339ns) {LogHandshakeFail:472(5229205ns) {UpdateErrorStateLog:545(9226ns)} DropOutgoingHandshake:457(554ns) ProcessPendingSessionEvents:495(28821467ns) {unordered DropSessionEvent:512 num# 100 sum# 28772915ns max# 7004961ns}}}
2025-11-27T18:19:12.692115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471440784178489:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-27T18:19:12.699070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471440784178480:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:12.706653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471436489211111:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2943) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471440784178480:2121] held: no
2025-11-27T18:19:12.707275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471440784178488:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31580
2025-11-27T18:19:12.718315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471440784178471:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:12.722263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471436489211111:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.734117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471436489211118:2071] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.734204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:12.734262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471436489211118:2071] [node 4] ICP22 created new session: [6:7577471440784178490:2071]
2025-11-27T18:19:12.734310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471440784178490:2071] [node 4] ICS09 handshake done sender: [6:7577471440784178471:2117] self: [6:7577471440784178484:0] peer: [4:7577471439935536010:0] socket: 102 qp: -1
2025-11-27T18:19:12.734353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471440784178490:2071] [node 4] ICS10 traffic start
2025-11-27T18:19:12.734394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:12.735326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:12.740184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471436489211111:2064] [node 5] ICP22 created new session: [6:7577471440784178492:2064]
2025-11-27T18:19:12.742417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471440784178492:2064] [node 5] ICS09 handshake done sender: [6:7577471440784178480:2121] self: [6:7577471440784178487:0] peer: [5:7577471439386903275:0] socket: 123 qp: -1
2025-11-27T18:19:12.742461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471440784178492:2064] [node 5] ICS10 traffic start
2025-11-27T18:19:12.678848Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471435091935906:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:12.742509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:12.742721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471436489211111:2064] [node 5] ICP35 event processing took too much time PendingConnection:279(25717455ns) {HandleHandshakeStatus:329(25714586ns) {DropIncomingHandshake:442(1534ns) DropOutgoingHandshake:457(462ns) SwitchToState:179(681ns) ProcessPendingSessionEvents:495(1344ns)}}
2025-11-27T18:19:12.680892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471435091935951:2095] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.713927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:12.716682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471435091935951:2095] [node 6] ICP22 created new session: [5:7577471439386903304:2095]
2025-11-27T18:19:12.721537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471439386903304:2095] [node 6] ICS09 handshake done sender: [5:7577471439386903276:2114] self: [5:7577471439386903275:0] peer: [6:7577471440784178487:0] socket: 87 qp: -1
2025-11-27T18:19:12.721596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471439386903304:2095] [node 6] ICS10 traffic start
2025-11-27T18:19:12.721669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:12.531372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471435640568640:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.531448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:12.531502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471435640568640:2048] [node 1] ICP22 created new session: [4:7577471439935536026:2048]
2025-11-27T18:19:12.531547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471439935536026:2048] [node 1] ICS09 handshake done sender: [4:7577471439935536005:2112] self: [4:7577471439935536019:0] peer: [1:7577471441106341024:0] socket: 88 qp: -1
2025-11-27T18:19:12.531582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471439935536026:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:12.531644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:12.536158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471435640568641:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:12.541799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471439935536023:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.541988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471439935536024:2107] [node 5] ICS09 handshake done sender: [4:7577471439935536008:2114] self: [4:7577471439935536007:0] peer: [5:7577471439386903284:0] socket: 93 qp: -1
2025-11-27T18:19:12.547278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471439935536024:2107] [node 5] ICS10 traffic start
2025-11-27T18:19:12.550691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471435640568642:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:12.562918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:12.565290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471439935536032:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50766
2025-11-27T18:19:12.567951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471439935536022:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:12.570093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471435640568641:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.581648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471439935536030:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50766
2025-11-27T18:19:12.586312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:12.586383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471435640568641:2049] [node 2] ICP22 created new session: [4:7577471439935536033:2049]
2025-11-27T18:19:12.586442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471439935536033:2049] [node 2] ICS09 handshake done sender: [4:7577471439935536022:2119] self: [4:7577471439935536028:0] peer: [2:7577471437691605248:0] socket: 126 qp: -1
2025-11-27T18:19:12.586482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471439935536033:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:12.599515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:12.604546Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471439935535999:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(107300875ns) {HandleHandshakeStatus:329(107296960ns) {DropIncomingHandshake:442(21014ns) DropOutgoingHandshake:457(2313ns) SwitchToState:179(1134ns) ProcessPendingSessionEvents:495(21948327ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21878920ns max# 21443538ns}}}
2025-11-27T18:19:12.604717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471439935536011:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:12.604741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471439935536011:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:12.620279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471435640568642:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4720) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471439935536023:2120] held: no
2025-11-27T18:19:12.621498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471439935536011:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:12.622079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471439935536023:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:12.643484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471435640568641:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(72509915ns) {HandleHandshakeStatus:329(72507135ns) {DropIncomingHandshake:442(962ns) DropOutgoingHandshake:457(6124ns) SwitchToState:179(315ns) ProcessPendingSessionEvents:495(3891056ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3836469ns max# 3436260ns}}}
2025-11-27T18:19:12.643546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471435640568642:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:12.643617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:12.643672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471435640568642:2050] [node 3] ICP22 created new session: [4:7577471439935536036:2050]
2025-11-27T18:19:12.643717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471439935536036:2050] [node 3] ICS09 handshake done sender: [4:7577471439935536023:2120] self: [4:7577471439935536029:0] peer: [3:7577471438757928170:0] socket: 127 qp: -1
2025-11-27T18:19:12.643754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471439935536036:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:12.643790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:12.653679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471439935536000:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:12.653760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:12.653810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471439935536000:2108] [node 6] ICP22 created new session: [4:7577471439935536038:2108]
2025-11-27T18:19:12.670825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471439935536038:2108] [node 6] ICS09 handshake done sender: [4:7577471439935536011:2115] self: [4:7577471439935536010:0] peer: [6:7577471440784178484:0] socket: 100 qp: -1
2025-11-27T18:19:12.672195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471439935536038:2108] [node 6] ICS10 traffic start
2025-11-27T18:19:12.675173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:12.750295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471439935536000:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(108773215ns) {HandleHandshakeStatus:329(108770684ns) {DropIncomingHandshake:442(28060ns) DropOutgoingHandshake:457(538ns) SwitchToState:179(934ns) ProcessPendingSessionEvents:495(49978136ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 43875544ns max# 13494006ns}}}
2025-11-27T18:19:12.751585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471433396637872:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(403058879ns) {HandleHandshakeStatus:329(403055790ns) {DropIncomingHandshake:442(2870ns) DropOutgoingHandshake:457(2303ns) SwitchToState:179(707ns) ProcessPendingSessionEvents:495(319262967ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 294883031ns max# 19761576ns}}}
2025-11-27T18:19:12.751680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471433396637872:2049] [node 3] ICP18 handshake (actor [2:7577471437691605243:2114]) is too old
2025-11-27T18:19:12.752638Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471433396637934:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(294838703ns) {HandleHandshakeStatus:329(294834856ns) {DropIncomingHandshake:442(1139ns) DropOutgoingHandshake:457(611ns) SwitchToState:179(358ns) ProcessPendingSessionEvents:495(198450430ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 198288043ns max# 20785129ns}}}
2025-11-27T18:19:12.745983Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471435091935951:2095] [node 6] ICP35 event processing took too much time PendingConnection:279(52707546ns) {HandleHandshakeStatus:329(52704194ns) {DropIncomingHandshake:442(27908ns) DropOutgoingHandshake:457(659ns) SwitchToState:179(1098ns) ProcessPendingSessionEvents:495(11761914ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11687477ns max# 5909665ns}}}
2025-11-27T18:19:12.746567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471438757928155:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(257133824ns) {HandleHandshakeStatus:329(257130957ns) {DropIncomingHandshake:442(10442ns) DropOutgoingHandshake:457(395ns) SwitchToState:179(456ns) ProcessPendingSessionEvents:495(199610494ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 199417583ns max# 19707936ns}}}
2025-11-27T18:19:12.749282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471435091935964:2103] [node 4] ICP35 event processing took too much time PendingConnection:279(289451207ns) {HandleHandshakeStatus:329(289447140ns) {DropIncomingHandshake:442(1396ns) DropOutgoingHandshake:457(1451041ns) SwitchToState:179(1366ns) ProcessPendingSessionEvents:495(218702462ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 218535485ns max# 22560304ns}}}
2025-11-27T18:19:12.817755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:12.818029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471437691605267:2048] [node 1] ICS01 socket: 82 reason# EndOfStream
2025-11-27T18:19:12.818079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471433396637871:2048] [node 1] ICP30 unregister session Session# [2:7577471437691605267:2048] VirtualId# [2:7577471437691605253:0]
2025-11-27T18:19:12.818114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471437691605267:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:12.818193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:12.818238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:12.818911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:12.819116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471438757928194:2048] [node 1] ICS01 socket: 107 reason# EndOfStream
2025-11-27T18:19:12.819156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471434462960792:2048] [node 1] ICP30 unregister session Session# [3:7577471438757928194:2048] VirtualId# [3:7577471438757928189:0]
2025-11-27T18:19:12.819184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471438757928194:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:12.819245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:12.819280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:12.825365Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:12.825584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471439386903301:2048] [node 1] ICS07 socket disconnect 110 reason# EndOfStream
2025-11-27T18:19:12.825619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471439386903301:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:12.825681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:12.825726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471439386903301:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:12.825412Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:12.825617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471439935536026:2048] [node 1] ICS07 socket disconnect 88 reason# EndOfStream
2025-11-27T18:19:12.825644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471439935536026:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:12.825727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:12.825798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471439935536026:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:12.826703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471435091935906:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12558) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:12.826780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471439386903301:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:12.826813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471435091935906:2048] [node 1] ICP30 unregister session Session# [5:7577471439386903301:2048] VirtualId# [5:7577471439386903295:0]
2025-11-27T18:19:12.826925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:12.826956Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471435091935906:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:12.826970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:12.828712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:12.828933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471440784178483:2048] [node 1] ICS07 socket disconnect 86 reason# EndOfStream
2025-11-27T18:19:12.828974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471440784178483:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:12.829047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:12.829097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471440784178483:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:12.826562Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471435640568640:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12558) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:12.826664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471439935536026:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:12.826697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471435640568640:2048] [node 1] ICP30 unregister session Session# [4:7577471439935536026:2048] VirtualId# [4:7577471439935536019:0]
2025-11-27T18:19:12.826832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:12.826864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471435640568640:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:12.826899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:12.830153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471436489211094:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12558) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:12.830214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471440784178483:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:12.830248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471436489211094:2048] [node 1] ICP30 unregister session Session# [6:7577471440784178483:2048] VirtualId# [6:7577471440784178474:0]
2025-11-27T18:19:12.830359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:12.830392Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471436489211094:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:12.830419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:12.835344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471435640568640:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:12.841375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471436489211094:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:12.841408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471435091935906:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:12.921694Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:12.921933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471438757928176:2049] [node 2] ICS01 socket: 67 reason# EndOfStream
2025-11-27T18:19:12.921978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471434462960793:2049] [node 2] ICP30 unregister session Session# [3:7577471438757928176:2049] VirtualId# [3:7577471438757928167:0]
2025-11-27T18:19:12.922013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471438757928176:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:12.922114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:12.922164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:12.928712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:12.928958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471439935536033:2049] [node 2] ICS07 socket disconnect 126 reason# EndOfStream
2025-11-27T18:19:12.928989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471439935536033:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:12.929062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:12.929101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471439935536033:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:12.929665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:12.929853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471439386903291:2049] [node 2] ICS01 socket: 63 reason# EndOfStream
2025-11-27T18:19:12.929894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471435091935907:2049] [node 2] ICP30 unregister session Session# [5:7577471439386903291:2049] VirtualId# [5:7577471439386903285:0]
2025-11-27T18:19:12.929926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471439386903291:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:12.929980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:12.930024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:12.930112Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:12.930292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471440784178482:2049] [node 2] ICS07 socket disconnect 106 reason# EndOfStream
2025-11-27T18:19:12.930318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471440784178482:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:12.930359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:12.930391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471440784178482:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:12.932068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471436489211095:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4996) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:12.932132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471440784178482:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:12.932298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471436489211095:2049] [node 2] ICP30 unregister session Session# [6:7577471440784178482:2049] VirtualId# [6:7577471440784178477:0]
2025-11-27T18:19:12.932468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:12.932508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471436489211095:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4996) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:12.932523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4996) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:12.932444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471435640568641:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4996) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:12.932500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471439935536033:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:12.932532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471435640568641:2049] [node 2] ICP30 unregister session Session# [4:7577471439935536033:2049] VirtualId# [4:7577471439935536028:0]
2025-11-27T18:19:12.932644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:12.932672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471435640568641:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4996) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:12.932683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4996) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:12.943545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471435640568641:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:12.942834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471436489211095:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:13.003703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:13.003905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471439386903298:2050] [node 3] ICS01 socket: 81 reason# EndOfStream
2025-11-27T18:19:13.003942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471435091935908:2050] [node 3] ICP30 unregister session Session# [5:7577471439386903298:2050] VirtualId# [5:7577471439386903287:0]
2025-11-27T18:19:13.003975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471439386903298:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.004050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:13.004085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:13.004178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:13.004334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471440784178478:2050] [node 3] ICS01 socket: 71 reason# EndOfStream
2025-11-27T18:19:13.004364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471436489211096:2050] [node 3] ICP30 unregister session Session# [6:7577471440784178478:2050] VirtualId# [6:7577471440784178469:0]
2025-11-27T18:19:13.004393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471440784178478:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.004435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:13.004465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:13.005018Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:13.005234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471439935536036:2050] [node 3] ICS01 socket: 127 reason# EndOfStream
2025-11-27T18:19:13.005290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471435640568642:2050] [node 3] ICP30 unregister session Session# [4:7577471439935536036:2050] VirtualId# [4:7577471439935536029:0]
2025-11-27T18:19:13.005318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471439935536036:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.005401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:13.005448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:13.093484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:13.093719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471439386903294:2103] [node 4] ICS07 socket disconnect 94 reason# EndOfStream
2025-11-27T18:19:13.093753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471439386903294:2103] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.093851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:13.093900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471439386903294:2103] [node 4] ICS15 start handshake
2025-11-27T18:19:13.094481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:13.094681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471440784178490:2071] [node 4] ICS01 socket: 102 reason# EndOfStream
2025-11-27T18:19:13.094717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471436489211118:2071] [node 4] ICP30 unregister session Session# [6:7577471440784178490:2071] VirtualId# [6:7577471440784178484:0]
2025-11-27T18:19:13.094763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471440784178490:2071] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.094852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:13.094890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:13.096254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471435091935964:2103] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20166) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:13.096312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471439386903294:2103] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:13.096358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471435091935964:2103] [node 4] ICP30 unregister session Session# [5:7577471439386903294:2103] VirtualId# [5:7577471439386903284:0]
2025-11-27T18:19:13.096470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:13.096515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471435091935964:2103] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20166) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:13.096552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20166) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:13.167132Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:13.167388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471440784178492:2064] [node 5] ICS01 socket: 123 reason# EndOfStream
2025-11-27T18:19:13.167435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471436489211111:2064] [node 5] ICP30 unregister session Session# [6:7577471440784178492:2064] VirtualId# [6:7577471440784178487:0]
2025-11-27T18:19:13.167472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471440784178492:2064] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.167535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:13.167572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:13.301555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471445439136231:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:13.301623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471445439136232:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:13.333773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471442604086264:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:13.333871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471442604086265:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:13.343638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471445538485252:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:13.343710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471445538485253:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:13.366353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471444762102575:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:13.366442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471444762102576:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:13.366486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471444762102577:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:13.386782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471442373431215:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:13.386840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471442373431216:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:13.386879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471442373431217:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:13.407103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471443291579811:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:13.407143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471443291579812:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:13.407169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471443291579813:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:13.483241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471443291579874:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:13.540655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471445538485315:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:13.539015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471445439136304:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:13.540160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471442604086328:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:13.542615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471445439136305:2118] [node 5] ICP01 ready to work
2025-11-27T18:19:13.544109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471444762102638:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:13.544740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471442604086329:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:13.543768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471442373431279:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:13.545232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471442604086330:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:13.544270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471445538485316:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:13.546995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471445538485317:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:13.547909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471442373431280:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:13.549500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471445439136306:2119] [node 6] ICP01 ready to work
2025-11-27T18:19:13.557988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471444762102639:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:13.581568Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471443291579874:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(74047550ns) {Configure:104(68396075ns) {ProcessConfigured:135(11813210ns) {StartInitialHandshake:157(11810782ns) {DropHandshakes:470(2105ns) {DropIncomingHandshake:442(439ns) DropOutgoingHandshake:457(349ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:477(4791ns) {SwitchToState:179(2783ns)}}}}}
2025-11-27T18:19:13.581747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471442373431215:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33102101ns) {Configure:104(33096463ns) {ProcessConfigured:135(48648ns) {StartInitialHandshake:157(47142ns) {DropHandshakes:470(1255ns) {DropIncomingHandshake:442(307ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(1041ns) {SwitchToState:179(469ns)}}}}}
2025-11-27T18:19:13.591181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471443291579887:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.604679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471445538485328:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11039
2025-11-27T18:19:13.604755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471445538485329:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11039
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.617731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471445439136306:2119] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(53371414ns) {Configure:104(53366515ns) {ProcessConfigured:135(81627ns) {StartInitialHandshake:157(78194ns) {DropHandshakes:470(3244ns) {DropIncomingHandshake:442(917ns) DropOutgoingHandshake:457(474ns)} GenerateSessionVirtualId:988(867ns) PrepareNewSessionHandshake:477(2061ns) {SwitchToState:179(719ns)}}}}}
2025-11-27T18:19:13.620675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471445439136317:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
2025-11-27T18:19:13.620733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471445439136318:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
2025-11-27T18:19:13.620882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471445439136319:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
2025-11-27T18:19:13.645438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471443291579811:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33777743ns) {Configure:104(33774254ns) {ProcessConfigured:135(14049606ns) {StartInitialHandshake:157(14047091ns) {DropHandshakes:470(2584ns) {DropIncomingHandshake:442(692ns) DropOutgoingHandshake:457(493ns)} GenerateSessionVirtualId:988(337ns) PrepareNewSessionHandshake:477(5142ns) {SwitchToState:179(2470ns)}}}}}
2025-11-27T18:19:13.639385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471444762102638:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31186433ns) {Configure:104(31182321ns) {ProcessConfigured:135(8959765ns) {StartInitialHandshake:157(8956659ns) {DropHandshakes:470(939ns) {DropIncomingHandshake:442(368ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(294ns) PrepareNewSessionHandshake:477(4667ns) {SwitchToState:179(2596ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.652724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471445439136318:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.653475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471442604086344:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46452
2025-11-27T18:19:13.656968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471445538485329:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.657225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471442604086344:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.657445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471442604086345:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46452
2025-11-27T18:19:13.657604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471442604086346:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46452
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.658709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471443291579896:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54632
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.659163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471442373431294:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55626
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.669891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471445439136323:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.672074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471445439136317:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.672217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471445439136308:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.672247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471445439136308:2120] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.672400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471444762102648:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7704
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.672564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471444762102649:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7704
2025-11-27T18:19:13.675147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471444762102648:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.671304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471442604086346:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.675327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471444762102649:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.671497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471442604086348:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46452
2025-11-27T18:19:13.675539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471444762102576:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.671710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471442604086264:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.675587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471444762102575:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.676520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471445439136308:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:13.676897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471443291579896:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.677318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471442604086348:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.677414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471442604086350:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46452
2025-11-27T18:19:13.677923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471443291579813:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.678012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471445439136323:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.678440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471442604086351:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46452
2025-11-27T18:19:13.676353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471442373431294:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.676482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471442373431295:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55626
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.678899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471445439136319:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.676350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471445538485333:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11039
2025-11-27T18:19:13.678947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471445439136231:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.679008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:13.679073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471445439136231:2048] [node 2] ICP22 created new session: [1:7577471445439136328:2048]
2025-11-27T18:19:13.679119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471445439136328:2048] [node 2] ICS09 handshake done sender: [1:7577471445439136308:2120] self: [1:7577471445439136307:0] peer: [2:7577471442604086349:0] socket: 83 qp: -1
2025-11-27T18:19:13.679155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471445439136328:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:13.679189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:13.681454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471445439136327:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15125
2025-11-27T18:19:13.681666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471445439136312:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.681686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471445439136312:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.676549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471445538485328:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.698805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471442373431295:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.678604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471445538485253:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.705455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471442373431279:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.704293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471445439136312:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:13.706115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471445439136327:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.706164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471445439136304:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.678747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471445538485333:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.713726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471442604086345:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.681147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471445538485319:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.681178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471445538485319:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.706220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-11-27T18:19:13.713922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471444762102657:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7704
2025-11-27T18:19:13.706270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471445439136304:2107] [node 4] ICP22 created new session: [1:7577471445439136334:2107]
::ffff:127.1.0.0
2025-11-27T18:19:13.706334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471445439136334:2107] [node 4] ICS09 handshake done sender: [1:7577471445439136312:2122] self: [1:7577471445439136311:0] peer: [4:7577471444762102656:0] socket: 85 qp: -1
2025-11-27T18:19:13.706369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471445439136334:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:13.706415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:13.709253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471445439136304:2107] [node 4] ICP18 handshake (actor [1:7577471445439136327:2129]) is too old
2025-11-27T18:19:13.714224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471443291579901:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54632
2025-11-27T18:19:13.714218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471442604086336:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.714876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471445538485252:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.715195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471444762102645:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.716173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471445538485319:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:13.715224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471444762102645:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.714253Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471442604086336:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.715876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471444762102649:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:13.721227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471445439136310:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.724748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471443291579901:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.721301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471445439136310:2121] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.716473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471444762102575:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.727833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471442373431298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55626
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.716527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:13.716579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471444762102575:2048] [node 1] ICP22 created new session: [4:7577471444762102659:2048]
2025-11-27T18:19:13.716634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471444762102659:2048] [node 1] ICS09 handshake done sender: [4:7577471444762102649:2115] self: [4:7577471444762102656:0] peer: [1:7577471445439136311:0] socket: 100 qp: -1
2025-11-27T18:19:13.716668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471444762102659:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:13.716719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: 2025-11-27T18:19:13.728223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471442373431299:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55626
::ffff:127.1.0.0
2025-11-27T18:19:13.728399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471443291579902:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54632
2025-11-27T18:19:13.728471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471442604086334:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.728512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471442604086334:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.730414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471445439136310:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:13.730367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471445538485317:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.732583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471443291579811:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.729636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471444762102645:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:13.730468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:13.732483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471445538485317:2109] [node 6] ICP22 created new session: [3:7577471445538485338:2109]
2025-11-27T18:19:13.732543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471445538485338:2109] [node 6] ICS09 handshake done sender: [3:7577471445538485319:2110] self: [3:7577471445538485318:0] peer: [6:7577471443291579900:0] socket: 76 qp: -1
2025-11-27T18:19:13.732580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471445538485338:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:13.732627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:13.730131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471442604086346:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:13.733497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471442373431295:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:13.741711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471445439136232:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.741788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:13.730699Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471442604086264:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471442604086346:2118] held: no
2025-11-27T18:19:13.741838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471445439136232:2049] [node 3] ICP22 created new session: [1:7577471445439136338:2049]
2025-11-27T18:19:13.732830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471443291579896:2125] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:13.747418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471442373431299:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.747520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471442373431279:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.747551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:13.747595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471442373431279:2108] [node 4] ICP22 created new session: [5:7577471442373431301:2108]
2025-11-27T18:19:13.747626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471442373431301:2108] [node 4] ICS09 handshake done sender: [5:7577471442373431295:2117] self: [5:7577471442373431297:0] peer: [4:7577471444762102644:0] socket: 106 qp: -1
2025-11-27T18:19:13.747650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471442373431301:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:13.747670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:13.741887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471445439136338:2049] [node 3] ICS09 handshake done sender: [1:7577471445439136310:2121] self: [1:7577471445439136309:0] peer: [3:7577471445538485337:0] socket: 84 qp: -1
2025-11-27T18:19:13.731195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471442604086264:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.741930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471445439136338:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:13.741960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:13.743334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471445439136314:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.743360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471445439136314:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.731232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:13.749752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471442373431300:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55626
2025-11-27T18:19:13.735719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471443291579813:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.750227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471444762102638:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.735800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:13.731288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471442604086264:2048] [node 1] ICP22 created new session: [2:7577471442604086353:2048]
2025-11-27T18:19:13.749826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471442373431215:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.750287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:13.753539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471445439136316:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.756286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471442373431300:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.756426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471442373431216:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.735845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471443291579813:2050] [node 3] ICP22 created new session: [6:7577471443291579904:2050]
2025-11-27T18:19:13.731344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471442604086353:2048] [node 1] ICS09 handshake done sender: [2:7577471442604086346:2118] self: [2:7577471442604086349:0] peer: [1:7577471445439136307:0] socket: 87 qp: -1
2025-11-27T18:19:13.758288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471442373431306:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55626
2025-11-27T18:19:13.750753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471444762102638:2108] [node 5] ICP22 created new session: [4:7577471444762102663:2108]
2025-11-27T18:19:13.753577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471445439136316:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.735903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471443291579904:2050] [node 3] ICS09 handshake done sender: [6:7577471443291579896:2125] self: [6:7577471443291579900:0] peer: [3:7577471445538485318:0] socket: 91 qp: -1
2025-11-27T18:19:13.735937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471443291579904:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:13.735967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:13.731380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471442604086353:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:13.731478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:13.735535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471442604086336:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:13.744593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471442604086328:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.744638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:13.744686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471442604086328:2107] [node 4] ICP22 created new session: [2:7577471442604086359:2107]
2025-11-27T18:19:13.744728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471442604086359:2107] [node 4] ICS09 handshake done sender: [2:7577471442604086336:2113] self: [2:7577471442604086335:0] peer: [4:7577471444762102655:0] socket: 68 qp: -1
2025-11-27T18:19:13.744760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471442604086359:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:13.753777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471444762102663:2108] [node 5] ICS09 handshake done sender: [4:7577471444762102645:2112] self: [4:7577471444762102644:0] peer: [5:7577471442373431297:0] socket: 105 qp: -1
2025-11-27T18:19:13.744792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:13.749572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471442604086334:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:13.753815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471444762102663:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:13.763165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471445439136314:2123] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:13.768186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471442373431300:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:13.753853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:13.750998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471442604086350:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.763581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471445439136316:2124] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.771844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471444762102661:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7704
2025-11-27T18:19:13.772593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471444762102662:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7704
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.773857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471445538485340:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11039
2025-11-27T18:19:13.772791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471444762102576:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29877) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471444762102648:2114] held: no
2025-11-27T18:19:13.776796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471442373431307:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55626
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.777659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471445538485341:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11039
2025-11-27T18:19:13.777541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471442373431216:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.777608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:13.777660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471442373431216:2049] [node 2] ICP22 created new session: [5:7577471442373431308:2049]
2025-11-27T18:19:13.777714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471442373431308:2049] [node 2] ICS09 handshake done sender: [5:7577471442373431300:2120] self: [5:7577471442373431305:0] peer: [2:7577471442604086337:0] socket: 124 qp: -1
2025-11-27T18:19:13.777749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471442373431308:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:13.777786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:13.751056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471442604086265:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.766648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471445439136306:2119] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.766696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.793617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471443291579907:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54632
2025-11-27T18:19:13.793827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471443291579908:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54632
2025-11-27T18:19:13.793938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471443291579909:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54632
2025-11-27T18:19:13.751125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:13.766725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471445439136306:2119] [node 6] ICP22 created new session: [1:7577471445439136342:2119]
2025-11-27T18:19:13.796652Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471443291579811:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471443291579901:2126] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.801368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471445538485252:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471445538485333:2117] held: no
2025-11-27T18:19:13.751173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471442604086265:2049] [node 3] ICP22 created new session: [2:7577471442604086362:2049]
2025-11-27T18:19:13.809014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471445538485253:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29877) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471445538485328:2115] held: no
2025-11-27T18:19:13.766761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471445439136342:2119] [node 6] ICS09 handshake done sender: [1:7577471445439136314:2123] self: [1:7577471445439136313:0] peer: [6:7577471443291579903:0] socket: 110 qp: -1
2025-11-27T18:19:13.766790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471445439136342:2119] [node 6] ICS10 traffic start
2025-11-27T18:19:13.797489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471443291579901:2126] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:13.766815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:13.804412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471443291579910:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54632
2025-11-27T18:19:13.804468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471443291579811:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.804566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:13.804629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471443291579811:2048] [node 1] ICP22 created new session: [6:7577471443291579911:2048]
2025-11-27T18:19:13.804681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471443291579911:2048] [node 1] ICS09 handshake done sender: [6:7577471443291579901:2126] self: [6:7577471443291579903:0] peer: [1:7577471445439136313:0] socket: 112 qp: -1
2025-11-27T18:19:13.804719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471443291579911:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:13.804774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:13.767536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471445439136305:2118] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.751213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471442604086362:2049] [node 3] ICS09 handshake done sender: [2:7577471442604086334:2112] self: [2:7577471442604086333:0] peer: [3:7577471445538485335:0] socket: 67 qp: -1
2025-11-27T18:19:13.751244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471442604086362:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:13.751275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:13.756006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471442604086328:2107] [node 4] ICP18 handshake (actor [2:7577471442604086350:2120]) is too old
2025-11-27T18:19:13.811845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471442373431299:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:13.767581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:13.767618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471445439136305:2118] [node 5] ICP22 created new session: [1:7577471445439136344:2118]
2025-11-27T18:19:13.768369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471445439136344:2118] [node 5] ICS09 handshake done sender: [1:7577471445439136316:2124] self: [1:7577471445439136315:0] peer: [5:7577471442373431303:0] socket: 107 qp: -1
2025-11-27T18:19:13.769420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471445439136344:2118] [node 5] ICS10 traffic start
2025-11-27T18:19:13.769927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:13.757136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471442604086338:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.757152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471442604086338:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.762827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471442604086338:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:13.769824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471442604086329:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.777208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:13.805387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471444762102648:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:13.781114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471442604086329:2108] [node 5] ICP22 created new session: [2:7577471442604086365:2108]
2025-11-27T18:19:13.781155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471442604086365:2108] [node 5] ICS09 handshake done sender: [2:7577471442604086338:2114] self: [2:7577471442604086337:0] peer: [5:7577471442373431305:0] socket: 122 qp: -1
2025-11-27T18:19:13.781182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471442604086365:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:13.781217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.815264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471445538485328:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:13.815909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471445538485333:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:13.812366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471444762102662:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.829562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471445538485253:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.829627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:13.829672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471445538485253:2049] [node 2] ICP22 created new session: [3:7577471445538485345:2049]
2025-11-27T18:19:13.829719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471445538485345:2049] [node 2] ICS09 handshake done sender: [3:7577471445538485328:2115] self: [3:7577471445538485335:0] peer: [2:7577471442604086333:0] socket: 74 qp: -1
2025-11-27T18:19:13.829751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471445538485345:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:13.829792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:13.830628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471445538485252:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.830676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:13.830710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471445538485252:2048] [node 1] ICP22 created new session: [3:7577471445538485347:2048]
2025-11-27T18:19:13.830747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471445538485347:2048] [node 1] ICS09 handshake done sender: [3:7577471445538485333:2117] self: [3:7577471445538485337:0] peer: [1:7577471445439136309:0] socket: 88 qp: -1
2025-11-27T18:19:13.830776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471445538485347:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:13.830800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.812411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471444762102576:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.820399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471443291579908:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.812454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:13.826109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471443291579874:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.845526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471445538485343:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11039
2025-11-27T18:19:13.826355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471443291579910:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.856463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471442604086329:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(67990757ns) {HandleHandshakeStatus:329(67988654ns) {DropIncomingHandshake:442(11305ns) DropOutgoingHandshake:457(372ns) SwitchToState:179(343ns) ProcessPendingSessionEvents:495(56423784ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 55927096ns max# 6567038ns}}}
2025-11-27T18:19:13.812505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471444762102576:2049] [node 2] ICP22 created new session: [4:7577471444762102668:2049]
2025-11-27T18:19:13.814184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471444762102668:2049] [node 2] ICS09 handshake done sender: [4:7577471444762102648:2114] self: [4:7577471444762102655:0] peer: [2:7577471442604086335:0] socket: 99 qp: -1
2025-11-27T18:19:13.826628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471443291579887:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.853488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471445538485343:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.814679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471444762102668:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:13.815856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:13.817426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471444762102667:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7704
2025-11-27T18:19:13.826967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471444762102651:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.826999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471444762102651:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.827908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471444762102651:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:13.828301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471444762102639:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.828352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:13.828396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471444762102639:2109] [node 6] ICP22 created new session: [4:7577471444762102672:2109]
2025-11-27T18:19:13.828440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471444762102672:2109] [node 6] ICS09 handshake done sender: [4:7577471444762102651:2116] self: [4:7577471444762102650:0] peer: [6:7577471443291579914:0] socket: 132 qp: -1
2025-11-27T18:19:13.828471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471444762102672:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:13.828505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:13.846977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471444762102667:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.848440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471444762102577:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.837815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471443291579915:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54632
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.866109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471442373431215:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.866177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:13.866232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471442373431215:2048] [node 1] ICP22 created new session: [5:7577471442373431310:2048]
2025-11-27T18:19:13.866283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471442373431310:2048] [node 1] ICS09 handshake done sender: [5:7577471442373431299:2119] self: [5:7577471442373431303:0] peer: [1:7577471445439136315:0] socket: 101 qp: -1
2025-11-27T18:19:13.866325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471442373431310:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:13.866380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:13.867019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471442373431215:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:13.868093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471442373431290:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.868117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471442373431290:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.876606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471442373431312:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55626
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.853677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471445538485325:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.853702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471445538485325:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.863892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471445538485325:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:13.880981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471444762102675:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7704
2025-11-27T18:19:13.881493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471444762102667:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:13.881672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471444762102577:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.881730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:13.881767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471444762102577:2050] [node 3] ICP22 created new session: [4:7577471444762102676:2050]
2025-11-27T18:19:13.881799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471444762102676:2050] [node 3] ICS09 handshake done sender: [4:7577471444762102667:2120] self: [4:7577471444762102674:0] peer: [3:7577471445538485324:0] socket: 81 qp: -1
2025-11-27T18:19:13.881824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471444762102676:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:13.881870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:13.885625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471443291579887:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6174) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471443291579910:2131] held: no
2025-11-27T18:19:13.886398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471443291579907:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.886470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471443291579910:2131] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:13.886562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471445538485315:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.886640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471442373431290:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:13.886633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:13.886677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471445538485315:2107] [node 4] ICP22 created new session: [3:7577471445538485351:2107]
2025-11-27T18:19:13.886720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471445538485351:2107] [node 4] ICS09 handshake done sender: [3:7577471445538485325:2113] self: [3:7577471445538485324:0] peer: [4:7577471444762102674:0] socket: 79 qp: -1
2025-11-27T18:19:13.886753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471445538485351:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:13.886796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:13.889734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471442373431312:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.889792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471442373431280:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.889840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:13.889875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471442373431280:2109] [node 6] ICP22 created new session: [5:7577471442373431314:2109]
2025-11-27T18:19:13.889916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471442373431314:2109] [node 6] ICS09 handshake done sender: [5:7577471442373431290:2115] self: [5:7577471442373431289:0] peer: [6:7577471443291579913:0] socket: 82 qp: -1
2025-11-27T18:19:13.889946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471442373431314:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:13.889977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:13.891960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471443291579887:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.893193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.893348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471443291579887:2110] [node 4] ICP22 created new session: [6:7577471443291579917:2110]
2025-11-27T18:19:13.893410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471443291579917:2110] [node 4] ICS09 handshake done sender: [6:7577471443291579910:2131] self: [6:7577471443291579914:0] peer: [4:7577471444762102650:0] socket: 133 qp: -1
2025-11-27T18:19:13.893443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471443291579917:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:13.893491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:13.894549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471443291579916:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54632
2025-11-27T18:19:13.895986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471442373431217:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.896394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471445538485327:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.896429Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471445538485327:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.897208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471445538485327:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.897641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471442373431317:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55626
2025-11-27T18:19:13.898094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471443291579812:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:13.899127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471442373431312:2123] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:13.899314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471442373431217:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.899359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:13.899405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471442373431217:2050] [node 3] ICP22 created new session: [5:7577471442373431318:2050]
2025-11-27T18:19:13.899448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471442373431318:2050] [node 3] ICS09 handshake done sender: [5:7577471442373431312:2123] self: [5:7577471442373431316:0] peer: [3:7577471445538485326:0] socket: 73 qp: -1
2025-11-27T18:19:13.899498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471442373431318:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:13.899535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:13.904602Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471443291579874:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19161) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471443291579908:2129] held: no
2025-11-27T18:19:13.902805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471445538485316:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.902870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:13.902916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471445538485316:2108] [node 5] ICP22 created new session: [3:7577471445538485354:2108]
2025-11-27T18:19:13.902958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471445538485354:2108] [node 5] ICS09 handshake done sender: [3:7577471445538485327:2114] self: [3:7577471445538485326:0] peer: [5:7577471442373431316:0] socket: 134 qp: -1
2025-11-27T18:19:13.902989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471445538485354:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:13.903014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:13.905846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471442604086340:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:13.905883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471442604086340:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:13.905321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471443291579908:2129] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:13.906810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471443291579874:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.906879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:13.906932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471443291579874:2108] [node 5] ICP22 created new session: [6:7577471443291579920:2108]
2025-11-27T18:19:13.907015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471443291579920:2108] [node 5] ICS09 handshake done sender: [6:7577471443291579908:2129] self: [6:7577471443291579913:0] peer: [5:7577471442373431289:0] socket: 129 qp: -1
2025-11-27T18:19:13.907054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471442604086340:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:13.907058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471443291579920:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:13.907128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:13.908900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471442604086330:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:13.908953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:13.908990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471442604086330:2109] [node 6] ICP22 created new session: [2:7577471442604086368:2109]
2025-11-27T18:19:13.909034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471442604086368:2109] [node 6] ICS09 handshake done sender: [2:7577471442604086340:2115] self: [2:7577471442604086339:0] peer: [6:7577471443291579919:0] socket: 123 qp: -1
2025-11-27T18:19:13.909068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471442604086368:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:13.909099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:13.909311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471443291579922:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54632
2025-11-27T18:19:13.910216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471443291579907:2128] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:13.910373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471443291579812:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:13.910427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:13.910473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471443291579812:2049] [node 2] ICP22 created new session: [6:7577471443291579923:2049]
2025-11-27T18:19:13.910514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471443291579923:2049] [node 2] ICS09 handshake done sender: [6:7577471443291579907:2128] self: [6:7577471443291579919:0] peer: [2:7577471442604086339:0] socket: 102 qp: -1
2025-11-27T18:19:13.910551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471443291579923:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:13.910583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:13.935294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:13.935549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471442604086353:2048] [node 1] ICS01 socket: 87 reason# EndOfStream
2025-11-27T18:19:13.935590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471442604086264:2048] [node 1] ICP30 unregister session Session# [2:7577471442604086353:2048] VirtualId# [2:7577471442604086349:0]
2025-11-27T18:19:13.935634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471442604086353:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.935695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:13.935734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:13.939341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:13.939537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471445538485347:2048] [node 1] ICS01 socket: 88 reason# EndOfStream
2025-11-27T18:19:13.939589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471445538485252:2048] [node 1] ICP30 unregister session Session# [3:7577471445538485347:2048] VirtualId# [3:7577471445538485337:0]
2025-11-27T18:19:13.939627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471445538485347:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.939692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:13.939730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:13.941059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:13.941310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471442373431310:2048] [node 1] ICS01 socket: 101 reason# EndOfStream
2025-11-27T18:19:13.941347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471442373431215:2048] [node 1] ICP30 unregister session Session# [5:7577471442373431310:2048] VirtualId# [5:7577471442373431303:0]
2025-11-27T18:19:13.941385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471442373431310:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.941456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:13.941494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:13.941842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:13.945873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:13.949785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471443291579911:2048] [node 1] ICS01 socket: 112 reason# EndOfStream
2025-11-27T18:19:13.949845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471443291579811:2048] [node 1] ICP30 unregister session Session# [6:7577471443291579911:2048] VirtualId# [6:7577471443291579903:0]
2025-11-27T18:19:13.949882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471443291579911:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.949860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471444762102659:2048] [node 1] ICS07 socket disconnect 100 reason# EndOfStream
2025-11-27T18:19:13.949924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471444762102659:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.949976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:13.950046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:13.950017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:13.950071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471444762102659:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:13.960594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471444762102575:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5435) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:13.960666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471444762102659:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:13.960693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471444762102575:2048] [node 1] ICP30 unregister session Session# [4:7577471444762102659:2048] VirtualId# [4:7577471444762102656:0]
2025-11-27T18:19:13.960740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:13.960773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471444762102575:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:13.960788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:13.970712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471444762102575:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:13.983200Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:13.983426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471445538485345:2049] [node 2] ICS01 socket: 74 reason# EndOfStream
2025-11-27T18:19:13.983475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471445538485253:2049] [node 2] ICP30 unregister session Session# [3:7577471445538485345:2049] VirtualId# [3:7577471445538485335:0]
2025-11-27T18:19:13.983513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471445538485345:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.983589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:13.983632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:13.986498Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:13.986719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471444762102668:2049] [node 2] ICS01 socket: 99 reason# EndOfStream
2025-11-27T18:19:13.986764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471444762102576:2049] [node 2] ICP30 unregister session Session# [4:7577471444762102668:2049] VirtualId# [4:7577471444762102655:0]
2025-11-27T18:19:13.986801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471444762102668:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.986896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:13.986957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:13.987373Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:13.987585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471442373431308:2049] [node 2] ICS01 socket: 124 reason# EndOfStream
2025-11-27T18:19:13.987642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471442373431216:2049] [node 2] ICP30 unregister session Session# [5:7577471442373431308:2049] VirtualId# [5:7577471442373431305:0]
2025-11-27T18:19:13.987672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471442373431308:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.987725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:13.987764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:13.988492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:13.988719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471443291579923:2049] [node 2] ICS01 socket: 102 reason# EndOfStream
2025-11-27T18:19:13.988759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471443291579812:2049] [node 2] ICP30 unregister session Session# [6:7577471443291579923:2049] VirtualId# [6:7577471443291579919:0]
2025-11-27T18:19:13.988791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471443291579923:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:13.988861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:13.988910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:14.064934Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:14.065172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471442373431318:2050] [node 3] ICS01 socket: 73 reason# EndOfStream
2025-11-27T18:19:14.065239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471442373431217:2050] [node 3] ICP30 unregister session Session# [5:7577471442373431318:2050] VirtualId# [5:7577471442373431316:0]
2025-11-27T18:19:14.065302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471442373431318:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:14.065381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:14.065424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:14.066270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:14.066470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471443291579904:2050] [node 3] ICS01 socket: 91 reason# EndOfStream
2025-11-27T18:19:14.066507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471443291579813:2050] [node 3] ICP30 unregister session Session# [6:7577471443291579904:2050] VirtualId# [6:7577471443291579900:0]
2025-11-27T18:19:14.066535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471443291579904:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:14.066597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:14.066638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:14.066457Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:14.066663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471444762102676:2050] [node 3] ICS07 socket disconnect 81 reason# EndOfStream
2025-11-27T18:19:14.066706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471444762102676:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:14.066781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:14.066827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471444762102676:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:14.096830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:14.097086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471442373431301:2108] [node 4] ICS01 socket: 106 reason# EndOfStream
2025-11-27T18:19:14.097134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471442373431279:2108] [node 4] ICP30 unregister session Session# [5:7577471442373431301:2108] VirtualId# [5:7577471442373431297:0]
2025-11-27T18:19:14.097174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471442373431301:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:14.097272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:14.097332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:14.098537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:14.098744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471443291579917:2110] [node 4] ICS01 socket: 133 reason# EndOfStream
2025-11-27T18:19:14.098794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471443291579887:2110] [node 4] ICP30 unregister session Session# [6:7577471443291579917:2110] VirtualId# [6:7577471443291579914:0]
2025-11-27T18:19:14.098832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471443291579917:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:14.098923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:14.098984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:14.175629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:14.175861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471443291579920:2108] [node 5] ICS01 socket: 129 reason# EndOfStream
2025-11-27T18:19:14.175910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471443291579874:2108] [node 5] ICP30 unregister session Session# [6:7577471443291579920:2108] VirtualId# [6:7577471443291579913:0]
2025-11-27T18:19:14.175952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471443291579920:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:14.176041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:14.176092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:14.290448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471446934208442:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:14.290507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471446934208443:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:14.336100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471449773119240:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:14.336161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471449773119241:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:14.364473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471450060328506:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:14.364527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471450060328507:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:14.392687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471449036470689:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:14.392751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471449036470690:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:14.392787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471449036470691:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:14.418159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471446042293376:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:14.418208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471446042293377:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:14.418238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471446042293378:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:14.454239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471447697840132:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:14.454283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471447697840133:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:14.454314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471447697840134:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:14.775722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471447697840190:2104] [node 5] ICP01 ready to work
2025-11-27T18:19:14.793127Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471447697840190:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17046124ns) {Configure:104(17040312ns) {ProcessConfigured:135(74060ns) {StartInitialHandshake:157(71663ns) {DropHandshakes:470(3577ns) {DropIncomingHandshake:442(745ns) DropOutgoingHandshake:457(500ns)} GenerateSessionVirtualId:988(1057ns) PrepareNewSessionHandshake:477(4404ns) {SwitchToState:179(479ns)}}}}}
2025-11-27T18:19:14.824985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471449773119321:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:14.925758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471447697840197:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:14.847153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471446934208505:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:14.926058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471449773119323:2111] [node 6] ICP01 ready to work
2025-11-27T18:19:14.942958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471449773119322:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:14.920793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471450060328569:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:14.922303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471446042293439:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:14.952955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471446042293443:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:14.920658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471446934208507:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:14.958881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471449773119240:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23030107ns) {Configure:104(23022197ns) {ProcessConfigured:135(39177ns) {StartInitialHandshake:157(37785ns) {DropHandshakes:470(1833ns) {DropIncomingHandshake:442(349ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(1121ns) PrepareNewSessionHandshake:477(735ns) {SwitchToState:179(349ns)}}}}}
2025-11-27T18:19:14.953763Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471447697840133:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(128658146ns) {Configure:104(128652367ns) {ProcessConfigured:135(75702ns) {StartInitialHandshake:157(72820ns) {DropHandshakes:470(3175ns) {DropIncomingHandshake:442(962ns) DropOutgoingHandshake:457(367ns)} GenerateSessionVirtualId:988(844ns) PrepareNewSessionHandshake:477(1478ns) {SwitchToState:179(498ns)}}}}}
2025-11-27T18:19:14.929203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471446934208506:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:14.952999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471450060328570:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:14.960034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471450060328571:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:14.964106Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471446042293378:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32826427ns) {Configure:104(32822701ns) {ProcessConfigured:135(21136480ns) {StartInitialHandshake:157(21133504ns) {DropHandshakes:470(3434ns) {DropIncomingHandshake:442(887ns) DropOutgoingHandshake:457(469ns)} GenerateSessionVirtualId:988(949ns) PrepareNewSessionHandshake:477(3158ns) {SwitchToState:179(1762ns)}}}}}
2025-11-27T18:19:14.986041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471449036470690:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(161778212ns) {Configure:104(161774421ns) {ProcessConfigured:135(72667ns) {StartInitialHandshake:157(70765ns) {DropHandshakes:470(3040ns) {DropIncomingHandshake:442(1005ns) DropOutgoingHandshake:457(407ns)} GenerateSessionVirtualId:988(1403ns) PrepareNewSessionHandshake:477(1874ns) {SwitchToState:179(711ns)}}}}}
2025-11-27T18:19:14.986776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471449036470754:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:14.987295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471449036470755:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.051367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471451992807506:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21065
2025-11-27T18:19:15.051441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471451992807507:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21065
2025-11-27T18:19:15.010355Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471449036470689:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(187275606ns) {Configure:104(187271754ns) {ProcessConfigured:135(13397861ns) {StartInitialHandshake:157(13395123ns) {DropHandshakes:470(3587ns) {DropIncomingHandshake:442(768ns) DropOutgoingHandshake:457(432ns)} GenerateSessionVirtualId:988(651ns) PrepareNewSessionHandshake:477(5237ns) {SwitchToState:179(2595ns)}}}}}
2025-11-27T18:19:15.021533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471449773119321:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31849171ns) {Configure:104(31843034ns) {ProcessConfigured:135(7963531ns) {StartInitialHandshake:157(7961544ns) {DropHandshakes:470(2783ns) {DropIncomingHandshake:442(662ns) DropOutgoingHandshake:457(539ns)} GenerateSessionVirtualId:988(683ns) PrepareNewSessionHandshake:477(2379ns) {SwitchToState:179(204ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.033636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471446042293377:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(43949131ns) {Configure:104(43943790ns) {ProcessConfigured:135(19945735ns) {StartInitialHandshake:157(19943035ns) {DropHandshakes:470(1573ns) {DropIncomingHandshake:442(466ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(596ns) PrepareNewSessionHandshake:477(5446ns) {SwitchToState:179(829ns)}}}}}
2025-11-27T18:19:15.038278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471454068086628:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35685
2025-11-27T18:19:15.038441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471454068086629:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35685
2025-11-27T18:19:15.081349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471454068086628:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.058094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471446934208443:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(60882945ns) {Configure:104(60878749ns) {ProcessConfigured:135(44138503ns) {StartInitialHandshake:157(44135737ns) {DropHandshakes:470(2869ns) {DropIncomingHandshake:442(738ns) DropOutgoingHandshake:457(640ns)} GenerateSessionVirtualId:988(847ns) PrepareNewSessionHandshake:477(21121567ns) {SwitchToState:179(6252ns)}}}}}
2025-11-27T18:19:15.034205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471446042293445:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.104829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471453331438064:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-27T18:19:15.119050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471453331438063:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-27T18:19:15.068289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471451229175810:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60521
2025-11-27T18:19:15.049520Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471446042293443:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(39350960ns) {Configure:104(39345469ns) {ProcessConfigured:135(37703720ns) {StartInitialHandshake:157(37701697ns) {DropHandshakes:470(2994ns) {DropIncomingHandshake:442(777ns) DropOutgoingHandshake:457(513ns)} GenerateSessionVirtualId:988(909ns) PrepareNewSessionHandshake:477(6696ns) {SwitchToState:179(3491ns)}}}}}
2025-11-27T18:19:15.111014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471450060328506:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(60237380ns) {Configure:104(60230966ns) {ProcessConfigured:135(73541ns) {StartInitialHandshake:157(70631ns) {DropHandshakes:470(3360ns) {DropIncomingHandshake:442(888ns) DropOutgoingHandshake:457(542ns)} GenerateSessionVirtualId:988(1049ns) PrepareNewSessionHandshake:477(2174ns) {SwitchToState:179(1051ns)}}}}}
2025-11-27T18:19:15.142781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471454068086629:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.093951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471451992807508:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21065
2025-11-27T18:19:15.068453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471451229175811:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60521
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.153622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471453331438058:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.153655Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471453331438058:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.154338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471453331438064:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.154754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471449036470689:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.154834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471453331438065:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-27T18:19:15.069417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471446042293445:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.121423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471451992807506:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.068572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471451229175812:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60521
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.159335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471454068086637:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35685
2025-11-27T18:19:15.130831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471450060328569:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(117024295ns) {Configure:104(117020581ns) {ProcessConfigured:135(63374ns) {StartInitialHandshake:157(61577ns) {DropHandshakes:470(3563ns) {DropIncomingHandshake:442(1217ns) DropOutgoingHandshake:457(475ns)} GenerateSessionVirtualId:988(605ns) PrepareNewSessionHandshake:477(1859ns) {SwitchToState:179(559ns)}}}}}
2025-11-27T18:19:15.086211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471450337260751:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
2025-11-27T18:19:15.121646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471447697840190:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.158596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471453331438058:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:15.135178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471451229175811:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.169095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471453331438056:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.169133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471453331438056:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.086308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471450337260752:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.150879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471454355295876:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26868
2025-11-27T18:19:15.138833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471451992807507:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.135285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471451229175810:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.150066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471447697840197:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.151045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471454355295877:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26868
2025-11-27T18:19:15.129574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471450337260752:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.162742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471451229175809:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.157737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471454355295876:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.153326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471446042293448:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.193900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471453331438063:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.161334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471450060328570:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(47968519ns) {Configure:104(47965101ns) {ProcessConfigured:135(2315482ns) {StartInitialHandshake:157(2312968ns) {DropHandshakes:470(2619ns) {DropIncomingHandshake:442(343ns) DropOutgoingHandshake:457(406ns)} GenerateSessionVirtualId:988(568ns) PrepareNewSessionHandshake:477(2229ns) {SwitchToState:179(747ns)}}}}}
2025-11-27T18:19:15.185594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471454355295877:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.198322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471450060328506:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.153621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471451992807508:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.162769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471451229175809:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.163518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471451229175812:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.163896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471451229175820:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60521
2025-11-27T18:19:15.172263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471451229175807:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.172289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471451229175807:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.173165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471451229175809:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:15.181916Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471446934208442:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36542193ns) {Configure:104(36536216ns) {ProcessConfigured:135(10529927ns) {StartInitialHandshake:157(10526801ns) {DropHandshakes:470(1979ns) {DropIncomingHandshake:442(455ns) DropOutgoingHandshake:457(355ns)} GenerateSessionVirtualId:988(560ns) PrepareNewSessionHandshake:477(2907ns) {SwitchToState:179(1422ns)}}}}}
2025-11-27T18:19:15.182149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471446934208506:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.182186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:15.182231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471446934208506:2107] [node 4] ICP22 created new session: [1:7577471451229175825:2107]
2025-11-27T18:19:15.182279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471451229175825:2107] [node 4] ICS09 handshake done sender: [1:7577471451229175809:2112] self: [1:7577471451229175808:0] peer: [4:7577471453331438067:0] socket: 100 qp: -1
2025-11-27T18:19:15.182307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471451229175825:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:15.182357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:15.193713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471451229175807:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.204430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471454068086637:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.202178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471453331438056:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:15.161407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471447697840132:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.153372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471446042293448:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.229841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471454068086627:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.229868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471454068086627:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.253592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471449036470755:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.229785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471451229175805:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.229828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471451229175805:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.249958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471446934208507:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.250009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:15.250062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471446934208507:2109] [node 6] ICP22 created new session: [1:7577471451229175829:2109]
2025-11-27T18:19:15.250108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471451229175829:2109] [node 6] ICS09 handshake done sender: [1:7577471451229175807:2111] self: [1:7577471451229175806:0] peer: [6:7577471451992807511:0] socket: 99 qp: -1
2025-11-27T18:19:15.250137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471451229175829:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:15.250172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:15.251364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471451229175820:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.153640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471446042293443:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.170721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471451992807513:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21065
2025-11-27T18:19:15.256974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471454068086627:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:15.164520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471446042293448:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:15.266474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:15.171552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471451992807512:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21065
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.280343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471454355295885:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26868
2025-11-27T18:19:15.171270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471450337260751:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.171988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471446042293439:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.172066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:15.174529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471446042293439:2109] [node 6] ICP22 created new session: [5:7577471450337260756:2109]
2025-11-27T18:19:15.181784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471450337260756:2109] [node 6] ICS09 handshake done sender: [5:7577471446042293448:2113] self: [5:7577471446042293447:0] peer: [6:7577471451992807509:0] socket: 83 qp: -1
2025-11-27T18:19:15.200428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471450337260756:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:15.200497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:15.202663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471446042293377:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.213280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471450337260757:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
2025-11-27T18:19:15.228766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471450337260758:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
2025-11-27T18:19:15.229139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471450337260752:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:15.281687Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471454355295877:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:15.281966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471450060328506:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.282043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:15.282095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471450060328506:2048] [node 1] ICP22 created new session: [3:7577471454355295886:2048]
2025-11-27T18:19:15.282138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471454355295886:2048] [node 1] ICS09 handshake done sender: [3:7577471454355295877:2115] self: [3:7577471454355295884:0] peer: [1:7577471446934208508:0] socket: 106 qp: -1
2025-11-27T18:19:15.282179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471454355295886:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:15.282215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:15.266794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471449036470755:2109] [node 6] ICP22 created new session: [4:7577471453331438070:2109]
2025-11-27T18:19:15.266850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471453331438070:2109] [node 6] ICS09 handshake done sender: [4:7577471453331438058:2113] self: [4:7577471453331438057:0] peer: [6:7577471451992807510:0] socket: 90 qp: -1
2025-11-27T18:19:15.266892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471453331438070:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:15.266938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:15.272000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471449036470754:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.272048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:15.272109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471449036470754:2108] [node 5] ICP22 created new session: [4:7577471453331438073:2108]
2025-11-27T18:19:15.272156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471453331438073:2108] [node 5] ICS09 handshake done sender: [4:7577471453331438056:2112] self: [4:7577471453331438055:0] peer: [5:7577471450337260754:0] socket: 89 qp: -1
2025-11-27T18:19:15.272201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471453331438073:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:15.272240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:15.276815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471453331438065:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.277137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471449036470690:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.277584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471453331438069:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.290893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471454355295889:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26868
2025-11-27T18:19:15.263806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471446934208506:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(80776495ns) {HandleHandshakeStatus:329(80773448ns) {DropIncomingHandshake:442(221ns) DropOutgoingHandshake:457(562ns) SwitchToState:179(547ns) ProcessPendingSessionEvents:495(80423522ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 75283472ns max# 13932593ns}}}
2025-11-27T18:19:15.173783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471451992807507:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:15.296673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471449773119322:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.296726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:15.248764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471446042293439:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(75856389ns) {HandleHandshakeStatus:329(75852427ns) {DropIncomingHandshake:442(30109ns) DropOutgoingHandshake:457(591ns) SwitchToState:179(196ns) ProcessPendingSessionEvents:495(3522121ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3457306ns max# 3050472ns}}}
2025-11-27T18:19:15.174346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471447697840197:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.297151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471449773119322:2108] [node 5] ICP22 created new session: [2:7577471454068086642:2108]
2025-11-27T18:19:15.275016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471451229175805:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:15.297212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471454068086642:2108] [node 5] ICS09 handshake done sender: [2:7577471454068086627:2131] self: [2:7577471454068086626:0] peer: [5:7577471450337260761:0] socket: 86 qp: -1
2025-11-27T18:19:15.294689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471446934208443:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.295752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:15.296059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471446934208443:2049] [node 3] ICP22 created new session: [1:7577471451229175834:2049]
2025-11-27T18:19:15.297602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471451229175834:2049] [node 3] ICS09 handshake done sender: [1:7577471451229175805:2110] self: [1:7577471446934208508:0] peer: [3:7577471454355295884:0] socket: 98 qp: -1
2025-11-27T18:19:15.297878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471451229175834:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.179722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:15.179788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471447697840197:2108] [node 4] ICP22 created new session: [6:7577471451992807514:2108]
2025-11-27T18:19:15.248998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471446042293443:2108] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.299996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471454355295890:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26868
2025-11-27T18:19:15.179841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471451992807514:2108] [node 4] ICS09 handshake done sender: [6:7577471451992807507:2116] self: [6:7577471451992807510:0] peer: [4:7577471453331438057:0] socket: 94 qp: -1
2025-11-27T18:19:15.179885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471451992807514:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:15.179931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:15.181115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471451992807506:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:15.198606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471447697840190:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.214138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471451992807516:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21065
2025-11-27T18:19:15.219235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:15.220082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471447697840190:2104] [node 5] ICP22 created new session: [6:7577471451992807517:2104]
2025-11-27T18:19:15.247054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471451992807517:2104] [node 5] ICS09 handshake done sender: [6:7577471451992807506:2115] self: [6:7577471451992807509:0] peer: [5:7577471446042293447:0] socket: 93 qp: -1
2025-11-27T18:19:15.247114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471451992807517:2104] [node 5] ICS10 traffic start
2025-11-27T18:19:15.247172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:15.291433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471451992807508:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:15.291732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471447697840132:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27116) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471451992807508:2117] held: no
2025-11-27T18:19:15.297247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471454068086642:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.299427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:15.292292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471447697840132:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.305612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471451229175823:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.305650Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471451229175823:2118] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.297319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:15.249042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.305166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471454355295889:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.308543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471449036470755:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(53058137ns) {HandleHandshakeStatus:329(53052973ns) {DropIncomingHandshake:442(1598ns) DropOutgoingHandshake:457(1242ns) SwitchToState:179(716ns) ProcessPendingSessionEvents:495(31792942ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 31701981ns max# 9679111ns}}}
2025-11-27T18:19:15.310383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471453331438072:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-27T18:19:15.292334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:15.298801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471454068086640:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35685
2025-11-27T18:19:15.249096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471446042293443:2108] [node 4] ICP22 created new session: [5:7577471450337260763:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.308301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471454355295875:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.308454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471451229175823:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:15.292396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471447697840132:2048] [node 1] ICP22 created new session: [6:7577471451992807520:2048]
2025-11-27T18:19:15.298860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471454068086641:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35685
2025-11-27T18:19:15.249140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471450337260763:2108] [node 4] ICS09 handshake done sender: [5:7577471450337260752:2117] self: [5:7577471450337260754:0] peer: [4:7577471453331438055:0] socket: 96 qp: -1
2025-11-27T18:19:15.313328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471453331438064:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:15.308766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471446934208442:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.308335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471454355295875:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.313610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471449036470691:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.292440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471451992807520:2048] [node 1] ICS09 handshake done sender: [6:7577471451992807508:2117] self: [6:7577471451992807511:0] peer: [1:7577471451229175806:0] socket: 101 qp: -1
2025-11-27T18:19:15.299849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471454068086641:2137] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.308818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:15.249172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471450337260763:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.317857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471454355295890:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.319154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471449036470689:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.292479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471451992807520:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:15.249208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:15.304831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471449773119240:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.308864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471446934208442:2048] [node 2] ICP22 created new session: [1:7577471451229175837:2048]
2025-11-27T18:19:15.318505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471450060328507:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.307738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471454068086646:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35685
2025-11-27T18:19:15.292514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:15.293568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471450337260762:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
2025-11-27T18:19:15.319225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:15.308914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471451229175837:2048] [node 2] ICS09 handshake done sender: [1:7577471451229175823:2118] self: [1:7577471451229175821:0] peer: [2:7577471454068086645:0] socket: 127 qp: -1
2025-11-27T18:19:15.328330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471454068086625:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.319277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471449036470689:2048] [node 1] ICP22 created new session: [4:7577471453331438079:2048]
2025-11-27T18:19:15.318611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471454355295892:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26868
2025-11-27T18:19:15.303264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471451992807522:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21065
2025-11-27T18:19:15.299034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471450337260758:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.308946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471451229175837:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:15.319319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471453331438079:2048] [node 1] ICS09 handshake done sender: [4:7577471453331438064:2116] self: [4:7577471453331438067:0] peer: [1:7577471451229175808:0] socket: 103 qp: -1
2025-11-27T18:19:15.319353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471453331438079:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:15.319391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:15.333854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471453331438081:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-27T18:19:15.336087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471453331438065:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:15.338088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471449036470691:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.338128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:15.338184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471449036470691:2050] [node 3] ICP22 created new session: [4:7577471453331438082:2050]
2025-11-27T18:19:15.338226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471453331438082:2050] [node 3] ICS09 handshake done sender: [4:7577471453331438065:2117] self: [4:7577471453331438078:0] peer: [3:7577471454355295872:0] socket: 111 qp: -1
2025-11-27T18:19:15.338257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471453331438082:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:15.338289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:15.323649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471454355295875:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:15.299123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471450337260751:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.328367Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471454068086625:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.303546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471451992807522:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.308983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:15.353793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471451229175814:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.353836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471451229175814:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.358496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471451229175839:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60521
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.303669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471447697840134:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.354769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471454068086631:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.352580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471447697840190:2104] [node 5] ICP35 event processing took too much time PendingConnection:279(133057551ns) {HandleHandshakeStatus:329(133054582ns) {DropIncomingHandshake:442(614ns) DropOutgoingHandshake:457(5899ns) SwitchToState:179(238ns) ProcessPendingSessionEvents:495(64894095ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60868879ns max# 13895987ns}}}
2025-11-27T18:19:15.360138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471447697840190:2104] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:15.303998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471446042293376:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.305121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471446042293377:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.305398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:15.307166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471446042293377:2049] [node 2] ICP22 created new session: [5:7577471450337260767:2049]
2025-11-27T18:19:15.323980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471450337260767:2049] [node 2] ICS09 handshake done sender: [5:7577471450337260751:2116] self: [5:7577471450337260761:0] peer: [2:7577471454068086626:0] socket: 88 qp: -1
2025-11-27T18:19:15.324732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471450337260767:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:15.324779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:15.361712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471450337260769:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
2025-11-27T18:19:15.362105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471450337260770:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
2025-11-27T18:19:15.326754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471454355295873:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.326796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471454355295873:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.328749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471450060328571:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.328822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:15.328876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471450060328571:2109] [node 6] ICP22 created new session: [3:7577471454355295895:2109]
2025-11-27T18:19:15.328928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471454355295895:2109] [node 6] ICS09 handshake done sender: [3:7577471454355295875:2113] self: [3:7577471454355295874:0] peer: [6:7577471451992807523:0] socket: 107 qp: -1
2025-11-27T18:19:15.328969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471454355295895:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:15.329002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:15.334474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471454355295873:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:15.337074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471454355295892:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.337122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471450060328569:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.337158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:15.337215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471450060328569:2107] [node 4] ICP22 created new session: [3:7577471454355295899:2107]
2025-11-27T18:19:15.337273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471454355295899:2107] [node 4] ICS09 handshake done sender: [3:7577471454355295873:2112] self: [3:7577471454355295872:0] peer: [4:7577471453331438078:0] socket: 109 qp: -1
2025-11-27T18:19:15.337314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471454355295899:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:15.337381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:15.343135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471450060328569:2107] [node 4] ICP18 handshake (actor [3:7577471454355295892:2120]) is too old
2025-11-27T18:19:15.354800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471454068086631:2134] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.362559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471454068086641:2137] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.370256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471447697840134:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62568) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471451992807522:2121] held: no
2025-11-27T18:19:15.372393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471451992807525:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21065
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.373059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471451992807524:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21065
2025-11-27T18:19:15.364625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471446042293377:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(59719286ns) {HandleHandshakeStatus:329(59717018ns) {DropIncomingHandshake:442(808ns) DropOutgoingHandshake:457(5376ns) SwitchToState:179(187ns) ProcessPendingSessionEvents:495(33711725ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28986148ns max# 4728636ns}}}
2025-11-27T18:19:15.369121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471446934208443:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(73856216ns) {HandleHandshakeStatus:329(73853181ns) {DropIncomingHandshake:442(22590ns) DropOutgoingHandshake:457(404ns) SwitchToState:179(450ns) ProcessPendingSessionEvents:495(62928240ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 58267826ns max# 8856746ns}}}
2025-11-27T18:19:15.373982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471450337260769:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.375905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471450337260758:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:15.376824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471451992807522:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:15.366174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471449773119322:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(68730711ns) {HandleHandshakeStatus:329(68728618ns) {DropIncomingHandshake:442(508ns) DropOutgoingHandshake:457(454ns) SwitchToState:179(400ns) ProcessPendingSessionEvents:495(67944000ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 67811428ns max# 8855076ns}}}
2025-11-27T18:19:15.381788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471447697840134:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.381840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:15.381883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471447697840134:2050] [node 3] ICP22 created new session: [6:7577471451992807526:2050]
2025-11-27T18:19:15.381924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471451992807526:2050] [node 3] ICS09 handshake done sender: [6:7577471451992807522:2121] self: [6:7577471451992807523:0] peer: [3:7577471454355295874:0] socket: 113 qp: -1
2025-11-27T18:19:15.381963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471451992807526:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:15.381991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:15.382806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471446042293378:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.382867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471446042293376:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.382912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:15.382952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471446042293376:2048] [node 1] ICP22 created new session: [5:7577471450337260772:2048]
2025-11-27T18:19:15.382995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471450337260772:2048] [node 1] ICS09 handshake done sender: [5:7577471450337260758:2119] self: [5:7577471450337260766:0] peer: [1:7577471451229175813:0] socket: 123 qp: -1
2025-11-27T18:19:15.383026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471450337260772:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:15.383056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:15.370046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471451229175814:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:15.381063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471446934208505:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.366277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471449773119240:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27116) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471454068086641:2137] held: no
2025-11-27T18:19:15.381123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:15.381167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471446934208505:2108] [node 5] ICP22 created new session: [1:7577471451229175841:2108]
2025-11-27T18:19:15.381212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471451229175841:2108] [node 5] ICS09 handshake done sender: [1:7577471451229175814:2116] self: [1:7577471451229175813:0] peer: [5:7577471450337260766:0] socket: 121 qp: -1
2025-11-27T18:19:15.381244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471451229175841:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:15.382559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:15.394261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471451992807525:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.399402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471454355295882:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.399459Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471454355295882:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.399940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471447697840133:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:15.367995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471449773119322:2108] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.417714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471451992807529:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21065
2025-11-27T18:19:15.404715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471450060328507:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471454355295890:2119] held: no
2025-11-27T18:19:15.369773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471454068086625:2130] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:15.405409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471449036470690:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471453331438063:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.422220Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471449036470690:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(18929041ns) {HandleHandshakeStatus:395(18925187ns) {LogHandshakeFail:472(3641558ns) {UpdateErrorStateLog:545(6105ns)} DropOutgoingHandshake:457(2514ns) ProcessPendingSessionEvents:495(12864450ns) {unordered DropSessionEvent:512 num# 100 sum# 476539ns max# 19720ns}}}
2025-11-27T18:19:15.422350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471453331438084:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28958
2025-11-27T18:19:15.413625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471454355295901:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26868
2025-11-27T18:19:15.424639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471454355295890:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:15.373874Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471454068086631:2134] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:15.374105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471449773119240:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.374145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:15.376166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471449773119240:2048] [node 1] ICP22 created new session: [2:7577471454068086650:2048]
2025-11-27T18:19:15.376203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471454068086650:2048] [node 1] ICS09 handshake done sender: [2:7577471454068086641:2137] self: [2:7577471454068086645:0] peer: [1:7577471451229175821:0] socket: 76 qp: -1
2025-11-27T18:19:15.376240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471454068086650:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:15.376276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:15.377679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471449773119321:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.377741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:15.377786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471449773119321:2107] [node 4] ICP22 created new session: [2:7577471454068086652:2107]
2025-11-27T18:19:15.377830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471454068086652:2107] [node 4] ICS09 handshake done sender: [2:7577471454068086625:2130] self: [2:7577471454068086624:0] peer: [4:7577471453331438075:0] socket: 85 qp: -1
2025-11-27T18:19:15.377862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471454068086652:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:15.377889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:15.378874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471449773119241:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.378926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:15.378984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471449773119241:2049] [node 3] ICP22 created new session: [2:7577471454068086654:2049]
2025-11-27T18:19:15.379028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471454068086654:2049] [node 3] ICS09 handshake done sender: [2:7577471454068086631:2134] self: [2:7577471454068086630:0] peer: [3:7577471454355295894:0] socket: 133 qp: -1
2025-11-27T18:19:15.379054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471454068086654:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:15.379084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:15.414084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471449773119327:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:15.414127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471449773119327:2129] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:15.415937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471449773119327:2129] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:15.416201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471449773119323:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.416290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:15.416347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471449773119323:2111] [node 6] ICP22 created new session: [2:7577471454068086657:2111]
2025-11-27T18:19:15.416391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471454068086657:2111] [node 6] ICS09 handshake done sender: [2:7577471449773119327:2129] self: [2:7577471449773119326:0] peer: [6:7577471451992807528:0] socket: 140 qp: -1
2025-11-27T18:19:15.416421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471454068086657:2111] [node 6] ICS10 traffic start
2025-11-27T18:19:15.416455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:15.423529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471453331438063:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:15.430838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471450337260774:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
2025-11-27T18:19:15.431055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471454355295882:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:15.432952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471451992807525:2123] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:15.432338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471450060328570:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:15.432435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:15.432497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471450060328570:2108] [node 5] ICP22 created new session: [3:7577471454355295903:2108]
2025-11-27T18:19:15.432546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471454355295903:2108] [node 5] ICS09 handshake done sender: [3:7577471454355295882:2116] self: [3:7577471454355295881:0] peer: [5:7577471450337260771:0] socket: 95 qp: -1
2025-11-27T18:19:15.432593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471454355295903:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:15.434150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471447697840133:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471451992807525:2123] held: no
2025-11-27T18:19:15.434784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471447697840133:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.434823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:15.434873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471447697840133:2049] [node 2] ICP22 created new session: [6:7577471451992807530:2049]
2025-11-27T18:19:15.434920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471451992807530:2049] [node 2] ICS09 handshake done sender: [6:7577471451992807525:2123] self: [6:7577471451992807528:0] peer: [2:7577471449773119326:0] socket: 142 qp: -1
2025-11-27T18:19:15.434956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471451992807530:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:15.434990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:15.436945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471450060328507:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.437009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:15.437054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471450060328507:2049] [node 2] ICP22 created new session: [3:7577471454355295905:2049]
2025-11-27T18:19:15.437099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471454355295905:2049] [node 2] ICS09 handshake done sender: [3:7577471454355295890:2119] self: [3:7577471454355295894:0] peer: [2:7577471454068086630:0] socket: 134 qp: -1
2025-11-27T18:19:15.437130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471454355295905:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:15.437166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:15.437847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:15.432676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471449036470690:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.445797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471446042293378:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62568) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471450337260769:2121] held: no
2025-11-27T18:19:15.446415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471450337260769:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:15.432726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:15.432768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471449036470690:2049] [node 2] ICP22 created new session: [4:7577471453331438085:2049]
2025-11-27T18:19:15.432802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471453331438085:2049] [node 2] ICS09 handshake done sender: [4:7577471453331438063:2115] self: [4:7577471453331438075:0] peer: [2:7577471454068086624:0] socket: 91 qp: -1
2025-11-27T18:19:15.432826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471453331438085:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:15.438655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:15.442946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471449036470690:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16325776ns) {HandleHandshakeStatus:329(16323771ns) {DropIncomingHandshake:442(2291ns) DropOutgoingHandshake:457(223ns) SwitchToState:179(384ns) ProcessPendingSessionEvents:495(1314ns)}}
2025-11-27T18:19:15.449945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471446042293378:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:15.450000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:15.450055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471446042293378:2050] [node 3] ICP22 created new session: [5:7577471450337260775:2050]
2025-11-27T18:19:15.450103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471450337260775:2050] [node 3] ICS09 handshake done sender: [5:7577471450337260769:2121] self: [5:7577471450337260771:0] peer: [3:7577471454355295881:0] socket: 137 qp: -1
2025-11-27T18:19:15.450141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471450337260775:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:15.450172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:15.515062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:15.515360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471454355295886:2048] [node 1] ICS01 socket: 106 reason# EndOfStream
2025-11-27T18:19:15.515413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471450060328506:2048] [node 1] ICP30 unregister session Session# [3:7577471454355295886:2048] VirtualId# [3:7577471454355295884:0]
2025-11-27T18:19:15.515450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471454355295886:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.515528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:15.515571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:15.518870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:15.519153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471454068086650:2048] [node 1] ICS01 socket: 76 reason# EndOfStream
2025-11-27T18:19:15.519204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471449773119240:2048] [node 1] ICP30 unregister session Session# [2:7577471454068086650:2048] VirtualId# [2:7577471454068086645:0]
2025-11-27T18:19:15.519252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471454068086650:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.519332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:15.519385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:15.525717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:15.525968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471450337260772:2048] [node 1] ICS01 socket: 123 reason# EndOfStream
2025-11-27T18:19:15.526008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471446042293376:2048] [node 1] ICP30 unregister session Session# [5:7577471450337260772:2048] VirtualId# [5:7577471450337260766:0]
2025-11-27T18:19:15.526044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471450337260772:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.526129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:15.526178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:15.526269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:15.526454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471451992807520:2048] [node 1] ICS01 socket: 101 reason# EndOfStream
2025-11-27T18:19:15.526493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471447697840132:2048] [node 1] ICP30 unregister session Session# [6:7577471451992807520:2048] VirtualId# [6:7577471451992807511:0]
2025-11-27T18:19:15.526548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471451992807520:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.526615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:15.526662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:15.526758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:15.526960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471453331438079:2048] [node 1] ICS07 socket disconnect 103 reason# EndOfStream
2025-11-27T18:19:15.526989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471453331438079:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.527065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:15.527104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471453331438079:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:15.529855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471449036470689:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27116) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:15.529918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471453331438079:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:15.529952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471449036470689:2048] [node 1] ICP30 unregister session Session# [4:7577471453331438079:2048] VirtualId# [4:7577471453331438067:0]
2025-11-27T18:19:15.529996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:15.530025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471449036470689:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27116) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:15.530040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27116) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:15.540381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471449036470689:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:15.656809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:15.657044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471454355295905:2049] [node 2] ICS01 socket: 134 reason# EndOfStream
2025-11-27T18:19:15.657091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471450060328507:2049] [node 2] ICP30 unregister session Session# [3:7577471454355295905:2049] VirtualId# [3:7577471454355295894:0]
2025-11-27T18:19:15.657128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471454355295905:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.657202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:15.657245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:15.661816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:15.662041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471453331438085:2049] [node 2] ICS01 socket: 91 reason# EndOfStream
2025-11-27T18:19:15.662081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471449036470690:2049] [node 2] ICP30 unregister session Session# [4:7577471453331438085:2049] VirtualId# [4:7577471453331438075:0]
2025-11-27T18:19:15.662118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471453331438085:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.662196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:15.662237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:15.662741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:15.662909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471451992807530:2049] [node 2] ICS01 socket: 142 reason# EndOfStream
2025-11-27T18:19:15.662937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471447697840133:2049] [node 2] ICP30 unregister session Session# [6:7577471451992807530:2049] VirtualId# [6:7577471451992807528:0]
2025-11-27T18:19:15.662958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471451992807530:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.663011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:15.663043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:15.662322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:15.662502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471450337260767:2049] [node 2] ICS01 socket: 88 reason# EndOfStream
2025-11-27T18:19:15.662539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471446042293377:2049] [node 2] ICP30 unregister session Session# [5:7577471450337260767:2049] VirtualId# [5:7577471450337260761:0]
2025-11-27T18:19:15.662565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471450337260767:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.662624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:15.662658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:15.697017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:15.697309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471451992807526:2050] [node 3] ICS01 socket: 113 reason# EndOfStream
2025-11-27T18:19:15.697355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471447697840134:2050] [node 3] ICP30 unregister session Session# [6:7577471451992807526:2050] VirtualId# [6:7577471451992807523:0]
2025-11-27T18:19:15.697391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471451992807526:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.697470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:15.697508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:15.697216Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:15.697419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471450337260775:2050] [node 3] ICS01 socket: 137 reason# EndOfStream
2025-11-27T18:19:15.697460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471446042293378:2050] [node 3] ICP30 unregister session Session# [5:7577471450337260775:2050] VirtualId# [5:7577471450337260771:0]
2025-11-27T18:19:15.697486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471450337260775:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.697797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:15.697855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:15.700164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:15.700418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471453331438082:2050] [node 3] ICS07 socket disconnect 111 reason# EndOfStream
2025-11-27T18:19:15.700444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471453331438082:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.700519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:15.700557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471453331438082:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:15.707658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471449036470691:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62568) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:15.707722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471453331438082:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:15.707752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471449036470691:2050] [node 3] ICP30 unregister session Session# [4:7577471453331438082:2050] VirtualId# [4:7577471453331438078:0]
2025-11-27T18:19:15.707796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:15.707832Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471449036470691:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:15.707854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:15.721488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471449036470691:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:15.763128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:15.763323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471450337260763:2108] [node 4] ICS07 socket disconnect 96 reason# EndOfStream
2025-11-27T18:19:15.763352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471450337260763:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.763434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:15.763474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471450337260763:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:15.763291Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:15.763491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471451992807514:2108] [node 4] ICS01 socket: 94 reason# EndOfStream
2025-11-27T18:19:15.763545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471447697840197:2108] [node 4] ICP30 unregister session Session# [6:7577471451992807514:2108] VirtualId# [6:7577471451992807510:0]
2025-11-27T18:19:15.763587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471451992807514:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.763692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:15.763738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:15.840500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:15.840764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471451992807517:2104] [node 5] ICS07 socket disconnect 93 reason# EndOfStream
2025-11-27T18:19:15.840797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471451992807517:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:15.840868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:15.840924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471451992807517:2104] [node 5] ICS15 start handshake
2025-11-27T18:19:15.841781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471447697840190:2104] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:15.841845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471451992807517:2104] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:15.841883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471447697840190:2104] [node 5] ICP30 unregister session Session# [6:7577471451992807517:2104] VirtualId# [6:7577471451992807509:0]
2025-11-27T18:19:15.841986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:19:15.842019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471447697840190:2104] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:15.842032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:15.973945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471450400264016:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:15.974013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471450400264017:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:15.987972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471453956151678:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:15.988012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471453956151679:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:16.011354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471458567836317:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:16.011407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471458567836318:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:16.037673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471458678641758:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:16.037721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471458678641759:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:16.037750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471458678641760:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:16.067084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471455915328799:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:16.067138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471455915328800:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:16.067169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471455915328801:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:16.118066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471457924134607:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:16.118106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471457924134608:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:16.118131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471457924134609:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:16.291016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471457924134668:2105] [node 5] ICP01 ready to work
2025-11-27T18:19:16.385042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471458251119038:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:16.417881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471458251119039:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:16.370312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471454695231377:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:16.370552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471454695231378:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:16.422327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471458567836380:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:16.425538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471458678641821:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:16.434277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471455915328862:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:16.438314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471455915328799:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(81754634ns) {Configure:104(81746968ns) {ProcessConfigured:135(80477ns) {StartInitialHandshake:157(78506ns) {DropHandshakes:470(3499ns) {DropIncomingHandshake:442(754ns) DropOutgoingHandshake:457(741ns)} GenerateSessionVirtualId:988(1110ns) PrepareNewSessionHandshake:477(1769ns) {SwitchToState:179(623ns)}}}}}
2025-11-27T18:19:16.434101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471458678641822:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:16.422514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471458567836381:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:16.434327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471457924134670:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:16.417388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471454695231376:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:16.462991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471458678641758:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(123799857ns) {Configure:104(123792747ns) {ProcessConfigured:135(68651ns) {StartInitialHandshake:157(66786ns) {DropHandshakes:470(2926ns) {DropIncomingHandshake:442(574ns) DropOutgoingHandshake:457(503ns)} GenerateSessionVirtualId:988(787ns) PrepareNewSessionHandshake:477(2044ns) {SwitchToState:179(724ns)}}}}}
2025-11-27T18:19:16.448472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471458251119042:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:16.465777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471455915328867:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:16.444498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471458567836382:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:16.484239Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471458251119038:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(63701689ns) {Configure:104(63696229ns) {ProcessConfigured:135(17853571ns) {StartInitialHandshake:157(17851193ns) {DropHandshakes:470(2092ns) {DropIncomingHandshake:442(594ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:477(5200ns) {SwitchToState:179(2585ns)}}}}}
2025-11-27T18:19:16.545940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471450400264017:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20073509ns) {Configure:104(20065762ns) {ProcessConfigured:135(77858ns) {StartInitialHandshake:157(75895ns) {DropHandshakes:470(1102ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(143ns)} GenerateSessionVirtualId:988(525ns) PrepareNewSessionHandshake:477(2225ns) {SwitchToState:179(964ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.570340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471458251119054:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-27T18:19:16.570429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471458251119055:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-27T18:19:16.570863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471450400264016:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(280752040ns) {Configure:104(280744979ns) {ProcessConfigured:135(19553588ns) {StartInitialHandshake:157(19550537ns) {DropHandshakes:470(1495ns) {DropIncomingHandshake:442(326ns) DropOutgoingHandshake:457(398ns)} GenerateSessionVirtualId:988(340ns) PrepareNewSessionHandshake:477(4931ns) {SwitchToState:179(2434ns)}}}}}
2025-11-27T18:19:16.571942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471454695231379:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20308
2025-11-27T18:19:16.572100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471454695231380:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20308
2025-11-27T18:19:16.572218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471454695231381:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20308
2025-11-27T18:19:16.572470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471454695231389:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20308
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.522225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471458678641759:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(96292214ns) {Configure:104(96287389ns) {ProcessConfigured:135(66286ns) {StartInitialHandshake:157(63094ns) {DropHandshakes:470(3593ns) {DropIncomingHandshake:442(775ns) DropOutgoingHandshake:457(688ns)} GenerateSessionVirtualId:988(915ns) PrepareNewSessionHandshake:477(2307ns) {SwitchToState:179(854ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.563754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471457924134672:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32813
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.590713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471458567836396:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-27T18:19:16.575486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471458251119056:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-27T18:19:16.525077Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471458678641821:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(37178972ns) {Configure:104(37175532ns) {ProcessConfigured:135(55529ns) {StartInitialHandshake:157(53621ns) {DropHandshakes:470(2552ns) {DropIncomingHandshake:442(827ns) DropOutgoingHandshake:457(337ns)} GenerateSessionVirtualId:988(380ns) PrepareNewSessionHandshake:477(1810ns) {SwitchToState:179(716ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.592950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471454695231377:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(44774400ns) {Configure:104(44770276ns) {ProcessConfigured:135(39911102ns) {StartInitialHandshake:157(39908149ns) {DropHandshakes:470(1142ns) {DropIncomingHandshake:442(411ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(367ns) PrepareNewSessionHandshake:477(2658ns) {SwitchToState:179(1072ns)}}}}}
2025-11-27T18:19:16.563946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471457924134673:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32813
2025-11-27T18:19:16.601188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471458251119055:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.599268Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471458678641822:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(122980568ns) {Configure:104(122972350ns) {ProcessConfigured:135(77650ns) {StartInitialHandshake:157(75929ns) {DropHandshakes:470(3411ns) {DropIncomingHandshake:442(590ns) DropOutgoingHandshake:457(450ns)} GenerateSessionVirtualId:988(670ns) PrepareNewSessionHandshake:477(1189ns) {SwitchToState:179(419ns)}}}}}
2025-11-27T18:19:16.585003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471457924134672:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.602085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471458251119046:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.593109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471454695231379:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.615833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471455915328801:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(149862239ns) {Configure:104(149858217ns) {ProcessConfigured:135(45361773ns) {StartInitialHandshake:157(45357928ns) {DropHandshakes:470(3690ns) {DropIncomingHandshake:442(1125ns) DropOutgoingHandshake:457(624ns)} GenerateSessionVirtualId:988(1267ns) PrepareNewSessionHandshake:477(5496ns) {SwitchToState:179(2568ns)}}}}}
2025-11-27T18:19:16.616337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471458567836381:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(127196511ns) {Configure:104(127191876ns) {ProcessConfigured:135(68704508ns) {StartInitialHandshake:157(68701111ns) {DropHandshakes:470(2861ns) {DropIncomingHandshake:442(730ns) DropOutgoingHandshake:457(529ns)} GenerateSessionVirtualId:988(436ns) PrepareNewSessionHandshake:477(2382ns) {SwitchToState:179(1047ns)}}}}}
2025-11-27T18:19:16.614881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471458678641836:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4399
2025-11-27T18:19:16.602111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471458251119046:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.593195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471454695231380:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.585138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471457924134673:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.626337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471458567836396:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.647218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471458678641836:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.665664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471458678641838:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4399
2025-11-27T18:19:16.649394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471455915328867:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(57531490ns) {Configure:104(57525296ns) {ProcessConfigured:135(66547ns) {StartInitialHandshake:157(64286ns) {DropHandshakes:470(3012ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(374ns)} GenerateSessionVirtualId:988(922ns) PrepareNewSessionHandshake:477(1388ns) {SwitchToState:179(643ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.587052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471457924134684:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32813
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.620634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471454695231389:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.614409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471458251119046:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:16.629048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471458567836386:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.649893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471455915328869:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22303
2025-11-27T18:19:16.689909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471458678641839:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4399
2025-11-27T18:19:16.696624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471458678641839:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.705952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471455915328870:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22303
2025-11-27T18:19:16.592536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471457924134608:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:16.627973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471458251119042:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:16.652372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471454695231383:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.628049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:16.628817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471458251119042:2110] [node 6] ICP22 created new session: [2:7577471458251119059:2110]
2025-11-27T18:19:16.663359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471458251119059:2110] [node 6] ICS09 handshake done sender: [2:7577471458251119046:2113] self: [2:7577471458251119045:0] peer: [6:7577471457924134685:0] socket: 83 qp: -1
2025-11-27T18:19:16.673916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471458251119059:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:16.673977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:16.685722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471458251119054:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.699866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471458251119056:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.708456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471458251119062:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-27T18:19:16.629102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471458567836386:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.597112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471457924134609:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(79629006ns) {Configure:104(79623387ns) {ProcessConfigured:135(14657899ns) {StartInitialHandshake:157(14655367ns) {DropHandshakes:470(3996ns) {DropIncomingHandshake:442(1375ns) DropOutgoingHandshake:457(627ns)} GenerateSessionVirtualId:988(839ns) PrepareNewSessionHandshake:477(12022ns) {SwitchToState:179(10388ns)}}}}}
2025-11-27T18:19:16.652401Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471454695231383:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.641581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471458567836395:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-27T18:19:16.708617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471458678641758:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.658620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471454695231383:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:16.726128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471453956151678:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:16.600403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471457924134609:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:16.642442Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471458567836386:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:16.742243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471455915328869:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.742361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471455915328870:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.742504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471455915328801:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:16.742564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471455915328800:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.757153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471455915328881:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22303
2025-11-27T18:19:16.752170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471458678641841:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4399
2025-11-27T18:19:16.739837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471458251119042:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(86014140ns) {HandleHandshakeStatus:329(86011402ns) {DropIncomingHandshake:442(27304ns) DropOutgoingHandshake:457(417ns) SwitchToState:179(622ns) ProcessPendingSessionEvents:495(619967ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 556236ns max# 84146ns}}}
2025-11-27T18:19:16.664110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471457924134668:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(171538649ns) {Configure:104(171531795ns) {ProcessConfigured:135(62380ns) {StartInitialHandshake:157(60609ns) {DropHandshakes:470(5705ns) {DropIncomingHandshake:442(964ns) DropOutgoingHandshake:457(396ns)} GenerateSessionVirtualId:988(1476ns) PrepareNewSessionHandshake:477(1846ns) {SwitchToState:179(611ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.658971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471450400264017:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:16.752249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471458678641842:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4399
2025-11-27T18:19:16.645848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471458567836317:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.748499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471458251119050:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.659042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:16.764103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471455915328870:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.696040Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471457924134608:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2233) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471457924134673:2111] held: no
2025-11-27T18:19:16.723135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471458567836317:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21583) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471458567836396:2115] held: no
2025-11-27T18:19:16.748528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471458251119050:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.775113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471455915328800:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:16.712931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471457924134684:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.659116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471450400264017:2049] [node 3] ICP22 created new session: [1:7577471454695231401:2049]
2025-11-27T18:19:16.776923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471455915328883:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22303
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.771226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471457924134607:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:16.723817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471458567836395:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.794569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471458678641841:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.756584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471458251119041:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.659166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471454695231401:2049] [node 3] ICS09 handshake done sender: [1:7577471454695231383:2114] self: [1:7577471454695231382:0] peer: [3:7577471458567836399:0] socket: 103 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.777205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:16.723862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471458567836382:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:16.771308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471457924134688:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32813
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.756634Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471458251119041:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.659209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471454695231401:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:16.797529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471458678641839:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.788614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471455915328800:2049] [node 2] ICP22 created new session: [5:7577471455915328882:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.723911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:16.659257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:16.814244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471458678641758:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21583) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471458678641839:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.785560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471457924134691:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32813
2025-11-27T18:19:16.756836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471458251119050:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:16.795451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471455915328882:2049] [node 2] ICS09 handshake done sender: [5:7577471455915328870:2113] self: [5:7577471455915328880:0] peer: [2:7577471458251119049:0] socket: 97 qp: -1
2025-11-27T18:19:16.723960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471458567836382:2109] [node 6] ICP22 created new session: [3:7577471458567836402:2109]
2025-11-27T18:19:16.798267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471457924134673:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:16.819619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471458678641838:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.666113Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471454695231376:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(148121175ns) {Configure:104(148116443ns) {ProcessConfigured:135(22654480ns) {StartInitialHandshake:157(22650116ns) {DropHandshakes:470(4144ns) {DropIncomingHandshake:442(1153ns) DropOutgoingHandshake:457(511ns)} GenerateSessionVirtualId:988(1150ns) PrepareNewSessionHandshake:477(6073981ns) {SwitchToState:179(4475ns)}}}}}
2025-11-27T18:19:16.757458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471458251119062:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.757663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471458251119066:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-27T18:19:16.765748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471458251119039:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:16.770395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471458251119066:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.770568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471458251119041:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:16.771345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:16.775035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471458251119039:2109] [node 5] ICP22 created new session: [2:7577471458251119069:2109]
2025-11-27T18:19:16.785474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471458251119069:2109] [node 5] ICS09 handshake done sender: [2:7577471458251119050:2115] self: [2:7577471458251119049:0] peer: [5:7577471455915328880:0] socket: 85 qp: -1
2025-11-27T18:19:16.789466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471458251119069:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:16.795881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:16.797991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471453956151679:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:16.798044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:16.798106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471453956151679:2049] [node 3] ICP22 created new session: [2:7577471458251119072:2049]
2025-11-27T18:19:16.799101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471458251119073:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-27T18:19:16.803571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471458251119039:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(36962196ns) {HandleHandshakeStatus:329(36959772ns) {DropIncomingHandshake:442(14123ns) DropOutgoingHandshake:457(577ns) SwitchToState:179(722ns) ProcessPendingSessionEvents:495(562791ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 508196ns max# 63498ns}}}
2025-11-27T18:19:16.815677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471458251119072:2049] [node 3] ICS09 handshake done sender: [2:7577471458251119041:2111] self: [2:7577471458251119040:0] peer: [3:7577471458567836407:0] socket: 116 qp: -1
2025-11-27T18:19:16.821887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471458251119072:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:16.829610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:16.846947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471453956151679:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(48055722ns) {HandleHandshakeStatus:329(48046718ns) {DropIncomingHandshake:442(681ns) DropOutgoingHandshake:457(751ns) SwitchToState:179(1528ns) ProcessPendingSessionEvents:495(534122ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 468021ns max# 77989ns}}}
2025-11-27T18:19:16.858080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471453956151678:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21583) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471458251119056:2118] held: no
2025-11-27T18:19:16.858514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471453956151679:2049] [node 3] ICP18 handshake (actor [2:7577471458251119066:2120]) is too old
2025-11-27T18:19:16.864505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471458251119056:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.795512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471455915328882:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:16.724011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471458567836402:2109] [node 6] ICS09 handshake done sender: [3:7577471458567836386:2111] self: [3:7577471458567836385:0] peer: [6:7577471457924134687:0] socket: 78 qp: -1
2025-11-27T18:19:16.819701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471458678641760:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:16.820047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471458678641843:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4399
2025-11-27T18:19:16.836944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471458678641846:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4399
2025-11-27T18:19:16.869967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471458678641841:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:16.871746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471458678641760:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:16.871797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:16.871851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471458678641760:2050] [node 3] ICP22 created new session: [4:7577471458678641848:2050]
2025-11-27T18:19:16.871906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471458678641848:2050] [node 3] ICS09 handshake done sender: [4:7577471458678641841:2118] self: [4:7577471458678641844:0] peer: [3:7577471458567836394:0] socket: 130 qp: -1
2025-11-27T18:19:16.871942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471458678641848:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:16.871978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:16.934294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471458678641843:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.811073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471457924134608:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:16.898829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471453956151678:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:16.681650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471454695231398:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20308
2025-11-27T18:19:16.724046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471458567836402:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:16.811131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:16.804457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:16.724081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:16.726268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471458567836400:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-27T18:19:16.735477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471458567836401:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-27T18:19:16.737753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471458567836400:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.738189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471458567836318:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:16.751682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471458567836396:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:16.760822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471458567836390:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.760858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471458567836390:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.761408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471458567836317:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:16.761456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:16.764146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471458567836317:2048] [node 1] ICP22 created new session: [3:7577471458567836408:2048]
2025-11-27T18:19:16.770081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471458567836408:2048] [node 1] ICS09 handshake done sender: [3:7577471458567836396:2115] self: [3:7577471458567836399:0] peer: [1:7577471454695231382:0] socket: 107 qp: -1
2025-11-27T18:19:16.773386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471458567836408:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:16.773435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:16.774748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471458567836410:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-27T18:19:16.775575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471458567836409:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63332
2025-11-27T18:19:16.785176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471458567836400:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:16.789438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471458567836390:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:16.814763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471458567836318:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:16.814814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:16.818881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471458567836318:2049] [node 2] ICP22 created new session: [3:7577471458567836413:2049]
2025-11-27T18:19:16.826189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471458567836380:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:16.826236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:16.826286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471458567836380:2108] [node 5] ICP22 created new session: [3:7577471458567836414:2108]
2025-11-27T18:19:16.826336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471458567836414:2108] [node 5] ICS09 handshake done sender: [3:7577471458567836390:2113] self: [3:7577471458567836389:0] peer: [5:7577471455915328879:0] socket: 80 qp: -1
2025-11-27T18:19:16.826369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471458567836414:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:16.826404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:16.833386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471458567836397:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.833431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471458567836397:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.834398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471458567836397:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:16.842983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471458567836413:2049] [node 2] ICS09 handshake done sender: [3:7577471458567836400:2117] self: [3:7577471458567836407:0] peer: [2:7577471458251119040:0] socket: 125 qp: -1
2025-11-27T18:19:16.852881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471458567836413:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:16.864158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:16.938362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471458567836381:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:16.938431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:16.938475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471458567836381:2107] [node 4] ICP22 created new session: [3:7577471458567836418:2107]
2025-11-27T18:19:16.938519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471458567836418:2107] [node 4] ICS09 handshake done sender: [3:7577471458567836397:2116] self: [3:7577471458567836394:0] peer: [4:7577471458678641844:0] socket: 127 qp: -1
2025-11-27T18:19:16.938552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471458567836418:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:16.938581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:16.948676Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471458567836382:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:16.969930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471458678641759:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:16.898887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:16.690206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471454695231381:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.733436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471454695231393:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.733480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471454695231393:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.743128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471454695231388:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.743165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471454695231388:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.753593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471454695231393:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:16.753650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471454695231398:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.768647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471454695231377:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:16.768713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:16.768771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471454695231377:2107] [node 4] ICP22 created new session: [1:7577471454695231407:2107]
2025-11-27T18:19:16.768822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471454695231407:2107] [node 4] ICS09 handshake done sender: [1:7577471454695231393:2118] self: [1:7577471454695231387:0] peer: [4:7577471458678641840:0] socket: 100 qp: -1
2025-11-27T18:19:16.769328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471454695231407:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:16.775153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:16.813989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471454695231388:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:16.820347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471450400264016:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:16.820400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:16.820443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471450400264016:2048] [node 2] ICP22 created new session: [1:7577471454695231411:2048]
2025-11-27T18:19:16.820482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471454695231411:2048] [node 2] ICS09 handshake done sender: [1:7577471454695231388:2116] self: [1:7577471454695231386:0] peer: [2:7577471458251119064:0] socket: 105 qp: -1
2025-11-27T18:19:16.820513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471454695231411:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:16.820546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:16.842775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471454695231385:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.842828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471454695231385:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.897682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471454695231385:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:16.929436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471454695231378:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:16.929502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:16.936720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471454695231378:2110] [node 6] ICP22 created new session: [1:7577471454695231414:2110]
2025-11-27T18:19:16.960045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471454695231397:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.836566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471455915328869:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:16.811208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471457924134608:2049] [node 2] ICP22 created new session: [6:7577471457924134694:2049]
2025-11-27T18:19:16.898944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471453956151678:2048] [node 1] ICP22 created new session: [2:7577471458251119075:2048]
2025-11-27T18:19:16.960077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471454695231397:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.811259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471457924134694:2049] [node 2] ICS09 handshake done sender: [6:7577471457924134673:2111] self: [6:7577471457924134685:0] peer: [2:7577471458251119045:0] socket: 87 qp: -1
2025-11-27T18:19:16.849388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471455915328801:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:16.998632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471458678641833:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.965381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471454695231414:2110] [node 6] ICS09 handshake done sender: [1:7577471454695231385:2115] self: [1:7577471454695231384:0] peer: [6:7577471457924134693:0] socket: 104 qp: -1
2025-11-27T18:19:16.899036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471458251119075:2048] [node 1] ICS09 handshake done sender: [2:7577471458251119056:2118] self: [2:7577471458251119064:0] peer: [1:7577471454695231386:0] socket: 108 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:16.974000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471454695231414:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:16.985363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:16.996938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471454695231397:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:16.997716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471454695231377:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(173522798ns) {HandleHandshakeStatus:329(173520083ns) {DropIncomingHandshake:442(15373ns) DropOutgoingHandshake:457(627ns) SwitchToState:179(955ns) ProcessPendingSessionEvents:495(158065104ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 157916478ns max# 18492649ns}}}
2025-11-27T18:19:17.002826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471454695231376:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:17.002921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:17.002979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471454695231376:2109] [node 5] ICP22 created new session: [1:7577471458990198713:2109]
2025-11-27T18:19:17.003031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471458990198713:2109] [node 5] ICS09 handshake done sender: [1:7577471454695231397:2119] self: [1:7577471454695231396:0] peer: [5:7577471455915328893:0] socket: 118 qp: -1
2025-11-27T18:19:17.003073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471458990198713:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:17.003106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:17.008915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471454695231378:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(72877284ns) {HandleHandshakeStatus:329(72874280ns) {DropIncomingHandshake:442(16306ns) DropOutgoingHandshake:457(606ns) SwitchToState:179(453ns) ProcessPendingSessionEvents:495(8490161ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8411800ns max# 7945076ns}}}
2025-11-27T18:19:16.811303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471457924134694:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:16.899073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471458251119075:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:16.811340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:16.812416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471457924134672:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:16.899117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:16.834546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471457924134609:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:16.995572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471458251119044:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.834596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:16.834649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471457924134609:2050] [node 3] ICP22 created new session: [6:7577471457924134696:2050]
2025-11-27T18:19:16.995623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471458251119044:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.834700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471457924134696:2050] [node 3] ICS09 handshake done sender: [6:7577471457924134672:2110] self: [6:7577471457924134687:0] peer: [3:7577471458567836385:0] socket: 82 qp: -1
2025-11-27T18:19:16.834737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471457924134696:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:16.834769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:16.869068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471457924134699:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32813
2025-11-27T18:19:16.871259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471457924134699:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.871403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471457924134668:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:16.998685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471458678641833:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.915297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471457924134701:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32813
2025-11-27T18:19:16.933769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471457924134702:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32813
2025-11-27T18:19:17.005829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471458678641833:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:16.949108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471457924134668:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8023) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471457924134699:2120] held: no
2025-11-27T18:19:17.013669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471458678641758:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(65362063ns) {HandleHandshakeStatus:395(65354717ns) {LogHandshakeFail:472(18237ns) {UpdateErrorStateLog:545(6018ns)} DropOutgoingHandshake:457(544ns) ProcessPendingSessionEvents:495(65279742ns) {unordered DropSessionEvent:512 num# 100 sum# 65222872ns max# 15200773ns}}}
2025-11-27T18:19:16.988372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471457924134699:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:17.013769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471458678641758:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:17.006317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471457924134607:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21583) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471457924134684:2115] held: no
2025-11-27T18:19:16.849457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:17.010253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471457924134684:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:17.011427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471457924134668:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(61610423ns) {HandleHandshakeStatus:395(61608108ns) {LogHandshakeFail:472(9858ns) {UpdateErrorStateLog:545(4284ns)} DropOutgoingHandshake:457(565ns) ProcessPendingSessionEvents:495(61533193ns) {unordered DropSessionEvent:512 num# 100 sum# 10214460ns max# 4170493ns}}}
2025-11-27T18:19:17.011751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471457924134668:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:17.011821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:17.011868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471457924134668:2105] [node 5] ICP22 created new session: [6:7577471462219102000:2105]
2025-11-27T18:19:17.011910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471462219102000:2105] [node 5] ICS09 handshake done sender: [6:7577471457924134699:2120] self: [6:7577471457924134700:0] peer: [5:7577471455915328875:0] socket: 79 qp: -1
2025-11-27T18:19:17.011949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471462219102000:2105] [node 5] ICS10 traffic start
2025-11-27T18:19:17.011982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:17.017882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471462219101999:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32813
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:17.013809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:17.013864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471458678641758:2048] [node 1] ICP22 created new session: [4:7577471462973609149:2048]
2025-11-27T18:19:17.013909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471462973609149:2048] [node 1] ICS09 handshake done sender: [4:7577471458678641839:2117] self: [4:7577471458678641840:0] peer: [1:7577471454695231387:0] socket: 124 qp: -1
2025-11-27T18:19:17.013941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471462973609149:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:17.013975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:17.016204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471458678641821:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:17.023862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:17.023923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471458678641821:2108] [node 5] ICP22 created new session: [4:7577471462973609151:2108]
2025-11-27T18:19:17.023970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471462973609151:2108] [node 5] ICS09 handshake done sender: [4:7577471458678641833:2113] self: [4:7577471458678641832:0] peer: [5:7577471455915328896:0] socket: 84 qp: -1
2025-11-27T18:19:17.024008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471462973609151:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:17.024044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:17.033792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471462973609153:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4399
2025-11-27T18:19:17.023438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471458251119044:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:17.034215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471462219101999:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:17.037024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471457924134670:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:16.849510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471455915328801:2050] [node 3] ICP22 created new session: [5:7577471455915328887:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:17.040567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471462219102003:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32813
2025-11-27T18:19:17.042605Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471457924134607:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(40629438ns) {HandleHandshakeStatus:395(40627557ns) {LogHandshakeFail:472(15497ns) {UpdateErrorStateLog:545(6223ns)} DropOutgoingHandshake:457(889ns) ProcessPendingSessionEvents:495(33168672ns) {unordered DropSessionEvent:512 num# 100 sum# 10525580ns max# 4566151ns}}}
2025-11-27T18:19:17.042674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471457924134607:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:17.042720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:17.042765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471457924134607:2048] [node 1] ICP22 created new session: [6:7577471462219102004:2048]
2025-11-27T18:19:17.042817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471462219102004:2048] [node 1] ICS09 handshake done sender: [6:7577471457924134684:2115] self: [6:7577471457924134693:0] peer: [1:7577471454695231384:0] socket: 106 qp: -1
2025-11-27T18:19:17.042857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471462219102004:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:17.042905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:16.849570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471455915328887:2050] [node 3] ICS09 handshake done sender: [5:7577471455915328869:2112] self: [5:7577471455915328879:0] peer: [3:7577471458567836389:0] socket: 95 qp: -1
2025-11-27T18:19:17.029738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471458251119038:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:17.051257Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471462219101999:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:17.034050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:17.034114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471458251119038:2107] [node 4] ICP22 created new session: [2:7577471462546086374:2107]
2025-11-27T18:19:17.044495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471462546086374:2107] [node 4] ICS09 handshake done sender: [2:7577471458251119044:2112] self: [2:7577471458251119043:0] peer: [4:7577471458678641851:0] socket: 94 qp: -1
2025-11-27T18:19:17.047632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471462546086374:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:17.047687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:17.050255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471458251119038:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(25738469ns) {HandleHandshakeStatus:329(25734868ns) {DropIncomingHandshake:442(21602ns) DropOutgoingHandshake:457(820ns) SwitchToState:179(542ns) ProcessPendingSessionEvents:495(546090ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 483931ns max# 60665ns}}}
2025-11-27T18:19:17.037457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471458678641835:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:17.057927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471458567836318:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(240047630ns) {HandleHandshakeStatus:329(240041378ns) {DropIncomingHandshake:442(646ns) DropOutgoingHandshake:457(5046ns) SwitchToState:179(540ns) ProcessPendingSessionEvents:495(165189092ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 159828221ns max# 25804857ns}}}
2025-11-27T18:19:16.849613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471455915328887:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:17.037509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471458678641835:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:17.059870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471457924134670:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:16.849653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:17.048062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471458678641835:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:17.052366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471458678641759:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2233) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471458678641843:2120] held: no
2025-11-27T18:19:17.053006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471458678641822:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:17.053109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:17.053163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471458678641822:2109] [node 6] ICP22 created new session: [4:7577471462973609155:2109]
2025-11-27T18:19:17.053219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471462973609155:2109] [node 6] ICS09 handshake done sender: [4:7577471458678641835:2114] self: [4:7577471458678641834:0] peer: [6:7577471462219102002:0] socket: 93 qp: -1
2025-11-27T18:19:17.053267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471462973609155:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:17.053306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:17.054360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471458678641843:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:17.063792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471458678641759:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:17.063841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:17.063901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471458678641759:2049] [node 2] ICP22 created new session: [4:7577471462973609157:2049]
2025-11-27T18:19:17.063950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471462973609157:2049] [node 2] ICS09 handshake done sender: [4:7577471458678641843:2120] self: [4:7577471458678641851:0] peer: [2:7577471458251119043:0] socket: 89 qp: -1
2025-11-27T18:19:17.063988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471462973609157:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:17.064019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:16.856592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471455915328889:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22303
2025-11-27T18:19:16.870515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471455915328890:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22303
2025-11-27T18:19:16.879430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471455915328885:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22303
2025-11-27T18:19:16.879440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471455915328889:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.905833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471455915328876:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.905877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471455915328876:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:16.918778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471455915328885:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.929331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471455915328799:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:16.929600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471455915328876:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:16.931182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471455915328867:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:16.931245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:16.931299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471455915328867:2109] [node 6] ICP22 created new session: [5:7577471455915328894:2109]
2025-11-27T18:19:16.931347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471455915328894:2109] [node 6] ICS09 handshake done sender: [5:7577471455915328876:2116] self: [5:7577471455915328875:0] peer: [6:7577471457924134700:0] socket: 143 qp: -1
2025-11-27T18:19:16.931390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471455915328894:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:16.931430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:16.978154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471455915328890:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:16.978325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471455915328862:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:17.015666Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471455915328799:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21583) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471455915328885:2119] held: no
2025-11-27T18:19:17.055719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471455915328799:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(39362640ns) {HandleHandshakeStatus:395(39359685ns) {LogHandshakeFail:472(14369ns) {UpdateErrorStateLog:545(5506ns)} DropOutgoingHandshake:457(423ns) ProcessPendingSessionEvents:495(39282262ns) {unordered DropSessionEvent:512 num# 100 sum# 4688603ns max# 2376422ns}}}
2025-11-27T18:19:17.055909Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471455915328862:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12049) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471455915328890:2121] held: no
2025-11-27T18:19:17.056708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471455915328897:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22303
2025-11-27T18:19:17.062408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471460210296194:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22303
2025-11-27T18:19:17.063118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471455915328885:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:17.067009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471455915328800:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(294745546ns) {HandleHandshakeStatus:329(294743113ns) {DropIncomingHandshake:442(3616ns) DropOutgoingHandshake:457(17724ns) SwitchToState:179(635ns) ProcessPendingSessionEvents:495(223010778ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 222822355ns max# 36294153ns}}}
2025-11-27T18:19:17.067096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471455915328890:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:17.067390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:17.072380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471457924134670:2108] [node 4] ICP22 created new session: [6:7577471462219102006:2108]
2025-11-27T18:19:17.072445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471462219102006:2108] [node 4] ICS09 handshake done sender: [6:7577471462219101999:2123] self: [6:7577471462219102002:0] peer: [4:7577471458678641834:0] socket: 121 qp: -1
2025-11-27T18:19:17.072484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471462219102006:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:17.072532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:17.077445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471455915328799:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:17.077505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:17.077575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471455915328799:2048] [node 1] ICP22 created new session: [5:7577471460210296195:2048]
2025-11-27T18:19:17.077631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471460210296195:2048] [node 1] ICS09 handshake done sender: [5:7577471455915328885:2119] self: [5:7577471455915328893:0] peer: [1:7577471454695231396:0] socket: 141 qp: -1
2025-11-27T18:19:17.077668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471460210296195:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:17.077704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:17.078608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471455915328862:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:17.078641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:17.078684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471455915328862:2108] [node 4] ICP22 created new session: [5:7577471460210296197:2108]
2025-11-27T18:19:17.078734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471460210296197:2108] [node 4] ICS09 handshake done sender: [5:7577471455915328890:2121] self: [5:7577471455915328896:0] peer: [4:7577471458678641832:0] socket: 146 qp: -1
2025-11-27T18:19:17.078766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471460210296197:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:17.078795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:17.134159Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:17.134413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471458251119075:2048] [node 1] ICS01 socket: 108 reason# EndOfStream
2025-11-27T18:19:17.134455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471453956151678:2048] [node 1] ICP30 unregister session Session# [2:7577471458251119075:2048] VirtualId# [2:7577471458251119064:0]
2025-11-27T18:19:17.134502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471458251119075:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.134583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:17.134623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:17.136386Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:17.136613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471458567836408:2048] [node 1] ICS01 socket: 107 reason# EndOfStream
2025-11-27T18:19:17.136670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471458567836317:2048] [node 1] ICP30 unregister session Session# [3:7577471458567836408:2048] VirtualId# [3:7577471458567836399:0]
2025-11-27T18:19:17.136708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471458567836408:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.136782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:17.136824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:17.140262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:17.141464Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:17.141668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471462973609149:2048] [node 1] ICS01 socket: 124 reason# EndOfStream
2025-11-27T18:19:17.141711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471458678641758:2048] [node 1] ICP30 unregister session Session# [4:7577471462973609149:2048] VirtualId# [4:7577471458678641840:0]
2025-11-27T18:19:17.141753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471462973609149:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.141849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:17.141910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:17.140499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471462219102004:2048] [node 1] ICS01 socket: 106 reason# EndOfStream
2025-11-27T18:19:17.140541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471457924134607:2048] [node 1] ICP30 unregister session Session# [6:7577471462219102004:2048] VirtualId# [6:7577471457924134693:0]
2025-11-27T18:19:17.140572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471462219102004:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.140644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:17.140679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:17.141378Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:17.145987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471460210296195:2048] [node 1] ICS01 socket: 141 reason# EndOfStream
2025-11-27T18:19:17.146037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471455915328799:2048] [node 1] ICP30 unregister session Session# [5:7577471460210296195:2048] VirtualId# [5:7577471455915328893:0]
2025-11-27T18:19:17.146077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471460210296195:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.146114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:17.146181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:17.229078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:17.229379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471458567836413:2049] [node 2] ICS07 socket disconnect 125 reason# EndOfStream
2025-11-27T18:19:17.229414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471458567836413:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.229509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:17.229555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471458567836413:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:17.233358Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:17.233646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471462973609157:2049] [node 2] ICS01 socket: 89 reason# EndOfStream
2025-11-27T18:19:17.233699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471458678641759:2049] [node 2] ICP30 unregister session Session# [4:7577471462973609157:2049] VirtualId# [4:7577471458678641851:0]
2025-11-27T18:19:17.233746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471462973609157:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.233826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:17.233877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:17.234323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471458567836318:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2233) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:17.234393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471458567836413:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:17.235154Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471458567836318:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2233) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:17.235234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471458567836413:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:17.235292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471458567836318:2049] [node 2] ICP30 unregister session Session# [3:7577471458567836413:2049] VirtualId# [3:7577471458567836407:0]
2025-11-27T18:19:17.235422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:17.235483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471458567836318:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:17.235508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:17.237656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:17.237911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471455915328882:2049] [node 2] ICS07 socket disconnect 97 reason# EndOfStream
2025-11-27T18:19:17.237947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471455915328882:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.238014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:17.238060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471455915328882:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:17.239114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471455915328800:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2233) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:17.239171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471455915328882:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:17.239213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471455915328800:2049] [node 2] ICP30 unregister session Session# [5:7577471455915328882:2049] VirtualId# [5:7577471455915328880:0]
2025-11-27T18:19:17.239310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:17.239351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471455915328800:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:17.239371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:17.245289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471458567836318:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:17.245495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:17.245853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471457924134694:2049] [node 2] ICS01 socket: 87 reason# EndOfStream
2025-11-27T18:19:17.245898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471457924134608:2049] [node 2] ICP30 unregister session Session# [6:7577471457924134694:2049] VirtualId# [6:7577471457924134685:0]
2025-11-27T18:19:17.245943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471457924134694:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.246056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:17.246111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:17.249631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471455915328800:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:17.309817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:17.310115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471458678641848:2050] [node 3] ICS01 socket: 130 reason# EndOfStream
2025-11-27T18:19:17.310176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471458678641760:2050] [node 3] ICP30 unregister session Session# [4:7577471458678641848:2050] VirtualId# [4:7577471458678641844:0]
2025-11-27T18:19:17.310219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471458678641848:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.310313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:17.310365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:17.310484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:17.310722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471455915328887:2050] [node 3] ICS07 socket disconnect 95 reason# EndOfStream
2025-11-27T18:19:17.310761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471455915328887:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.310818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:17.310864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471455915328887:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:17.312326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471455915328801:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25847) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:17.312388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471455915328887:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:17.312428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471455915328801:2050] [node 3] ICP30 unregister session Session# [5:7577471455915328887:2050] VirtualId# [5:7577471455915328879:0]
2025-11-27T18:19:17.312476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:17.312513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471455915328801:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25847) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:17.312533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25847) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:17.315343Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:17.315587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471457924134696:2050] [node 3] ICS07 socket disconnect 82 reason# EndOfStream
2025-11-27T18:19:17.315616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471457924134696:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.315705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:17.315748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471457924134696:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:17.318314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471457924134609:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25847) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:17.318390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471457924134696:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:17.318433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471457924134609:2050] [node 3] ICP30 unregister session Session# [6:7577471457924134696:2050] VirtualId# [6:7577471457924134687:0]
2025-11-27T18:19:17.318527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:17.318578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471457924134609:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25847) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:17.318598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25847) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:17.323727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471455915328801:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:17.329572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471457924134609:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:17.381490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:17.381714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471460210296197:2108] [node 4] ICS01 socket: 146 reason# EndOfStream
2025-11-27T18:19:17.381764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471455915328862:2108] [node 4] ICP30 unregister session Session# [5:7577471460210296197:2108] VirtualId# [5:7577471455915328896:0]
2025-11-27T18:19:17.381812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471460210296197:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.381883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:17.381933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:17.385411Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:17.385752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471462219102006:2108] [node 4] ICS07 socket disconnect 121 reason# EndOfStream
2025-11-27T18:19:17.385813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471462219102006:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.385921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:17.385986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471462219102006:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:17.387201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471457924134670:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12049) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:17.387335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471462219102006:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:17.387396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471457924134670:2108] [node 4] ICP30 unregister session Session# [6:7577471462219102006:2108] VirtualId# [6:7577471462219102002:0]
2025-11-27T18:19:17.387450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:17.387503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471457924134670:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12049) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:17.387530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12049) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:17.397823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471457924134670:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:19:17.457538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:17.457777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471462219102000:2105] [node 5] ICS01 socket: 79 reason# EndOfStream
2025-11-27T18:19:17.457816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471457924134668:2105] [node 5] ICP30 unregister session Session# [6:7577471462219102000:2105] VirtualId# [6:7577471457924134700:0]
2025-11-27T18:19:17.457853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471462219102000:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:17.457935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:17.457996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:17.661094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471462439897164:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:17.661155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471462439897165:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:17.681447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471461998749033:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:17.681504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471461998749034:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:17.710564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471459764690178:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:17.710610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471459764690179:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:17.735638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471459395470030:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:17.735682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471459395470031:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:17.735708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471459395470032:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:17.760342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471461529565859:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:17.760391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471461529565860:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:17.760426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471461529565861:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:17.785556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471462655565584:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:17.785609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471462655565586:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:17.785621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471462655565585:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:17.868475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471462655565643:2105] [node 5] ICP01 ready to work
2025-11-27T18:19:17.899030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471459764690239:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:17.988969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471461998749097:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:17.989720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471461529565932:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:18.046110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471465824533229:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:18.050837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471459764690239:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(151489065ns) {Configure:104(151485501ns) {ProcessConfigured:135(86648ns) {StartInitialHandshake:157(83301ns) {DropHandshakes:470(3417ns) {DropIncomingHandshake:442(708ns) DropOutgoingHandshake:457(691ns)} GenerateSessionVirtualId:988(958ns) PrepareNewSessionHandshake:477(2148ns) {SwitchToState:179(708ns)}}}}}
2025-11-27T18:19:18.039658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471462655565651:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:18.061470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471461529565920:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53333
2025-11-27T18:19:18.073661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471462655565586:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21736712ns) {Configure:104(21731474ns) {ProcessConfigured:135(20223975ns) {StartInitialHandshake:157(20221063ns) {DropHandshakes:470(3379ns) {DropIncomingHandshake:442(823ns) DropOutgoingHandshake:457(484ns)} GenerateSessionVirtualId:988(929ns) PrepareNewSessionHandshake:477(3426989ns) {SwitchToState:179(4760ns)}}}}}
2025-11-27T18:19:18.118067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471466734864524:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:18.118524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471466734864527:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:18.117794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471466293716394:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:18.127315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471466293716395:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:18.136678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471463690437390:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:18.121036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471464059657546:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.136595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471466734864523:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:18.145312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471463690437389:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.134642Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471459764690179:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(94001106ns) {Configure:104(93996540ns) {ProcessConfigured:135(74411ns) {StartInitialHandshake:157(71551ns) {DropHandshakes:470(3412ns) {DropIncomingHandshake:442(694ns) DropOutgoingHandshake:457(419ns)} GenerateSessionVirtualId:988(960ns) PrepareNewSessionHandshake:477(2114ns) {SwitchToState:179(678ns)}}}}}
2025-11-27T18:19:18.149911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471462655565651:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(65220803ns) {Configure:104(65216668ns) {ProcessConfigured:135(11918303ns) {StartInitialHandshake:157(11914260ns) {DropHandshakes:470(3128ns) {DropIncomingHandshake:442(872ns) DropOutgoingHandshake:457(490ns)} GenerateSessionVirtualId:988(1217ns) PrepareNewSessionHandshake:477(6987ns) {SwitchToState:179(2249ns)}}}}}
2025-11-27T18:19:18.153704Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471462439897164:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(200603385ns) {Configure:104(200596271ns) {ProcessConfigured:135(77086ns) {StartInitialHandshake:157(73994ns) {DropHandshakes:470(3613ns) {DropIncomingHandshake:442(759ns) DropOutgoingHandshake:457(494ns)} GenerateSessionVirtualId:988(825ns) PrepareNewSessionHandshake:477(2791ns) {SwitchToState:179(979ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.161676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471461529565859:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(115675257ns) {Configure:104(115671220ns) {ProcessConfigured:135(67823ns) {StartInitialHandshake:157(65874ns) {DropHandshakes:470(2813ns) {DropIncomingHandshake:442(921ns) DropOutgoingHandshake:457(548ns)} GenerateSessionVirtualId:988(608ns) PrepareNewSessionHandshake:477(1706ns) {SwitchToState:179(513ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.152862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471462655565585:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(81412574ns) {Configure:104(81407975ns) {ProcessConfigured:135(22839115ns) {StartInitialHandshake:157(22836910ns) {DropHandshakes:470(3489ns) {DropIncomingHandshake:442(773ns) DropOutgoingHandshake:457(360ns)} GenerateSessionVirtualId:988(924ns) PrepareNewSessionHandshake:477(3160ns) {SwitchToState:179(1417ns)}}}}}
2025-11-27T18:19:18.153306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471464059657547:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.177025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471466734864528:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61496
2025-11-27T18:19:18.201716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471461529565920:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.162259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471464059657554:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4968
2025-11-27T18:19:18.245985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471461998749097:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(200005701ns) {Configure:104(199998992ns) {ProcessConfigured:135(55759ns) {StartInitialHandshake:157(51272ns) {DropHandshakes:470(1359ns) {DropIncomingHandshake:442(326ns) DropOutgoingHandshake:457(304ns)} GenerateSessionVirtualId:988(1081ns) PrepareNewSessionHandshake:477(1767ns) {SwitchToState:179(716ns)}}}}}
2025-11-27T18:19:18.225541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471466950532958:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8216
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.252681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471459395470032:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(144265870ns) {Configure:104(144255848ns) {ProcessConfigured:135(24164196ns) {StartInitialHandshake:157(24161404ns) {DropHandshakes:470(3840ns) {DropIncomingHandshake:442(1083ns) DropOutgoingHandshake:457(454ns)} GenerateSessionVirtualId:988(363ns) PrepareNewSessionHandshake:477(4021ns) {SwitchToState:179(1580ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.177172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471466734864529:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61496
2025-11-27T18:19:18.214995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471461529565860:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(89414101ns) {Configure:104(89407595ns) {ProcessConfigured:135(73628ns) {StartInitialHandshake:157(71338ns) {DropHandshakes:470(3044ns) {DropIncomingHandshake:442(742ns) DropOutgoingHandshake:457(510ns)} GenerateSessionVirtualId:988(1596ns) PrepareNewSessionHandshake:477(1998ns) {SwitchToState:179(910ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.213675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471464059657553:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4968
2025-11-27T18:19:18.251697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471466293716396:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26736
2025-11-27T18:19:18.185841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471466734864524:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24293960ns) {Configure:104(24288929ns) {ProcessConfigured:135(22228897ns) {StartInitialHandshake:157(22227229ns) {DropHandshakes:470(3347ns) {DropIncomingHandshake:442(824ns) DropOutgoingHandshake:457(429ns)} GenerateSessionVirtualId:988(888ns) PrepareNewSessionHandshake:477(2336ns) {SwitchToState:179(1184ns)}}}}}
2025-11-27T18:19:18.259880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471459395470030:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(198040933ns) {Configure:104(198031945ns) {ProcessConfigured:135(14241007ns) {StartInitialHandshake:157(14238100ns) {DropHandshakes:470(7293096ns) {DropIncomingHandshake:442(942ns) DropOutgoingHandshake:457(1680ns)} GenerateSessionVirtualId:988(1649ns) PrepareNewSessionHandshake:477(3442ns) {SwitchToState:179(995ns)}}}}}
2025-11-27T18:19:18.245438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471464059657553:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.225866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471465824533244:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53333
2025-11-27T18:19:18.251859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471466293716397:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26736
2025-11-27T18:19:18.273929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471464059657555:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4968
2025-11-27T18:19:18.293459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471464059657554:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.262220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471463690437392:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4381
2025-11-27T18:19:18.210074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471466734864529:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.289590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471466293716396:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.281903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471465824533245:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53333
2025-11-27T18:19:18.314285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471466950532958:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.298315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471463690437389:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(69224096ns) {Configure:104(69220326ns) {ProcessConfigured:135(78547ns) {StartInitialHandshake:157(76226ns) {DropHandshakes:470(3570ns) {DropIncomingHandshake:442(923ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(681ns) PrepareNewSessionHandshake:477(1846ns) {SwitchToState:179(640ns)}}}}}
2025-11-27T18:19:18.226255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471466734864528:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.349551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471464059657555:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.353681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471459764690245:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.353733Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471459764690245:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.230953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471466734864527:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(65141045ns) {Configure:104(65134762ns) {ProcessConfigured:135(66611ns) {StartInitialHandshake:157(63939ns) {DropHandshakes:470(3009ns) {DropIncomingHandshake:442(924ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(1004ns) PrepareNewSessionHandshake:477(2103ns) {SwitchToState:179(817ns)}}}}}
2025-11-27T18:19:18.281969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471465824533246:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53333
2025-11-27T18:19:18.295940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471466293716395:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(86125839ns) {Configure:104(86117048ns) {ProcessConfigured:135(61365ns) {StartInitialHandshake:157(55576ns) {DropHandshakes:470(2064ns) {DropIncomingHandshake:442(543ns) DropOutgoingHandshake:457(482ns)} GenerateSessionVirtualId:988(480ns) PrepareNewSessionHandshake:477(1304ns) {SwitchToState:179(567ns)}}}}}
2025-11-27T18:19:18.301936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471463690437394:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4381
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.320571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471462655565643:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.302107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471463690437407:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4381
2025-11-27T18:19:18.309216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471463690437394:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.310143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471463690437392:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.315102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471459395470032:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.349469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471463690437404:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4381
2025-11-27T18:19:18.353652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471463690437407:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.369655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471459395470031:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.362510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471466950532960:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8216
2025-11-27T18:19:18.367499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471466293716397:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.277672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471462439897165:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(382341696ns) {Configure:104(382338412ns) {ProcessConfigured:135(11876460ns) {StartInitialHandshake:157(11873055ns) {DropHandshakes:470(3578ns) {DropIncomingHandshake:442(857ns) DropOutgoingHandshake:457(813ns)} GenerateSessionVirtualId:988(1022ns) PrepareNewSessionHandshake:477(3319ns) {SwitchToState:179(1744ns)}}}}}
2025-11-27T18:19:18.288810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471465824533244:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.367848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471459764690245:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:18.374456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471463690437411:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4381
2025-11-27T18:19:18.373886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471461998749033:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.288951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471461529565859:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.288498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471466734864543:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61496
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.413926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471462655565643:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21968) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471466950532958:2115] held: no
2025-11-27T18:19:18.425826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471466950532961:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8216
2025-11-27T18:19:18.397858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471459764690239:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(18090844ns) {IncomingHandshake:261(6775649ns) {DropIncomingHandshake:442(577ns) PrepareNewSessionHandshake:477(1056ns) {SwitchToState:179(198ns)} GenerateSessionVirtualId:988(270ns) IssueIncomingHandshakeReply:193(8807ns)}}
2025-11-27T18:19:18.437923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471463690437396:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.374142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471466293716409:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26736
2025-11-27T18:19:18.448540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471462655565643:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(33896023ns) {HandleHandshakeStatus:395(33889812ns) {LogHandshakeFail:472(20218ns) {UpdateErrorStateLog:545(6881ns)} DropOutgoingHandshake:457(683ns) ProcessPendingSessionEvents:495(33777411ns) {unordered DropSessionEvent:512 num# 100 sum# 33731668ns max# 18551574ns}}}
2025-11-27T18:19:18.448772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471466950532958:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:18.298171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471466734864533:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.437966Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471463690437396:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.439145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471459395470032:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471463690437392:2110] held: no
2025-11-27T18:19:18.439840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471463690437392:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:18.441473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471463690437404:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.459307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471463690437412:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4381
2025-11-27T18:19:18.459561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471459395470030:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.385905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471466293716399:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.385934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471466293716399:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.387950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471461998749034:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(208170981ns) {Configure:104(208165184ns) {ProcessConfigured:135(18783950ns) {StartInitialHandshake:157(18781214ns) {DropHandshakes:470(3178ns) {DropIncomingHandshake:442(842ns) DropOutgoingHandshake:457(441ns)} GenerateSessionVirtualId:988(1055ns) PrepareNewSessionHandshake:477(4186ns) {SwitchToState:179(2636ns)}}}}}
2025-11-27T18:19:18.392541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471466293716405:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.392602Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471466293716405:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.397747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471466293716399:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:18.449269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471462655565643:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.409692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471464059657547:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(139630993ns) {Configure:104(139624649ns) {ProcessConfigured:135(41019315ns) {StartInitialHandshake:157(41016529ns) {DropHandshakes:470(727ns) {DropIncomingHandshake:442(282ns) DropOutgoingHandshake:457(107ns)} GenerateSessionVirtualId:988(326ns) PrepareNewSessionHandshake:477(2064ns) {SwitchToState:179(876ns)}}}}}
2025-11-27T18:19:18.321166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471465824533248:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53333
2025-11-27T18:19:18.298214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471466734864533:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.414126Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471466293716409:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.321809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471465824533235:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.461635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471459395470032:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.310380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471466734864533:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:18.449333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.420655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471459764690239:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.321835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471465824533235:2122] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.414638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471466293716412:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26736
2025-11-27T18:19:18.470224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471463690437396:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.420727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:18.449392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471462655565643:2105] [node 5] ICP22 created new session: [6:7577471466950532962:2105]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.380423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471466734864523:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.333997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471465824533235:2122] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:18.334122Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471465824533244:2125] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:18.358705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471465824533245:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.365347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471461529565859:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.365428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:18.365484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471461529565859:2048] [node 1] ICP22 created new session: [5:7577471465824533250:2048]
2025-11-27T18:19:18.365535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471465824533250:2048] [node 1] ICS09 handshake done sender: [5:7577471465824533244:2125] self: [5:7577471465824533247:0] peer: [1:7577471466734864532:0] socket: 109 qp: -1
2025-11-27T18:19:18.365581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471465824533250:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:18.365623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:18.375716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471461529565860:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.377498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471465824533229:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.377575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:18.377623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471465824533229:2110] [node 6] ICP22 created new session: [5:7577471465824533254:2110]
2025-11-27T18:19:18.377685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471465824533254:2110] [node 6] ICS09 handshake done sender: [5:7577471465824533235:2122] self: [5:7577471465824533234:0] peer: [6:7577471466950532959:0] socket: 99 qp: -1
2025-11-27T18:19:18.377722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471465824533254:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:18.377753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:18.381456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471465824533246:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.394183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471461529565932:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.394453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471465824533256:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53333
2025-11-27T18:19:18.429484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471465824533245:2126] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:18.429864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471461529565860:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.429931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:18.429986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471461529565860:2049] [node 2] ICP22 created new session: [5:7577471465824533258:2049]
2025-11-27T18:19:18.430032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471465824533258:2049] [node 2] ICS09 handshake done sender: [5:7577471465824533245:2126] self: [5:7577471465824533253:0] peer: [2:7577471466293716398:0] socket: 112 qp: -1
2025-11-27T18:19:18.430067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471465824533258:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:18.430103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:18.459978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471465824533260:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53333
2025-11-27T18:19:18.470096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471465824533260:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.479781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471461529565861:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.519801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471461529565932:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7441) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471465824533246:2127] held: no
2025-11-27T18:19:18.521582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471465824533263:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53333
2025-11-27T18:19:18.477609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.449436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471466950532962:2105] [node 5] ICS09 handshake done sender: [6:7577471466950532958:2115] self: [6:7577471466950532959:0] peer: [5:7577471465824533234:0] socket: 101 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.479381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471466293716405:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:18.380485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:18.543269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471461529565861:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471465824533260:2130] held: no
2025-11-27T18:19:18.420789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471459764690239:2107] [node 4] ICP22 created new session: [3:7577471464059657562:2107]
2025-11-27T18:19:18.449478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471466950532962:2105] [node 5] ICS10 traffic start
2025-11-27T18:19:18.480027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471459395470032:2050] [node 3] ICP22 created new session: [4:7577471463690437416:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.380598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471466734864542:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61496
2025-11-27T18:19:18.498724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471463690437390:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.420839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471464059657562:2107] [node 4] ICS09 handshake done sender: [3:7577471459764690245:2112] self: [3:7577471459764690244:0] peer: [4:7577471463690437409:0] socket: 86 qp: -1
2025-11-27T18:19:18.498786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:18.386785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471466734864526:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.386808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471466734864526:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.387739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471466734864526:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:18.388167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471466734864543:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.397561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471466734864523:2108] [node 5] ICP22 created new session: [1:7577471466734864546:2108]
2025-11-27T18:19:18.397589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471462439897164:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.397812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:18.418579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471466734864546:2108] [node 5] ICS09 handshake done sender: [1:7577471466734864533:2114] self: [1:7577471466734864532:0] peer: [5:7577471465824533247:0] socket: 108 qp: -1
2025-11-27T18:19:18.418646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471466734864546:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:18.434070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471466734864542:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.434387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471462439897164:2048] [node 2] ICP22 created new session: [1:7577471466734864549:2048]
2025-11-27T18:19:18.437687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:18.457576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471466734864549:2048] [node 2] ICS09 handshake done sender: [1:7577471466734864526:2110] self: [1:7577471466734864525:0] peer: [2:7577471466293716410:0] socket: 103 qp: -1
2025-11-27T18:19:18.461796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471466734864549:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:18.461840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:18.481334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471466734864531:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.481377Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471466734864531:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.514310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471466734864531:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:18.522441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471466734864538:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.522477Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471466734864538:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.526247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471466734864524:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.526317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:18.526371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471466734864524:2107] [node 4] ICP22 created new session: [1:7577471466734864555:2107]
2025-11-27T18:19:18.526421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471466734864555:2107] [node 4] ICS09 handshake done sender: [1:7577471466734864531:2113] self: [1:7577471466734864530:0] peer: [4:7577471463690437414:0] socket: 107 qp: -1
2025-11-27T18:19:18.526466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471466734864555:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:18.526502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:18.535743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471466734864538:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:18.563378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471462439897164:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(174209128ns) {HandleHandshakeStatus:329(174206820ns) {DropIncomingHandshake:442(2299ns) DropOutgoingHandshake:457(724ns) SwitchToState:179(712ns) ProcessPendingSessionEvents:495(20957611ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20892355ns max# 20488661ns}}}
2025-11-27T18:19:18.563444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471466734864527:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.563506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:18.563565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471466734864527:2109] [node 6] ICP22 created new session: [1:7577471466734864558:2109]
2025-11-27T18:19:18.563607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471466734864558:2109] [node 6] ICS09 handshake done sender: [1:7577471466734864538:2115] self: [1:7577471466734864537:0] peer: [6:7577471466950532965:0] socket: 136 qp: -1
2025-11-27T18:19:18.563650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471466734864558:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:18.563685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:18.564810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471462439897164:2048] [node 2] ICP18 handshake (actor [1:7577471466734864542:2117]) is too old
2025-11-27T18:19:18.577496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471466734864541:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.577540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471466734864541:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.481983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471466293716395:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.461574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:18.420876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471464059657562:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:18.544033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471465824533260:2130] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:18.420921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:18.462829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471466950532961:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.498839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471463690437390:2108] [node 5] ICP22 created new session: [4:7577471463690437418:2108]
2025-11-27T18:19:18.606070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471466734864541:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:18.483874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471462655565584:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.482238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471466293716397:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:18.440199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471464059657547:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18541298ns) {IncomingHandshake:261(18539517ns) {DropIncomingHandshake:442(650ns) PrepareNewSessionHandshake:477(2707ns) {SwitchToState:179(1171ns)} GenerateSessionVirtualId:988(892ns) IssueIncomingHandshakeReply:193(8320ns)}}
2025-11-27T18:19:18.498890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471463690437418:2108] [node 5] ICS09 handshake done sender: [4:7577471463690437396:2113] self: [4:7577471463690437395:0] peer: [5:7577471465824533257:0] socket: 117 qp: -1
2025-11-27T18:19:18.498928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471463690437418:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:18.498957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:18.500641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471463690437407:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:18.500960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471463690437417:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4381
2025-11-27T18:19:18.505206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471459395470031:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.573562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471461529565861:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.505314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:18.484291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:18.495339Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471462655565643:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(24217747ns) {HandleHandshakeStatus:329(24214451ns) {DropIncomingHandshake:442(1343ns) DropOutgoingHandshake:457(592ns) SwitchToState:179(723ns) ProcessPendingSessionEvents:495(1775ns)}}
2025-11-27T18:19:18.625677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471462439897165:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.609458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471465824533261:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53333
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.457834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471464059657565:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4968
2025-11-27T18:19:18.505360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471459395470031:2049] [node 2] ICP22 created new session: [4:7577471463690437421:2049]
2025-11-27T18:19:18.630903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:18.501580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471466950532966:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8216
2025-11-27T18:19:18.484355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471466293716395:2107] [node 4] ICP22 created new session: [2:7577471466293716417:2107]
2025-11-27T18:19:18.609535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:18.613691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471461529565861:2050] [node 3] ICP22 created new session: [5:7577471465824533264:2050]
2025-11-27T18:19:18.629442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471465824533264:2050] [node 3] ICS09 handshake done sender: [5:7577471465824533260:2130] self: [5:7577471465824533262:0] peer: [3:7577471464059657550:0] socket: 142 qp: -1
2025-11-27T18:19:18.633412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471465824533264:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:18.649428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471465824533246:2127] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:18.649433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:18.668673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471461529565932:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.668727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:18.677615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471461529565932:2109] [node 4] ICP22 created new session: [5:7577471465824533266:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.495611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471464059657565:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.505408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471463690437421:2049] [node 2] ICS09 handshake done sender: [4:7577471463690437407:2118] self: [4:7577471463690437410:0] peer: [2:7577471466293716404:0] socket: 125 qp: -1
2025-11-27T18:19:18.484418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471466293716417:2107] [node 4] ICS09 handshake done sender: [2:7577471466293716405:2115] self: [2:7577471466293716404:0] peer: [4:7577471463690437410:0] socket: 124 qp: -1
2025-11-27T18:19:18.497662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471464059657551:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.497693Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471464059657551:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.501278Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471464059657551:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:18.501987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471464059657546:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.502043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:18.502090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471464059657546:2108] [node 5] ICP22 created new session: [3:7577471464059657568:2108]
2025-11-27T18:19:18.502134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471464059657568:2108] [node 5] ICS09 handshake done sender: [3:7577471464059657551:2115] self: [3:7577471464059657550:0] peer: [5:7577471465824533262:0] socket: 140 qp: -1
2025-11-27T18:19:18.502167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471464059657568:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:18.502216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:18.533466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471464059657567:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4968
2025-11-27T18:19:18.549344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471459764690178:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.569564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471464059657567:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.569759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471459764690179:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.573376Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471459764690178:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27131808ns) {IncomingHandshake:261(27127343ns) {DropIncomingHandshake:442(763ns) PrepareNewSessionHandshake:477(1632ns) {SwitchToState:179(631ns)} GenerateSessionVirtualId:988(634ns) IssueIncomingHandshakeReply:193(11190122ns) {SwitchToState:179(2005ns)}}}
2025-11-27T18:19:18.658940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471464059657574:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4968
2025-11-27T18:19:18.677559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471459764690179:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28776) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471464059657567:2121] held: no
2025-11-27T18:19:18.515858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471462655565584:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20962204ns) {IncomingHandshake:261(20959951ns) {DropIncomingHandshake:442(1409ns) PrepareNewSessionHandshake:477(3021ns) {SwitchToState:179(2173ns)} GenerateSessionVirtualId:988(953ns) IssueIncomingHandshakeReply:193(10162941ns) {SwitchToState:179(1872ns)}}}
2025-11-27T18:19:18.637840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471462439897165:2049] [node 3] ICP22 created new session: [1:7577471466734864561:2049]
2025-11-27T18:19:18.484466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471466293716417:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:18.484502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:18.485630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471461998749097:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.485682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:18.485740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471461998749097:2108] [node 5] ICP22 created new session: [2:7577471466293716419:2108]
2025-11-27T18:19:18.485781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471466293716419:2108] [node 5] ICS09 handshake done sender: [2:7577471466293716399:2113] self: [2:7577471466293716398:0] peer: [5:7577471465824533253:0] socket: 110 qp: -1
2025-11-27T18:19:18.485807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471466293716419:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:18.485829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:18.488719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471466293716416:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26736
2025-11-27T18:19:18.491183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471466293716394:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(103103167ns) {Configure:104(103093841ns) {ProcessConfigured:135(55470ns) {StartInitialHandshake:157(52146ns) {DropHandshakes:470(3301ns) {DropIncomingHandshake:442(831ns) DropOutgoingHandshake:457(386ns)} GenerateSessionVirtualId:988(869ns) PrepareNewSessionHandshake:477(1784ns) {SwitchToState:179(730ns)}}}}}
2025-11-27T18:19:18.491285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471461998749033:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.491333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:18.491384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471461998749033:2048] [node 1] ICP22 created new session: [2:7577471466293716424:2048]
2025-11-27T18:19:18.491434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471466293716424:2048] [node 1] ICS09 handshake done sender: [2:7577471466293716397:2112] self: [2:7577471466293716410:0] peer: [1:7577471466734864525:0] socket: 106 qp: -1
2025-11-27T18:19:18.491485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471466293716424:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:18.491517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:18.510866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471466293716416:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.511324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471466293716395:2107] [node 4] ICP18 handshake (actor [2:7577471466293716416:2119]) is too old
2025-11-27T18:19:18.595665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471466293716408:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.595702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471466293716408:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.613327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471466293716408:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:18.621436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471461998749034:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.629361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:18.630411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471461998749034:2049] [node 3] ICP22 created new session: [2:7577471466293716428:2049]
2025-11-27T18:19:18.649230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471466293716428:2049] [node 3] ICS09 handshake done sender: [2:7577471466293716408:2116] self: [2:7577471466293716407:0] peer: [3:7577471464059657572:0] socket: 147 qp: -1
2025-11-27T18:19:18.649491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471466293716428:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:18.661645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:18.661697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471466293716423:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.661726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471466293716423:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.663169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471466293716423:2120] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:18.663415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471466293716394:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.663458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:18.663510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471466293716394:2109] [node 6] ICP22 created new session: [2:7577471466293716431:2109]
2025-11-27T18:19:18.663564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471466293716431:2109] [node 6] ICS09 handshake done sender: [2:7577471466293716423:2120] self: [2:7577471466293716422:0] peer: [6:7577471466950532972:0] socket: 100 qp: -1
2025-11-27T18:19:18.663600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471466293716431:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:18.663641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:18.685843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471466293716433:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26736
2025-11-27T18:19:18.517503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471466950532966:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.690027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471465824533266:2109] [node 4] ICS09 handshake done sender: [5:7577471465824533246:2127] self: [5:7577471465824533257:0] peer: [4:7577471463690437395:0] socket: 119 qp: -1
2025-11-27T18:19:18.513120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471463690437416:2050] [node 3] ICS09 handshake done sender: [4:7577471463690437392:2110] self: [4:7577471463690437409:0] peer: [3:7577471459764690244:0] socket: 97 qp: -1
2025-11-27T18:19:18.641595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471466734864561:2049] [node 3] ICS09 handshake done sender: [1:7577471466734864541:2116] self: [1:7577471466734864540:0] peer: [3:7577471464059657570:0] socket: 141 qp: -1
2025-11-27T18:19:18.716277Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471461998749034:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(101641637ns) {HandleHandshakeStatus:329(101636812ns) {DropIncomingHandshake:442(33334ns) DropOutgoingHandshake:457(701ns) SwitchToState:179(6932ns) ProcessPendingSessionEvents:495(17963118ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17900811ns max# 17498517ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.698777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471465824533266:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:18.526003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471466950532967:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8216
2025-11-27T18:19:18.513572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471463690437421:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:18.734870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471459764690178:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14576) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471464059657565:2120] held: no
2025-11-27T18:19:18.707362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471461529565861:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(123251976ns) {HandleHandshakeStatus:329(123246909ns) {DropIncomingHandshake:442(4604ns) DropOutgoingHandshake:457(460ns) SwitchToState:179(4728ns) ProcessPendingSessionEvents:495(1982ns)}}
2025-11-27T18:19:18.651192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471466734864523:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(276926587ns) {HandleHandshakeStatus:329(276922959ns) {DropIncomingHandshake:442(2678ns) DropOutgoingHandshake:457(1189ns) SwitchToState:179(711ns) ProcessPendingSessionEvents:495(170949063ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 170763624ns max# 16941422ns}}}
2025-11-27T18:19:18.735634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471464059657559:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.735663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471464059657559:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.737166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471464059657573:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4968
2025-11-27T18:19:18.738495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471459764690179:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(72364160ns) {HandleHandshakeStatus:395(72361426ns) {LogHandshakeFail:472(15329ns) {UpdateErrorStateLog:545(4751ns)} DropOutgoingHandshake:457(2427ns) ProcessPendingSessionEvents:495(60114454ns) {unordered DropSessionEvent:512 num# 100 sum# 60062376ns max# 11301551ns}}}
2025-11-27T18:19:18.738867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471464059657567:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:18.757655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471459764690179:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.757719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:18.757777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471459764690179:2049] [node 2] ICP22 created new session: [3:7577471464059657576:2049]
2025-11-27T18:19:18.757819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471464059657576:2049] [node 2] ICS09 handshake done sender: [3:7577471464059657567:2121] self: [3:7577471464059657572:0] peer: [2:7577471466293716407:0] socket: 148 qp: -1
2025-11-27T18:19:18.757855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471464059657576:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:18.757891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:18.758822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471464059657565:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:18.759760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471464059657559:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:18.514740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471463690437422:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4381
2025-11-27T18:19:18.538710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471462655565651:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.769434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471459764690178:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.769499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:18.769552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471459764690178:2048] [node 1] ICP22 created new session: [3:7577471464059657578:2048]
2025-11-27T18:19:18.769605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471464059657578:2048] [node 1] ICS09 handshake done sender: [3:7577471464059657565:2120] self: [3:7577471464059657570:0] peer: [1:7577471466734864540:0] socket: 122 qp: -1
2025-11-27T18:19:18.769642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471464059657578:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:18.769679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:18.517628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:18.664640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471466734864561:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:18.716857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:18.522359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471463690437416:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:18.546186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471466950532969:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8216
2025-11-27T18:19:18.664687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:18.522407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:18.544461Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471459395470030:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14576) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471463690437404:2116] held: no
2025-11-27T18:19:18.545046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471463690437406:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.545069Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471463690437406:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:18.545756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471463690437404:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:18.582086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471463690437417:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.582140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471459395470030:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.582179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:18.582230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471459395470030:2048] [node 1] ICP22 created new session: [4:7577471463690437426:2048]
2025-11-27T18:19:18.582270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471463690437426:2048] [node 1] ICS09 handshake done sender: [4:7577471463690437404:2116] self: [4:7577471463690437414:0] peer: [1:7577471466734864530:0] socket: 115 qp: -1
2025-11-27T18:19:18.582308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471463690437426:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:18.582344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:18.593608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471463690437406:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:18.605687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471459395470032:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(133981035ns) {HandleHandshakeStatus:329(133978501ns) {DropIncomingHandshake:442(847ns) DropOutgoingHandshake:457(524ns) SwitchToState:179(431ns) ProcessPendingSessionEvents:495(970ns)}}
2025-11-27T18:19:18.609559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471463690437389:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.737364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:18.741813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471463690437389:2109] [node 6] ICP22 created new session: [4:7577471463690437429:2109]
2025-11-27T18:19:18.741890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471463690437429:2109] [node 6] ICS09 handshake done sender: [4:7577471463690437406:2117] self: [4:7577471463690437405:0] peer: [6:7577471466950532968:0] socket: 149 qp: -1
2025-11-27T18:19:18.741943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471463690437429:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:18.742020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:18.743559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471463690437389:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(133193753ns) {HandleHandshakeStatus:329(133190843ns) {DropIncomingHandshake:442(28471ns) DropOutgoingHandshake:457(610ns) SwitchToState:179(697ns) ProcessPendingSessionEvents:495(519560ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 463487ns max# 53992ns}}}
2025-11-27T18:19:18.766170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471461529565932:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(86921012ns) {HandleHandshakeStatus:329(86917420ns) {DropIncomingHandshake:442(1326ns) DropOutgoingHandshake:457(974ns) SwitchToState:179(714ns) ProcessPendingSessionEvents:495(1168ns)}}
2025-11-27T18:19:18.549465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471466950532967:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.801385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471464059657547:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:18.778817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471462439897165:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(152212130ns) {HandleHandshakeStatus:329(152207428ns) {DropIncomingHandshake:442(17929ns) DropOutgoingHandshake:457(439ns) SwitchToState:179(698ns) ProcessPendingSessionEvents:495(79233180ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 79129657ns max# 18164648ns}}}
2025-11-27T18:19:18.813451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:18.817358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471464059657547:2109] [node 6] ICP22 created new session: [3:7577471464059657580:2109]
2025-11-27T18:19:18.570879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471466950532970:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8216
2025-11-27T18:19:18.601441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471466950532961:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:18.630264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471462655565585:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.647406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471466950532970:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:18.658660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471462655565584:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.661480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471462655565585:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(28689364ns) {IncomingHandshake:261(28687064ns) {DropIncomingHandshake:442(847ns) PrepareNewSessionHandshake:477(7793502ns) {SwitchToState:179(8353ns)} GenerateSessionVirtualId:988(952ns) IssueIncomingHandshakeReply:193(15599728ns) {SwitchToState:179(3300ns)}}}
2025-11-27T18:19:18.664794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471462655565651:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7441) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471466950532966:2118] held: no
2025-11-27T18:19:18.681862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471466950532971:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8216
2025-11-27T18:19:18.682475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471462655565586:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:18.683165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471466950532973:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8216
2025-11-27T18:19:18.683608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471466950532967:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:18.683815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471462655565585:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.683860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:18.683903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471462655565585:2049] [node 2] ICP22 created new session: [6:7577471466950532976:2049]
2025-11-27T18:19:18.683953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471466950532976:2049] [node 2] ICS09 handshake done sender: [6:7577471466950532967:2119] self: [6:7577471466950532972:0] peer: [2:7577471466293716422:0] socket: 153 qp: -1
2025-11-27T18:19:18.683985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471466950532976:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:18.684017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:18.730724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471462655565651:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(68323933ns) {HandleHandshakeStatus:395(68321215ns) {LogHandshakeFail:472(3172391ns) {UpdateErrorStateLog:545(5778ns)} DropOutgoingHandshake:457(520ns) ProcessPendingSessionEvents:495(65089589ns) {unordered DropSessionEvent:512 num# 100 sum# 31694096ns max# 10913321ns}}}
2025-11-27T18:19:18.731017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471466950532966:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:18.737330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:18.741598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471462655565651:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.753134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471462655565584:2048] [node 1] ICP22 created new session: [6:7577471466950532978:2048]
2025-11-27T18:19:18.753297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:18.753366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471462655565651:2110] [node 4] ICP22 created new session: [6:7577471466950532979:2110]
2025-11-27T18:19:18.753423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471466950532979:2110] [node 4] ICS09 handshake done sender: [6:7577471466950532966:2118] self: [6:7577471466950532968:0] peer: [4:7577471463690437405:0] socket: 150 qp: -1
2025-11-27T18:19:18.753468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471466950532979:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:18.753521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:18.755269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471466950532981:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8216
2025-11-27T18:19:18.760499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471466950532978:2048] [node 1] ICS09 handshake done sender: [6:7577471466950532961:2117] self: [6:7577471466950532965:0] peer: [1:7577471466734864537:0] socket: 85 qp: -1
2025-11-27T18:19:18.765891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471466950532970:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:18.768306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471466950532978:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:18.770642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:18.781676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471462655565586:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:18.781766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:18.781821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471462655565586:2050] [node 3] ICP22 created new session: [6:7577471466950532983:2050]
2025-11-27T18:19:18.781868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471466950532983:2050] [node 3] ICS09 handshake done sender: [6:7577471466950532970:2121] self: [6:7577471466950532974:0] peer: [3:7577471464059657557:0] socket: 96 qp: -1
2025-11-27T18:19:18.781910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471466950532983:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:18.781953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:18.830735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471464059657580:2109] [node 6] ICS09 handshake done sender: [3:7577471464059657559:2119] self: [3:7577471464059657557:0] peer: [6:7577471466950532974:0] socket: 123 qp: -1
2025-11-27T18:19:18.830813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471464059657580:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:18.830864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:18.832531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471464059657547:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(46419469ns) {HandleHandshakeStatus:329(46416131ns) {DropIncomingHandshake:442(71635ns) DropOutgoingHandshake:457(559ns) SwitchToState:179(622ns) ProcessPendingSessionEvents:495(724833ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 664477ns max# 274475ns}}}
2025-11-27T18:19:18.835111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471462655565584:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(174702731ns) {HandleHandshakeStatus:329(174698872ns) {DropIncomingHandshake:442(1043ns) DropOutgoingHandshake:457(6975ns) SwitchToState:179(688ns) ProcessPendingSessionEvents:495(11829195ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11755282ns max# 8454692ns}}}
2025-11-27T18:19:18.835204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471462655565584:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:18.869484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:18.869722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471466293716424:2048] [node 1] ICS01 socket: 106 reason# EndOfStream
2025-11-27T18:19:18.869767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471461998749033:2048] [node 1] ICP30 unregister session Session# [2:7577471466293716424:2048] VirtualId# [2:7577471466293716410:0]
2025-11-27T18:19:18.869811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471466293716424:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:18.869891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:18.869941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:18.884841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:18.885072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471464059657578:2048] [node 1] ICS01 socket: 122 reason# EndOfStream
2025-11-27T18:19:18.885113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471459764690178:2048] [node 1] ICP30 unregister session Session# [3:7577471464059657578:2048] VirtualId# [3:7577471464059657570:0]
2025-11-27T18:19:18.885148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471464059657578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:18.885217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:18.885275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:18.886164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:18.886330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471465824533250:2048] [node 1] ICS01 socket: 109 reason# EndOfStream
2025-11-27T18:19:18.886371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471461529565859:2048] [node 1] ICP30 unregister session Session# [5:7577471465824533250:2048] VirtualId# [5:7577471465824533247:0]
2025-11-27T18:19:18.886398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471465824533250:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:18.886455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:18.886488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:18.888011Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:18.888245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471466950532978:2048] [node 1] ICS07 socket disconnect 85 reason# EndOfStream
2025-11-27T18:19:18.888273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471466950532978:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:18.888348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:18.888395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471466950532978:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:18.886187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:18.886394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471463690437426:2048] [node 1] ICS01 socket: 115 reason# EndOfStream
2025-11-27T18:19:18.886428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471459395470030:2048] [node 1] ICP30 unregister session Session# [4:7577471463690437426:2048] VirtualId# [4:7577471463690437414:0]
2025-11-27T18:19:18.886458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471463690437426:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:18.886533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:18.886577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:18.898180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471462655565584:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14576) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:18.898248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471466950532978:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:18.898282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471462655565584:2048] [node 1] ICP30 unregister session Session# [6:7577471466950532978:2048] VirtualId# [6:7577471466950532965:0]
2025-11-27T18:19:18.898409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:18.898459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471462655565584:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14576) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:18.898477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14576) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:18.908512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471462655565584:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:18.973571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:18.973831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471464059657576:2049] [node 2] ICS01 socket: 148 reason# EndOfStream
2025-11-27T18:19:18.973908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471459764690179:2049] [node 2] ICP30 unregister session Session# [3:7577471464059657576:2049] VirtualId# [3:7577471464059657572:0]
2025-11-27T18:19:18.973947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471464059657576:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:18.974046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:18.974102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:18.979256Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:18.979498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471463690437421:2049] [node 2] ICS07 socket disconnect 125 reason# EndOfStream
2025-11-27T18:19:18.979530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471463690437421:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:18.979619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:18.979677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471463690437421:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:18.980385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:18.991023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471459395470031:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28776) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:18.991075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471463690437421:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:18.991099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471459395470031:2049] [node 2] ICP30 unregister session Session# [4:7577471463690437421:2049] VirtualId# [4:7577471463690437410:0]
2025-11-27T18:19:18.991134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:18.991152Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471459395470031:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:18.991161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:18.980576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471465824533258:2049] [node 2] ICS07 socket disconnect 112 reason# EndOfStream
2025-11-27T18:19:18.980611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471465824533258:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:18.980717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:18.980785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471465824533258:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:18.989918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471461529565860:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28776) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:18.989987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471465824533258:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:18.990027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471461529565860:2049] [node 2] ICP30 unregister session Session# [5:7577471465824533258:2049] VirtualId# [5:7577471465824533253:0]
2025-11-27T18:19:18.990084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:18.990123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471461529565860:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:18.990140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:18.987905Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:18.990660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471466950532976:2049] [node 2] ICS01 socket: 153 reason# EndOfStream
2025-11-27T18:19:18.992073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471462655565585:2049] [node 2] ICP30 unregister session Session# [6:7577471466950532976:2049] VirtualId# [6:7577471466950532972:0]
2025-11-27T18:19:18.992148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471466950532976:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:18.992291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:18.996223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:19.000866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471461529565860:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:19.005354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471459395470031:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:19.147083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:19.147328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471465824533264:2050] [node 3] ICS01 socket: 142 reason# EndOfStream
2025-11-27T18:19:19.147370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471461529565861:2050] [node 3] ICP30 unregister session Session# [5:7577471465824533264:2050] VirtualId# [5:7577471465824533262:0]
2025-11-27T18:19:19.147405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471465824533264:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:19.147489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:19.147567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:19.148497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:19.148718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471466950532983:2050] [node 3] ICS01 socket: 96 reason# EndOfStream
2025-11-27T18:19:19.148760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471462655565586:2050] [node 3] ICP30 unregister session Session# [6:7577471466950532983:2050] VirtualId# [6:7577471466950532974:0]
2025-11-27T18:19:19.148787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471466950532983:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:19.148858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:19.148894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:19.149231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:19.149430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471463690437416:2050] [node 3] ICS01 socket: 97 reason# EndOfStream
2025-11-27T18:19:19.149528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471459395470032:2050] [node 3] ICP30 unregister session Session# [4:7577471463690437416:2050] VirtualId# [4:7577471463690437409:0]
2025-11-27T18:19:19.149565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471463690437416:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:19.149635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:19.149678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:19.209195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:19.209456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471465824533266:2109] [node 4] ICS01 socket: 119 reason# EndOfStream
2025-11-27T18:19:19.209505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471461529565932:2109] [node 4] ICP30 unregister session Session# [5:7577471465824533266:2109] VirtualId# [5:7577471465824533257:0]
2025-11-27T18:19:19.209544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471465824533266:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:19.209613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:19.209663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:19.211435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:19.211660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471466950532979:2110] [node 4] ICS01 socket: 150 reason# EndOfStream
2025-11-27T18:19:19.211701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471462655565651:2110] [node 4] ICP30 unregister session Session# [6:7577471466950532979:2110] VirtualId# [6:7577471466950532968:0]
2025-11-27T18:19:19.211749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471466950532979:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:19.211828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:19.211869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:19.301544Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:19.303609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471466950532962:2105] [node 5] ICS01 socket: 101 reason# EndOfStream
2025-11-27T18:19:19.303656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471462655565643:2105] [node 5] ICP30 unregister session Session# [6:7577471466950532962:2105] VirtualId# [6:7577471466950532959:0]
2025-11-27T18:19:19.303684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471466950532962:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:19.303773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:19.303812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:19.452698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471469136583420:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:19.452821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471469136583419:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:19.497627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471469759237264:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:19.497684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471469759237265:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:19.557472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471471425881773:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:19.557530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471471425881774:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:19.591640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471467917568105:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:19.591684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471467917568106:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:19.591716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471467917568107:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:19.630667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471470119095547:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:19.630697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471470119095548:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:19.630718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471470119095549:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:19.682734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471470801345758:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:19.682779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471470801345759:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:19.682805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471470801345760:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:20.223077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471473431550778:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:20.341743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471474054204627:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:20.290367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471473431550779:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:20.369603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471474054204626:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:20.373332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471475096313098:2080] [node 4] ICP01 ready to work
2025-11-27T18:19:20.341742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471474414062906:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:20.421614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471475720849159:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:20.431205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471472212535464:2108] [node 6] ICP01 ready to work
2025-11-27T18:19:20.421648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471475096313101:2092] [node 5] ICP01 ready to work
2025-11-27T18:19:20.422067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471473431550781:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:20.449435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471474054204628:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:20.458024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471475720849161:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:20.445631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471472212535465:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:20.464746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471474414062910:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:20.468741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471475720849160:2121] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.508445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471473431550778:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(284880755ns) {Configure:104(284876635ns) {ProcessConfigured:135(7184174ns) {StartInitialHandshake:157(7182167ns) {DropHandshakes:470(2598ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(399ns)} GenerateSessionVirtualId:988(553ns) PrepareNewSessionHandshake:477(3422ns) {SwitchToState:179(2084ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.486846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471470119095549:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(186780242ns) {Configure:104(186774113ns) {ProcessConfigured:135(13458299ns) {StartInitialHandshake:157(13454549ns) {DropHandshakes:470(2640ns) {DropIncomingHandshake:442(690ns) DropOutgoingHandshake:457(365ns)} GenerateSessionVirtualId:988(653ns) PrepareNewSessionHandshake:477(2186ns) {SwitchToState:179(498ns)}}}}}
2025-11-27T18:19:20.502923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471467917568107:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(69207198ns) {Configure:104(69199680ns) {ProcessConfigured:135(66710ns) {StartInitialHandshake:157(64765ns) {DropHandshakes:470(1605ns) {DropIncomingHandshake:442(400ns) DropOutgoingHandshake:457(419ns)} GenerateSessionVirtualId:988(363ns) PrepareNewSessionHandshake:477(1529ns) {SwitchToState:179(677ns)}}}}}
2025-11-27T18:19:20.515993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471475720849165:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15378
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.493839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471474414062912:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42824
2025-11-27T18:19:20.503425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471472212535468:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3836
2025-11-27T18:19:20.528831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471473431550780:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34411
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.554673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471470801345760:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(183733207ns) {Configure:104(183726448ns) {ProcessConfigured:135(58790ns) {StartInitialHandshake:157(55623ns) {DropHandshakes:470(2497ns) {DropIncomingHandshake:442(602ns) DropOutgoingHandshake:457(294ns)} GenerateSessionVirtualId:988(566ns) PrepareNewSessionHandshake:477(1871ns) {SwitchToState:179(534ns)}}}}}
2025-11-27T18:19:20.537585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471469136583419:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(258592643ns) {Configure:104(258587240ns) {ProcessConfigured:135(20756209ns) {StartInitialHandshake:157(20753695ns) {DropHandshakes:470(1741ns) {DropIncomingHandshake:442(353ns) DropOutgoingHandshake:457(534ns)} GenerateSessionVirtualId:988(403ns) PrepareNewSessionHandshake:477(5609ns) {SwitchToState:179(2790ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.498598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471474414062912:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.540873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471467917568105:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(111040267ns) {Configure:104(111034751ns) {ProcessConfigured:135(11252407ns) {StartInitialHandshake:157(11249137ns) {DropHandshakes:470(1467ns) {DropIncomingHandshake:442(532ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:477(4229ns) {SwitchToState:179(1845ns)}}}}}
2025-11-27T18:19:20.516071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471475720849166:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15378
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.553342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471475720849163:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.577476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471473431550780:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.498738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471470119095549:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.541054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471472212535465:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(51328345ns) {Configure:104(51324446ns) {ProcessConfigured:135(25695950ns) {StartInitialHandshake:157(25693820ns) {DropHandshakes:470(1394ns) {DropIncomingHandshake:442(357ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(386ns) PrepareNewSessionHandshake:477(2531ns) {SwitchToState:179(134ns)}}}}}
2025-11-27T18:19:20.586514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471475096313101:2092] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(121408700ns) {Configure:104(121401079ns) {ProcessConfigured:135(63822ns) {StartInitialHandshake:157(61988ns) {DropHandshakes:470(2510ns) {DropIncomingHandshake:442(478ns) DropOutgoingHandshake:457(524ns)} GenerateSessionVirtualId:988(380ns) PrepareNewSessionHandshake:477(1135ns) {SwitchToState:179(567ns)}}}}}
2025-11-27T18:19:20.553395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471475720849163:2137] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.660719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471474054204634:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62493
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.504338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471474414062916:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42824
2025-11-27T18:19:20.563932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471474414062918:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42824
2025-11-27T18:19:20.575066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471474414062917:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42824
2025-11-27T18:19:20.612054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471474414062918:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.571703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471472212535468:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.602535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471470801345759:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(328484900ns) {Configure:104(328471707ns) {ProcessConfigured:135(35349409ns) {StartInitialHandshake:157(35346218ns) {DropHandshakes:470(3253ns) {DropIncomingHandshake:442(724ns) DropOutgoingHandshake:457(455ns)} GenerateSessionVirtualId:988(1558ns) PrepareNewSessionHandshake:477(4123ns) {SwitchToState:179(2331ns)}}}}}
2025-11-27T18:19:20.631104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471473431550783:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.664050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471474054204633:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62493
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.580773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471475720849166:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.603001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471475096313126:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43053
2025-11-27T18:19:20.665548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471475096313098:2080] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(68692219ns) {Configure:104(68685569ns) {ProcessConfigured:135(59718222ns) {StartInitialHandshake:157(59713306ns) {DropHandshakes:470(4076ns) {DropIncomingHandshake:442(762ns) DropOutgoingHandshake:457(489ns)} GenerateSessionVirtualId:988(3470ns) PrepareNewSessionHandshake:477(4141ns) {SwitchToState:179(1621ns)}}}}}
2025-11-27T18:19:20.677823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471475096313131:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43053
2025-11-27T18:19:20.678040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471475096313126:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.572481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471472212535479:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3836
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.689890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471475096313132:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43053
2025-11-27T18:19:20.690430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471475096313132:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.690561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471470801345759:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.631146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471473431550783:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:20.697880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471475096313131:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.680954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471474054204632:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62493
2025-11-27T18:19:20.669418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471474414062917:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.596198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471472212535479:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.703877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471475096313135:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43053
2025-11-27T18:19:20.632186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471473431550783:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:20.580892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471475720849165:2138] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.609169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471472212535481:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3836
2025-11-27T18:19:20.621461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471467917568105:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.633938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471472212535482:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3836
2025-11-27T18:19:20.654063Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471467917568105:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25170563ns) {IncomingHandshake:261(25168420ns) {DropIncomingHandshake:442(475ns) PrepareNewSessionHandshake:477(2641ns) {SwitchToState:179(1202ns)} GenerateSessionVirtualId:988(530ns) IssueIncomingHandshakeReply:193(11651016ns) {SwitchToState:179(4355ns)}}}
2025-11-27T18:19:20.703413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471472212535479:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:20.587848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471475720849175:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15378
2025-11-27T18:19:20.691873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471474414062910:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(41959340ns) {Configure:104(41947769ns) {ProcessConfigured:135(9189773ns) {StartInitialHandshake:157(9186563ns) {DropHandshakes:470(3962ns) {DropIncomingHandshake:442(749ns) DropOutgoingHandshake:457(535ns)} GenerateSessionVirtualId:988(957ns) PrepareNewSessionHandshake:477(3327ns) {SwitchToState:179(1768ns)}}}}}
2025-11-27T18:19:20.704504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471470801345760:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.711570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471467917568105:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:20.719060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471472212535481:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.719379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471467917568107:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.632685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471473431550778:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.682100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471474054204633:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.692000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471474414062916:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.632728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:20.692227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471470119095549:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471474414062912:2111] held: no
2025-11-27T18:19:20.711837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471474414062920:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42824
2025-11-27T18:19:20.733633Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471470119095547:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(203420682ns) {Configure:104(187430375ns) {ProcessConfigured:135(8885531ns) {StartInitialHandshake:157(8299221ns) {DropHandshakes:470(3511ns) {DropIncomingHandshake:442(1029ns) DropOutgoingHandshake:457(467ns)} GenerateSessionVirtualId:988(996ns) PrepareNewSessionHandshake:477(3712ns) {SwitchToState:179(1436ns)}}}}}
2025-11-27T18:19:20.734096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471470119095547:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.741547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471474414062906:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.689919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471474054204632:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.637591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471475720849163:2137] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.735408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471470801345758:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(229853117ns) {Configure:104(229848598ns) {ProcessConfigured:135(8524666ns) {StartInitialHandshake:157(8521605ns) {DropHandshakes:470(623ns) {DropIncomingHandshake:442(211ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:477(2401ns) {SwitchToState:179(1667ns)}}}}}
2025-11-27T18:19:20.730246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:20.632778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471473431550778:2108] [node 4] ICP22 created new session: [1:7577471473431550796:2108]
2025-11-27T18:19:20.632823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471473431550796:2108] [node 4] ICS09 handshake done sender: [1:7577471473431550783:2111] self: [1:7577471473431550782:0] peer: [4:7577471472212535480:0] socket: 111 qp: -1
2025-11-27T18:19:20.632865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471473431550796:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:20.632905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:20.693939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471473431550793:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34411
2025-11-27T18:19:20.703539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471473431550793:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.719848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471469136583420:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(220073010ns) {Configure:104(220066404ns) {ProcessConfigured:135(13673511ns) {StartInitialHandshake:157(13670986ns) {DropHandshakes:470(16703ns) {DropIncomingHandshake:442(14962ns) DropOutgoingHandshake:457(806ns)} GenerateSessionVirtualId:988(480ns) PrepareNewSessionHandshake:477(3097ns) {SwitchToState:179(1798ns)}}}}}
2025-11-27T18:19:20.722736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471473431550792:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.722759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471473431550792:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:20.725245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471473431550792:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:20.743809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471469136583419:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.743862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:20.743868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471473431550788:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.743891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471473431550788:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:20.743900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471469136583419:2048] [node 2] ICP22 created new session: [1:7577471473431550802:2048]
2025-11-27T18:19:20.743929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471473431550802:2048] [node 2] ICS09 handshake done sender: [1:7577471473431550792:2114] self: [1:7577471473431550784:0] peer: [2:7577471474054204645:0] socket: 132 qp: -1
2025-11-27T18:19:20.748608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471473431550802:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:20.750397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471473431550788:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:20.753690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471473431550786:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.753737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471473431550786:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.690837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471474054204636:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.642969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471475720849159:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.643048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:20.643101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471475720849159:2109] [node 5] ICP22 created new session: [3:7577471475720849179:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.730318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471467917568105:2048] [node 1] ICP22 created new session: [4:7577471472212535487:2048]
2025-11-27T18:19:20.735616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471470801345758:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.690859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471474054204636:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.749914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471474414062912:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:20.643158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471475720849179:2109] [node 5] ICS09 handshake done sender: [3:7577471475720849163:2137] self: [3:7577471475720849162:0] peer: [5:7577471474414062915:0] socket: 95 qp: -1
2025-11-27T18:19:20.769951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.643196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471475720849179:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:20.643251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:20.684533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471475720849175:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.711079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471471425881773:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(185457674ns) {Configure:104(185453030ns) {ProcessConfigured:135(87617ns) {StartInitialHandshake:157(85507ns) {DropHandshakes:470(12502ns) {DropIncomingHandshake:442(656ns) DropOutgoingHandshake:457(265ns)} GenerateSessionVirtualId:988(470ns) PrepareNewSessionHandshake:477(2233ns) {SwitchToState:179(529ns)}}}}}
2025-11-27T18:19:20.720605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471475720849171:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.720634Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471475720849171:2141] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:20.741592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471475720849177:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.741630Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471475720849177:2143] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:20.742758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471475720849171:2141] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:20.745231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471475720849186:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15378
2025-11-27T18:19:20.785669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471475720849161:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.785735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:20.785854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471475720849177:2143] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:20.796518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471475720849161:2107] [node 4] ICP22 created new session: [3:7577471475720849188:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.734082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471472212535472:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.782108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471473431550806:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34411
2025-11-27T18:19:20.701556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471469759237265:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(195990070ns) {Configure:104(195986992ns) {ProcessConfigured:135(7331096ns) {StartInitialHandshake:157(7329086ns) {DropHandshakes:470(3180ns) {DropIncomingHandshake:442(977ns) DropOutgoingHandshake:457(404ns)} GenerateSessionVirtualId:988(702ns) PrepareNewSessionHandshake:477(4231ns) {SwitchToState:179(163ns)}}}}}
2025-11-27T18:19:20.846541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471475720849188:2107] [node 4] ICS09 handshake done sender: [3:7577471475720849171:2141] self: [3:7577471475720849170:0] peer: [4:7577471472212535485:0] socket: 121 qp: -1
2025-11-27T18:19:20.753949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471470119095548:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(211263736ns) {Configure:104(211254414ns) {ProcessConfigured:135(9982827ns) {StartInitialHandshake:157(9980230ns) {DropHandshakes:470(1649ns) {DropIncomingHandshake:442(419ns) DropOutgoingHandshake:457(435ns)} GenerateSessionVirtualId:988(353ns) PrepareNewSessionHandshake:477(1889ns) {SwitchToState:179(181ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.734109Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471472212535472:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:20.709502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471474054204634:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.754131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471470119095548:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.777379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471475096313139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43053
2025-11-27T18:19:20.853583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471475720849188:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:20.790876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471469136583419:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(46237675ns) {HandleHandshakeStatus:329(46234731ns) {DropIncomingHandshake:442(12066ns) DropOutgoingHandshake:457(364ns) SwitchToState:179(102ns) ProcessPendingSessionEvents:495(524351ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 465734ns max# 65939ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.739035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471472212535472:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:20.777537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471475096313140:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43053
2025-11-27T18:19:20.791081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471473431550786:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.791779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471473431550779:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.712063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471474054204636:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:20.721476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471469759237264:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.810713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471474054204626:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.810788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:20.810834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471474054204626:2110] [node 6] ICP22 created new session: [2:7577471474054204647:2110]
2025-11-27T18:19:20.810886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471474054204647:2110] [node 6] ICS09 handshake done sender: [2:7577471474054204636:2115] self: [2:7577471474054204635:0] peer: [6:7577471475096313133:0] socket: 135 qp: -1
2025-11-27T18:19:20.810925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471474054204647:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:20.810962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:20.812523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471474054204630:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.812560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471474054204630:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:20.818176Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471469759237264:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27526) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471474054204634:2114] held: no
2025-11-27T18:19:20.829596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471474054204646:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62493
2025-11-27T18:19:20.841922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471474054204649:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62493
2025-11-27T18:19:20.864151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471474054204649:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.867692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471474054204630:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:20.885737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471474054204654:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62493
2025-11-27T18:19:20.743142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471467917568106:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(248213931ns) {Configure:104(248208135ns) {ProcessConfigured:135(16546124ns) {StartInitialHandshake:157(16543602ns) {DropHandshakes:470(2406ns) {DropIncomingHandshake:442(441ns) DropOutgoingHandshake:457(682ns)} GenerateSessionVirtualId:988(565ns) PrepareNewSessionHandshake:477(1799ns) {SwitchToState:179(128ns)}}}}}
2025-11-27T18:19:20.853628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:20.755695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471474414062930:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42824
2025-11-27T18:19:20.777627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471475096313141:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43053
2025-11-27T18:19:20.791377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471475096313132:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:20.797066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471475096313139:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.805979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471475096313142:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43053
2025-11-27T18:19:20.815032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471470801345759:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:20.815071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:20.815100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471470801345759:2049] [node 2] ICP22 created new session: [6:7577471475096313144:2049]
2025-11-27T18:19:20.815144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471475096313144:2049] [node 2] ICS09 handshake done sender: [6:7577471475096313132:2116] self: [6:7577471475096313133:0] peer: [2:7577471474054204635:0] socket: 136 qp: -1
2025-11-27T18:19:20.815172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471475096313144:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:20.815209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:20.819091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471475096313098:2080] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.846017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471475096313126:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:20.857832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471475096313131:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:20.861518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471470801345760:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:20.865603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471470801345758:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:20.873532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471475096313140:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.876123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:20.881474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:20.884506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471470801345758:2048] [node 1] ICP22 created new session: [6:7577471475096313147:2048]
2025-11-27T18:19:20.892133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471470801345760:2050] [node 3] ICP22 created new session: [6:7577471475096313148:2050]
2025-11-27T18:19:20.897821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471475096313147:2048] [node 1] ICS09 handshake done sender: [6:7577471475096313126:2112] self: [6:7577471475096313138:0] peer: [1:7577471473431550785:0] socket: 120 qp: -1
2025-11-27T18:19:20.897870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471475096313147:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:20.913039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471475096313149:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43053
2025-11-27T18:19:20.915811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471475096313101:2092] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.916073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471475096313148:2050] [node 3] ICS09 handshake done sender: [6:7577471475096313131:2115] self: [6:7577471475096313134:0] peer: [3:7577471475720849176:0] socket: 100 qp: -1
2025-11-27T18:19:20.917790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471475096313139:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:20.918015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471475096313098:2080] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:20.918075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:20.918120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471475096313098:2080] [node 4] ICP22 created new session: [6:7577471475096313151:2080]
2025-11-27T18:19:20.918168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471475096313151:2080] [node 4] ICS09 handshake done sender: [6:7577471475096313139:2119] self: [6:7577471475096313143:0] peer: [4:7577471472212535466:0] socket: 149 qp: -1
2025-11-27T18:19:20.918205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471475096313151:2080] [node 4] ICS10 traffic start
2025-11-27T18:19:20.918239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:20.929568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471475096313148:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.854843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471475720849186:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.791829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:20.745575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471472212535487:2048] [node 1] ICS09 handshake done sender: [4:7577471472212535479:2115] self: [4:7577471472212535480:0] peer: [1:7577471473431550782:0] socket: 117 qp: -1
2025-11-27T18:19:20.763944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471474414062919:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42824
2025-11-27T18:19:20.917514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471474054204627:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.941375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:20.854889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471475720849160:2121] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.917596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:20.791881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471473431550779:2107] [node 5] ICP22 created new session: [1:7577471473431550808:2107]
2025-11-27T18:19:20.754652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471472212535489:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3836
2025-11-27T18:19:20.764113Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471474414062906:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(19814069ns) {IncomingHandshake:261(12272948ns) {DropIncomingHandshake:442(426ns) PrepareNewSessionHandshake:477(1652ns) {SwitchToState:179(723ns)} GenerateSessionVirtualId:988(353ns) IssueIncomingHandshakeReply:193(12232693ns) {SwitchToState:179(2441ns)}}}
2025-11-27T18:19:20.777858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471470119095549:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:20.777936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:20.777989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471470119095549:2050] [node 3] ICP22 created new session: [5:7577471474414062934:2050]
2025-11-27T18:19:20.778044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471474414062934:2050] [node 3] ICS09 handshake done sender: [5:7577471474414062912:2111] self: [5:7577471474414062915:0] peer: [3:7577471475720849162:0] socket: 97 qp: -1
2025-11-27T18:19:20.778082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471474414062934:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:20.778127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:20.779126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471474414062916:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:20.797121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471474414062931:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42824
2025-11-27T18:19:20.797176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471474414062906:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:20.797235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:20.807569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471474414062906:2108] [node 4] ICP22 created new session: [5:7577471474414062936:2108]
2025-11-27T18:19:20.818742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471474414062936:2108] [node 4] ICS09 handshake done sender: [5:7577471474414062916:2113] self: [5:7577471474414062925:0] peer: [4:7577471472212535470:0] socket: 104 qp: -1
2025-11-27T18:19:20.830098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471474414062919:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.831521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471474414062918:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:20.850078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471470119095547:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:20.853453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471474414062936:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:20.853515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:20.857687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:20.860122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471474414062940:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42824
2025-11-27T18:19:20.863834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471470119095547:2048] [node 1] ICP22 created new session: [5:7577471474414062939:2048]
2025-11-27T18:19:20.890933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471474414062939:2048] [node 1] ICS09 handshake done sender: [5:7577471474414062918:2115] self: [5:7577471474414062927:0] peer: [1:7577471473431550787:0] socket: 115 qp: -1
2025-11-27T18:19:20.890986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471474414062939:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:20.894969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471474414062917:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:20.895292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471470119095548:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:20.895357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:20.895406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471470119095548:2049] [node 2] ICP22 created new session: [5:7577471474414062942:2049]
2025-11-27T18:19:20.895460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471474414062942:2049] [node 2] ICS09 handshake done sender: [5:7577471474414062917:2114] self: [5:7577471474414062933:0] peer: [2:7577471474054204629:0] socket: 106 qp: -1
2025-11-27T18:19:20.895505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471474414062942:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:20.895544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:20.898006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:20.908888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471474414062906:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(95832359ns) {HandleHandshakeStatus:329(95829641ns) {DropIncomingHandshake:442(3119ns) DropOutgoingHandshake:457(7676ns) SwitchToState:179(631ns) ProcessPendingSessionEvents:495(529649ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 471458ns max# 71288ns}}}
2025-11-27T18:19:20.937246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471474414062922:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.937302Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471474414062922:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:20.954500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471474414062922:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:20.946581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:20.854950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:20.791939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471473431550808:2107] [node 5] ICS09 handshake done sender: [1:7577471473431550788:2113] self: [1:7577471473431550787:0] peer: [5:7577471474414062927:0] socket: 114 qp: -1
2025-11-27T18:19:20.791975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471473431550808:2107] [node 5] ICS10 traffic start
2025-11-27T18:19:20.792014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:20.798491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471473431550778:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(136096772ns) {HandleHandshakeStatus:329(136093536ns) {DropIncomingHandshake:442(472ns) DropOutgoingHandshake:457(762ns) SwitchToState:179(636ns) ProcessPendingSessionEvents:495(103390715ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 93226595ns max# 13903861ns}}}
2025-11-27T18:19:20.798567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471473431550781:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.798600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:20.798642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471473431550781:2109] [node 6] ICP22 created new session: [1:7577471473431550810:2109]
2025-11-27T18:19:20.798688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471473431550810:2109] [node 6] ICS09 handshake done sender: [1:7577471473431550786:2112] self: [1:7577471473431550785:0] peer: [6:7577471475096313138:0] socket: 112 qp: -1
2025-11-27T18:19:20.798735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471473431550810:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:20.798775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:20.800191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471473431550778:2108] [node 4] ICP18 handshake (actor [1:7577471473431550793:2115]) is too old
2025-11-27T18:19:20.847179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471473431550812:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34411
2025-11-27T18:19:20.865202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471473431550806:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.869897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471473431550779:2107] [node 5] ICP18 handshake (actor [1:7577471473431550806:2117]) is too old
2025-11-27T18:19:20.893581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471473431550812:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.913421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471473431550800:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.913460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471473431550800:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:20.926933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471473431550800:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:20.927244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471469136583420:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.927309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:20.927358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471469136583420:2049] [node 3] ICP22 created new session: [1:7577471473431550815:2049]
2025-11-27T18:19:20.927411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471473431550815:2049] [node 3] ICS09 handshake done sender: [1:7577471473431550800:2116] self: [1:7577471473431550799:0] peer: [3:7577471475720849193:0] socket: 143 qp: -1
2025-11-27T18:19:20.927442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471473431550815:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:20.927480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:20.979150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471474414062910:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.768280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471472212535487:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:20.768334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:20.779547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471472212535481:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:20.782309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471472212535465:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.814913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471467917568105:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(109651249ns) {HandleHandshakeStatus:329(109649258ns) {DropIncomingHandshake:442(1726ns) DropOutgoingHandshake:457(12445ns) SwitchToState:179(662ns) ProcessPendingSessionEvents:495(519449ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 468161ns max# 59869ns}}}
2025-11-27T18:19:20.818696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:20.821589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471467917568107:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:20.821666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:20.821716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471467917568107:2050] [node 3] ICP22 created new session: [4:7577471472212535493:2050]
2025-11-27T18:19:20.821760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471472212535493:2050] [node 3] ICS09 handshake done sender: [4:7577471472212535481:2116] self: [4:7577471472212535485:0] peer: [3:7577471475720849170:0] socket: 124 qp: -1
2025-11-27T18:19:20.821796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471472212535493:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:20.821830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:20.822480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471467917568107:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:20.832280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471472212535465:2109] [node 5] ICP22 created new session: [4:7577471472212535491:2109]
2025-11-27T18:19:20.841702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471472212535495:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3836
2025-11-27T18:19:20.853412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471472212535491:2109] [node 5] ICS09 handshake done sender: [4:7577471472212535472:2113] self: [4:7577471472212535470:0] peer: [5:7577471474414062925:0] socket: 102 qp: -1
2025-11-27T18:19:20.857756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471472212535495:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.861082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471472212535491:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:20.865453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471472212535467:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.865480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471472212535467:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:20.869334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:20.881491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471467917568106:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.890297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471472212535498:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3836
2025-11-27T18:19:20.890700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471472212535467:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:20.892584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471472212535464:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.892624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:20.901839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471472212535464:2108] [node 6] ICP22 created new session: [4:7577471472212535500:2108]
2025-11-27T18:19:20.901897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471472212535500:2108] [node 6] ICS09 handshake done sender: [4:7577471472212535467:2110] self: [4:7577471472212535466:0] peer: [6:7577471475096313143:0] socket: 101 qp: -1
2025-11-27T18:19:20.901939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471472212535500:2108] [node 6] ICS10 traffic start
2025-11-27T18:19:20.901996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.917647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471474054204627:2109] [node 5] ICP22 created new session: [2:7577471474054204655:2109]
2025-11-27T18:19:20.913400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471472212535498:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.855000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471475720849160:2121] [node 6] ICP22 created new session: [3:7577471475720849191:2121]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.994042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:20.941669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471472212535464:2108] [node 6] ICP18 handshake (actor [4:7577471472212535498:2121]) is too old
2025-11-27T18:19:20.933347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471474054204641:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.995782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471474414062910:2109] [node 6] ICP22 created new session: [5:7577471474414062946:2109]
2025-11-27T18:19:20.855048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471475720849191:2121] [node 6] ICS09 handshake done sender: [3:7577471475720849177:2143] self: [3:7577471475720849176:0] peer: [6:7577471475096313134:0] socket: 133 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:20.933396Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471474054204641:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:21.005765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471474414062946:2109] [node 6] ICS09 handshake done sender: [5:7577471474414062922:2118] self: [5:7577471474414062921:0] peer: [6:7577471475096313150:0] socket: 141 qp: -1
2025-11-27T18:19:21.013365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471475096313101:2092] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18599) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471475096313140:2120] held: no
2025-11-27T18:19:20.855074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471475720849191:2121] [node 6] ICS10 traffic start
2025-11-27T18:19:20.945585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471472212535502:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3836
2025-11-27T18:19:20.941796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471474054204655:2109] [node 5] ICS09 handshake done sender: [2:7577471474054204630:2111] self: [2:7577471474054204629:0] peer: [5:7577471474414062933:0] socket: 105 qp: -1
2025-11-27T18:19:21.012024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471474414062946:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:20.941842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471474054204655:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:20.941883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:20.943408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471474054204627:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(25117440ns) {HandleHandshakeStatus:329(25114578ns) {DropIncomingHandshake:442(462ns) DropOutgoingHandshake:457(574ns) SwitchToState:179(751ns) ProcessPendingSessionEvents:495(597564ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 533065ns max# 64642ns}}}
2025-11-27T18:19:20.953778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471474054204641:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:20.965413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471474054204642:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.965463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471474054204642:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:20.966190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471474054204628:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:20.983735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:20.995502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471474054204628:2108] [node 4] ICP22 created new session: [2:7577471474054204658:2108]
2025-11-27T18:19:21.002615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471474054204658:2108] [node 4] ICS09 handshake done sender: [2:7577471474054204641:2116] self: [2:7577471474054204640:0] peer: [4:7577471472212535497:0] socket: 113 qp: -1
2025-11-27T18:19:21.007930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471474054204642:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:21.010070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471474054204658:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:21.013916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471469759237265:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:21.014005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:21.014076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471469759237265:2049] [node 3] ICP22 created new session: [2:7577471478349171956:2049]
2025-11-27T18:19:21.022177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:21.022418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471478349171956:2049] [node 3] ICS09 handshake done sender: [2:7577471474054204642:2117] self: [2:7577471474054204638:0] peer: [3:7577471475720849196:0] socket: 127 qp: -1
2025-11-27T18:19:21.022461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471478349171956:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:21.022500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:21.055302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471474054204628:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(88903570ns) {HandleHandshakeStatus:329(88900391ns) {DropIncomingHandshake:442(18640ns) DropOutgoingHandshake:457(714ns) SwitchToState:179(1100ns) ProcessPendingSessionEvents:495(7970954ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7911456ns max# 7480359ns}}}
2025-11-27T18:19:20.855119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:20.981848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471472212535495:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:21.025905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471475096313155:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43053
2025-11-27T18:19:21.017448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:21.035879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471475096313140:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:21.083275Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471469759237265:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(68518767ns) {HandleHandshakeStatus:329(68514715ns) {DropIncomingHandshake:442(17617ns) DropOutgoingHandshake:457(276ns) SwitchToState:179(4211ns) ProcessPendingSessionEvents:495(59811405ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 59373174ns max# 13214714ns}}}
2025-11-27T18:19:21.084092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471469759237264:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(171958062ns) {HandleHandshakeStatus:395(171954711ns) {LogHandshakeFail:472(20683ns) {UpdateErrorStateLog:545(10702ns)} DropOutgoingHandshake:457(2578ns) ProcessPendingSessionEvents:495(128378080ns) {unordered DropSessionEvent:512 num# 100 sum# 31851122ns max# 10716912ns}}}
2025-11-27T18:19:20.877932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471475720849161:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(103612099ns) {HandleHandshakeStatus:329(103608108ns) {DropIncomingHandshake:442(16348ns) DropOutgoingHandshake:457(522ns) SwitchToState:179(685ns) ProcessPendingSessionEvents:495(9501640ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9435550ns max# 8936515ns}}}
2025-11-27T18:19:20.878052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471471425881773:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:20.890694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471475720849194:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15378
2025-11-27T18:19:20.913498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471475720849194:2146] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:20.929526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471471425881774:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:21.084265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471474054204634:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:20.938343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471471425881773:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27526) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471475720849186:2145] held: no
2025-11-27T18:19:20.998660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471475720849197:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15378
2025-11-27T18:19:21.017876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471475720849195:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15378
2025-11-27T18:19:21.029356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471475720849186:2145] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:21.029598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471471425881773:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:21.029644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:21.029697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471471425881773:2048] [node 1] ICP22 created new session: [3:7577471480015816494:2048]
2025-11-27T18:19:21.029742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471480015816494:2048] [node 1] ICS09 handshake done sender: [3:7577471475720849186:2145] self: [3:7577471475720849193:0] peer: [1:7577471473431550799:0] socket: 144 qp: -1
2025-11-27T18:19:21.029779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471480015816494:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:21.029814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:21.035780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471475720849194:2146] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:21.053502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471471425881774:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471475720849194:2146] held: no
2025-11-27T18:19:21.005375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471467917568106:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471472212535495:2120] held: no
2025-11-27T18:19:21.083352Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471472212535465:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(315334454ns) {HandleHandshakeStatus:329(315330970ns) {DropIncomingHandshake:442(20260ns) DropOutgoingHandshake:457(681ns) SwitchToState:179(706ns) ProcessPendingSessionEvents:495(152772326ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 152643723ns max# 24687896ns}}}
2025-11-27T18:19:21.085676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471469759237264:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:21.020367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471474414062910:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(53330685ns) {HandleHandshakeStatus:329(53328184ns) {DropIncomingHandshake:442(29628ns) DropOutgoingHandshake:457(3147ns) SwitchToState:179(963ns) ProcessPendingSessionEvents:495(927128ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 865799ns max# 194701ns}}}
2025-11-27T18:19:21.042935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471470801345760:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(180542767ns) {HandleHandshakeStatus:329(180539781ns) {DropIncomingHandshake:442(759ns) DropOutgoingHandshake:457(5804ns) SwitchToState:179(745ns) ProcessPendingSessionEvents:495(86593304ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 86481251ns max# 13385973ns}}}
2025-11-27T18:19:21.089612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471471425881774:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(49281127ns) {HandleHandshakeStatus:395(49277947ns) {LogHandshakeFail:472(14854ns) {UpdateErrorStateLog:545(4957ns)} DropOutgoingHandshake:457(1911ns) ProcessPendingSessionEvents:495(35273702ns) {unordered DropSessionEvent:512 num# 100 sum# 7582663ns max# 3255944ns}}}
2025-11-27T18:19:21.089684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471471425881774:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:21.089736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:21.089784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471471425881774:2049] [node 2] ICP22 created new session: [3:7577471480015816496:2049]
2025-11-27T18:19:21.089830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471480015816496:2049] [node 2] ICS09 handshake done sender: [3:7577471475720849194:2146] self: [3:7577471475720849196:0] peer: [2:7577471474054204638:0] socket: 159 qp: -1
2025-11-27T18:19:21.089864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471480015816496:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:21.089915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:21.093327Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471467917568106:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(55026384ns) {HandleHandshakeStatus:395(51164898ns) {LogHandshakeFail:472(21963ns) {UpdateErrorStateLog:545(5631ns)} DropOutgoingHandshake:457(594ns) ProcessPendingSessionEvents:495(51066630ns) {unordered DropSessionEvent:512 num# 100 sum# 8175538ns max# 5137469ns}}}
2025-11-27T18:19:21.093407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471467917568106:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:21.093475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:21.093521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471467917568106:2049] [node 2] ICP22 created new session: [4:7577471476507502799:2049]
2025-11-27T18:19:21.093563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471476507502799:2049] [node 2] ICS09 handshake done sender: [4:7577471472212535495:2120] self: [4:7577471472212535497:0] peer: [2:7577471474054204640:0] socket: 156 qp: -1
2025-11-27T18:19:21.093594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471476507502799:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:21.093633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:21.074228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471470119095547:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(111164452ns) {HandleHandshakeStatus:329(110790326ns) {DropIncomingHandshake:442(3972ns) DropOutgoingHandshake:457(13006ns) SwitchToState:179(647ns) ProcessPendingSessionEvents:495(7422854ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7377661ns max# 7040723ns}}}
2025-11-27T18:19:21.056897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471475096313101:2092] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:21.069545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:21.069615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471475096313101:2092] [node 5] ICP22 created new session: [6:7577471479391280452:2092]
2025-11-27T18:19:21.069665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471479391280452:2092] [node 5] ICS09 handshake done sender: [6:7577471475096313140:2120] self: [6:7577471475096313150:0] peer: [5:7577471474414062921:0] socket: 151 qp: -1
2025-11-27T18:19:21.069707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471479391280452:2092] [node 5] ICS10 traffic start
2025-11-27T18:19:21.069751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:21.090514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471475096313101:2092] [node 5] ICP35 event processing took too much time PendingConnection:279(35830648ns) {HandleHandshakeStatus:329(35826992ns) {DropIncomingHandshake:442(3231ns) DropOutgoingHandshake:457(421ns) SwitchToState:179(743ns) ProcessPendingSessionEvents:495(924ns)}}
2025-11-27T18:19:21.093411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471470801345758:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(226955671ns) {HandleHandshakeStatus:329(226952067ns) {DropIncomingHandshake:442(619ns) DropOutgoingHandshake:457(11069ns) SwitchToState:179(776ns) ProcessPendingSessionEvents:495(113412286ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 113284632ns max# 16308632ns}}}
2025-11-27T18:19:21.090338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:21.099805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471469759237264:2048] [node 1] ICP22 created new session: [2:7577471478349171959:2048]
2025-11-27T18:19:21.099875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471478349171959:2048] [node 1] ICS09 handshake done sender: [2:7577471474054204634:2114] self: [2:7577471474054204645:0] peer: [1:7577471473431550784:0] socket: 134 qp: -1
2025-11-27T18:19:21.099914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471478349171959:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:21.099969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:21.144885Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:21.145118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471480015816494:2048] [node 1] ICS01 socket: 144 reason# EndOfStream
2025-11-27T18:19:21.145181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471471425881773:2048] [node 1] ICP30 unregister session Session# [3:7577471480015816494:2048] VirtualId# [3:7577471475720849193:0]
2025-11-27T18:19:21.145218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471480015816494:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.145472Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:21.145573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:21.145637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:21.145685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471478349171959:2048] [node 1] ICS01 socket: 134 reason# EndOfStream
2025-11-27T18:19:21.145730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471469759237264:2048] [node 1] ICP30 unregister session Session# [2:7577471478349171959:2048] VirtualId# [2:7577471474054204645:0]
2025-11-27T18:19:21.145760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471478349171959:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.145846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:21.145892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:21.149358Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:21.149562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471474414062939:2048] [node 1] ICS07 socket disconnect 115 reason# EndOfStream
2025-11-27T18:19:21.149585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471474414062939:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.149636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:21.149692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471474414062939:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:21.151392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:21.151623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471475096313147:2048] [node 1] ICS07 socket disconnect 120 reason# EndOfStream
2025-11-27T18:19:21.151651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471475096313147:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.151708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:21.151742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471475096313147:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:21.152103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:21.151043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471470119095547:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27526) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:21.151088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471474414062939:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:21.151124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471470119095547:2048] [node 1] ICP30 unregister session Session# [5:7577471474414062939:2048] VirtualId# [5:7577471474414062927:0]
2025-11-27T18:19:21.151209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:21.152288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471472212535487:2048] [node 1] ICS07 socket disconnect 117 reason# EndOfStream
2025-11-27T18:19:21.152327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471472212535487:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.152384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:21.157397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471470801345758:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27526) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:21.152420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471472212535487:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:21.157520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471475096313147:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:21.157561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471470801345758:2048] [node 1] ICP30 unregister session Session# [6:7577471475096313147:2048] VirtualId# [6:7577471475096313138:0]
2025-11-27T18:19:21.153374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471467917568105:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27526) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:21.161833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:21.161446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471472212535487:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:21.161527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471467917568105:2048] [node 1] ICP30 unregister session Session# [4:7577471472212535487:2048] VirtualId# [4:7577471472212535480:0]
2025-11-27T18:19:21.161881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471470801345758:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27526) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.151241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471470119095547:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27526) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.151254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27526) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.161898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27526) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.169753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:21.169835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471467917568105:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27526) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.169854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27526) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.170287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [4:7577471467917568105:2048] [node 1] ICP35 event processing took too much time StateWork:294(16754770ns) {HandleHandshakeStatus:395(16747263ns) {LogHandshakeFail:472(26450ns) {UpdateErrorStateLog:545(2091ns)} DropOutgoingHandshake:457(146ns) ProcessPendingSessionEvents:495(211ns) UnregisterSession:538(77315ns) {ProcessPendingSessionEvents:495(265ns) SwitchToInitialState:202(2242ns) {SwitchToState:179(996ns)}} TransitToErrorState:752(252255ns) {SwitchToState:179(6900ns) ProcessPendingSessionEvents:495(328ns) DropHandshakes:470(185179ns) {DropIncomingHandshake:442(872ns) DropOutgoingHandshake:457(387ns)}}}}
2025-11-27T18:19:21.173438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471470119095547:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:21.178162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471470801345758:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:21.180459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471467917568105:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:21.235338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:21.235570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471480015816496:2049] [node 2] ICS01 socket: 159 reason# EndOfStream
2025-11-27T18:19:21.235617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471471425881774:2049] [node 2] ICP30 unregister session Session# [3:7577471480015816496:2049] VirtualId# [3:7577471475720849196:0]
2025-11-27T18:19:21.235654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471480015816496:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.235725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:21.235767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:21.240000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:21.240281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471474414062942:2049] [node 2] ICS01 socket: 106 reason# EndOfStream
2025-11-27T18:19:21.240332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471470119095548:2049] [node 2] ICP30 unregister session Session# [5:7577471474414062942:2049] VirtualId# [5:7577471474414062933:0]
2025-11-27T18:19:21.240377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471474414062942:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.240446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:21.240494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:21.241141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:21.241514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:21.241700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471475096313144:2049] [node 2] ICS07 socket disconnect 136 reason# EndOfStream
2025-11-27T18:19:21.241731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471475096313144:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.241787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:21.241819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471475096313144:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:21.243393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471470801345759:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7668) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:21.243456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471475096313144:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:21.243491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471470801345759:2049] [node 2] ICP30 unregister session Session# [6:7577471475096313144:2049] VirtualId# [6:7577471475096313133:0]
2025-11-27T18:19:21.243525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:21.243572Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471470801345759:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.243593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.241345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471476507502799:2049] [node 2] ICS01 socket: 156 reason# EndOfStream
2025-11-27T18:19:21.241380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471467917568106:2049] [node 2] ICP30 unregister session Session# [4:7577471476507502799:2049] VirtualId# [4:7577471472212535497:0]
2025-11-27T18:19:21.241415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471476507502799:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.241497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:21.241549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:21.255404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471470801345759:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:21.333078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:21.333298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471474414062934:2050] [node 3] ICS01 socket: 97 reason# EndOfStream
2025-11-27T18:19:21.333334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471470119095549:2050] [node 3] ICP30 unregister session Session# [5:7577471474414062934:2050] VirtualId# [5:7577471474414062915:0]
2025-11-27T18:19:21.334046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471474414062934:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.334121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:21.334161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:21.331983Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:21.332250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471475096313148:2050] [node 3] ICS07 socket disconnect 100 reason# EndOfStream
2025-11-27T18:19:21.332284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471475096313148:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.332360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:21.332399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471475096313148:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:21.333236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471470801345760:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4668) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:21.333431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471475096313148:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:21.333469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471470801345760:2050] [node 3] ICP30 unregister session Session# [6:7577471475096313148:2050] VirtualId# [6:7577471475096313134:0]
2025-11-27T18:19:21.333580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:21.333617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471470801345760:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.333634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4668) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.337416Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:21.337673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471472212535493:2050] [node 3] ICS01 socket: 124 reason# EndOfStream
2025-11-27T18:19:21.337712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471467917568107:2050] [node 3] ICP30 unregister session Session# [4:7577471472212535493:2050] VirtualId# [4:7577471472212535485:0]
2025-11-27T18:19:21.337746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471472212535493:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.337832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:21.337876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:21.341302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471470801345760:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:21.463886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:21.464267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471474414062936:2108] [node 4] ICS01 socket: 104 reason# EndOfStream
2025-11-27T18:19:21.464349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471474414062906:2108] [node 4] ICP30 unregister session Session# [5:7577471474414062936:2108] VirtualId# [5:7577471474414062925:0]
2025-11-27T18:19:21.464456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471474414062936:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.464639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:21.464752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:21.465370Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:21.465589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471475096313151:2080] [node 4] ICS07 socket disconnect 149 reason# EndOfStream
2025-11-27T18:19:21.465650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471475096313151:2080] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.465771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:21.465859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471475096313151:2080] [node 4] ICS15 start handshake
2025-11-27T18:19:21.498110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471475096313098:2080] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64526) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:21.498189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471475096313151:2080] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:21.498229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471475096313098:2080] [node 4] ICP30 unregister session Session# [6:7577471475096313151:2080] VirtualId# [6:7577471475096313143:0]
2025-11-27T18:19:21.498319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:21.498352Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471475096313098:2080] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64526) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.498369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64526) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:21.509705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471475096313098:2080] [node 4] ICP33 wake up from error state
2025-11-27T18:19:21.549443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:21.549706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471479391280452:2092] [node 5] ICS01 socket: 151 reason# EndOfStream
2025-11-27T18:19:21.549752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471475096313101:2092] [node 5] ICP30 unregister session Session# [6:7577471479391280452:2092] VirtualId# [6:7577471475096313150:0]
2025-11-27T18:19:21.549789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471479391280452:2092] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:21.549869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:21.549911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:21.745521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471476098969989:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:21.745567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471476098969990:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:21.786077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471475953837997:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:21.786137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471475953837998:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:21.845657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471477453638408:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:21.845734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471477453638409:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:21.881833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471478186271022:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:21.881897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471478186271023:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:21.881927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471478186271024:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:21.916420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471478230642851:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:21.916488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471478230642852:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:21.916522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471478230642853:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:21.941097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471475896901874:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:21.941154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471475896901875:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:21.943038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471475896901876:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:22.303667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471482525610163:2062] [node 6] ICP01 ready to work
2025-11-27T18:19:22.391710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471480393937348:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:22.394892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471481748605765:2098] [node 6] ICP01 ready to work
2025-11-27T18:19:22.433724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471482481238374:2088] [node 5] ICP01 ready to work
2025-11-27T18:19:22.458284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471480248805383:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:22.456708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471480393937349:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:22.497537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471482481238382:2090] [node 6] ICP01 ready to work
2025-11-27T18:19:22.470250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471481748605768:2100] [node 4] ICP01 ready to work
2025-11-27T18:19:22.538056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471481748605769:2102] [node 5] ICP01 ready to work
2025-11-27T18:19:22.525796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471480248805384:2134] [node 4] ICP01 ready to work
2025-11-27T18:19:22.542381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471480393937350:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:22.553163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471478186271023:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(50419110ns) {Configure:104(50415643ns) {ProcessConfigured:135(10112578ns) {StartInitialHandshake:157(10110918ns) {DropHandshakes:470(2484ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(638ns) PrepareNewSessionHandshake:477(2437ns) {SwitchToState:179(119ns)}}}}}
2025-11-27T18:19:22.525408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471482525610163:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(141858870ns) {Configure:104(141851886ns) {ProcessConfigured:135(44118647ns) {StartInitialHandshake:157(44115138ns) {DropHandshakes:470(2955ns) {DropIncomingHandshake:442(721ns) DropOutgoingHandshake:457(359ns)} GenerateSessionVirtualId:988(754ns) PrepareNewSessionHandshake:477(11914335ns) {SwitchToState:179(6626ns)}}}}}
2025-11-27T18:19:22.549747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471475896901876:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(166275301ns) {Configure:104(166269332ns) {ProcessConfigured:135(39902ns) {StartInitialHandshake:157(38693ns) {DropHandshakes:470(925ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(312ns) PrepareNewSessionHandshake:477(1008ns) {SwitchToState:179(265ns)}}}}}
2025-11-27T18:19:22.560323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471480248805385:2112] [node 6] ICP01 ready to work
2025-11-27T18:19:22.568410Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471477453638408:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27516651ns) {Configure:104(27513086ns) {ProcessConfigured:135(47879ns) {StartInitialHandshake:157(45143ns) {DropHandshakes:470(1660ns) {DropIncomingHandshake:442(607ns) DropOutgoingHandshake:457(393ns)} GenerateSessionVirtualId:988(347ns) PrepareNewSessionHandshake:477(884ns) {SwitchToState:179(436ns)}}}}}
2025-11-27T18:19:22.543466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471482525610213:2064] [node 4] ICP01 ready to work
2025-11-27T18:19:22.578774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471478230642851:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19937326ns) {Configure:104(19933648ns) {ProcessConfigured:135(49317ns) {StartInitialHandshake:157(47164ns) {DropHandshakes:470(963ns) {DropIncomingHandshake:442(141ns) DropOutgoingHandshake:457(114ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:477(945ns) {SwitchToState:179(260ns)}}}}}
2025-11-27T18:19:22.550208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471480191869235:2061] [node 5] ICP01 ready to work
2025-11-27T18:19:22.552741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471480191869238:2071] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.589663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471482481238393:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53302
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.611040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471481748605780:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42038
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.608382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471480393937359:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5446
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.619488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471480393937365:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5446
2025-11-27T18:19:22.628051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471482481238397:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53302
2025-11-27T18:19:22.625350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471475896901875:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(62730434ns) {Configure:104(62726857ns) {ProcessConfigured:135(13274779ns) {StartInitialHandshake:157(13271767ns) {DropHandshakes:470(1156ns) {DropIncomingHandshake:442(652ns) DropOutgoingHandshake:457(103ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:477(3741ns) {SwitchToState:179(1900ns)}}}}}
2025-11-27T18:19:22.626363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471480393937365:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.632737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471482525610225:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19213
2025-11-27T18:19:22.631237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471480248805384:2134] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(66201846ns) {Configure:104(66197995ns) {ProcessConfigured:135(4994234ns) {StartInitialHandshake:157(4991312ns) {DropHandshakes:470(2966ns) {DropIncomingHandshake:442(625ns) DropOutgoingHandshake:457(382ns)} GenerateSessionVirtualId:988(577ns) PrepareNewSessionHandshake:477(4070ns) {SwitchToState:179(2650ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.635784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471480191869245:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56325
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.636470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471481748605784:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42038
2025-11-27T18:19:22.637065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471480393937367:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5446
2025-11-27T18:19:22.637619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471482525610225:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.638059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471478230642851:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.645534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471480393937352:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.650132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471482481238397:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.645571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471480393937352:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.647125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471480393937352:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:22.647577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471480393937348:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.647633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:22.647685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471480393937348:2107] [node 5] ICP22 created new session: [1:7577471480393937369:2107]
2025-11-27T18:19:22.647747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471480393937369:2107] [node 5] ICS09 handshake done sender: [1:7577471480393937352:2110] self: [1:7577471480393937351:0] peer: [5:7577471482525610226:0] socket: 105 qp: -1
2025-11-27T18:19:22.647799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471480393937369:2107] [node 5] ICS10 traffic start
2025-11-27T18:19:22.647841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:22.642263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471480191869245:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.653966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471481748605780:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.654395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471481748605785:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42038
2025-11-27T18:19:22.653909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471480393937359:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.653434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471480191869246:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56325
2025-11-27T18:19:22.656770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471481748605785:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.657198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471480393937354:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.657229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471480393937354:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.657851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471480393937358:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.657870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471480393937358:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.656079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471478186271022:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:22.660686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471482481238393:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.657140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471480191869246:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.656888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471477453638408:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.670397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471480191869235:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:22.671469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471480248805396:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53304
2025-11-27T18:19:22.671951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471480248805397:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53304
2025-11-27T18:19:22.672956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471480248805398:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53304
2025-11-27T18:19:22.673113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471480248805399:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53304
2025-11-27T18:19:22.670831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471482525610227:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19213
2025-11-27T18:19:22.685863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471481748605784:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.685642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471480248805397:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.676075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471475896901876:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:22.696892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471480393937375:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5446
2025-11-27T18:19:22.697498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471482525610189:2087] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.697531Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471482525610189:2087] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.698399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471480393937354:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:22.698824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471480393937358:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:22.695675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471482481238400:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53302
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.702738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471480191869252:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56325
2025-11-27T18:19:22.700474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471480393937367:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.701318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471481748605771:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.712986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471482525610228:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19213
2025-11-27T18:19:22.700368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471482481238400:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.700529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471480393937349:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.685774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471480248805399:2145] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.699596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471480248805396:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.699687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471480248805398:2144] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.700002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471475953837997:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:22.725358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471482481238401:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53302
2025-11-27T18:19:22.701365Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471481748605771:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.700597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:22.700649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471480393937349:2108] [node 4] ICP22 created new session: [1:7577471480393937376:2108]
2025-11-27T18:19:22.700697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471480393937376:2108] [node 4] ICS09 handshake done sender: [1:7577471480393937354:2111] self: [1:7577471480393937353:0] peer: [4:7577471482481238398:0] socket: 106 qp: -1
2025-11-27T18:19:22.700735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471480393937376:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:22.700771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:22.710162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471480393937375:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.710279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471480393937356:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.710304Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471480393937356:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.711021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471476098969990:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.711088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:22.711146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471476098969990:2049] [node 3] ICP22 created new session: [1:7577471480393937378:2049]
2025-11-27T18:19:22.711195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471480393937378:2049] [node 3] ICS09 handshake done sender: [1:7577471480393937358:2113] self: [1:7577471480393937357:0] peer: [3:7577471481748605787:0] socket: 128 qp: -1
2025-11-27T18:19:22.711229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471480393937378:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:22.711264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:22.713154Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471482525610225:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:22.702359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471481748605789:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42038
2025-11-27T18:19:22.726230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471481748605771:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.737859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471482525610228:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.737931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471478230642851:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.738001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:22.738069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471478230642851:2048] [node 1] ICP22 created new session: [5:7577471482525610231:2048]
2025-11-27T18:19:22.738152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471482525610231:2048] [node 1] ICS09 handshake done sender: [5:7577471482525610225:2115] self: [5:7577471482525610226:0] peer: [1:7577471480393937351:0] socket: 119 qp: -1
2025-11-27T18:19:22.738192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471482525610231:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:22.738230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:22.741413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471482525610229:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19213
2025-11-27T18:19:22.745602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471480191869246:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:22.738487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471478186271023:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:22.746389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471481748605765:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.746472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:22.746547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471481748605765:2098] [node 6] ICP22 created new session: [3:7577471481748605791:2098]
2025-11-27T18:19:22.749838Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471478186271022:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17941) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471482481238397:2116] held: no
2025-11-27T18:19:22.750626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471482481238397:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.751002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471480248805391:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.751043Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471480248805391:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.752365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471480248805407:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53304
2025-11-27T18:19:22.752722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471480248805391:2139] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:22.746612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471481748605791:2098] [node 6] ICS09 handshake done sender: [3:7577471481748605771:2110] self: [3:7577471481748605770:0] peer: [6:7577471480191869251:0] socket: 109 qp: -1
2025-11-27T18:19:22.753075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471480248805384:2134] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.746655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471481748605791:2098] [node 6] ICS10 traffic start
2025-11-27T18:19:22.753145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:22.757363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471482525610229:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.757589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471482525610189:2087] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:22.753195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471480248805384:2134] [node 4] ICP22 created new session: [2:7577471480248805409:2134]
2025-11-27T18:19:22.753242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471480248805409:2134] [node 4] ICS09 handshake done sender: [2:7577471480248805391:2139] self: [2:7577471480248805390:0] peer: [4:7577471482481238402:0] socket: 137 qp: -1
2025-11-27T18:19:22.753293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471480248805409:2134] [node 4] ICS10 traffic start
2025-11-27T18:19:22.753350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:22.770036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471475896901876:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.770114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:22.770188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471475896901876:2050] [node 3] ICP22 created new session: [6:7577471480191869254:2050]
2025-11-27T18:19:22.770239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471480191869254:2050] [node 3] ICS09 handshake done sender: [6:7577471480191869246:2116] self: [6:7577471480191869251:0] peer: [3:7577471481748605770:0] socket: 118 qp: -1
2025-11-27T18:19:22.770285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471480191869254:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:22.770324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:22.773838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471480393937356:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:22.746691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:22.769511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471481748605785:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.776798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471476098969989:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.781128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471480191869253:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56325
2025-11-27T18:19:22.776452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471478186271022:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.774407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471482525610163:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.774506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:22.774582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471482525610163:2062] [node 6] ICP22 created new session: [5:7577471482525610235:2062]
2025-11-27T18:19:22.774706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471482525610235:2062] [node 6] ICS09 handshake done sender: [5:7577471482525610189:2087] self: [5:7577471482525610172:0] peer: [6:7577471480191869250:0] socket: 96 qp: -1
2025-11-27T18:19:22.775239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471482525610235:2062] [node 6] ICS10 traffic start
2025-11-27T18:19:22.775277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:22.776535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:22.776602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471478186271022:2048] [node 1] ICP22 created new session: [4:7577471482481238404:2048]
2025-11-27T18:19:22.776648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471482481238404:2048] [node 1] ICS09 handshake done sender: [4:7577471482481238397:2116] self: [4:7577471482481238398:0] peer: [1:7577471480393937353:0] socket: 113 qp: -1
2025-11-27T18:19:22.778157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471482481238404:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:22.778201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:22.783583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471480248805398:2144] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:22.778136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:22.781571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471476098969989:2048] [node 2] ICP22 created new session: [1:7577471480393937384:2048]
2025-11-27T18:19:22.781631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471480393937384:2048] [node 2] ICS09 handshake done sender: [1:7577471480393937356:2112] self: [1:7577471480393937355:0] peer: [2:7577471480248805406:0] socket: 107 qp: -1
2025-11-27T18:19:22.781664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471480393937384:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:22.781704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.784442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471477453638408:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17941) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471481748605785:2117] held: no
2025-11-27T18:19:22.785034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471477453638408:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.785076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:22.785121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471477453638408:2048] [node 1] ICP22 created new session: [3:7577471481748605794:2048]
2025-11-27T18:19:22.785188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471481748605794:2048] [node 1] ICS09 handshake done sender: [3:7577471481748605785:2117] self: [3:7577471481748605787:0] peer: [1:7577471480393937357:0] socket: 129 qp: -1
2025-11-27T18:19:22.785220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471481748605794:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:22.785247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:22.787814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471481748605793:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42038
2025-11-27T18:19:22.788653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471475953837997:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.794144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471478230642852:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.797215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471481748605796:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42038
2025-11-27T18:19:22.800502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471481748605793:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.788792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:22.788858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471475953837997:2048] [node 1] ICP22 created new session: [2:7577471480248805412:2048]
2025-11-27T18:19:22.788904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471480248805412:2048] [node 1] ICS09 handshake done sender: [2:7577471480248805398:2144] self: [2:7577471480248805406:0] peer: [1:7577471480393937355:0] socket: 126 qp: -1
2025-11-27T18:19:22.788932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471480248805412:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:22.788960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.810918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471478186271023:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14544) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471482481238400:2117] held: no
2025-11-27T18:19:22.811725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471482481238403:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53302
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.812299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471482525610238:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19213
2025-11-27T18:19:22.812442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471482525610239:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19213
2025-11-27T18:19:22.812034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471480248805389:2138] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.812077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471480248805389:2138] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.815213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471482525610238:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.815341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471478230642853:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.817144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471480191869256:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56325
2025-11-27T18:19:22.817287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471480191869245:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:22.816666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471482481238400:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:22.816399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471481748605779:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.816438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471481748605779:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.817921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471480248805389:2138] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.818209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471480191869235:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3403) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471480191869245:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.829074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471481748605779:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.838349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471480248805383:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.838419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:22.838469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471480248805383:2107] [node 5] ICP22 created new session: [2:7577471480248805417:2107]
2025-11-27T18:19:22.838516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471480248805417:2107] [node 5] ICS09 handshake done sender: [2:7577471480248805389:2138] self: [2:7577471480248805388:0] peer: [5:7577471482525610237:0] socket: 136 qp: -1
2025-11-27T18:19:22.838550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471480248805417:2107] [node 5] ICS10 traffic start
2025-11-27T18:19:22.838585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.847640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471480248805416:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53304
2025-11-27T18:19:22.818875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471480191869235:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.826508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471480393937361:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.826545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471480393937361:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.837518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471480393937361:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:22.841057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471480393937350:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.841112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:22.841162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471480393937350:2109] [node 6] ICP22 created new session: [1:7577471480393937387:2109]
2025-11-27T18:19:22.841203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471480393937387:2109] [node 6] ICS09 handshake done sender: [1:7577471480393937361:2115] self: [1:7577471480393937360:0] peer: [6:7577471480191869262:0] socket: 110 qp: -1
2025-11-27T18:19:22.841236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471480393937387:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:22.841279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.821167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471482525610241:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19213
2025-11-27T18:19:22.856700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471482481238407:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53302
2025-11-27T18:19:22.856999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471478186271023:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.857039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:22.857086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471478186271023:2049] [node 2] ICP22 created new session: [4:7577471482481238409:2049]
2025-11-27T18:19:22.857157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471482481238409:2049] [node 2] ICS09 handshake done sender: [4:7577471482481238400:2117] self: [4:7577471482481238402:0] peer: [2:7577471480248805390:0] socket: 138 qp: -1
2025-11-27T18:19:22.857187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471482481238409:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:22.857214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.862517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471482481238407:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.864179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471482481238411:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53302
2025-11-27T18:19:22.818919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:22.837922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471482525610242:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19213
2025-11-27T18:19:22.858002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471481748605796:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.818962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471480191869235:2061] [node 5] ICP22 created new session: [6:7577471480191869257:2061]
2025-11-27T18:19:22.819007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471480191869257:2061] [node 5] ICS09 handshake done sender: [6:7577471480191869245:2115] self: [6:7577471480191869250:0] peer: [5:7577471482525610172:0] socket: 117 qp: -1
2025-11-27T18:19:22.819041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471480191869257:2061] [node 5] ICS10 traffic start
2025-11-27T18:19:22.819086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:22.824976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471480191869256:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.826075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471480191869259:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56325
2025-11-27T18:19:22.826165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471475896901874:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:22.858059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471481748605769:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.858110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:22.858156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471481748605769:2102] [node 5] ICP22 created new session: [3:7577471481748605801:2102]
2025-11-27T18:19:22.858217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471481748605801:2102] [node 5] ICS09 handshake done sender: [3:7577471481748605779:2114] self: [3:7577471481748605778:0] peer: [5:7577471482525610240:0] socket: 102 qp: -1
2025-11-27T18:19:22.858255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471481748605801:2102] [node 5] ICS10 traffic start
2025-11-27T18:19:22.858282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:22.842846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471482525610229:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:22.876784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471481748605777:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.876822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471481748605777:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.843013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471478230642852:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.843069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:22.843116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471478230642852:2049] [node 2] ICP22 created new session: [5:7577471482525610243:2049]
2025-11-27T18:19:22.843156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471482525610243:2049] [node 2] ICS09 handshake done sender: [5:7577471482525610229:2118] self: [5:7577471482525610237:0] peer: [2:7577471480248805388:0] socket: 140 qp: -1
2025-11-27T18:19:22.843195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471482525610243:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:22.843239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:22.844787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471482525610238:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:22.854911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471478230642853:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.854992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:22.855044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471478230642853:2050] [node 3] ICP22 created new session: [5:7577471482525610245:2050]
2025-11-27T18:19:22.855089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471482525610245:2050] [node 3] ICS09 handshake done sender: [5:7577471482525610238:2119] self: [5:7577471482525610240:0] peer: [3:7577471481748605778:0] socket: 133 qp: -1
2025-11-27T18:19:22.855125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471482525610245:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:22.855162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:22.874300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471478186271024:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.880158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471480191869263:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56325
2025-11-27T18:19:22.880584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471480248805416:2147] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.882418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471477453638409:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:22.883098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471480248805393:2140] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.883130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471480248805393:2140] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.884215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471482481238390:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.884243Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471482481238390:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.884949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471482481238411:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.885402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471482481238413:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53302
2025-11-27T18:19:22.885791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471480248805393:2140] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:22.886057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471475953837998:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.886119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:22.886170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471475953837998:2049] [node 3] ICP22 created new session: [2:7577471480248805421:2049]
2025-11-27T18:19:22.886213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471480248805421:2049] [node 3] ICS09 handshake done sender: [2:7577471480248805393:2140] self: [2:7577471480248805392:0] peer: [3:7577471481748605803:0] socket: 120 qp: -1
2025-11-27T18:19:22.886246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471480248805421:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:22.886278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:22.887733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471481748605777:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:22.888051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471481748605768:2100] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.888108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:22.888156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471481748605768:2100] [node 4] ICP22 created new session: [3:7577471481748605806:2100]
2025-11-27T18:19:22.888196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471481748605806:2100] [node 4] ICS09 handshake done sender: [3:7577471481748605777:2113] self: [3:7577471481748605776:0] peer: [4:7577471482481238412:0] socket: 153 qp: -1
2025-11-27T18:19:22.888231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471481748605806:2100] [node 4] ICS10 traffic start
2025-11-27T18:19:22.888262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:22.881509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471475896901874:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17941) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471480191869256:2119] held: no
2025-11-27T18:19:22.882288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471480191869259:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.881595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471482525610239:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.881740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471482525610213:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.890093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471480191869256:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:22.890151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471482481238390:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.895636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471475896901874:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.895705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:22.895763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471475896901874:2048] [node 1] ICP22 created new session: [6:7577471480191869265:2048]
2025-11-27T18:19:22.895825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471480191869265:2048] [node 1] ICS09 handshake done sender: [6:7577471480191869256:2119] self: [6:7577471480191869262:0] peer: [1:7577471480393937360:0] socket: 152 qp: -1
2025-11-27T18:19:22.895869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471480191869265:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:22.895909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:22.896966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471480191869264:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56325
2025-11-27T18:19:22.897429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471475896901875:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.900037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471481748605805:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42038
2025-11-27T18:19:22.890301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471482525610248:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19213
2025-11-27T18:19:22.900480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471477453638409:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14544) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471481748605796:2120] held: no
2025-11-27T18:19:22.903113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471480248805395:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.903150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471480248805395:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.901942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471482525610239:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:22.903073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471482525610213:2064] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.902291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471481748605796:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:22.905859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471480191869268:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56325
2025-11-27T18:19:22.909802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471480191869259:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:22.911613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471475896901875:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.911668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:22.911719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471475896901875:2049] [node 2] ICP22 created new session: [6:7577471480191869269:2049]
2025-11-27T18:19:22.911759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471480191869269:2049] [node 2] ICS09 handshake done sender: [6:7577471480191869259:2120] self: [6:7577471480191869267:0] peer: [2:7577471480248805394:0] socket: 159 qp: -1
2025-11-27T18:19:22.911794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471480191869269:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:22.911823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:22.905876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:22.913795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471482525610213:2064] [node 4] ICP22 created new session: [5:7577471482525610249:2064]
2025-11-27T18:19:22.908763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471480248805395:2141] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:22.910122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471480248805385:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.910173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:22.910219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471480248805385:2112] [node 6] ICP22 created new session: [2:7577471480248805424:2112]
2025-11-27T18:19:22.910263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471480248805424:2112] [node 6] ICS09 handshake done sender: [2:7577471480248805395:2141] self: [2:7577471480248805394:0] peer: [6:7577471480191869267:0] socket: 156 qp: -1
2025-11-27T18:19:22.910303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471480248805424:2112] [node 6] ICS10 traffic start
2025-11-27T18:19:22.910337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:22.904612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471477453638409:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.918135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471482525610249:2064] [node 4] ICS09 handshake done sender: [5:7577471482525610239:2120] self: [5:7577471482525610247:0] peer: [4:7577471482481238389:0] socket: 141 qp: -1
2025-11-27T18:19:22.918600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471482525610249:2064] [node 4] ICS10 traffic start
2025-11-27T18:19:22.918666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:22.920052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471482525610213:2064] [node 4] ICP35 event processing took too much time PendingConnection:279(16161312ns) {HandleHandshakeStatus:329(16158374ns) {DropIncomingHandshake:442(497ns) DropOutgoingHandshake:457(3827ns) SwitchToState:179(483ns) ProcessPendingSessionEvents:495(457675ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 406209ns max# 49977ns}}}
2025-11-27T18:19:22.904666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:22.904715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471477453638409:2049] [node 2] ICP22 created new session: [3:7577471481748605808:2049]
2025-11-27T18:19:22.904755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471481748605808:2049] [node 2] ICS09 handshake done sender: [3:7577471481748605796:2120] self: [3:7577471481748605803:0] peer: [2:7577471480248805392:0] socket: 122 qp: -1
2025-11-27T18:19:22.904797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471481748605808:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:22.904834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:22.922333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471482481238374:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.922405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:22.922450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471482481238374:2088] [node 5] ICP22 created new session: [4:7577471482481238416:2088]
2025-11-27T18:19:22.922504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471482481238416:2088] [node 5] ICS09 handshake done sender: [4:7577471482481238390:2113] self: [4:7577471482481238389:0] peer: [5:7577471482525610247:0] socket: 135 qp: -1
2025-11-27T18:19:22.922537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471482481238416:2088] [node 5] ICS10 traffic start
2025-11-27T18:19:22.922571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:22.923626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471478186271024:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13988) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471482481238407:2120] held: no
2025-11-27T18:19:22.929253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471480191869264:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.929685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471480191869238:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:22.931933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471482481238407:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:22.933546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471482481238392:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:22.933585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471482481238392:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:22.934816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471478186271024:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.934864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:22.934916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471478186271024:2050] [node 3] ICP22 created new session: [4:7577471482481238418:2050]
2025-11-27T18:19:22.934981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471482481238418:2050] [node 3] ICS09 handshake done sender: [4:7577471482481238407:2120] self: [4:7577471482481238412:0] peer: [3:7577471481748605776:0] socket: 154 qp: -1
2025-11-27T18:19:22.935019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471482481238418:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:22.935063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:22.954279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471480191869272:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56325
2025-11-27T18:19:22.954592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471482481238392:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:22.955381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471480191869264:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:22.955654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471480191869238:2071] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:22.955720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:22.955775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471480191869238:2071] [node 4] ICP22 created new session: [6:7577471480191869273:2071]
2025-11-27T18:19:22.955825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471480191869273:2071] [node 4] ICS09 handshake done sender: [6:7577471480191869264:2122] self: [6:7577471480191869271:0] peer: [4:7577471482481238391:0] socket: 114 qp: -1
2025-11-27T18:19:22.955863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471480191869273:2071] [node 4] ICS10 traffic start
2025-11-27T18:19:22.955908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:22.958032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471482481238382:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:22.958103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:22.958148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471482481238382:2090] [node 6] ICP22 created new session: [4:7577471482481238421:2090]
2025-11-27T18:19:22.958205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471482481238421:2090] [node 6] ICS09 handshake done sender: [4:7577471482481238392:2114] self: [4:7577471482481238391:0] peer: [6:7577471480191869271:0] socket: 97 qp: -1
2025-11-27T18:19:22.958247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471482481238421:2090] [node 6] ICS10 traffic start
2025-11-27T18:19:22.958287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:22.985526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:22.985756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471480248805412:2048] [node 1] ICS01 socket: 126 reason# EndOfStream
2025-11-27T18:19:22.985801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471475953837997:2048] [node 1] ICP30 unregister session Session# [2:7577471480248805412:2048] VirtualId# [2:7577471480248805406:0]
2025-11-27T18:19:22.985838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471480248805412:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:22.985958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:22.986011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:22.990572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:22.990821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471481748605794:2048] [node 1] ICS01 socket: 129 reason# EndOfStream
2025-11-27T18:19:22.990861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471477453638408:2048] [node 1] ICP30 unregister session Session# [3:7577471481748605794:2048] VirtualId# [3:7577471481748605787:0]
2025-11-27T18:19:22.990889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471481748605794:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:22.990964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:22.991001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:22.997497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:22.997737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471482525610231:2048] [node 1] ICS07 socket disconnect 119 reason# EndOfStream
2025-11-27T18:19:22.997774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471482525610231:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:22.997852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:22.997892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471482525610231:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:22.998368Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:22.998523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471482481238404:2048] [node 1] ICS01 socket: 113 reason# EndOfStream
2025-11-27T18:19:22.998563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471478186271022:2048] [node 1] ICP30 unregister session Session# [4:7577471482481238404:2048] VirtualId# [4:7577471482481238398:0]
2025-11-27T18:19:22.998592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471482481238404:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:22.998652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:22.998692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:22.998782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:22.998947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471480191869265:2048] [node 1] ICS01 socket: 152 reason# EndOfStream
2025-11-27T18:19:22.998977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471475896901874:2048] [node 1] ICP30 unregister session Session# [6:7577471480191869265:2048] VirtualId# [6:7577471480191869262:0]
2025-11-27T18:19:22.999003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471480191869265:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:22.999047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:22.999078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:23.000364Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471478230642851:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17941) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:23.000427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471482525610231:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:23.000461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471478230642851:2048] [node 1] ICP30 unregister session Session# [5:7577471482525610231:2048] VirtualId# [5:7577471482525610226:0]
2025-11-27T18:19:23.000496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:23.000553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471478230642851:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17941) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:23.000582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17941) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:23.013436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471478230642851:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:23.072357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:23.072619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471481748605808:2049] [node 2] ICS01 socket: 122 reason# EndOfStream
2025-11-27T18:19:23.072668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471477453638409:2049] [node 2] ICP30 unregister session Session# [3:7577471481748605808:2049] VirtualId# [3:7577471481748605803:0]
2025-11-27T18:19:23.072713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471481748605808:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:23.072786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:23.072822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:23.084859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:23.085138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471482525610243:2049] [node 2] ICS01 socket: 140 reason# EndOfStream
2025-11-27T18:19:23.085182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471478230642852:2049] [node 2] ICP30 unregister session Session# [5:7577471482525610243:2049] VirtualId# [5:7577471482525610237:0]
2025-11-27T18:19:23.085220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471482525610243:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:23.085325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:23.085371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:23.085766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:23.085952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471482481238409:2049] [node 2] ICS01 socket: 138 reason# EndOfStream
2025-11-27T18:19:23.085987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471478186271023:2049] [node 2] ICP30 unregister session Session# [4:7577471482481238409:2049] VirtualId# [4:7577471482481238402:0]
2025-11-27T18:19:23.086016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471482481238409:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:23.086097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:23.086141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:23.086224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:23.086413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471480191869269:2049] [node 2] ICS01 socket: 159 reason# EndOfStream
2025-11-27T18:19:23.086446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471475896901875:2049] [node 2] ICP30 unregister session Session# [6:7577471480191869269:2049] VirtualId# [6:7577471480191869267:0]
2025-11-27T18:19:23.086472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471480191869269:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:23.086523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:23.086559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:23.129494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:23.129832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471482481238418:2050] [node 3] ICS01 socket: 154 reason# EndOfStream
2025-11-27T18:19:23.129872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471478186271024:2050] [node 3] ICP30 unregister session Session# [4:7577471482481238418:2050] VirtualId# [4:7577471482481238412:0]
2025-11-27T18:19:23.129902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471482481238418:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:23.129973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:23.130012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:23.129494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:23.129732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471480191869254:2050] [node 3] ICS01 socket: 118 reason# EndOfStream
2025-11-27T18:19:23.129769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471475896901876:2050] [node 3] ICP30 unregister session Session# [6:7577471480191869254:2050] VirtualId# [6:7577471480191869251:0]
2025-11-27T18:19:23.129807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471480191869254:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:23.129886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:23.129930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:23.130022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:23.130206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471482525610245:2050] [node 3] ICS07 socket disconnect 133 reason# EndOfStream
2025-11-27T18:19:23.130235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471482525610245:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:23.130283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:23.130317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471482525610245:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:23.131890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471478230642853:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13988) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:23.131941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471482525610245:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:23.131974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471478230642853:2050] [node 3] ICP30 unregister session Session# [5:7577471482525610245:2050] VirtualId# [5:7577471482525610240:0]
2025-11-27T18:19:23.132009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:23.132044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471478230642853:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13988) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:23.132059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13988) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:23.142910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471478230642853:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:23.165452Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:23.165457Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:23.165695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471482525610249:2064] [node 4] ICS07 socket disconnect 141 reason# EndOfStream
2025-11-27T18:19:23.165727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471482525610249:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:23.165818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:23.165866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471482525610249:2064] [node 4] ICS15 start handshake
2025-11-27T18:19:23.165686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471480191869273:2071] [node 4] ICS01 socket: 114 reason# EndOfStream
2025-11-27T18:19:23.165716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471480191869238:2071] [node 4] ICP30 unregister session Session# [6:7577471480191869273:2071] VirtualId# [6:7577471480191869271:0]
2025-11-27T18:19:23.165741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471480191869273:2071] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:23.165835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:23.165882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:23.174056Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471482525610213:2064] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14032) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:23.174117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471482525610249:2064] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:23.174160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471482525610213:2064] [node 4] ICP30 unregister session Session# [5:7577471482525610249:2064] VirtualId# [5:7577471482525610247:0]
2025-11-27T18:19:23.174262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:23.174301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471482525610213:2064] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14032) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:23.174316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14032) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:23.183618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471482525610213:2064] [node 4] ICP33 wake up from error state
2025-11-27T18:19:23.234800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:23.235051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471480191869257:2061] [node 5] ICS01 socket: 117 reason# EndOfStream
2025-11-27T18:19:23.235098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471480191869235:2061] [node 5] ICP30 unregister session Session# [6:7577471480191869257:2061] VirtualId# [6:7577471480191869250:0]
2025-11-27T18:19:23.235138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471480191869257:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:23.235214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:23.235265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:23.334005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471488118888968:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:23.334064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471488118888969:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:23.365908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471484615162481:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:23.366015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471484615162482:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:23.390197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471485566305990:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:23.390233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471485566305991:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:23.419982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471486708285247:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:23.420039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471486708285248:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:23.420492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471486708285249:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:23.447807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471484491005015:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:23.447859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471484491005017:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:23.448118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471484491005016:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:23.477609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471488313171566:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:23.477652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471488313171567:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:23.477878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471488313171568:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:23.962192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471488313171622:2102] [node 4] ICP01 ready to work
2025-11-27T18:19:23.965388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471488313171625:2104] [node 5] ICP01 ready to work
2025-11-27T18:19:24.124448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471492413856330:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:24.193706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471491003252607:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:24.170851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471488910129840:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:24.182448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471489861273349:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:24.194128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471488785972392:2127] [node 4] ICP01 ready to work
2025-11-27T18:19:24.153892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471492413856329:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:24.197567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471492413856331:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:24.217640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471489861273350:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:24.191297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471484615162481:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(201008810ns) {Configure:104(200999429ns) {ProcessConfigured:135(111818ns) {StartInitialHandshake:157(108615ns) {DropHandshakes:470(2665ns) {DropIncomingHandshake:442(592ns) DropOutgoingHandshake:457(396ns)} GenerateSessionVirtualId:988(1032ns) PrepareNewSessionHandshake:477(1996ns) {SwitchToState:179(578ns)}}}}}
2025-11-27T18:19:24.217834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471491003252608:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:24.197962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471488785972393:2108] [node 6] ICP01 ready to work
2025-11-27T18:19:24.191724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471488910129843:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:24.198261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471488910129844:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:24.241776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471489861273352:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:24.258953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471485566305990:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(226132534ns) {Configure:104(226126825ns) {ProcessConfigured:135(12639837ns) {StartInitialHandshake:157(12636030ns) {DropHandshakes:470(1859ns) {DropIncomingHandshake:442(519ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(686ns) PrepareNewSessionHandshake:477(4514ns) {SwitchToState:179(2372ns)}}}}}
2025-11-27T18:19:24.277872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471488313171568:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(218608628ns) {Configure:104(202221038ns) {ProcessConfigured:135(60485ns) {StartInitialHandshake:157(54498ns) {DropHandshakes:470(2185ns) {DropIncomingHandshake:442(403ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(834ns) PrepareNewSessionHandshake:477(1249ns) {SwitchToState:179(736ns)}}}}}
2025-11-27T18:19:24.258453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471484615162482:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(58292924ns) {Configure:104(58288331ns) {ProcessConfigured:135(4748881ns) {StartInitialHandshake:157(4745604ns) {DropHandshakes:470(2691ns) {DropIncomingHandshake:442(708ns) DropOutgoingHandshake:457(351ns)} GenerateSessionVirtualId:988(1172ns) PrepareNewSessionHandshake:477(4288ns) {SwitchToState:179(2162ns)}}}}}
2025-11-27T18:19:24.289564Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471488785972393:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(61538020ns) {Configure:104(61534872ns) {ProcessConfigured:135(20188023ns) {StartInitialHandshake:157(20185296ns) {DropHandshakes:470(1320ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(328ns)} GenerateSessionVirtualId:988(676ns) PrepareNewSessionHandshake:477(1954ns) {SwitchToState:179(716ns)}}}}}
2025-11-27T18:19:24.278237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471486708285248:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41166735ns) {Configure:104(41163942ns) {ProcessConfigured:135(14290525ns) {StartInitialHandshake:157(14288705ns) {DropHandshakes:470(1889ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(842ns)} GenerateSessionVirtualId:988(540ns) PrepareNewSessionHandshake:477(3716ns) {SwitchToState:179(1829ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.310282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471488910129843:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(39607398ns) {Configure:104(39602045ns) {ProcessConfigured:135(75427ns) {StartInitialHandshake:157(73392ns) {DropHandshakes:470(2958ns) {DropIncomingHandshake:442(738ns) DropOutgoingHandshake:457(489ns)} GenerateSessionVirtualId:988(844ns) PrepareNewSessionHandshake:477(2331ns) {SwitchToState:179(927ns)}}}}}
2025-11-27T18:19:24.315658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471488785972403:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59933
2025-11-27T18:19:24.327748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471488785972404:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59933
2025-11-27T18:19:24.305688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471489861273365:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1350
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.352567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471488785972403:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.355902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471489861273366:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1350
2025-11-27T18:19:24.361375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471486708285308:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16948
2025-11-27T18:19:24.374225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471492413856331:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(104784539ns) {Configure:104(104780080ns) {ProcessConfigured:135(52864ns) {StartInitialHandshake:157(51173ns) {DropHandshakes:470(2550ns) {DropIncomingHandshake:442(653ns) DropOutgoingHandshake:457(365ns)} GenerateSessionVirtualId:988(713ns) PrepareNewSessionHandshake:477(2115ns) {SwitchToState:179(756ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.370308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471489861273366:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.358726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471488910129840:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(45686377ns) {Configure:104(45682194ns) {ProcessConfigured:135(64663ns) {StartInitialHandshake:157(62278ns) {DropHandshakes:470(3187ns) {DropIncomingHandshake:442(702ns) DropOutgoingHandshake:457(491ns)} GenerateSessionVirtualId:988(1068ns) PrepareNewSessionHandshake:477(2004ns) {SwitchToState:179(855ns)}}}}}
2025-11-27T18:19:24.365904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471492608138938:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36948
2025-11-27T18:19:24.393637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471489861273367:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1350
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.409794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471492608138938:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.409940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471488313171567:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.393342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471488785972404:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.359447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471488910129851:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-27T18:19:24.408855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471489861273369:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1350
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.405378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471484491005017:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:24.423179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471492413856343:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16461
2025-11-27T18:19:24.417550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471489861273367:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.359505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471488910129852:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-27T18:19:24.410327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471488910129848:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.410355Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471488910129848:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.412737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471488910129852:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.423997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471488910129848:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:24.437562Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471488313171566:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(149431416ns) {Configure:104(149425100ns) {ProcessConfigured:135(14580188ns) {StartInitialHandshake:157(14577227ns) {DropHandshakes:470(653134ns) {DropIncomingHandshake:442(2191ns) DropOutgoingHandshake:457(345ns)} GenerateSessionVirtualId:988(1168ns) PrepareNewSessionHandshake:477(3174ns) {SwitchToState:179(1824ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.431361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471492413856344:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16461
2025-11-27T18:19:24.434152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471492413856343:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.438096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471488785972411:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59933
2025-11-27T18:19:24.438367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471491003252607:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(78874888ns) {Configure:104(78870680ns) {ProcessConfigured:135(9743525ns) {StartInitialHandshake:157(9741123ns) {DropHandshakes:470(1496ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(558ns) PrepareNewSessionHandshake:477(4234ns) {SwitchToState:179(2175ns)}}}}}
2025-11-27T18:19:24.423247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471489861273359:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.446338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471488785972404:2133] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.446650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471486708285308:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.457968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471484491005017:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.457439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471488910129851:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.423287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471489861273359:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.458048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:24.449758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471491003252613:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16948
2025-11-27T18:19:24.457678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471492413856344:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.458720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471484491005017:2050] [node 3] ICP22 created new session: [5:7577471488785972412:2050]
2025-11-27T18:19:24.451421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471491003252621:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16948
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.426818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471485566305991:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:24.457546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471488910129843:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.462348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471488785972412:2050] [node 3] ICS09 handshake done sender: [5:7577471488785972404:2133] self: [5:7577471488785972410:0] peer: [3:7577471489861273358:0] socket: 111 qp: -1
2025-11-27T18:19:24.462479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471491003252621:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.431209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471489861273359:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:24.433942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471489861273365:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.464741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471492413856346:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16461
2025-11-27T18:19:24.463719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471488785972412:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:24.465425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471488118889029:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.458202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471488910129846:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.462302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471489861273350:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.463751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471491003252613:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.458241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471488910129846:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.464415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:24.465472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471488118889029:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.462378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:24.462393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471489861273369:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.462422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471489861273350:2108] [node 5] ICP22 created new session: [3:7577471489861273374:2108]
2025-11-27T18:19:24.464662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471489861273374:2108] [node 5] ICS09 handshake done sender: [3:7577471489861273359:2113] self: [3:7577471489861273358:0] peer: [5:7577471488785972410:0] socket: 109 qp: -1
2025-11-27T18:19:24.465571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471489861273374:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:24.467187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:24.458571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:24.469303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471491003252614:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16948
2025-11-27T18:19:24.469361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471488118889029:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:24.470269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471486708285247:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:24.470346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471486708285249:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:24.467832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471492608138942:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36948
2025-11-27T18:19:24.460294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471488910129843:2109] [node 6] ICP22 created new session: [2:7577471488910129860:2109]
2025-11-27T18:19:24.483474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471491003252614:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.462419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471488910129860:2109] [node 6] ICS09 handshake done sender: [2:7577471488910129848:2112] self: [2:7577471488910129847:0] peer: [6:7577471492608138939:0] socket: 127 qp: -1
2025-11-27T18:19:24.463248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471484615162481:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:24.463451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471488910129861:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-27T18:19:24.463574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471488910129862:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-27T18:19:24.463775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471488910129860:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:24.485476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471492608138938:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.463987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471488910129846:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:24.464468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.489012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471492413856346:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.490350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471488313171567:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.465287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471484615162482:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.495103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471491003252626:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16948
2025-11-27T18:19:24.495143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471492413856334:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.494052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:24.495143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471489861273357:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.465336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:24.495177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471492413856334:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.465392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471484615162482:2049] [node 3] ICP22 created new session: [2:7577471488910129867:2049]
2025-11-27T18:19:24.494978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471488313171567:2049] [node 2] ICP22 created new session: [6:7577471492608138943:2049]
2025-11-27T18:19:24.495192Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471489861273357:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.465640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471488910129867:2049] [node 3] ICS09 handshake done sender: [2:7577471488910129846:2111] self: [2:7577471488910129845:0] peer: [3:7577471489861273370:0] socket: 126 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.495319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471492608138943:2049] [node 2] ICS09 handshake done sender: [6:7577471492608138938:2114] self: [6:7577471492608138939:0] peer: [2:7577471488910129847:0] socket: 130 qp: -1
2025-11-27T18:19:24.466492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471488910129867:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:24.504831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471491003252626:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.502015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471488118888968:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.466537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:24.502555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:24.506261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471488785972415:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59933
2025-11-27T18:19:24.502765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471488118888968:2048] [node 2] ICP22 created new session: [1:7577471492413856352:2048]
2025-11-27T18:19:24.482630Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471488910129843:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(24256192ns) {HandleHandshakeStatus:329(24252010ns) {DropIncomingHandshake:442(594ns) DropOutgoingHandshake:457(17730ns) SwitchToState:179(639ns) ProcessPendingSessionEvents:495(16229740ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16158266ns max# 7033434ns}}}
2025-11-27T18:19:24.485063Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471484615162482:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(19039936ns) {HandleHandshakeStatus:329(19036678ns) {DropIncomingHandshake:442(10535ns) DropOutgoingHandshake:457(312ns) SwitchToState:179(298ns) ProcessPendingSessionEvents:495(16112676ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16046738ns max# 7129974ns}}}
2025-11-27T18:19:24.505283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471492413856352:2048] [node 2] ICS09 handshake done sender: [1:7577471488118889029:2107] self: [1:7577471488118889028:0] peer: [2:7577471488910129865:0] socket: 106 qp: -1
2025-11-27T18:19:24.495578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471492608138943:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:24.495618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:24.505326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471492413856352:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:24.505087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471486708285248:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:24.485206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471488910129861:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.488625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471488910129869:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6491
2025-11-27T18:19:24.506204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.493949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471488910129862:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.495173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471488910129851:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:24.505368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471484615162481:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.509335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471489861273373:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1350
2025-11-27T18:19:24.508941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471492608138946:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36948
2025-11-27T18:19:24.509709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471485566305990:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.509935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471492413856350:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16461
2025-11-27T18:19:24.510161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471492413856334:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.510454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471492608138947:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36948
2025-11-27T18:19:24.510648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471484491005017:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(49132836ns) {HandleHandshakeStatus:329(49130428ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(4654ns) SwitchToState:179(336ns) ProcessPendingSessionEvents:495(39450643ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39323762ns max# 7191642ns}}}
2025-11-27T18:19:24.505415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.510786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471489861273350:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(46811272ns) {HandleHandshakeStatus:329(44818597ns) {DropIncomingHandshake:442(15776ns) DropOutgoingHandshake:457(337ns) SwitchToState:179(688ns) ProcessPendingSessionEvents:495(37160633ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 37053907ns max# 7678403ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.510948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471489861273373:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.511365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471485566305991:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471489861273367:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.506506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471484615162481:2048] [node 1] ICP22 created new session: [2:7577471488910129872:2048]
2025-11-27T18:19:24.506563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471488910129872:2048] [node 1] ICS09 handshake done sender: [2:7577471488910129851:2114] self: [2:7577471488910129865:0] peer: [1:7577471488118889028:0] socket: 117 qp: -1
2025-11-27T18:19:24.506892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471488910129872:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:24.511743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471491003252631:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16948
2025-11-27T18:19:24.507082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:24.511821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471491003252632:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16948
2025-11-27T18:19:24.511919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471489861273377:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1350
2025-11-27T18:19:24.512782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471492413856350:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.513025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471491003252621:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:24.514846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471492413856336:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.514880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471492413856336:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.515134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471486708285247:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.514969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471492413856330:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.515724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.515645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471488910129854:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.515036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:24.515157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471492413856330:2108] [node 4] ICP22 created new session: [1:7577471492413856355:2108]
2025-11-27T18:19:24.522165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471492413856355:2108] [node 4] ICS09 handshake done sender: [1:7577471492413856334:2111] self: [1:7577471492413856333:0] peer: [4:7577471491003252627:0] socket: 133 qp: -1
2025-11-27T18:19:24.515781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471486708285247:2048] [node 1] ICP22 created new session: [4:7577471491003252633:2048]
2025-11-27T18:19:24.522554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471491003252633:2048] [node 1] ICS09 handshake done sender: [4:7577471491003252621:2118] self: [4:7577471491003252627:0] peer: [1:7577471492413856333:0] socket: 140 qp: -1
2025-11-27T18:19:24.530751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471489861273357:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:24.533854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471491003252633:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:24.515685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471488910129854:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.534945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471492608138947:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.535117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471488313171566:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:24.522608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471492413856355:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:24.525466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:24.529130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471492413856357:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16461
2025-11-27T18:19:24.531214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471492413856336:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.533949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:24.535495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471488785972415:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.544368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471486708285247:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(28472002ns) {HandleHandshakeStatus:329(28468941ns) {DropIncomingHandshake:442(788ns) DropOutgoingHandshake:457(4480ns) SwitchToState:179(461ns) ProcessPendingSessionEvents:495(9322015ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9251347ns max# 4628942ns}}}
2025-11-27T18:19:24.537428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471488785972416:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59933
2025-11-27T18:19:24.537498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471484491005016:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:24.542146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471488910129854:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:24.557180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471488910129850:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.557222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471488910129850:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.557811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471488910129840:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.557868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:24.557935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471488910129840:2107] [node 4] ICP22 created new session: [2:7577471488910129876:2107]
2025-11-27T18:19:24.557987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471488910129876:2107] [node 4] ICS09 handshake done sender: [2:7577471488910129854:2116] self: [2:7577471488910129853:0] peer: [4:7577471491003252630:0] socket: 150 qp: -1
2025-11-27T18:19:24.559333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471488910129876:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:24.559387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:24.542150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471488118888969:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.542657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:24.542725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471488118888969:2049] [node 3] ICP22 created new session: [1:7577471492413856361:2049]
2025-11-27T18:19:24.556839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471489861273367:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.558606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471489861273349:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.558645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:24.558683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471489861273349:2107] [node 4] ICP22 created new session: [3:7577471489861273382:2107]
2025-11-27T18:19:24.558731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471489861273382:2107] [node 4] ICS09 handshake done sender: [3:7577471489861273357:2112] self: [3:7577471489861273356:0] peer: [4:7577471491003252628:0] socket: 108 qp: -1
2025-11-27T18:19:24.558769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471489861273382:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:24.558802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:24.542788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471492413856361:2049] [node 3] ICS09 handshake done sender: [1:7577471492413856336:2112] self: [1:7577471492413856335:0] peer: [3:7577471489861273379:0] socket: 134 qp: -1
2025-11-27T18:19:24.542823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471492413856361:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:24.571629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471488910129850:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:24.563430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471492608138949:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36948
2025-11-27T18:19:24.566453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471492608138947:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:24.574049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471488313171566:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.545714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471492413856338:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.579929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471485566305991:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.579991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:24.580049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471485566305991:2049] [node 2] ICP22 created new session: [3:7577471489861273385:2049]
2025-11-27T18:19:24.580098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471489861273385:2049] [node 2] ICS09 handshake done sender: [3:7577471489861273367:2117] self: [3:7577471489861273370:0] peer: [2:7577471488910129845:0] socket: 129 qp: -1
2025-11-27T18:19:24.580135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471489861273385:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:24.580167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.573993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471488910129844:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.545737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471492413856338:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.545870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:24.562946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471492413856338:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:24.589927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471488785972418:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59933
2025-11-27T18:19:24.578930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:24.584975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471488313171566:2048] [node 1] ICP22 created new session: [6:7577471492608138950:2048]
2025-11-27T18:19:24.589856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471492608138950:2048] [node 1] ICS09 handshake done sender: [6:7577471492608138947:2118] self: [6:7577471492608138948:0] peer: [1:7577471492413856337:0] socket: 160 qp: -1
2025-11-27T18:19:24.590451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471492608138950:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:24.590522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:24.579072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:24.582517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471492413856331:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.589678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471488910129844:2108] [node 5] ICP22 created new session: [2:7577471488910129879:2108]
2025-11-27T18:19:24.596141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471488910129879:2108] [node 5] ICS09 handshake done sender: [2:7577471488910129850:2113] self: [2:7577471488910129849:0] peer: [5:7577471488785972417:0] socket: 118 qp: -1
2025-11-27T18:19:24.596956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471488910129879:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:24.597035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:24.597687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471492608138946:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.582600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:24.595209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471488785972416:2137] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.602336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471488313171568:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.620171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471489861273361:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.620212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471489861273361:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.621691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471489861273384:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1350
2025-11-27T18:19:24.606006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471491003252613:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:24.601396Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471484491005016:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471488785972415:2136] held: no
2025-11-27T18:19:24.602170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471488785972415:2136] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:24.582654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471492413856331:2110] [node 6] ICP22 created new session: [1:7577471492413856364:2110]
2025-11-27T18:19:24.582708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471492413856364:2110] [node 6] ICS09 handshake done sender: [1:7577471492413856338:2113] self: [1:7577471492413856337:0] peer: [6:7577471492608138948:0] socket: 159 qp: -1
2025-11-27T18:19:24.616482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471488313171566:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(41562392ns) {HandleHandshakeStatus:329(41558497ns) {DropIncomingHandshake:442(682ns) DropOutgoingHandshake:457(5266ns) SwitchToState:179(489ns) ProcessPendingSessionEvents:495(9048586ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8982020ns max# 8540702ns}}}
2025-11-27T18:19:24.582745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471492413856364:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:24.582777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:24.632962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471484491005016:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.633047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:24.633109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471484491005016:2049] [node 2] ICP22 created new session: [5:7577471488785972419:2049]
2025-11-27T18:19:24.633166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471488785972419:2049] [node 2] ICS09 handshake done sender: [5:7577471488785972415:2136] self: [5:7577471488785972417:0] peer: [2:7577471488910129849:0] socket: 154 qp: -1
2025-11-27T18:19:24.633210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471488785972419:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:24.633249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:24.633641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471484491005015:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:24.630853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471488910129844:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(33159077ns) {HandleHandshakeStatus:329(33155123ns) {DropIncomingHandshake:442(15933ns) DropOutgoingHandshake:457(452ns) SwitchToState:179(495ns) ProcessPendingSessionEvents:495(9902638ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9829403ns max# 8434446ns}}}
2025-11-27T18:19:24.641761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471486708285249:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.641830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.641890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471486708285249:2050] [node 3] ICP22 created new session: [4:7577471491003252636:2050]
2025-11-27T18:19:24.661348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471492413856340:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.669345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471489861273369:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:24.685711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471489861273349:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(39463906ns) {HandleHandshakeStatus:329(39459621ns) {DropIncomingHandshake:442(5386ns) DropOutgoingHandshake:457(240ns) SwitchToState:179(126ns) ProcessPendingSessionEvents:495(39133445ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39001071ns max# 7306726ns}}}
2025-11-27T18:19:24.641939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471491003252636:2050] [node 3] ICS09 handshake done sender: [4:7577471491003252613:2113] self: [4:7577471491003252628:0] peer: [3:7577471489861273356:0] socket: 121 qp: -1
2025-11-27T18:19:24.685855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471485566305990:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.685911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:24.685968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471485566305990:2048] [node 1] ICP22 created new session: [3:7577471489861273388:2048]
2025-11-27T18:19:24.686015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471489861273388:2048] [node 1] ICS09 handshake done sender: [3:7577471489861273369:2118] self: [3:7577471489861273379:0] peer: [1:7577471492413856335:0] socket: 135 qp: -1
2025-11-27T18:19:24.686056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471489861273388:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:24.686094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:24.688537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471489861273361:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:24.661412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471492413856340:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.697387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471492608138953:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36948
2025-11-27T18:19:24.697681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471492608138955:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36948
2025-11-27T18:19:24.698590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471489861273352:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.641981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471491003252636:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:24.698674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:24.695943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471488118888969:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(76864748ns) {HandleHandshakeStatus:329(76861166ns) {DropIncomingHandshake:442(1457ns) DropOutgoingHandshake:457(1148ns) SwitchToState:179(478ns) ProcessPendingSessionEvents:495(60427757ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60325531ns max# 9669077ns}}}
2025-11-27T18:19:24.698093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471492608138946:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:24.698733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471489861273352:2109] [node 6] ICP22 created new session: [3:7577471489861273390:2109]
2025-11-27T18:19:24.698795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471489861273390:2109] [node 6] ICS09 handshake done sender: [3:7577471489861273361:2114] self: [3:7577471489861273360:0] peer: [6:7577471492608138952:0] socket: 157 qp: -1
2025-11-27T18:19:24.698842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471489861273390:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:24.642016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:24.698887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:24.706577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471488313171568:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.707314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:24.707370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471488313171568:2050] [node 3] ICP22 created new session: [6:7577471492608138956:2050]
2025-11-27T18:19:24.707418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471492608138956:2050] [node 3] ICS09 handshake done sender: [6:7577471492608138946:2117] self: [6:7577471492608138952:0] peer: [3:7577471489861273360:0] socket: 158 qp: -1
2025-11-27T18:19:24.707454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471492608138956:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:24.642710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471486708285249:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.707497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.643334Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471486708285248:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471491003252626:2119] held: no
2025-11-27T18:19:24.714674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471492608138953:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.719275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471492413856340:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:24.717797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471492608138954:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36948
2025-11-27T18:19:24.719832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471488785972408:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.643978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471491003252635:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16948
2025-11-27T18:19:24.650429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471491003252626:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:24.719868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471488785972408:2134] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.718094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471488313171625:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.661780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471486708285248:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.719970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471488785972423:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59933
2025-11-27T18:19:24.718749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471492608138954:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.723456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471488785972423:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.719019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471488313171622:2102] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:24.661843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:24.723542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471488785972424:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59933
2025-11-27T18:19:24.662051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471486708285248:2049] [node 2] ICP22 created new session: [4:7577471491003252640:2049]
2025-11-27T18:19:24.662099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471491003252640:2049] [node 2] ICS09 handshake done sender: [4:7577471491003252626:2119] self: [4:7577471491003252630:0] peer: [2:7577471488910129853:0] socket: 151 qp: -1
No RDMA context found for GID: 2025-11-27T18:19:24.724446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471488785972416:2137] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:19:24.662134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471491003252640:2049] [node 2] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:19:24.662166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:24.724732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471488785972392:2127] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:24.720658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471491003252620:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.726164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471492608138960:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36948
2025-11-27T18:19:24.725903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471492413856329:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.720681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471491003252620:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.725838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471488785972408:2134] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:24.722607Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471491003252620:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:24.726173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471484491005015:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.726380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:24.726339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471492608138961:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36948
2025-11-27T18:19:24.726225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:24.726660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471492413856329:2109] [node 5] ICP22 created new session: [1:7577471492413856367:2109]
2025-11-27T18:19:24.726486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471484491005015:2048] [node 1] ICP22 created new session: [5:7577471488785972427:2048]
2025-11-27T18:19:24.723309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471491003252608:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.723399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:24.726862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471492413856367:2109] [node 5] ICS09 handshake done sender: [1:7577471492413856340:2114] self: [1:7577471492413856339:0] peer: [5:7577471488785972421:0] socket: 165 qp: -1
2025-11-27T18:19:24.726609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471488785972427:2048] [node 1] ICS09 handshake done sender: [5:7577471488785972416:2137] self: [5:7577471488785972421:0] peer: [1:7577471492413856339:0] socket: 166 qp: -1
2025-11-27T18:19:24.723449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471491003252608:2110] [node 6] ICP22 created new session: [4:7577471491003252643:2110]
2025-11-27T18:19:24.727797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471492413856367:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:24.726835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471488785972427:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:24.723500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471491003252643:2110] [node 6] ICS09 handshake done sender: [4:7577471491003252620:2117] self: [4:7577471491003252619:0] peer: [6:7577471492608138959:0] socket: 148 qp: -1
2025-11-27T18:19:24.727717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:24.723536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471491003252643:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:24.723568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:24.727967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:24.728089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471491003252618:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:24.732320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471488313171622:2102] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471492608138954:2121] held: no
2025-11-27T18:19:24.733498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471492608138954:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:24.737298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471488118888969:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:24.728117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471491003252618:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:24.733935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471492608138953:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:24.732770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471491003252618:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:24.735161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471488785972393:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.738434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471488313171622:2102] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.735239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:24.735286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471488785972393:2108] [node 6] ICP22 created new session: [5:7577471488785972429:2108]
2025-11-27T18:19:24.735285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471491003252607:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:24.738475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:24.735363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471488785972429:2108] [node 6] ICS09 handshake done sender: [5:7577471488785972408:2134] self: [5:7577471488785972400:0] peer: [6:7577471492608138958:0] socket: 113 qp: -1
2025-11-27T18:19:24.738513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471488313171622:2102] [node 4] ICP22 created new session: [6:7577471492608138962:2102]
2025-11-27T18:19:24.737006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:24.739180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471492608138962:2102] [node 4] ICS09 handshake done sender: [6:7577471492608138954:2121] self: [6:7577471492608138959:0] peer: [4:7577471491003252619:0] socket: 155 qp: -1
2025-11-27T18:19:24.737768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471491003252607:2109] [node 5] ICP22 created new session: [4:7577471491003252646:2109]
2025-11-27T18:19:24.739490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471492608138962:2102] [node 4] ICS10 traffic start
2025-11-27T18:19:24.735409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471488785972429:2108] [node 6] ICS10 traffic start
2025-11-27T18:19:24.739622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:24.754739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471488313171625:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.757556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:24.738575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471491003252646:2109] [node 5] ICS09 handshake done sender: [4:7577471491003252618:2116] self: [4:7577471491003252617:0] peer: [5:7577471488785972426:0] socket: 146 qp: -1
2025-11-27T18:19:24.758721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471488313171625:2104] [node 5] ICP22 created new session: [6:7577471492608138964:2104]
2025-11-27T18:19:24.759013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471492608138964:2104] [node 5] ICS09 handshake done sender: [6:7577471492608138953:2120] self: [6:7577471492608138958:0] peer: [5:7577471488785972400:0] socket: 142 qp: -1
2025-11-27T18:19:24.738887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471491003252646:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:24.739222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:24.762699Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471491003252607:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(26505505ns) {HandleHandshakeStatus:329(26502290ns) {DropIncomingHandshake:442(13836ns) DropOutgoingHandshake:457(238ns) SwitchToState:179(904ns) ProcessPendingSessionEvents:495(8378195ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8293633ns max# 5273978ns}}}
2025-11-27T18:19:24.735445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:24.759043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471488785972392:2127] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471488785972423:2139] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:24.759057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471492608138964:2104] [node 5] ICS10 traffic start
2025-11-27T18:19:24.764799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471488785972431:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59933
2025-11-27T18:19:24.759177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:24.771405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471488785972423:2139] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:24.771560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471488785972392:2127] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:24.771754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:24.771865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471488785972392:2127] [node 4] ICP22 created new session: [5:7577471488785972432:2127]
2025-11-27T18:19:24.771920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471488785972432:2127] [node 4] ICS09 handshake done sender: [5:7577471488785972423:2139] self: [5:7577471488785972426:0] peer: [4:7577471491003252617:0] socket: 156 qp: -1
2025-11-27T18:19:24.771969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471488785972432:2127] [node 4] ICS10 traffic start
2025-11-27T18:19:24.772020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:24.772578Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471488313171625:2104] [node 5] ICP35 event processing took too much time PendingConnection:279(16522061ns) {HandleHandshakeStatus:329(16518727ns) {DropIncomingHandshake:442(1178ns) DropOutgoingHandshake:457(11248ns) SwitchToState:179(783ns) ProcessPendingSessionEvents:495(7658709ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7570913ns max# 4215081ns}}}
2025-11-27T18:19:24.772670Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471488313171625:2104] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:24.819817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:24.820067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471488910129872:2048] [node 1] ICS07 socket disconnect 117 reason# EndOfStream
2025-11-27T18:19:24.820104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471488910129872:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:24.820208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:24.820253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471488910129872:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:24.821974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:24.824196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471489861273388:2048] [node 1] ICS01 socket: 135 reason# EndOfStream
2025-11-27T18:19:24.824253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471485566305990:2048] [node 1] ICP30 unregister session Session# [3:7577471489861273388:2048] VirtualId# [3:7577471489861273379:0]
2025-11-27T18:19:24.824290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471489861273388:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:24.824371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:24.824418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:24.825064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471484615162481:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19776) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:24.825277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471488910129872:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:24.826256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471484615162481:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19776) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:24.830189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471488910129872:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:24.830244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471484615162481:2048] [node 1] ICP30 unregister session Session# [2:7577471488910129872:2048] VirtualId# [2:7577471488910129865:0]
2025-11-27T18:19:24.830372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:24.830413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471484615162481:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:24.830430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:24.831472Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:24.833570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471491003252633:2048] [node 1] ICS07 socket disconnect 140 reason# EndOfStream
2025-11-27T18:19:24.833617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471491003252633:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:24.833699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:24.833742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471491003252633:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:24.841689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471484615162481:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:24.834834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471486708285247:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19776) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:24.834886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471491003252633:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:24.835302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:24.836700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471492608138950:2048] [node 1] ICS07 socket disconnect 160 reason# EndOfStream
2025-11-27T18:19:24.836736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471492608138950:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:24.836820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:24.836868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471492608138950:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:24.837197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:24.841617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471488785972427:2048] [node 1] ICS01 socket: 166 reason# EndOfStream
2025-11-27T18:19:24.841681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471484491005015:2048] [node 1] ICP30 unregister session Session# [5:7577471488785972427:2048] VirtualId# [5:7577471488785972421:0]
2025-11-27T18:19:24.841727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471488785972427:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:24.841836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:24.841886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:24.834919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471486708285247:2048] [node 1] ICP30 unregister session Session# [4:7577471491003252633:2048] VirtualId# [4:7577471491003252627:0]
2025-11-27T18:19:24.835029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:24.835060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471486708285247:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:24.835074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:24.846568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471486708285247:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:24.846207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471488313171566:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19776) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:24.846274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471492608138950:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:24.846898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471488313171566:2048] [node 1] ICP30 unregister session Session# [6:7577471492608138950:2048] VirtualId# [6:7577471492608138948:0]
2025-11-27T18:19:24.847018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:24.847056Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471488313171566:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:24.847077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:24.853844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471488313171566:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:24.929489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:24.929742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471489861273385:2049] [node 2] ICS01 socket: 129 reason# EndOfStream
2025-11-27T18:19:24.929788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471485566305991:2049] [node 2] ICP30 unregister session Session# [3:7577471489861273385:2049] VirtualId# [3:7577471489861273370:0]
2025-11-27T18:19:24.929826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471489861273385:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:24.929904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:24.929956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:24.933532Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:24.933761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471491003252640:2049] [node 2] ICS01 socket: 151 reason# EndOfStream
2025-11-27T18:19:24.933812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471486708285248:2049] [node 2] ICP30 unregister session Session# [4:7577471491003252640:2049] VirtualId# [4:7577471491003252630:0]
2025-11-27T18:19:24.933848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471491003252640:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:24.933923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:24.933983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:24.933569Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:24.933770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471488785972419:2049] [node 2] ICS01 socket: 154 reason# EndOfStream
2025-11-27T18:19:24.934021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:24.933808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471484491005016:2049] [node 2] ICP30 unregister session Session# [5:7577471488785972419:2049] VirtualId# [5:7577471488785972417:0]
2025-11-27T18:19:24.933844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471488785972419:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:24.933921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:24.933976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:24.934241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471492608138943:2049] [node 2] ICS07 socket disconnect 130 reason# EndOfStream
2025-11-27T18:19:24.934284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471492608138943:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:24.934354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:24.934406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471492608138943:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:24.935559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471488313171567:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23321) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:24.935663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471492608138943:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:24.935712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471488313171567:2049] [node 2] ICP30 unregister session Session# [6:7577471492608138943:2049] VirtualId# [6:7577471492608138939:0]
2025-11-27T18:19:24.935759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:24.935794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471488313171567:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23321) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:24.935814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23321) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:24.945772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471488313171567:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:24.999522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:24.999789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471491003252636:2050] [node 3] ICS01 socket: 121 reason# EndOfStream
2025-11-27T18:19:24.999865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471486708285249:2050] [node 3] ICP30 unregister session Session# [4:7577471491003252636:2050] VirtualId# [4:7577471491003252628:0]
2025-11-27T18:19:24.999907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471491003252636:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:24.999979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:25.000022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:25.000440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:25.000641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471488785972412:2050] [node 3] ICS01 socket: 111 reason# EndOfStream
2025-11-27T18:19:25.000676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471484491005017:2050] [node 3] ICP30 unregister session Session# [5:7577471488785972412:2050] VirtualId# [5:7577471488785972410:0]
2025-11-27T18:19:25.000705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471488785972412:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:25.000767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:25.000842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:25.005429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:25.005639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471492608138956:2050] [node 3] ICS07 socket disconnect 158 reason# EndOfStream
2025-11-27T18:19:25.005668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471492608138956:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:25.005774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:25.005832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471492608138956:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:25.015031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471488313171568:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17925) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:25.015095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471492608138956:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:25.015134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471488313171568:2050] [node 3] ICP30 unregister session Session# [6:7577471492608138956:2050] VirtualId# [6:7577471492608138952:0]
2025-11-27T18:19:25.015214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:25.015244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471488313171568:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17925) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:25.015258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17925) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:25.023578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471488313171568:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:25.069694Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:25.069960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471488785972432:2127] [node 4] ICS01 socket: 156 reason# EndOfStream
2025-11-27T18:19:25.070005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471488785972392:2127] [node 4] ICP30 unregister session Session# [5:7577471488785972432:2127] VirtualId# [5:7577471488785972426:0]
2025-11-27T18:19:25.070054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471488785972432:2127] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:25.070148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:25.070198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:25.073423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:25.073807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471492608138962:2102] [node 4] ICS01 socket: 155 reason# EndOfStream
2025-11-27T18:19:25.073865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471488313171622:2102] [node 4] ICP30 unregister session Session# [6:7577471492608138962:2102] VirtualId# [6:7577471492608138959:0]
2025-11-27T18:19:25.073901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471492608138962:2102] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:25.074004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:25.074099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:25.160927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:25.161174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471492608138964:2104] [node 5] ICS07 socket disconnect 142 reason# EndOfStream
2025-11-27T18:19:25.161210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471492608138964:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:25.161553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:25.161637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471492608138964:2104] [node 5] ICS15 start handshake
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:25.264189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471494037240022:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:25.264439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471494037240021:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:25.290948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471495948538484:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:25.291008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471495948538485:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:25.314334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471496773402626:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:25.314369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471496773402627:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:25.341178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471494559061581:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:25.341226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471494559061582:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:25.341497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471494559061583:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:25.370155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471495874232277:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:25.370387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471495874232278:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:25.370421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471495874232279:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:25.399435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471493740629618:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:25.399501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471493740629619:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:25.399534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471493740629620:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:25.582814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471493740629674:2101] [node 5] ICP01 ready to work
2025-11-27T18:19:25.657996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471495948538547:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:25.812196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471494559061644:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:25.658574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471495948538548:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:25.811964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471493740629682:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:25.817329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471495948538551:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:25.789418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471494037240102:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:25.805451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471496773402689:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:25.819104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471494037240103:2127] [node 6] ICP01 ready to work
2025-11-27T18:19:25.819045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471495874232340:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:25.822716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471496773402690:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:25.832734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471496773402691:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:25.820822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471494037240022:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(237799002ns) {Configure:104(237793768ns) {ProcessConfigured:135(83179ns) {StartInitialHandshake:157(80644ns) {DropHandshakes:470(3042ns) {DropIncomingHandshake:442(740ns) DropOutgoingHandshake:457(492ns)} GenerateSessionVirtualId:988(1291ns) PrepareNewSessionHandshake:477(1339ns) {SwitchToState:179(425ns)}}}}}
2025-11-27T18:19:25.821288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471494559061645:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:25.824230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471494037240106:2117] [node 5] ICP01 ready to work
2025-11-27T18:19:25.849782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471493740629674:2101] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24872440ns) {Configure:104(24865570ns) {ProcessConfigured:135(69238ns) {StartInitialHandshake:157(66798ns) {DropHandshakes:470(3096ns) {DropIncomingHandshake:442(711ns) DropOutgoingHandshake:457(499ns)} GenerateSessionVirtualId:988(1233ns) PrepareNewSessionHandshake:477(2734ns) {SwitchToState:179(966ns)}}}}}
2025-11-27T18:19:25.837634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471495874232341:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:25.858864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471494037240021:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(262570838ns) {Configure:104(262563485ns) {ProcessConfigured:135(18467266ns) {StartInitialHandshake:157(18464698ns) {DropHandshakes:470(3679ns) {DropIncomingHandshake:442(1029ns) DropOutgoingHandshake:457(535ns)} GenerateSessionVirtualId:988(729ns) PrepareNewSessionHandshake:477(3009ns) {SwitchToState:179(1471ns)}}}}}
2025-11-27T18:19:25.884737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471495948538547:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(223285906ns) {Configure:104(223282093ns) {ProcessConfigured:135(15821560ns) {StartInitialHandshake:157(15818584ns) {DropHandshakes:470(8791536ns) {DropIncomingHandshake:442(1477ns) DropOutgoingHandshake:457(3244ns)} GenerateSessionVirtualId:988(3715ns) PrepareNewSessionHandshake:477(6116ns) {SwitchToState:179(3841ns)}}}}}
2025-11-27T18:19:25.888589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471495948538485:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(281726020ns) {Configure:104(276445528ns) {ProcessConfigured:135(8417207ns) {StartInitialHandshake:157(8414487ns) {DropHandshakes:470(4020ns) {DropIncomingHandshake:442(1719ns) DropOutgoingHandshake:457(429ns)} GenerateSessionVirtualId:988(505ns) PrepareNewSessionHandshake:477(3694ns) {SwitchToState:179(2060ns)}}}}}
2025-11-27T18:19:25.896472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471495948538548:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(78245564ns) {Configure:104(78238863ns) {ProcessConfigured:135(62609ns) {StartInitialHandshake:157(60300ns) {DropHandshakes:470(2657ns) {DropIncomingHandshake:442(344ns) DropOutgoingHandshake:457(394ns)} GenerateSessionVirtualId:988(678ns) PrepareNewSessionHandshake:477(1779ns) {SwitchToState:179(547ns)}}}}}
2025-11-27T18:19:25.897735Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471493740629620:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19718999ns) {Configure:104(19713000ns) {ProcessConfigured:135(4398409ns) {StartInitialHandshake:157(4396074ns) {DropHandshakes:470(2652ns) {DropIncomingHandshake:442(614ns) DropOutgoingHandshake:457(364ns)} GenerateSessionVirtualId:988(812ns) PrepareNewSessionHandshake:477(3312ns) {SwitchToState:179(1944ns)}}}}}
2025-11-27T18:19:25.923838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471493740629619:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(246966995ns) {Configure:104(246960607ns) {ProcessConfigured:135(14930284ns) {StartInitialHandshake:157(14927835ns) {DropHandshakes:470(2252ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(376ns)} GenerateSessionVirtualId:988(302ns) PrepareNewSessionHandshake:477(6149ns) {SwitchToState:179(3280ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:25.933763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471495948538563:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25898
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:25.917912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471495874232340:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(53668917ns) {Configure:104(53664817ns) {ProcessConfigured:135(19354757ns) {StartInitialHandshake:157(19352589ns) {DropHandshakes:470(3339ns) {DropIncomingHandshake:442(821ns) DropOutgoingHandshake:457(544ns)} GenerateSessionVirtualId:988(826ns) PrepareNewSessionHandshake:477(4324ns) {SwitchToState:179(2984ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:25.946977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471495874232350:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21074
2025-11-27T18:19:25.953918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471495948538564:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25898
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:25.979189Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471494559061581:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(131543583ns) {Configure:104(131536499ns) {ProcessConfigured:135(52706ns) {StartInitialHandshake:157(46519ns) {DropHandshakes:470(1808ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(371ns) PrepareNewSessionHandshake:477(1692ns) {SwitchToState:179(722ns)}}}}}
2025-11-27T18:19:25.992509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471494037240114:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7428
2025-11-27T18:19:25.996763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471495874232351:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21074
2025-11-27T18:19:25.992600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471494037240115:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7428
2025-11-27T18:19:26.007129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471494037240115:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.004830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471496773402626:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(95925011ns) {Configure:104(95918669ns) {ProcessConfigured:135(14915266ns) {StartInitialHandshake:157(14912088ns) {DropHandshakes:470(3294ns) {DropIncomingHandshake:442(708ns) DropOutgoingHandshake:457(484ns)} GenerateSessionVirtualId:988(1362ns) PrepareNewSessionHandshake:477(3267ns) {SwitchToState:179(2320ns)}}}}}
2025-11-27T18:19:26.000725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471495874232350:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.004666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471495948538564:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.009767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471495874232278:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.008984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471495948538554:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.010620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471496773402627:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(102002080ns) {Configure:104(101994043ns) {ProcessConfigured:135(22416300ns) {StartInitialHandshake:157(22413111ns) {DropHandshakes:470(3915ns) {DropIncomingHandshake:442(1181ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(946ns) PrepareNewSessionHandshake:477(3042ns) {SwitchToState:179(1535ns)}}}}}
2025-11-27T18:19:26.012210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471496773402691:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19351470ns) {Configure:104(19347576ns) {ProcessConfigured:135(9595961ns) {StartInitialHandshake:157(9591909ns) {DropHandshakes:470(3060ns) {DropIncomingHandshake:442(644ns) DropOutgoingHandshake:457(376ns)} GenerateSessionVirtualId:988(1329ns) PrepareNewSessionHandshake:477(5369ns) {SwitchToState:179(2759ns)}}}}}
2025-11-27T18:19:26.010626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471495874232351:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.026786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471494559061582:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46068081ns) {Configure:104(46061173ns) {ProcessConfigured:135(41512ns) {StartInitialHandshake:157(39456ns) {DropHandshakes:470(1829ns) {DropIncomingHandshake:442(544ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(719ns) {SwitchToState:179(173ns)}}}}}
2025-11-27T18:19:26.009010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471495948538554:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.025340Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471494037240106:2117] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(47170190ns) {Configure:104(47149690ns) {ProcessConfigured:135(68199ns) {StartInitialHandshake:157(64901ns) {DropHandshakes:470(1815ns) {DropIncomingHandshake:442(416ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(997ns) PrepareNewSessionHandshake:477(2835ns) {SwitchToState:179(2282ns)}}}}}
2025-11-27T18:19:26.030956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471494037240114:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.030175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471494559061583:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(146762278ns) {Configure:104(146753991ns) {ProcessConfigured:135(7164848ns) {StartInitialHandshake:157(7161437ns) {DropHandshakes:470(6305555ns) {DropIncomingHandshake:442(479ns) DropOutgoingHandshake:457(1847ns)} GenerateSessionVirtualId:988(2358ns) PrepareNewSessionHandshake:477(2463ns) {SwitchToState:179(1273ns)}}}}}
2025-11-27T18:19:26.017865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471495948538563:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.031215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471494037240108:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.018045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471495948538554:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:26.031255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471494037240108:2129] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.038002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471500169199655:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21074
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.034399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471494559061648:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55403
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.029934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471495948538484:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.035356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471494559061649:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55403
2025-11-27T18:19:26.035479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471494559061650:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55403
2025-11-27T18:19:26.038427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471494559061649:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.038564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471494559061650:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.038930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471496773402696:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33344
2025-11-27T18:19:26.034432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471495948538547:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.039140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471494559061648:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.047027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471496773402697:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33344
2025-11-27T18:19:26.036227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:26.040553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471494559061581:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.047458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471496773402693:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33344
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.050583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471495874232350:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:26.050488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471494037240110:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.044420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471494559061582:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.036283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471495948538547:2108] [node 5] ICP22 created new session: [2:7577471500243505864:2108]
2025-11-27T18:19:26.051435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471493740629682:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(70083582ns) {Configure:104(70074637ns) {ProcessConfigured:135(9725269ns) {StartInitialHandshake:157(9721690ns) {DropHandshakes:470(1352ns) {DropIncomingHandshake:442(284ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:477(3458ns) {SwitchToState:179(2154ns)}}}}}
2025-11-27T18:19:26.049657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471496773402696:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.052282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471495874232278:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.050533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471494037240110:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.038448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471500243505865:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25898
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.070759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471498035596992:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24688
2025-11-27T18:19:26.053666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471494037240108:2129] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:26.054471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471498854028961:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55403
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.051527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471496773402697:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.039034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471500243505864:2108] [node 5] ICS09 handshake done sender: [2:7577471495948538554:2111] self: [2:7577471495948538552:0] peer: [5:7577471500169199653:0] socket: 108 qp: -1
2025-11-27T18:19:26.055957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.080196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471494037240110:2130] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:26.054396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471496773402626:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.055540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471494559061648:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:26.060118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471495874232278:2049] [node 2] ICP22 created new session: [5:7577471500169199656:2049]
2025-11-27T18:19:26.044873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471500243505864:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.102226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471498035596993:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24688
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.058064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471494559061582:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.058109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:26.058154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471494559061582:2049] [node 2] ICP22 created new session: [4:7577471498854028964:2049]
2025-11-27T18:19:26.058198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471498854028964:2049] [node 2] ICS09 handshake done sender: [4:7577471494559061648:2111] self: [4:7577471498854028960:0] peer: [2:7577471495948538556:0] socket: 132 qp: -1
2025-11-27T18:19:26.058238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471498854028964:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:26.058268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:26.086184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471498854028963:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55403
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.105812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471494037240102:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.064011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471500169199656:2049] [node 2] ICS09 handshake done sender: [5:7577471495874232350:2113] self: [5:7577471500169199653:0] peer: [2:7577471495948538552:0] socket: 114 qp: -1
2025-11-27T18:19:26.054717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471496773402693:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.044929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.109622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471498035596992:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.105869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:26.112342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471498854028963:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.046640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471495948538557:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.055000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471496773402627:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.069824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471500169199657:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21074
2025-11-27T18:19:26.133990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471498854028962:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55403
2025-11-27T18:19:26.046675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471495948538557:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.105914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471494037240102:2107] [node 4] ICP22 created new session: [1:7577471498332207423:2107]
2025-11-27T18:19:26.129750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471493740629674:2101] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.109396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471501068370008:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33344
2025-11-27T18:19:26.072211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471500169199656:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:26.105964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471498332207423:2107] [node 4] ICS09 handshake done sender: [1:7577471494037240110:2130] self: [1:7577471494037240109:0] peer: [4:7577471498854028958:0] socket: 113 qp: -1
2025-11-27T18:19:26.047767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471500243505865:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.112656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471501068370003:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33344
2025-11-27T18:19:26.072252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:26.156785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471498035596996:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24688
2025-11-27T18:19:26.160600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471494559061583:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.157795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471496773402693:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:26.106003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471498332207423:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:26.052519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471495948538557:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:26.160708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471494559061649:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:26.165893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471494559061581:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1053) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471494559061649:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.080570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471500169199657:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.056199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471495948538555:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.056220Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471495948538555:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.057172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471495948538551:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.057223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:26.057291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471495948538551:2107] [node 4] ICP22 created new session: [2:7577471500243505871:2107]
2025-11-27T18:19:26.057337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471500243505871:2107] [node 4] ICS09 handshake done sender: [2:7577471495948538557:2113] self: [2:7577471495948538556:0] peer: [4:7577471498854028960:0] socket: 110 qp: -1
2025-11-27T18:19:26.057369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471500243505871:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:26.057404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:26.084651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471500243505870:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25898
2025-11-27T18:19:26.087184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471495948538555:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:26.104558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471495948538563:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:26.117523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471495948538485:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.117572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:26.117631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471495948538485:2049] [node 3] ICP22 created new session: [2:7577471500243505875:2049]
2025-11-27T18:19:26.117683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471500243505875:2049] [node 3] ICS09 handshake done sender: [2:7577471495948538555:2112] self: [2:7577471495948538553:0] peer: [3:7577471501068370007:0] socket: 109 qp: -1
2025-11-27T18:19:26.117733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471500243505875:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:26.117775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:26.119133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471495948538484:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.119174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:26.119222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471495948538484:2048] [node 1] ICP22 created new session: [2:7577471500243505877:2048]
2025-11-27T18:19:26.119265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471500243505877:2048] [node 1] ICS09 handshake done sender: [2:7577471495948538563:2115] self: [2:7577471500243505863:0] peer: [1:7577471494037240107:0] socket: 115 qp: -1
2025-11-27T18:19:26.119289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471500243505877:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:26.119313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:26.161629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471495948538562:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.161649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471495948538562:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.166055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471495948538562:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:26.156879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471498035596997:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24688
2025-11-27T18:19:26.106041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:26.158365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471501068370003:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.159917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471496773402699:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.159949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471496773402699:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.160190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471496773402627:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.160240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:26.160291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471496773402627:2049] [node 2] ICP22 created new session: [3:7577471501068370010:2049]
2025-11-27T18:19:26.172324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471501068370010:2049] [node 2] ICS09 handshake done sender: [3:7577471496773402693:2110] self: [3:7577471501068370007:0] peer: [2:7577471495948538553:0] socket: 125 qp: -1
2025-11-27T18:19:26.175052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471496773402701:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.175076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471496773402701:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.178695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471501068370010:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:26.109459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471495874232279:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.158401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471498035596993:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.182252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:26.183209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471496773402626:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1053) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471496773402696:2112] held: no
2025-11-27T18:19:26.184447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471496773402699:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.107273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471498332207418:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7428
2025-11-27T18:19:26.181988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471495948538548:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.161230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471493740629619:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.133651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471495874232279:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(28372488ns) {IncomingHandshake:261(22093388ns) {DropIncomingHandshake:442(531ns) PrepareNewSessionHandshake:477(5194ns) {SwitchToState:179(2128ns)} GenerateSessionVirtualId:988(1501ns) IssueIncomingHandshakeReply:193(6897164ns) {SwitchToState:179(2024ns)}}}
2025-11-27T18:19:26.205660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471496773402689:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.205730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.205779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471496773402689:2108] [node 5] ICP22 created new session: [3:7577471501068370016:2108]
2025-11-27T18:19:26.205827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471501068370016:2108] [node 5] ICS09 handshake done sender: [3:7577471496773402699:2114] self: [3:7577471496773402698:0] peer: [5:7577471500169199660:0] socket: 149 qp: -1
2025-11-27T18:19:26.205866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471501068370016:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:26.205894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:26.107396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471498332207419:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7428
2025-11-27T18:19:26.149646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471495874232353:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.182079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:26.172174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471498035596999:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24688
2025-11-27T18:19:26.222110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471498854028969:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55403
2025-11-27T18:19:26.229386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471496773402627:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(41850836ns) {HandleHandshakeStatus:329(41847804ns) {DropIncomingHandshake:442(779ns) DropOutgoingHandshake:457(5796ns) SwitchToState:179(433ns) ProcessPendingSessionEvents:495(587143ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 522991ns max# 79425ns}}}
2025-11-27T18:19:26.230033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471496773402701:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:26.107656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471498332207422:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7428
2025-11-27T18:19:26.196320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471495948538548:2109] [node 6] ICP22 created new session: [2:7577471500243505880:2109]
2025-11-27T18:19:26.230520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471498854028963:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:26.149694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471495874232353:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.176912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471498035596992:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:26.218055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471500243505880:2109] [node 6] ICS09 handshake done sender: [2:7577471495948538562:2114] self: [2:7577471495948538561:0] peer: [6:7577471498035596998:0] socket: 161 qp: -1
2025-11-27T18:19:26.108259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471494037240021:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.232644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471498854028970:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55403
2025-11-27T18:19:26.181979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471498035596994:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24688
2025-11-27T18:19:26.160155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471495874232353:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:26.108324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:26.222227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471500243505880:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:26.182127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471495874232341:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.182187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:26.193891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471500169199661:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21074
2025-11-27T18:19:26.194309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471500169199664:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21074
2025-11-27T18:19:26.194393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471500169199665:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21074
2025-11-27T18:19:26.195714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471500169199661:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.195826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471500169199657:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:26.206517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471495874232341:2109] [node 6] ICP22 created new session: [5:7577471500169199663:2109]
2025-11-27T18:19:26.229527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471500169199663:2109] [node 6] ICS09 handshake done sender: [5:7577471495874232353:2115] self: [5:7577471495874232352:0] peer: [6:7577471498035596995:0] socket: 159 qp: -1
2025-11-27T18:19:26.230211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471500169199663:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:26.230249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:26.230305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471500169199664:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.230354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471495874232279:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.230449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:26.230503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471495874232279:2050] [node 3] ICP22 created new session: [5:7577471500169199667:2050]
2025-11-27T18:19:26.230548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471500169199667:2050] [node 3] ICS09 handshake done sender: [5:7577471500169199657:2118] self: [5:7577471500169199660:0] peer: [3:7577471496773402698:0] socket: 157 qp: -1
2025-11-27T18:19:26.230581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471500169199667:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:26.231363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:26.237286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471495874232341:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(54322072ns) {HandleHandshakeStatus:329(54318741ns) {DropIncomingHandshake:442(15396ns) DropOutgoingHandshake:457(311ns) SwitchToState:179(535ns) ProcessPendingSessionEvents:495(3566279ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3494253ns max# 3066507ns}}}
2025-11-27T18:19:26.237671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471495874232277:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.244976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471495874232278:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(176961953ns) {HandleHandshakeStatus:329(176958803ns) {DropIncomingHandshake:442(577ns) DropOutgoingHandshake:457(9971ns) SwitchToState:179(1538ns) ProcessPendingSessionEvents:495(137850697ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 135955606ns max# 16695876ns}}}
2025-11-27T18:19:26.247003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471495874232340:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.263809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471496773402690:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.233368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471494559061583:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.196235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471493740629674:2101] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.112567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471494037240021:2048] [node 2] ICP22 created new session: [1:7577471498332207427:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.229535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:26.202481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471498035596996:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.113183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471494037240105:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.233470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:26.273444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:26.113226Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471494037240105:2128] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.233519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471494559061583:2050] [node 3] ICP22 created new session: [4:7577471498854028971:2050]
No RDMA context found for GID: 2025-11-27T18:19:26.269936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471495948538551:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(139292711ns) {HandleHandshakeStatus:329(139289868ns) {DropIncomingHandshake:442(13138ns) DropOutgoingHandshake:457(492ns) SwitchToState:179(160ns) ProcessPendingSessionEvents:495(138931616ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 138756936ns max# 16214962ns}}}
::ffff:127.1.0.0
2025-11-27T18:19:26.206502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:26.273523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471496773402690:2107] [node 4] ICP22 created new session: [3:7577471501068370018:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.233580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471498854028971:2050] [node 3] ICS09 handshake done sender: [4:7577471498854028963:2120] self: [4:7577471498854028967:0] peer: [3:7577471496773402700:0] socket: 155 qp: -1
2025-11-27T18:19:26.206554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471493740629674:2101] [node 5] ICP22 created new session: [6:7577471498035597000:2101]
2025-11-27T18:19:26.273582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471501068370018:2107] [node 4] ICS09 handshake done sender: [3:7577471496773402701:2115] self: [3:7577471496773402700:0] peer: [4:7577471498854028967:0] socket: 154 qp: -1
2025-11-27T18:19:26.310175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471500169199671:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21074
2025-11-27T18:19:26.113362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471498332207427:2048] [node 2] ICS09 handshake done sender: [1:7577471494037240108:2129] self: [1:7577471494037240107:0] peer: [2:7577471500243505863:0] socket: 112 qp: -1
2025-11-27T18:19:26.233620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471498854028971:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:26.114556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471498332207422:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.210515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471498035596994:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.211850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471498035596993:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:26.212201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471493740629619:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10853) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471498035596993:2116] held: no
2025-11-27T18:19:26.212793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471493740629619:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.212838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:26.212890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471493740629619:2049] [node 2] ICP22 created new session: [6:7577471498035597001:2049]
2025-11-27T18:19:26.212930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471498035597001:2049] [node 2] ICS09 handshake done sender: [6:7577471498035596993:2116] self: [6:7577471498035596998:0] peer: [2:7577471495948538561:0] socket: 162 qp: -1
2025-11-27T18:19:26.212962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471498035597001:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:26.213004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:26.220814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471498035597000:2101] [node 5] ICS09 handshake done sender: [6:7577471498035596992:2115] self: [6:7577471498035596995:0] peer: [5:7577471495874232352:0] socket: 160 qp: -1
2025-11-27T18:19:26.225488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471498035597000:2101] [node 5] ICS10 traffic start
2025-11-27T18:19:26.225540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:26.230473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471493740629618:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.233234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471493740629682:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.294377Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471493740629674:2101] [node 5] ICP35 event processing took too much time PendingConnection:279(108735101ns) {HandleHandshakeStatus:329(108731721ns) {DropIncomingHandshake:442(3964ns) DropOutgoingHandshake:457(6149807ns) SwitchToState:179(1313ns) ProcessPendingSessionEvents:495(64247201ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 64134354ns max# 14499983ns}}}
2025-11-27T18:19:26.296237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471493740629674:2101] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:26.302105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471498035597009:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24688
2025-11-27T18:19:26.303448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471498035597006:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24688
2025-11-27T18:19:26.321563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471498035596996:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:26.273618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471501068370018:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:26.273654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:26.274625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471501068370009:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33344
2025-11-27T18:19:26.300185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471496773402689:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(23646516ns) {HandleHandshakeStatus:329(23643317ns) {DropIncomingHandshake:442(14479ns) DropOutgoingHandshake:457(405ns) SwitchToState:179(746ns) ProcessPendingSessionEvents:495(23277911ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23169931ns max# 7800720ns}}}
2025-11-27T18:19:26.233669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:26.122992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471498332207427:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:26.331720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471495874232340:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27608) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471500169199664:2120] held: no
2025-11-27T18:19:26.332414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471500169199664:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.265347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471498854028952:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.123070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:26.330827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471493740629682:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.330876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:26.330913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471493740629682:2108] [node 4] ICP22 created new session: [6:7577471498035597011:2108]
2025-11-27T18:19:26.330952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471498035597011:2108] [node 4] ICS09 handshake done sender: [6:7577471498035596996:2118] self: [6:7577471498035597008:0] peer: [4:7577471498854028953:0] socket: 169 qp: -1
2025-11-27T18:19:26.330987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471498035597011:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:26.331018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:26.334791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471498035597010:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24688
2025-11-27T18:19:26.265400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471498854028952:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.329893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471496773402696:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.124935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471494037240021:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(21613459ns) {HandleHandshakeStatus:329(21610361ns) {DropIncomingHandshake:442(2115ns) DropOutgoingHandshake:457(954ns) SwitchToState:179(509ns) ProcessPendingSessionEvents:495(925688ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 865152ns max# 447425ns}}}
2025-11-27T18:19:26.270416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471494559061581:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(59016880ns) {HandleHandshakeStatus:395(59014082ns) {LogHandshakeFail:472(12890ns) {UpdateErrorStateLog:545(4768ns)} DropOutgoingHandshake:457(1019ns) ProcessPendingSessionEvents:495(56040832ns) {unordered DropSessionEvent:512 num# 100 sum# 55982208ns max# 12472679ns}}}
2025-11-27T18:19:26.338350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471498035596994:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:26.127700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471498332207419:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.279674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471494559061581:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.338692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471501068370020:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33344
2025-11-27T18:19:26.347637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471493740629618:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.279753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:26.279815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471494559061581:2048] [node 1] ICP22 created new session: [4:7577471498854028973:2048]
2025-11-27T18:19:26.133979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471498332207418:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.279859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471498854028973:2048] [node 1] ICS09 handshake done sender: [4:7577471494559061649:2112] self: [4:7577471498854028958:0] peer: [1:7577471494037240109:0] socket: 133 qp: -1
2025-11-27T18:19:26.279903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471498854028973:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:26.343046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471496773402626:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.279939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:26.138012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471494037240021:2048] [node 2] ICP18 handshake (actor [1:7577471498332207419:2136]) is too old
2025-11-27T18:19:26.343141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:26.282878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471498854028954:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.349912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:26.282912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471498854028954:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.285144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471498854028952:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:26.300649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471498854028954:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:26.343200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471496773402626:2048] [node 1] ICP22 created new session: [3:7577471501068370021:2048]
2025-11-27T18:19:26.343270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471501068370021:2048] [node 1] ICS09 handshake done sender: [3:7577471496773402696:2112] self: [3:7577471501068370005:0] peer: [1:7577471494037240104:0] socket: 131 qp: -1
2025-11-27T18:19:26.343319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471501068370021:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:26.343386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:26.353636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471501068370020:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.309340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471494559061645:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.309700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:26.355033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471495874232279:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(114007128ns) {HandleHandshakeStatus:329(114004007ns) {DropIncomingHandshake:442(849ns) DropOutgoingHandshake:457(5141ns) SwitchToState:179(484ns) ProcessPendingSessionEvents:495(90511952ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 90412624ns max# 29204017ns}}}
2025-11-27T18:19:26.309750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471494559061645:2108] [node 5] ICP22 created new session: [4:7577471498854028977:2108]
2025-11-27T18:19:26.355115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471495874232340:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.313362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471498854028977:2108] [node 5] ICS09 handshake done sender: [4:7577471498854028952:2116] self: [4:7577471498854028951:0] peer: [5:7577471500169199670:0] socket: 172 qp: -1
2025-11-27T18:19:26.317866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471498854028977:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:26.321376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:26.355186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:26.325709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471498854028970:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.355243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471495874232340:2108] [node 4] ICP22 created new session: [5:7577471500169199673:2108]
2025-11-27T18:19:26.325783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471494559061644:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.355301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471500169199673:2108] [node 4] ICS09 handshake done sender: [5:7577471500169199664:2120] self: [5:7577471500169199670:0] peer: [4:7577471498854028951:0] socket: 153 qp: -1
2025-11-27T18:19:26.330681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:26.355335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471500169199673:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:26.332782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471494559061644:2109] [node 6] ICP22 created new session: [4:7577471498854028979:2109]
2025-11-27T18:19:26.355365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:26.335869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471498854028979:2109] [node 6] ICS09 handshake done sender: [4:7577471498854028954:2117] self: [4:7577471498854028953:0] peer: [6:7577471498035597008:0] socket: 143 qp: -1
2025-11-27T18:19:26.336013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471498854028979:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:26.338774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:26.351845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471493740629618:2048] [node 1] ICP22 created new session: [6:7577471498035597013:2048]
2025-11-27T18:19:26.145874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471494037240102:2107] [node 4] ICP18 handshake (actor [1:7577471498332207418:2135]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.158231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471494037240105:2128] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:26.351917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471498035597013:2048] [node 1] ICS09 handshake done sender: [6:7577471498035596994:2117] self: [6:7577471498035597007:0] peer: [1:7577471494037240116:0] socket: 166 qp: -1
2025-11-27T18:19:26.357295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471501068370000:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.357335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471500169199672:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21074
2025-11-27T18:19:26.352664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471498035597013:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:26.160986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471494037240022:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.357337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471501068370000:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.352704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:26.163218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:26.355230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471498035597006:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.355340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471493740629620:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:26.173326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471494037240022:2049] [node 3] ICP22 created new session: [1:7577471498332207431:2049]
2025-11-27T18:19:26.177431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471498332207431:2049] [node 3] ICS09 handshake done sender: [1:7577471494037240105:2128] self: [1:7577471494037240104:0] peer: [3:7577471501068370005:0] socket: 111 qp: -1
2025-11-27T18:19:26.182170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471498332207431:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:26.205374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:26.269822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471494037240117:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.269868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471494037240117:2133] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.274936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471498332207415:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:26.274984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471498332207415:2134] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:26.320506Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471498332207415:2134] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:26.325824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471494037240022:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(71112021ns) {HandleHandshakeStatus:329(71109800ns) {DropIncomingHandshake:442(10591ns) DropOutgoingHandshake:457(282ns) SwitchToState:179(150ns) ProcessPendingSessionEvents:495(14569370ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14490731ns max# 7373842ns}}}
2025-11-27T18:19:26.328394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471494037240117:2133] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:26.332720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471494037240103:2127] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.335673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471494037240106:2117] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.338301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:26.362088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471495874232277:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1053) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471500169199661:2119] held: no
2025-11-27T18:19:26.339056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:26.361773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471493740629618:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:26.339112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471494037240103:2127] [node 6] ICP22 created new session: [1:7577471498332207436:2127]
2025-11-27T18:19:26.339158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471498332207436:2127] [node 6] ICS09 handshake done sender: [1:7577471494037240117:2133] self: [1:7577471494037240116:0] peer: [6:7577471498035597007:0] socket: 140 qp: -1
2025-11-27T18:19:26.339226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471498332207436:2127] [node 6] ICS10 traffic start
2025-11-27T18:19:26.339262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:26.343760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471494037240106:2117] [node 5] ICP22 created new session: [1:7577471498332207435:2117]
2025-11-27T18:19:26.347785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471498332207435:2117] [node 5] ICS09 handshake done sender: [1:7577471498332207415:2134] self: [1:7577471498332207414:0] peer: [5:7577471500169199669:0] socket: 165 qp: -1
2025-11-27T18:19:26.347818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471498332207435:2117] [node 5] ICS10 traffic start
2025-11-27T18:19:26.348464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:26.362913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471494559061645:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(56716996ns) {HandleHandshakeStatus:329(56712675ns) {DropIncomingHandshake:442(21676ns) DropOutgoingHandshake:457(638ns) SwitchToState:179(821ns) ProcessPendingSessionEvents:495(32774280ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 30693216ns max# 4005217ns}}}
2025-11-27T18:19:26.363748Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471494559061644:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(35479898ns) {HandleHandshakeStatus:329(35477477ns) {DropIncomingHandshake:442(781ns) DropOutgoingHandshake:457(628ns) SwitchToState:179(485ns) ProcessPendingSessionEvents:495(13472480ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13390747ns max# 2153499ns}}}
2025-11-27T18:19:26.363828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471494559061644:2109] [node 6] ICP18 handshake (actor [4:7577471498854028970:2122]) is too old
2025-11-27T18:19:26.363896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471501068370000:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:26.364000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471495948538548:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(178368214ns) {HandleHandshakeStatus:329(175032527ns) {DropIncomingHandshake:442(21092ns) DropOutgoingHandshake:457(464ns) SwitchToState:179(614ns) ProcessPendingSessionEvents:495(125250167ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 125064286ns max# 10874472ns}}}
2025-11-27T18:19:26.365296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471496773402691:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:26.365389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:26.365647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471500169199661:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:26.370341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471496773402691:2109] [node 6] ICP22 created new session: [3:7577471501068370025:2109]
2025-11-27T18:19:26.377568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471495874232277:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.373450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471501068370025:2109] [node 6] ICS09 handshake done sender: [3:7577471501068370000:2117] self: [3:7577471496773402703:0] peer: [6:7577471498035597015:0] socket: 117 qp: -1
2025-11-27T18:19:26.379286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471493740629620:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16514) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471498035597006:2121] held: no
2025-11-27T18:19:26.378250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471501068370025:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:26.379364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:26.383103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471498035597016:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24688
2025-11-27T18:19:26.379733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471495874232277:2048] [node 1] ICP22 created new session: [5:7577471500169199675:2048]
2025-11-27T18:19:26.378570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:26.381172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471500169199675:2048] [node 1] ICS09 handshake done sender: [5:7577471500169199661:2119] self: [5:7577471500169199669:0] peer: [1:7577471498332207414:0] socket: 167 qp: -1
2025-11-27T18:19:26.383400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471500169199675:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:26.383459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:26.384920Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471496773402691:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18797885ns) {HandleHandshakeStatus:329(18795234ns) {DropIncomingHandshake:442(10995ns) DropOutgoingHandshake:457(255ns) SwitchToState:179(611ns) ProcessPendingSessionEvents:495(3218283ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3158338ns max# 2464701ns}}}
2025-11-27T18:19:26.385626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471498035597006:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:26.386044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471493740629620:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:26.386130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:26.386174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471493740629620:2050] [node 3] ICP22 created new session: [6:7577471498035597017:2050]
2025-11-27T18:19:26.386206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471498035597017:2050] [node 3] ICS09 handshake done sender: [6:7577471498035597006:2121] self: [6:7577471498035597015:0] peer: [3:7577471496773402703:0] socket: 119 qp: -1
2025-11-27T18:19:26.386229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471498035597017:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:26.386270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:26.411866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:26.412105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471500243505877:2048] [node 1] ICS01 socket: 115 reason# EndOfStream
2025-11-27T18:19:26.412167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471495948538484:2048] [node 1] ICP30 unregister session Session# [2:7577471500243505877:2048] VirtualId# [2:7577471500243505863:0]
2025-11-27T18:19:26.412208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471500243505877:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.412294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:26.412346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:26.414250Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:26.414481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471501068370021:2048] [node 1] ICS01 socket: 131 reason# EndOfStream
2025-11-27T18:19:26.414527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471496773402626:2048] [node 1] ICP30 unregister session Session# [3:7577471501068370021:2048] VirtualId# [3:7577471501068370005:0]
2025-11-27T18:19:26.414560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471501068370021:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.414630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:26.414673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:26.417406Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:26.417640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471498854028973:2048] [node 1] ICS01 socket: 133 reason# EndOfStream
2025-11-27T18:19:26.417679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471494559061581:2048] [node 1] ICP30 unregister session Session# [4:7577471498854028973:2048] VirtualId# [4:7577471498854028958:0]
2025-11-27T18:19:26.417710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471498854028973:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.417773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:26.417815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:26.421425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:26.421631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471500169199675:2048] [node 1] ICS01 socket: 167 reason# EndOfStream
2025-11-27T18:19:26.421668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471495874232277:2048] [node 1] ICP30 unregister session Session# [5:7577471500169199675:2048] VirtualId# [5:7577471500169199669:0]
2025-11-27T18:19:26.421698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471500169199675:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.421767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:26.421808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:26.421905Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:26.422096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471498035597013:2048] [node 1] ICS07 socket disconnect 166 reason# EndOfStream
2025-11-27T18:19:26.422125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471498035597013:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.422195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:26.422233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471498035597013:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:26.423243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471493740629618:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1053) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:26.423401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471498035597013:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:26.423445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471493740629618:2048] [node 1] ICP30 unregister session Session# [6:7577471498035597013:2048] VirtualId# [6:7577471498035597007:0]
2025-11-27T18:19:26.423545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:26.423580Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471493740629618:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1053) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:26.423598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1053) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:26.429498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471493740629618:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:26.491613Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:26.491842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471501068370010:2049] [node 2] ICS07 socket disconnect 125 reason# EndOfStream
2025-11-27T18:19:26.491879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471501068370010:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.491965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:26.492028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471501068370010:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:26.494385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471496773402627:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10853) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:26.494435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471501068370010:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:26.496393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:26.496463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471496773402627:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10853) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:26.496628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471501068370010:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:26.496675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471496773402627:2049] [node 2] ICP30 unregister session Session# [3:7577471501068370010:2049] VirtualId# [3:7577471501068370007:0]
2025-11-27T18:19:26.496805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:26.496847Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471496773402627:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10853) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:26.496646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471498854028964:2049] [node 2] ICS01 socket: 132 reason# EndOfStream
2025-11-27T18:19:26.496863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10853) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:26.496708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471494559061582:2049] [node 2] ICP30 unregister session Session# [4:7577471498854028964:2049] VirtualId# [4:7577471498854028960:0]
2025-11-27T18:19:26.496737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471498854028964:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.496817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:26.496857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:26.498142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:26.498339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471500169199656:2049] [node 2] ICS07 socket disconnect 114 reason# EndOfStream
2025-11-27T18:19:26.498375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471500169199656:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.498482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:26.498526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471500169199656:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:26.499469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471495874232278:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10853) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:26.499532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471500169199656:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:26.499570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471495874232278:2049] [node 2] ICP30 unregister session Session# [5:7577471500169199656:2049] VirtualId# [5:7577471500169199653:0]
2025-11-27T18:19:26.499734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:26.505446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471498035597001:2049] [node 2] ICS01 socket: 162 reason# EndOfStream
2025-11-27T18:19:26.505496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471493740629619:2049] [node 2] ICP30 unregister session Session# [6:7577471498035597001:2049] VirtualId# [6:7577471498035596998:0]
2025-11-27T18:19:26.505579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471498035597001:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.505734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:26.505824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:26.499985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:26.506209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471495874232278:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10853) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:26.506231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10853) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:26.517806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471495874232278:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:26.548029Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:26.548296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471498854028971:2050] [node 3] ICS01 socket: 155 reason# EndOfStream
2025-11-27T18:19:26.548345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471494559061583:2050] [node 3] ICP30 unregister session Session# [4:7577471498854028971:2050] VirtualId# [4:7577471498854028967:0]
2025-11-27T18:19:26.547854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:26.548381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471498854028971:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.548099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471500169199667:2050] [node 3] ICS07 socket disconnect 157 reason# EndOfStream
2025-11-27T18:19:26.548123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471500169199667:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.548209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:26.548480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:26.548280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471500169199667:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:26.548532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:26.549055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471495874232279:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16514) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:26.549143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471500169199667:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:26.549175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471495874232279:2050] [node 3] ICP30 unregister session Session# [5:7577471500169199667:2050] VirtualId# [5:7577471500169199660:0]
2025-11-27T18:19:26.549275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:26.549311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471495874232279:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16514) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:26.549325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16514) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:26.549729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:26.550019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471498035597017:2050] [node 3] ICS01 socket: 119 reason# EndOfStream
2025-11-27T18:19:26.550053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471493740629620:2050] [node 3] ICP30 unregister session Session# [6:7577471498035597017:2050] VirtualId# [6:7577471498035597015:0]
2025-11-27T18:19:26.550090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471498035597017:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.550148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:26.550191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:26.559608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471495874232279:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:26.624992Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:26.625314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471500169199673:2108] [node 4] ICS01 socket: 153 reason# EndOfStream
2025-11-27T18:19:26.625363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471495874232340:2108] [node 4] ICP30 unregister session Session# [5:7577471500169199673:2108] VirtualId# [5:7577471500169199670:0]
2025-11-27T18:19:26.625400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471500169199673:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.625488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:26.625531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:26.625681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:26.625924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471498035597011:2108] [node 4] ICS01 socket: 169 reason# EndOfStream
2025-11-27T18:19:26.625965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471493740629682:2108] [node 4] ICP30 unregister session Session# [6:7577471498035597011:2108] VirtualId# [6:7577471498035597008:0]
2025-11-27T18:19:26.625999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471498035597011:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.626075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:26.626134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:26.664227Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:26.664485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471498035597000:2101] [node 5] ICS07 socket disconnect 160 reason# EndOfStream
2025-11-27T18:19:26.664521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471498035597000:2101] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:26.664608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:26.664651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471498035597000:2101] [node 5] ICS15 start handshake
2025-11-27T18:19:26.665579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471493740629674:2101] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21074) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:26.665633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471498035597000:2101] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:26.665681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471493740629674:2101] [node 5] ICP30 unregister session Session# [6:7577471498035597000:2101] VirtualId# [6:7577471498035596995:0]
2025-11-27T18:19:26.665808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:19:26.665854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471493740629674:2101] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21074) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:26.665869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21074) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:26.675280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471493740629674:2101] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:26.783426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471499488681882:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:26.784111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471499488681881:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:26.803898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471500547992129:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:26.803946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471500547992130:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:26.833459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471497963121531:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:26.833500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471497963121532:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:26.855198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471497875815194:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:26.855238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471497875815195:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:26.855279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471497875815196:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:26.879321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471499665744294:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:26.879366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471499665744295:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:26.879404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471499665744296:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:26.903904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471497596065065:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:26.903958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471497596065066:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:26.904057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471497596065067:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:27.022109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471501891032421:2105] [node 5] ICP01 ready to work
2025-11-27T18:19:27.051519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471503783649243:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:27.063013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471503783649241:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:27.107764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471503960711653:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:27.113286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471502170782553:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:27.117925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471502258088892:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:27.120566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471503960711654:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:27.120664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471501891032425:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:27.117730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471502258088891:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:27.063032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471503783649242:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:27.121627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471502170782554:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:27.113528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471504842959488:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:27.129589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471503783649243:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(66504893ns) {Configure:104(66500729ns) {ProcessConfigured:135(7049033ns) {StartInitialHandshake:157(7045867ns) {DropHandshakes:470(2170ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(1051ns) PrepareNewSessionHandshake:477(3040ns) {SwitchToState:179(1333ns)}}}}}
2025-11-27T18:19:27.120100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471504842959489:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:27.121647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471502258088890:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:27.120619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471504842959490:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.205275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471503960711661:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10848
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.208122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471497963121532:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(82307761ns) {Configure:104(82298612ns) {ProcessConfigured:135(48873ns) {StartInitialHandshake:157(46389ns) {DropHandshakes:470(955ns) {DropIncomingHandshake:442(229ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(470ns) PrepareNewSessionHandshake:477(4832ns) {SwitchToState:179(470ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.213363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471503783649252:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
No RDMA context found for GID: 2025-11-27T18:19:27.218040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471504842959491:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7181
::ffff:127.1.0.0
2025-11-27T18:19:27.217975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471503783649252:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.218147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471503960711661:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.222973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471503783649245:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.220452Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471497596065065:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(94612583ns) {Configure:104(94608480ns) {ProcessConfigured:135(939575ns) {StartInitialHandshake:157(937831ns) {DropHandshakes:470(1387ns) {DropIncomingHandshake:442(403ns) DropOutgoingHandshake:457(263ns)} GenerateSessionVirtualId:988(405ns) PrepareNewSessionHandshake:477(2408ns) {SwitchToState:179(1163ns)}}}}}
2025-11-27T18:19:27.225859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471499665744294:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(80998516ns) {Configure:104(80990406ns) {ProcessConfigured:135(799009ns) {StartInitialHandshake:157(796400ns) {DropHandshakes:470(1213ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(9920ns) PrepareNewSessionHandshake:477(3830ns) {SwitchToState:179(749ns)}}}}}
2025-11-27T18:19:27.221317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471500547992129:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(118705043ns) {Configure:104(118694677ns) {ProcessConfigured:135(2300107ns) {StartInitialHandshake:157(2297676ns) {DropHandshakes:470(2470ns) {DropIncomingHandshake:442(385ns) DropOutgoingHandshake:457(342ns)} GenerateSessionVirtualId:988(403ns) PrepareNewSessionHandshake:477(2031ns) {SwitchToState:179(967ns)}}}}}
2025-11-27T18:19:27.221826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471500547992130:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20831470ns) {Configure:104(20827526ns) {ProcessConfigured:135(33145ns) {StartInitialHandshake:157(31885ns) {DropHandshakes:470(1399ns) {DropIncomingHandshake:442(409ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(539ns) PrepareNewSessionHandshake:477(513ns) {SwitchToState:179(175ns)}}}}}
2025-11-27T18:19:27.222062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471504842959491:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.222513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471500547992129:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.223017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471503783649245:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.220239Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471497963121531:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(95570946ns) {Configure:104(95563430ns) {ProcessConfigured:135(10035819ns) {StartInitialHandshake:157(10034384ns) {DropHandshakes:470(3019ns) {DropIncomingHandshake:442(844ns) DropOutgoingHandshake:457(675ns)} GenerateSessionVirtualId:988(939ns) PrepareNewSessionHandshake:477(1966ns) {SwitchToState:179(105ns)}}}}}
2025-11-27T18:19:27.225853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471501891032421:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(71962525ns) {Configure:104(71956438ns) {ProcessConfigured:135(1033143ns) {StartInitialHandshake:157(1031637ns) {DropHandshakes:470(946ns) {DropIncomingHandshake:442(287ns) DropOutgoingHandshake:457(164ns)} GenerateSessionVirtualId:988(340ns) PrepareNewSessionHandshake:477(1613ns) {SwitchToState:179(728ns)}}}}}
2025-11-27T18:19:27.229176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471502170782553:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(79655458ns) {Configure:104(79649559ns) {ProcessConfigured:135(2680037ns) {StartInitialHandshake:157(2678444ns) {DropHandshakes:470(1169ns) {DropIncomingHandshake:442(337ns) DropOutgoingHandshake:457(280ns)} GenerateSessionVirtualId:988(441ns) PrepareNewSessionHandshake:477(1958ns) {SwitchToState:179(732ns)}}}}}
2025-11-27T18:19:27.231714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471497596065066:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20707150ns) {Configure:104(20700984ns) {ProcessConfigured:135(508588ns) {StartInitialHandshake:157(507026ns) {DropHandshakes:470(1490ns) {DropIncomingHandshake:442(295ns) DropOutgoingHandshake:457(514ns)} GenerateSessionVirtualId:988(537ns) PrepareNewSessionHandshake:477(1976ns) {SwitchToState:179(905ns)}}}}}
2025-11-27T18:19:27.223904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471503783649245:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:27.228785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471499665744295:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(91560481ns) {Configure:104(91551280ns) {ProcessConfigured:135(4888659ns) {StartInitialHandshake:157(4243167ns) {DropHandshakes:470(1603ns) {DropIncomingHandshake:442(509ns) DropOutgoingHandshake:457(146ns)} GenerateSessionVirtualId:988(913ns) PrepareNewSessionHandshake:477(1220ns) {SwitchToState:179(542ns)}}}}}
2025-11-27T18:19:27.230537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471499488681881:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.231639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471497875815196:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(79769115ns) {Configure:104(79759636ns) {ProcessConfigured:135(7502905ns) {StartInitialHandshake:157(7499713ns) {DropHandshakes:470(924ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(323ns) PrepareNewSessionHandshake:477(2100ns) {SwitchToState:179(986ns)}}}}}
2025-11-27T18:19:27.220350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471502258088893:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-11-27T18:19:27.233227Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471497875815195:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(52422053ns) {Configure:104(52413010ns) {ProcessConfigured:135(1211892ns) {StartInitialHandshake:157(1210715ns) {DropHandshakes:470(1138ns) {DropIncomingHandshake:442(316ns) DropOutgoingHandshake:457(183ns)} GenerateSessionVirtualId:988(519ns) PrepareNewSessionHandshake:477(1189ns) {SwitchToState:179(479ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.230501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471502258088893:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.232650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:27.233135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471499488681881:2048] [node 2] ICP22 created new session: [1:7577471503783649259:2048]
2025-11-27T18:19:27.233009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471502258088897:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.236271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471504842959499:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7181
2025-11-27T18:19:27.233853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471503783649259:2048] [node 2] ICS09 handshake done sender: [1:7577471503783649245:2111] self: [1:7577471503783649244:0] peer: [2:7577471504842959498:0] socket: 114 qp: -1
2025-11-27T18:19:27.239972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471502258088891:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20551961ns) {Configure:104(20545407ns) {ProcessConfigured:135(54870ns) {StartInitialHandshake:157(53084ns) {DropHandshakes:470(1538ns) {DropIncomingHandshake:442(524ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(1402ns) {SwitchToState:179(414ns)}}}}}
2025-11-27T18:19:27.261583Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471504842959490:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21474181ns) {Configure:104(21469218ns) {ProcessConfigured:135(65913ns) {StartInitialHandshake:157(60809ns) {DropHandshakes:470(2733ns) {DropIncomingHandshake:442(585ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(925ns) PrepareNewSessionHandshake:477(1296ns) {SwitchToState:179(414ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.234369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471503783649259:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.263699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471504842959491:2110] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:27.268774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471503960711654:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(35375331ns) {Configure:104(35370030ns) {ProcessConfigured:135(70774ns) {StartInitialHandshake:157(65417ns) {DropHandshakes:470(1486ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:477(1425ns) {SwitchToState:179(583ns)}}}}}
2025-11-27T18:19:27.243818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471502258088897:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.234406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.268004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471500547992129:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.277651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471503960711654:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.287515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471501891032427:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59185
2025-11-27T18:19:27.252640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471502258088903:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.240359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471503783649258:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-11-27T18:19:27.297523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471502170782556:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.294411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471503960711667:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10848
2025-11-27T18:19:27.252725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471497963121531:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.287702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471501891032428:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59185
2025-11-27T18:19:27.268045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:27.268075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471500547992129:2048] [node 1] ICP22 created new session: [2:7577471504842959507:2048]
2025-11-27T18:19:27.268106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471504842959507:2048] [node 1] ICS09 handshake done sender: [2:7577471504842959491:2110] self: [2:7577471504842959498:0] peer: [1:7577471503783649244:0] socket: 116 qp: -1
2025-11-27T18:19:27.268149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471504842959507:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:27.268174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:27.289322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471504842959504:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7181
2025-11-27T18:19:27.299042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471504842959488:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(57381930ns) {Configure:104(57377272ns) {ProcessConfigured:135(11164760ns) {StartInitialHandshake:157(11159230ns) {DropHandshakes:470(1632ns) {DropIncomingHandshake:442(384ns) DropOutgoingHandshake:457(174ns)} GenerateSessionVirtualId:988(1139ns) PrepareNewSessionHandshake:477(3397ns) {SwitchToState:179(1963ns)}}}}}
2025-11-27T18:19:27.307369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471504842959510:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7181
2025-11-27T18:19:27.297564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471502170782556:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.244502Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471503783649242:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27646330ns) {Configure:104(21421189ns) {ProcessConfigured:135(1160100ns) {StartInitialHandshake:157(1158553ns) {DropHandshakes:470(1148ns) {DropIncomingHandshake:442(345ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(590ns) PrepareNewSessionHandshake:477(1633ns) {SwitchToState:179(440ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.338544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471504842959504:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.294760Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471501891032425:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(62719991ns) {Configure:104(62711422ns) {ProcessConfigured:135(17584938ns) {StartInitialHandshake:157(17581882ns) {DropHandshakes:470(1248ns) {DropIncomingHandshake:442(285ns) DropOutgoingHandshake:457(162ns)} GenerateSessionVirtualId:988(375ns) PrepareNewSessionHandshake:477(2953ns) {SwitchToState:179(1829ns)}}}}}
2025-11-27T18:19:27.316217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471503960711656:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.256244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471503783649258:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.253034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471502258088908:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-11-27T18:19:27.307969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471502170782556:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.300960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471497596065067:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(64264276ns) {Configure:104(64258590ns) {ProcessConfigured:135(48823ns) {StartInitialHandshake:157(46563ns) {DropHandshakes:470(1474ns) {DropIncomingHandshake:442(319ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(432ns) PrepareNewSessionHandshake:477(1043ns) {SwitchToState:179(302ns)}}}}}
2025-11-27T18:19:27.316260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471503960711656:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.275856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471502258088908:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.350750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471504842959495:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.261992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471503783649247:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.301067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471501891032427:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.319144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471503960711671:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10848
2025-11-27T18:19:27.357325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471502170782554:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.280173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471502258088913:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-11-27T18:19:27.350790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471504842959495:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.262019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471503783649247:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.305717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471501891032438:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59185
2025-11-27T18:19:27.322073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471503960711667:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.338232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471502258088893:2110] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:27.357791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:27.357439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471504842959510:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.279253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471503783649257:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-11-27T18:19:27.305841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471501891032428:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.341683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471497963121531:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.345527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471499665744294:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.367858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471502170782554:2108] [node 5] ICP22 created new session: [4:7577471502170782570:2108]
2025-11-27T18:19:27.288783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471503783649247:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:27.306195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471501891032421:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.357561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471504842959501:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.341782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:27.367507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471503960711661:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:27.306516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471497596065065:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.357588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471504842959501:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.371476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471502170782570:2108] [node 5] ICS09 handshake done sender: [4:7577471502170782556:2110] self: [4:7577471502170782555:0] peer: [5:7577471503960711670:0] socket: 117 qp: -1
2025-11-27T18:19:27.319132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471503783649251:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.341970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471502258088903:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.367863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471503960711656:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.316108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471501891032437:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59185
2025-11-27T18:19:27.371781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471502170782570:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:27.361555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471504842959495:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:27.319174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471503783649251:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.342212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471497963121531:2048] [node 1] ICP22 created new session: [3:7577471502258088914:2048]
2025-11-27T18:19:27.368824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471503960711672:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10848
2025-11-27T18:19:27.372264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:27.372027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471500547992130:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.331079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471501891032438:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.333983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471499488681882:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.375190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471502170782569:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-27T18:19:27.370564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471503960711654:2108] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.394545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471502170782571:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-27T18:19:27.372088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:27.349436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471497963121532:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.334039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:27.352736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471497596065066:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.352685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471502258088914:2048] [node 1] ICS09 handshake done sender: [3:7577471502258088893:2110] self: [3:7577471502258088909:0] peer: [1:7577471503783649246:0] socket: 119 qp: -1
2025-11-27T18:19:27.370868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:27.395299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471502170782574:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-27T18:19:27.372316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471500547992130:2049] [node 3] ICP22 created new session: [2:7577471504842959517:2049]
2025-11-27T18:19:27.334411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471499488681882:2049] [node 3] ICP22 created new session: [1:7577471503783649268:2049]
2025-11-27T18:19:27.353708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471501891032437:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.353660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471502258088914:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:27.370925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471503960711654:2108] [node 4] ICP22 created new session: [5:7577471503960711675:2108]
2025-11-27T18:19:27.373434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471504842959501:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.371771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471503960711675:2108] [node 4] ICS09 handshake done sender: [5:7577471503960711661:2112] self: [5:7577471503960711670:0] peer: [4:7577471502170782555:0] socket: 126 qp: -1
2025-11-27T18:19:27.357889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471497596065067:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.335873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471499488681881:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(104635045ns) {HandleHandshakeStatus:329(104632300ns) {DropIncomingHandshake:442(265ns) DropOutgoingHandshake:457(317ns) SwitchToState:179(227ns) ProcessPendingSessionEvents:495(80306886ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 80197505ns max# 8820641ns}}}
2025-11-27T18:19:27.399587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471502170782569:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.381713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471504842959517:2049] [node 3] ICS09 handshake done sender: [2:7577471504842959495:2112] self: [2:7577471504842959494:0] peer: [3:7577471502258088915:0] socket: 132 qp: -1
2025-11-27T18:19:27.357456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:27.399604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471502170782571:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.371813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471503960711675:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.336611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471499488681881:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:27.385379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471501891032449:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59185
2025-11-27T18:19:27.363871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471502258088917:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-11-27T18:19:27.388414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471504842959517:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:27.372105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:27.402708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471497875815194:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.337047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471503783649269:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-11-27T18:19:27.388474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:27.389827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471501891032447:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59185
2025-11-27T18:19:27.370799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471502258088903:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:27.402788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471502170782574:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.372116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471503960711672:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.351981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471503783649251:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:27.393729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471501891032450:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59185
2025-11-27T18:19:27.398176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471504842959489:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.373579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471503960711653:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.403564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471497875815195:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.393315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:27.372403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471502258088899:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.352301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471503783649268:2049] [node 3] ICS09 handshake done sender: [1:7577471503783649247:2112] self: [1:7577471503783649246:0] peer: [3:7577471502258088909:0] socket: 115 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.400972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471501891032427:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:27.402225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:27.372424Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471502258088899:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.393815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471503960711653:2109] [node 6] ICP22 created new session: [5:7577471503960711678:2109]
2025-11-27T18:19:27.357796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471503783649266:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-11-27T18:19:27.401397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471501891032438:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:27.409509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471497875815196:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.373011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471497963121532:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.402134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471501891032428:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:27.445276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471502170782580:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-27T18:19:27.373069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:27.415627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471497596065066:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.373109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471497963121532:2049] [node 2] ICP22 created new session: [3:7577471502258088919:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.403580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471504842959489:2109] [node 6] ICP22 created new session: [2:7577471504842959521:2109]
2025-11-27T18:19:27.415682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:27.357838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471503783649268:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:27.447046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471502170782554:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(72545903ns) {HandleHandshakeStatus:329(72542899ns) {DropIncomingHandshake:442(1168ns) DropOutgoingHandshake:457(1475ns) SwitchToState:179(616ns) ProcessPendingSessionEvents:495(40364508ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 38936639ns max# 7450459ns}}}
2025-11-27T18:19:27.410247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471500547992130:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(18191576ns) {HandleHandshakeStatus:329(18189343ns) {DropIncomingHandshake:442(14580ns) DropOutgoingHandshake:457(203ns) SwitchToState:179(406ns) ProcessPendingSessionEvents:495(430167ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 384710ns max# 41956ns}}}
2025-11-27T18:19:27.373163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471502258088919:2049] [node 2] ICS09 handshake done sender: [3:7577471502258088903:2116] self: [3:7577471502258088915:0] peer: [2:7577471504842959494:0] socket: 139 qp: -1
2025-11-27T18:19:27.396546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471503960711678:2109] [node 6] ICS09 handshake done sender: [5:7577471503960711656:2110] self: [5:7577471503960711655:0] peer: [6:7577471501891032444:0] socket: 123 qp: -1
2025-11-27T18:19:27.396586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471503960711678:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:27.373187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471502258088919:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:27.447336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471502170782579:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-27T18:19:27.415725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471497596065066:2049] [node 2] ICP22 created new session: [6:7577471501891032452:2049]
2025-11-27T18:19:27.414316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471504842959521:2109] [node 6] ICS09 handshake done sender: [2:7577471504842959501:2114] self: [2:7577471504842959500:0] peer: [6:7577471501891032446:0] socket: 150 qp: -1
2025-11-27T18:19:27.373212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:27.357888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:27.415764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471501891032452:2049] [node 2] ICS09 handshake done sender: [6:7577471501891032438:2118] self: [6:7577471501891032446:0] peer: [2:7577471504842959500:0] socket: 154 qp: -1
2025-11-27T18:19:27.447763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471502170782578:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-27T18:19:27.419165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471504842959521:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:27.367801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471503783649255:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.397098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:27.415797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471501891032452:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:27.364404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471503783649243:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.401468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471503960711679:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10848
2025-11-27T18:19:27.420477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:27.398203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471502258088899:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:27.415857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:27.367839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471503783649255:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.448656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471502170782569:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:27.406631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471502258088890:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.420514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471504842959516:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7181
2025-11-27T18:19:27.413277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471499665744294:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471503960711667:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.452082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471497875815195:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.416616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471497596065066:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:27.367871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.414075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471503960711667:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:27.416990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471497596065065:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.422996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471500547992129:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(152718104ns) {HandleHandshakeStatus:329(152712403ns) {DropIncomingHandshake:442(417ns) DropOutgoingHandshake:457(4260ns) SwitchToState:179(441ns) ProcessPendingSessionEvents:495(121561025ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 121406122ns max# 12048304ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.415294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:27.453282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:27.370615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471503783649243:2110] [node 6] ICP22 created new session: [1:7577471503783649273:2110]
2025-11-27T18:19:27.419260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471502258088890:2109] [node 6] ICP22 created new session: [3:7577471502258088922:2109]
2025-11-27T18:19:27.421293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471499665744294:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.417061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:27.424337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471504842959519:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7181
2025-11-27T18:19:27.453324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471497875815195:2049] [node 2] ICP22 created new session: [4:7577471502170782582:2049]
2025-11-27T18:19:27.426857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:27.417110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471497596065065:2048] [node 1] ICP22 created new session: [6:7577471501891032454:2048]
2025-11-27T18:19:27.426910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471499665744294:2048] [node 1] ICP22 created new session: [5:7577471503960711681:2048]
2025-11-27T18:19:27.458011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471502170782582:2049] [node 2] ICS09 handshake done sender: [4:7577471502170782569:2115] self: [4:7577471502170782575:0] peer: [2:7577471504842959502:0] socket: 166 qp: -1
2025-11-27T18:19:27.417155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471501891032454:2048] [node 1] ICS09 handshake done sender: [6:7577471501891032428:2112] self: [6:7577471501891032445:0] peer: [1:7577471503783649250:0] socket: 129 qp: -1
2025-11-27T18:19:27.371894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471503783649273:2110] [node 6] ICS09 handshake done sender: [1:7577471503783649251:2113] self: [1:7577471503783649250:0] peer: [6:7577471501891032445:0] socket: 124 qp: -1
2025-11-27T18:19:27.428661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471503960711654:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(57492873ns) {HandleHandshakeStatus:329(57489843ns) {DropIncomingHandshake:442(1730ns) DropOutgoingHandshake:457(8502ns) SwitchToState:179(485ns) ProcessPendingSessionEvents:495(53413713ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 53291690ns max# 6924509ns}}}
2025-11-27T18:19:27.419568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471502258088922:2109] [node 6] ICS09 handshake done sender: [3:7577471502258088899:2113] self: [3:7577471502258088898:0] peer: [6:7577471501891032448:0] socket: 147 qp: -1
2025-11-27T18:19:27.417182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471501891032454:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:27.431025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471504842959519:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.461442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471502170782582:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:27.372083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471503783649273:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:27.432602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471504842959516:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.421241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471502258088905:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.433936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471503960711681:2048] [node 1] ICS09 handshake done sender: [5:7577471503960711667:2115] self: [5:7577471503960711673:0] peer: [1:7577471503783649254:0] socket: 143 qp: -1
2025-11-27T18:19:27.417216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:27.432807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471504842959503:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.372118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:27.439520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471503960711681:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:27.421288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471502258088905:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.461490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:27.372274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471503783649269:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.419128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471501891032421:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.432831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471504842959503:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.462183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471502170782574:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:27.421357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471502258088922:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:27.441873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:27.443602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471504842959503:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.444933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471504842959490:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.419178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:27.445424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:27.445761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471504842959490:2107] [node 4] ICP22 created new session: [2:7577471504842959526:2107]
2025-11-27T18:19:27.425158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:27.443552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471503960711684:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10848
2025-11-27T18:19:27.447516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471504842959526:2107] [node 4] ICS09 handshake done sender: [2:7577471504842959503:2115] self: [2:7577471504842959502:0] peer: [4:7577471502170782575:0] socket: 162 qp: -1
2025-11-27T18:19:27.389613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471503783649255:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:27.462521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471502170782571:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:27.394276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471503783649241:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.396773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:27.397030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471503783649241:2108] [node 5] ICP22 created new session: [1:7577471503783649276:2108]
2025-11-27T18:19:27.399767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471503783649276:2108] [node 5] ICS09 handshake done sender: [1:7577471503783649255:2115] self: [1:7577471503783649254:0] peer: [5:7577471503960711673:0] socket: 142 qp: -1
2025-11-27T18:19:27.401362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471503783649276:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:27.402270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:27.414350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471503783649263:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.414448Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471503783649263:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.427025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471503783649263:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:27.474897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471503783649243:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(85124879ns) {HandleHandshakeStatus:329(82462879ns) {DropIncomingHandshake:442(2707ns) DropOutgoingHandshake:457(1398ns) SwitchToState:179(488ns) ProcessPendingSessionEvents:495(57395378ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 57270960ns max# 6521245ns}}}
2025-11-27T18:19:27.475263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471503783649242:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.475317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:27.475206Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471499488681882:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(112800981ns) {HandleHandshakeStatus:329(112796997ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(525ns) SwitchToState:179(783ns) ProcessPendingSessionEvents:495(75882584ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 75721694ns max# 8614478ns}}}
2025-11-27T18:19:27.475667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471503783649242:2107] [node 4] ICP22 created new session: [1:7577471503783649279:2107]
2025-11-27T18:19:27.476479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471503783649279:2107] [node 4] ICS09 handshake done sender: [1:7577471503783649263:2118] self: [1:7577471503783649260:0] peer: [4:7577471502170782576:0] socket: 165 qp: -1
2025-11-27T18:19:27.476746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471503783649279:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:27.477056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:27.483116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471503783649243:2110] [node 6] ICP18 handshake (actor [1:7577471503783649269:2120]) is too old
2025-11-27T18:19:27.419220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471501891032421:2105] [node 5] ICP22 created new session: [6:7577471501891032456:2105]
2025-11-27T18:19:27.438305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471497963121531:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(95778969ns) {HandleHandshakeStatus:329(95775288ns) {DropIncomingHandshake:442(607ns) DropOutgoingHandshake:457(5595ns) SwitchToState:179(454ns) ProcessPendingSessionEvents:495(68646506ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 67457886ns max# 7541652ns}}}
2025-11-27T18:19:27.462843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471497875815196:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4162) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471502170782574:2117] held: no
2025-11-27T18:19:27.447555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471504842959526:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:27.444326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471503960711682:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10848
2025-11-27T18:19:27.488209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471503783649241:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(79506605ns) {HandleHandshakeStatus:329(79503703ns) {DropIncomingHandshake:442(15362ns) DropOutgoingHandshake:457(363ns) SwitchToState:179(426ns) ProcessPendingSessionEvents:495(58453543ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 55536410ns max# 5521208ns}}}
2025-11-27T18:19:27.419319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471501891032456:2105] [node 5] ICS09 handshake done sender: [6:7577471501891032427:2111] self: [6:7577471501891032444:0] peer: [5:7577471503960711655:0] socket: 128 qp: -1
2025-11-27T18:19:27.463413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471497875815196:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.419529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471501891032456:2105] [node 5] ICS10 traffic start
2025-11-27T18:19:27.447844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:27.495811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471499488681882:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:27.420735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:27.456721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471504842959509:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.456744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471504842959509:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.446006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471501891032451:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59185
2025-11-27T18:19:27.457794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471504842959509:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:27.448111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471501891032437:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:27.439240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471502258088905:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:27.450325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471497596065067:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.468273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471504842959488:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.444132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471502258088891:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.450796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:27.468334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:27.445374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:27.450838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471497596065067:2050] [node 3] ICP22 created new session: [6:7577471501891032460:2050]
2025-11-27T18:19:27.468378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471504842959488:2108] [node 5] ICP22 created new session: [2:7577471504842959529:2108]
2025-11-27T18:19:27.445729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471502258088891:2107] [node 4] ICP22 created new session: [3:7577471502258088926:2107]
2025-11-27T18:19:27.450899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471501891032460:2050] [node 3] ICS09 handshake done sender: [6:7577471501891032437:2117] self: [6:7577471501891032448:0] peer: [3:7577471502258088898:0] socket: 153 qp: -1
2025-11-27T18:19:27.468431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471504842959529:2108] [node 5] ICS09 handshake done sender: [2:7577471504842959509:2117] self: [2:7577471504842959506:0] peer: [5:7577471503960711685:0] socket: 131 qp: -1
2025-11-27T18:19:27.446798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471502258088926:2107] [node 4] ICS09 handshake done sender: [3:7577471502258088905:2117] self: [3:7577471502258088904:0] peer: [4:7577471502170782577:0] socket: 175 qp: -1
2025-11-27T18:19:27.450947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471501891032460:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:27.447181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471502258088926:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:27.468465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471504842959529:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:27.451000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:27.473610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471501891032462:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59185
2025-11-27T18:19:27.468496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:27.447236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:27.476433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471502258088902:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.496528Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471504842959489:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(87550308ns) {HandleHandshakeStatus:329(87547252ns) {DropIncomingHandshake:442(11913ns) DropOutgoingHandshake:457(376ns) SwitchToState:179(476ns) ProcessPendingSessionEvents:495(51933009ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 48690495ns max# 9858992ns}}}
2025-11-27T18:19:27.476716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471501891032462:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.480208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471501891032425:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.481727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471501891032421:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(61396274ns) {HandleHandshakeStatus:329(61391702ns) {DropIncomingHandshake:442(1442ns) DropOutgoingHandshake:457(6846ns) SwitchToState:179(377ns) ProcessPendingSessionEvents:495(42807017ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 40143414ns max# 6677656ns}}}
2025-11-27T18:19:27.445728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471499665744294:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23626889ns) {HandleHandshakeStatus:329(23624078ns) {DropIncomingHandshake:442(630ns) DropOutgoingHandshake:457(272ns) SwitchToState:179(5521ns) ProcessPendingSessionEvents:495(333ns)}}
2025-11-27T18:19:27.497499Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471497596065067:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(29025464ns) {HandleHandshakeStatus:329(29023390ns) {DropIncomingHandshake:442(1172ns) DropOutgoingHandshake:457(3768ns) SwitchToState:179(301ns) ProcessPendingSessionEvents:495(27446681ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26568439ns max# 9707315ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.476474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471502258088902:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.463468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:27.477064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471497963121532:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(75820984ns) {HandleHandshakeStatus:329(75818926ns) {DropIncomingHandshake:442(498ns) DropOutgoingHandshake:457(4030ns) SwitchToState:179(312ns) ProcessPendingSessionEvents:495(75046495ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 74896913ns max# 14498291ns}}}
2025-11-27T18:19:27.486126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471502258088928:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-11-27T18:19:27.451751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471503960711682:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.454337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471499665744295:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.463516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471497875815196:2050] [node 3] ICP22 created new session: [4:7577471502170782584:2050]
2025-11-27T18:19:27.502140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471504842959490:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(41488923ns) {HandleHandshakeStatus:329(41487030ns) {DropIncomingHandshake:442(12392ns) DropOutgoingHandshake:457(889ns) SwitchToState:179(181ns) ProcessPendingSessionEvents:495(37321364ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 37221710ns max# 4509921ns}}}
2025-11-27T18:19:27.501048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471501891032464:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59185
2025-11-27T18:19:27.463564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471502170782584:2050] [node 3] ICS09 handshake done sender: [4:7577471502170782574:2117] self: [4:7577471502170782577:0] peer: [3:7577471502258088904:0] socket: 176 qp: -1
2025-11-27T18:19:27.502499Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471502258088890:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(73581753ns) {HandleHandshakeStatus:329(73578578ns) {DropIncomingHandshake:442(1497ns) DropOutgoingHandshake:457(887ns) SwitchToState:179(830ns) ProcessPendingSessionEvents:495(42887796ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 41806179ns max# 8670662ns}}}
2025-11-27T18:19:27.456738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471503960711684:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.502018Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471501891032425:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30024) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471501891032462:2124] held: no
2025-11-27T18:19:27.463594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471502170782584:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:27.506151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471502258088891:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(52179862ns) {HandleHandshakeStatus:329(52177768ns) {DropIncomingHandshake:442(1197109ns) DropOutgoingHandshake:457(478ns) SwitchToState:179(339ns) ProcessPendingSessionEvents:495(46181037ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 45322046ns max# 7412297ns}}}
2025-11-27T18:19:27.463624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:27.464017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471497875815194:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.464074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:27.464112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471497875815194:2048] [node 1] ICP22 created new session: [4:7577471502170782586:2048]
2025-11-27T18:19:27.464147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471502170782586:2048] [node 1] ICS09 handshake done sender: [4:7577471502170782571:2116] self: [4:7577471502170782576:0] peer: [1:7577471503783649260:0] socket: 168 qp: -1
2025-11-27T18:19:27.464177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471502170782586:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:27.464201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:27.465089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471502170782581:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-27T18:19:27.477307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471502170782581:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.483342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471502170782564:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:27.483378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471502170782564:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:27.484317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471502170782564:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:27.496278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471502170782553:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.496401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:27.496448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471502170782553:2109] [node 6] ICP22 created new session: [4:7577471502170782591:2109]
2025-11-27T18:19:27.496489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471502170782591:2109] [node 6] ICS09 handshake done sender: [4:7577471502170782564:2113] self: [4:7577471502170782562:0] peer: [6:7577471501891032463:0] socket: 118 qp: -1
2025-11-27T18:19:27.496522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471502170782591:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:27.496578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:27.458737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471499665744296:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:27.459161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471503960711687:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10848
2025-11-27T18:19:27.474503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471503960711682:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:27.477042Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471499665744295:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471503960711682:2120] held: no
2025-11-27T18:19:27.501238Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471503960711653:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(104328861ns) {HandleHandshakeStatus:329(104326700ns) {DropIncomingHandshake:442(1012ns) DropOutgoingHandshake:457(280ns) SwitchToState:179(416ns) ProcessPendingSessionEvents:495(74302839ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 68829550ns max# 7910344ns}}}
2025-11-27T18:19:27.507633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471502258088902:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:27.514293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471502258088892:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:27.514410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:27.514460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471502258088892:2108] [node 5] ICP22 created new session: [3:7577471502258088930:2108]
2025-11-27T18:19:27.514503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471502258088930:2108] [node 5] ICS09 handshake done sender: [3:7577471502258088902:2115] self: [3:7577471502258088901:0] peer: [5:7577471503960711686:0] socket: 159 qp: -1
2025-11-27T18:19:27.514535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471502258088930:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:27.514574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:27.501606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577471503960711653:2109] [node 6] ICP18 handshake (actor [5:7577471503960711672:2118]) is too old
2025-11-27T18:19:27.514630Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471499665744295:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(19225318ns) {HandleHandshakeStatus:395(19222306ns) {LogHandshakeFail:472(11770ns) {UpdateErrorStateLog:545(3945ns)} DropOutgoingHandshake:457(989ns) ProcessPendingSessionEvents:495(18851411ns) {unordered DropSessionEvent:512 num# 100 sum# 18802943ns max# 3510387ns}}}
2025-11-27T18:19:27.515805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471499665744295:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.515862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:27.515909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471499665744295:2049] [node 2] ICP22 created new session: [5:7577471503960711689:2049]
2025-11-27T18:19:27.515951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471503960711689:2049] [node 2] ICS09 handshake done sender: [5:7577471503960711682:2120] self: [5:7577471503960711685:0] peer: [2:7577471504842959506:0] socket: 157 qp: -1
2025-11-27T18:19:27.515985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471503960711689:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:27.522340Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471497875815195:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(44737600ns) {HandleHandshakeStatus:329(44735435ns) {DropIncomingHandshake:442(4075ns) DropOutgoingHandshake:457(7848ns) SwitchToState:179(331ns) ProcessPendingSessionEvents:495(25738028ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24350673ns max# 3667639ns}}}
2025-11-27T18:19:27.516015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:27.523263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471497875815195:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:27.517132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471503960711688:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10848
2025-11-27T18:19:27.517356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471503960711684:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:27.519357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471499665744296:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.523393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:27.524374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471499665744296:2050] [node 3] ICP22 created new session: [5:7577471503960711691:2050]
2025-11-27T18:19:27.527096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471503960711691:2050] [node 3] ICS09 handshake done sender: [5:7577471503960711684:2121] self: [5:7577471503960711686:0] peer: [3:7577471502258088901:0] socket: 182 qp: -1
2025-11-27T18:19:27.527893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471503960711691:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:27.528095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:27.543286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471501891032425:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(19257460ns) {HandleHandshakeStatus:395(19039512ns) {LogHandshakeFail:472(15374ns) {UpdateErrorStateLog:545(4337ns)} DropOutgoingHandshake:457(490ns) ProcessPendingSessionEvents:495(18576997ns) {unordered DropSessionEvent:512 num# 100 sum# 6533806ns max# 2785820ns}}}
2025-11-27T18:19:27.543417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471501891032462:2124] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:27.543637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471501891032425:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:27.543858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:27.544047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471501891032425:2108] [node 4] ICP22 created new session: [6:7577471501891032465:2108]
2025-11-27T18:19:27.544099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471501891032465:2108] [node 4] ICS09 handshake done sender: [6:7577471501891032462:2124] self: [6:7577471501891032463:0] peer: [4:7577471502170782562:0] socket: 145 qp: -1
2025-11-27T18:19:27.544149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471501891032465:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:27.544207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:27.590128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:27.590370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471504842959507:2048] [node 1] ICS07 socket disconnect 116 reason# EndOfStream
2025-11-27T18:19:27.590411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471504842959507:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.590491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:27.590550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471504842959507:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:27.591570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:27.591787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471502258088914:2048] [node 1] ICS07 socket disconnect 119 reason# EndOfStream
2025-11-27T18:19:27.591818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471502258088914:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.591903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:27.591961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471502258088914:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:27.594601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471497963121531:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:27.594661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471502258088914:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:27.595465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471500547992129:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:27.595508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471504842959507:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:27.598148Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:27.598200Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471497963121531:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:27.598280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471502258088914:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:27.598322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471497963121531:2048] [node 1] ICP30 unregister session Session# [3:7577471502258088914:2048] VirtualId# [3:7577471502258088909:0]
2025-11-27T18:19:27.598368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:27.598708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471502170782586:2048] [node 1] ICS07 socket disconnect 168 reason# EndOfStream
2025-11-27T18:19:27.598398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471497963121531:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.598413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.599698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471502170782586:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.599782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:27.599835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471502170782586:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:27.599459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471500547992129:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:27.599520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471504842959507:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:27.600381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:27.600988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471503960711681:2048] [node 1] ICS01 socket: 143 reason# EndOfStream
2025-11-27T18:19:27.605302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:27.605543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471501891032454:2048] [node 1] ICS01 socket: 129 reason# EndOfStream
2025-11-27T18:19:27.605585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471497596065065:2048] [node 1] ICP30 unregister session Session# [6:7577471501891032454:2048] VirtualId# [6:7577471501891032445:0]
2025-11-27T18:19:27.605617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471501891032454:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.605691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:27.605739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:27.599562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471500547992129:2048] [node 1] ICP30 unregister session Session# [2:7577471504842959507:2048] VirtualId# [2:7577471504842959498:0]
2025-11-27T18:19:27.601091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471499665744294:2048] [node 1] ICP30 unregister session Session# [5:7577471503960711681:2048] VirtualId# [5:7577471503960711673:0]
2025-11-27T18:19:27.608205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471497963121531:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:27.601111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471503960711681:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.606648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:27.606718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:27.606834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471497875815194:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:27.608400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471502170782586:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:27.599689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:27.599722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471500547992129:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.599736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.610183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471497875815194:2048] [node 1] ICP30 unregister session Session# [4:7577471502170782586:2048] VirtualId# [4:7577471502170782576:0]
2025-11-27T18:19:27.610257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471500547992129:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:27.610293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:27.610369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471497875815194:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.610400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.620475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471497875815194:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:27.654402Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:27.654664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471502258088919:2049] [node 2] ICS07 socket disconnect 139 reason# EndOfStream
2025-11-27T18:19:27.654710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471502258088919:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.654797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:27.654851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471502258088919:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:27.658191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:27.658417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471502170782582:2049] [node 2] ICS07 socket disconnect 166 reason# EndOfStream
2025-11-27T18:19:27.658705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471502170782582:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.658774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:27.658827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471502170782582:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:27.658325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471497963121532:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3356) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:27.658376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471502258088919:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:27.674889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471497963121532:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3356) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:27.674798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471497875815195:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3356) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:27.674883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471502170782582:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:27.676390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:27.674962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471497875815195:2049] [node 2] ICP30 unregister session Session# [4:7577471502170782582:2049] VirtualId# [4:7577471502170782575:0]
2025-11-27T18:19:27.674987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471502258088919:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:27.675310Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:27.675284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:27.675320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471497875815195:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.675335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.676838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471501891032452:2049] [node 2] ICS01 socket: 154 reason# EndOfStream
2025-11-27T18:19:27.675020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471497963121532:2049] [node 2] ICP30 unregister session Session# [3:7577471502258088919:2049] VirtualId# [3:7577471502258088915:0]
2025-11-27T18:19:27.675289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:27.676892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471497596065066:2049] [node 2] ICP30 unregister session Session# [6:7577471501891032452:2049] VirtualId# [6:7577471501891032446:0]
2025-11-27T18:19:27.675322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471497963121532:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.676944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471501891032452:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.676604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.677045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:27.676901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471503960711689:2049] [node 2] ICS01 socket: 157 reason# EndOfStream
2025-11-27T18:19:27.685776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471497963121532:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:27.677120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:27.676959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471499665744295:2049] [node 2] ICP30 unregister session Session# [5:7577471503960711689:2049] VirtualId# [5:7577471503960711685:0]
2025-11-27T18:19:27.676995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471503960711689:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.677156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:27.677234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:27.685875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471497875815195:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:27.756989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:27.757306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471502170782584:2050] [node 3] ICS01 socket: 176 reason# EndOfStream
2025-11-27T18:19:27.757365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471497875815196:2050] [node 3] ICP30 unregister session Session# [4:7577471502170782584:2050] VirtualId# [4:7577471502170782577:0]
2025-11-27T18:19:27.757405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471502170782584:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.757486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:27.757545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:27.758160Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:27.758345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471503960711691:2050] [node 3] ICS07 socket disconnect 182 reason# EndOfStream
2025-11-27T18:19:27.758375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471503960711691:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.758448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:27.758494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471503960711691:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:27.759231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471499665744296:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4162) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:27.759284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471503960711691:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:27.759361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471499665744296:2050] [node 3] ICP30 unregister session Session# [5:7577471503960711691:2050] VirtualId# [5:7577471503960711686:0]
2025-11-27T18:19:27.759865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:27.759944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471499665744296:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4162) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.759974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4162) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.759550Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:27.759853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471501891032460:2050] [node 3] ICS07 socket disconnect 153 reason# EndOfStream
2025-11-27T18:19:27.759887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471501891032460:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.759980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:27.760021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471501891032460:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:27.760920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471497596065067:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4162) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:27.761147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471501891032460:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:27.761199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471497596065067:2050] [node 3] ICP30 unregister session Session# [6:7577471501891032460:2050] VirtualId# [6:7577471501891032448:0]
2025-11-27T18:19:27.761328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:27.761357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471497596065067:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4162) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.761371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4162) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:27.769925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471499665744296:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:27.770899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471497596065067:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:27.829721Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:27.830008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471503960711675:2108] [node 4] ICS01 socket: 126 reason# EndOfStream
2025-11-27T18:19:27.830055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471503960711654:2108] [node 4] ICP30 unregister session Session# [5:7577471503960711675:2108] VirtualId# [5:7577471503960711670:0]
2025-11-27T18:19:27.830087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471503960711675:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.830168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:27.830213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:27.831334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:27.831563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471501891032465:2108] [node 4] ICS01 socket: 145 reason# EndOfStream
2025-11-27T18:19:27.831596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471501891032425:2108] [node 4] ICP30 unregister session Session# [6:7577471501891032465:2108] VirtualId# [6:7577471501891032463:0]
2025-11-27T18:19:27.831627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471501891032465:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.831698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:27.831734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:27.917586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:27.918059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471501891032456:2105] [node 5] ICS01 socket: 128 reason# EndOfStream
2025-11-27T18:19:27.918097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471501891032421:2105] [node 5] ICP30 unregister session Session# [6:7577471501891032456:2105] VirtualId# [6:7577471501891032444:0]
2025-11-27T18:19:27.918128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471501891032456:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:27.918242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:27.918341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:28.074914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471509639329153:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:28.075128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471509639329154:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:28.095172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471506201514791:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:28.111185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471506201514790:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:28.140342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471509872020008:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:28.140390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471509872020009:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:28.167516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471509750873185:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:28.167561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471509750873187:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:28.167560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471509750873186:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:28.192140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471506906569462:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:28.192195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471506906569464:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:28.192872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471506906569463:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:28.228213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471506295862727:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:28.228249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471506295862729:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:28.228865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471506295862728:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:28.356739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471506295862781:2099] [node 5] ICP01 ready to work
2025-11-27T18:19:28.366151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471506295862787:2104] [node 4] ICP01 ready to work
2025-11-27T18:19:28.373196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471509639329216:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:28.379012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471509639329217:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:28.387611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471506201514857:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:28.453807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471506201514791:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(50318144ns) {Configure:104(50313003ns) {ProcessConfigured:135(45742455ns) {StartInitialHandshake:157(45739077ns) {DropHandshakes:470(3308ns) {DropIncomingHandshake:442(819ns) DropOutgoingHandshake:457(346ns)} GenerateSessionVirtualId:988(1184ns) PrepareNewSessionHandshake:477(4831ns) {SwitchToState:179(2740ns)}}}}}
2025-11-27T18:19:28.458000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471509750873266:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:28.387801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471509639329218:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:28.462724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471506906569526:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:28.463667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471509872020072:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:28.457980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471506201514859:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:28.474799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471506295862787:2104] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(100583863ns) {Configure:104(96119288ns) {ProcessConfigured:135(75078ns) {StartInitialHandshake:157(70105ns) {DropHandshakes:470(3292ns) {DropIncomingHandshake:442(877ns) DropOutgoingHandshake:457(360ns)} GenerateSessionVirtualId:988(999ns) PrepareNewSessionHandshake:477(2212ns) {SwitchToState:179(814ns)}}}}}
2025-11-27T18:19:28.464145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471509872020071:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:28.464746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471506201514861:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:28.466272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471509750873267:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:28.469377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471506906569527:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:28.463557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471509872020073:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.512844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471509872020074:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-11-27T18:19:28.519007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471509639329153:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(161691292ns) {Configure:104(161686125ns) {ProcessConfigured:135(12648338ns) {StartInitialHandshake:157(12646003ns) {DropHandshakes:470(1139ns) {DropIncomingHandshake:442(248ns) DropOutgoingHandshake:457(172ns)} GenerateSessionVirtualId:988(592ns) PrepareNewSessionHandshake:477(8022ns) {SwitchToState:179(1985ns)}}}}}
2025-11-27T18:19:28.513895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471506295862781:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(123841476ns) {Configure:104(123837728ns) {ProcessConfigured:135(15316028ns) {StartInitialHandshake:157(15313408ns) {DropHandshakes:470(3250ns) {DropIncomingHandshake:442(849ns) DropOutgoingHandshake:457(447ns)} GenerateSessionVirtualId:988(815ns) PrepareNewSessionHandshake:477(4436ns) {SwitchToState:179(2073ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.585756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471506295862798:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48431
2025-11-27T18:19:28.600141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471506295862729:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(44971200ns) {Configure:104(44966147ns) {ProcessConfigured:135(17296956ns) {StartInitialHandshake:157(17294893ns) {DropHandshakes:470(1909ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(333ns)} GenerateSessionVirtualId:988(579ns) PrepareNewSessionHandshake:477(2844ns) {SwitchToState:179(95ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.577358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471509750873185:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(114275054ns) {Configure:104(114267726ns) {ProcessConfigured:135(64808ns) {StartInitialHandshake:157(61617ns) {DropHandshakes:470(3376ns) {DropIncomingHandshake:442(838ns) DropOutgoingHandshake:457(747ns)} GenerateSessionVirtualId:988(873ns) PrepareNewSessionHandshake:477(2338ns) {SwitchToState:179(964ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.586379Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471506201514790:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(198417603ns) {Configure:104(198405286ns) {ProcessConfigured:135(34264897ns) {StartInitialHandshake:157(34261324ns) {DropHandshakes:470(3191ns) {DropIncomingHandshake:442(761ns) DropOutgoingHandshake:457(431ns)} GenerateSessionVirtualId:988(889ns) PrepareNewSessionHandshake:477(3398ns) {SwitchToState:179(1638ns)}}}}}
2025-11-27T18:19:28.615312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471506906569534:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54053
2025-11-27T18:19:28.587221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471509872020074:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.620870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471509750873272:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2889
2025-11-27T18:19:28.612688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471506201514858:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.611301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471509872020009:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(116690519ns) {Configure:104(116684355ns) {ProcessConfigured:135(63250ns) {StartInitialHandshake:157(61184ns) {DropHandshakes:470(2935ns) {DropIncomingHandshake:442(698ns) DropOutgoingHandshake:457(435ns)} GenerateSessionVirtualId:988(919ns) PrepareNewSessionHandshake:477(1778ns) {SwitchToState:179(667ns)}}}}}
2025-11-27T18:19:28.619774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471506295862798:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.646133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471506906569464:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(142863578ns) {Configure:104(142857437ns) {ProcessConfigured:135(61124ns) {StartInitialHandshake:157(58991ns) {DropHandshakes:470(2891ns) {DropIncomingHandshake:442(709ns) DropOutgoingHandshake:457(353ns)} GenerateSessionVirtualId:988(716ns) PrepareNewSessionHandshake:477(1923ns) {SwitchToState:179(556ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.612715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471506201514858:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.661936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471509639329154:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(198051264ns) {Configure:104(198041216ns) {ProcessConfigured:135(14664838ns) {StartInitialHandshake:157(14661823ns) {DropHandshakes:470(1956ns) {DropIncomingHandshake:442(478ns) DropOutgoingHandshake:457(426ns)} GenerateSessionVirtualId:988(535ns) PrepareNewSessionHandshake:477(4148ns) {SwitchToState:179(1973ns)}}}}}
2025-11-27T18:19:28.620052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471506295862805:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48431
2025-11-27T18:19:28.611560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471509872020009:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.670061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471509750873272:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.624541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471509872020008:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(109480952ns) {Configure:104(109472561ns) {ProcessConfigured:135(59878ns) {StartInitialHandshake:157(57008ns) {DropHandshakes:470(3813ns) {DropIncomingHandshake:442(1060ns) DropOutgoingHandshake:457(518ns)} GenerateSessionVirtualId:988(1134ns) PrepareNewSessionHandshake:477(2200ns) {SwitchToState:179(838ns)}}}}}
2025-11-27T18:19:28.680349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471506906569538:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54053
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.686636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471509639329218:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(82063665ns) {Configure:104(82058048ns) {ProcessConfigured:135(52648ns) {StartInitialHandshake:157(50831ns) {DropHandshakes:470(2231ns) {DropIncomingHandshake:442(522ns) DropOutgoingHandshake:457(417ns)} GenerateSessionVirtualId:988(911ns) PrepareNewSessionHandshake:477(844ns) {SwitchToState:179(394ns)}}}}}
2025-11-27T18:19:28.673269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471506201514865:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-11-27T18:19:28.620132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471506295862729:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:28.681252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471509750873186:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(203408992ns) {Configure:104(203400031ns) {ProcessConfigured:135(10552083ns) {StartInitialHandshake:157(10549865ns) {DropHandshakes:470(3131ns) {DropIncomingHandshake:442(547ns) DropOutgoingHandshake:457(435ns)} GenerateSessionVirtualId:988(757ns) PrepareNewSessionHandshake:477(3271ns) {SwitchToState:179(1839ns)}}}}}
2025-11-27T18:19:28.635136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471509872020076:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.704829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471506906569533:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54053
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.681571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471509750873271:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2889
2025-11-27T18:19:28.679543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471506201514859:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21679265ns) {Configure:104(21669186ns) {ProcessConfigured:135(64076ns) {StartInitialHandshake:157(61420ns) {DropHandshakes:470(6331ns) {DropIncomingHandshake:442(562ns) DropOutgoingHandshake:457(478ns)} GenerateSessionVirtualId:988(684ns) PrepareNewSessionHandshake:477(1997ns) {SwitchToState:179(618ns)}}}}}
2025-11-27T18:19:28.635179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471509872020076:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:28.625115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471506295862797:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48431
2025-11-27T18:19:28.662710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471509872020091:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-11-27T18:19:28.664141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471509872020076:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:28.682091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471506201514858:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:28.714305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471506906569534:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.681710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471509750873277:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.709635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471509872020091:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.670399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471506295862797:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.745529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471509639329225:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36731
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.682219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471509750873278:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2889
2025-11-27T18:19:28.725504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471506906569532:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.713390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471509872020087:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-11-27T18:19:28.745582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471509639329231:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36731
2025-11-27T18:19:28.709463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471506201514863:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.725530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471506906569532:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:28.682365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471506295862805:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.713838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471509872020092:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-11-27T18:19:28.689760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471509750873277:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.709511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471506201514863:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:28.714130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471506201514791:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:28.718580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471506201514868:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-11-27T18:19:28.720632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:28.720707Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471506201514865:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.720883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471506201514863:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:28.727091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471506201514791:2049] [node 3] ICP22 created new session: [2:7577471506201514873:2049]
2025-11-27T18:19:28.733840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471506201514790:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:28.733975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471506201514868:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.748281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471506201514873:2049] [node 3] ICS09 handshake done sender: [2:7577471506201514858:2113] self: [2:7577471506201514850:0] peer: [3:7577471509872020083:0] socket: 120 qp: -1
2025-11-27T18:19:28.748334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471506201514873:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:28.748396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:28.748456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471506201514857:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:28.748506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:28.748566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471506201514857:2107] [node 4] ICP22 created new session: [2:7577471506201514878:2107]
2025-11-27T18:19:28.749855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471506201514791:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(34965140ns) {HandleHandshakeStatus:329(34962069ns) {DropIncomingHandshake:442(1032ns) DropOutgoingHandshake:457(1060ns) SwitchToState:179(807ns) ProcessPendingSessionEvents:495(509339ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 457587ns max# 60463ns}}}
2025-11-27T18:19:28.749982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471506201514791:2049] [node 3] ICP18 handshake (actor [2:7577471506201514868:2117]) is too old
2025-11-27T18:19:28.751515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471506201514878:2107] [node 4] ICS09 handshake done sender: [2:7577471506201514863:2114] self: [2:7577471506201514862:0] peer: [4:7577471509750873284:0] socket: 128 qp: -1
2025-11-27T18:19:28.752863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471506201514878:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:28.764515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:28.687396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471506295862807:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48431
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.742210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471509639329216:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(180624402ns) {Configure:104(180617624ns) {ProcessConfigured:135(223516ns) {StartInitialHandshake:157(217042ns) {DropHandshakes:470(3625ns) {DropIncomingHandshake:442(1048ns) DropOutgoingHandshake:457(470ns)} GenerateSessionVirtualId:988(572ns) PrepareNewSessionHandshake:477(1782ns) {SwitchToState:179(641ns)}}}}}
2025-11-27T18:19:28.689904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471509750873186:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:28.726837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471509872020071:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:28.726057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471506906569533:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.751349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471509639329223:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.705270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471506295862781:2099] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:28.839641Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471506201514861:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(175287590ns) {Configure:104(175280305ns) {ProcessConfigured:135(72534ns) {StartInitialHandshake:157(70198ns) {DropHandshakes:470(2663ns) {DropIncomingHandshake:442(643ns) DropOutgoingHandshake:457(465ns)} GenerateSessionVirtualId:988(362ns) PrepareNewSessionHandshake:477(1480ns) {SwitchToState:179(551ns)}}}}}
2025-11-27T18:19:28.736568Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471506906569462:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(195194410ns) {Configure:104(195185670ns) {ProcessConfigured:135(13439261ns) {StartInitialHandshake:157(13435480ns) {DropHandshakes:470(1320ns) {DropIncomingHandshake:442(230ns) DropOutgoingHandshake:457(359ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(3971ns) {SwitchToState:179(888ns)}}}}}
2025-11-27T18:19:28.701869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471509750873271:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.737650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.709763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471506295862787:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.751387Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471509639329223:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:28.736773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471506906569462:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:28.740264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471509872020074:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:28.726698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471509750873269:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.737497Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471506295862787:2104] [node 4] ICP35 event processing took too much time PendingConnection:279(16231334ns) {IncomingHandshake:261(7477941ns) {DropIncomingHandshake:442(327ns) PrepareNewSessionHandshake:477(1150ns) {SwitchToState:179(610ns)} GenerateSessionVirtualId:988(388ns) IssueIncomingHandshakeReply:193(7439969ns) {SwitchToState:179(1957ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.756572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471509639329234:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36731
2025-11-27T18:19:28.726731Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471509750873269:2128] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:28.740106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471506906569532:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:28.740573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471506906569526:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:28.740620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:28.740652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471506906569526:2109] [node 6] ICP22 created new session: [5:7577471506906569547:2109]
2025-11-27T18:19:28.740686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471506906569547:2109] [node 6] ICS09 handshake done sender: [5:7577471506906569532:2112] self: [5:7577471506906569531:0] peer: [6:7577471506295862809:0] socket: 148 qp: -1
2025-11-27T18:19:28.740711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471506906569547:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:28.740738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:28.751240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471506906569463:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(218065360ns) {Configure:104(218056742ns) {ProcessConfigured:135(27387324ns) {StartInitialHandshake:157(27385104ns) {DropHandshakes:470(4872484ns) {DropIncomingHandshake:442(345ns) DropOutgoingHandshake:457(2035ns)} GenerateSessionVirtualId:988(1932ns) PrepareNewSessionHandshake:477(3129ns) {SwitchToState:179(1246ns)}}}}}
2025-11-27T18:19:28.773976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471506906569538:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.774119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471506906569527:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:28.853492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471506906569552:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54053
2025-11-27T18:19:28.861524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471506906569551:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54053
2025-11-27T18:19:28.882652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471506906569551:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.882817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471506906569553:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54053
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.742504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471509872020008:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.763946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471506295862798:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:28.875611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471506201514790:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31631) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471506201514865:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.727819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471509750873278:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.914290Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471506906569527:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18699) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471506906569538:2116] held: no
2025-11-27T18:19:28.751590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471509872020071:2109] [node 6] ICP22 created new session: [3:7577471509872020094:2109]
2025-11-27T18:19:28.756714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471509639329220:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.893912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471506201514880:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-11-27T18:19:28.766032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471506295862810:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48431
2025-11-27T18:19:28.742756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471509750873286:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2889
2025-11-27T18:19:28.757556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471509750873269:2128] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:28.773210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471509750873277:2133] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:28.778413Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471509750873187:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(205737091ns) {Configure:104(205728003ns) {ProcessConfigured:135(8858526ns) {StartInitialHandshake:157(8855266ns) {DropHandshakes:470(1697ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(508ns) PrepareNewSessionHandshake:477(3856ns) {SwitchToState:179(1959ns)}}}}}
2025-11-27T18:19:28.785821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471509750873187:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:28.796458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471509750873186:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:28.800385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:28.809722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471509750873186:2049] [node 2] ICP22 created new session: [4:7577471509750873291:2049]
2025-11-27T18:19:28.826027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471509750873276:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.826064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471509750873276:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:28.835207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471509750873267:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:28.837377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471509750873291:2049] [node 2] ICS09 handshake done sender: [4:7577471509750873277:2133] self: [4:7577471509750873284:0] peer: [2:7577471506201514862:0] socket: 140 qp: -1
2025-11-27T18:19:28.839269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471509750873291:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:28.842197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:28.847735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:28.847800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471509750873276:2132] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:28.854003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471509750873267:2109] [node 6] ICP22 created new session: [4:7577471509750873293:2109]
2025-11-27T18:19:28.857787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471509750873266:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:28.857887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:28.857952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471509750873266:2108] [node 5] ICP22 created new session: [4:7577471509750873296:2108]
2025-11-27T18:19:28.858008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471509750873296:2108] [node 5] ICS09 handshake done sender: [4:7577471509750873276:2132] self: [4:7577471509750873275:0] peer: [5:7577471506906569550:0] socket: 154 qp: -1
2025-11-27T18:19:28.858062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471509750873296:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:28.858096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:28.864733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471509750873293:2109] [node 6] ICS09 handshake done sender: [4:7577471509750873269:2128] self: [4:7577471509750873268:0] peer: [6:7577471506295862808:0] socket: 125 qp: -1
2025-11-27T18:19:28.875229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471509750873293:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:28.882400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:28.919209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471509750873300:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2889
2025-11-27T18:19:28.914329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471506906569533:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:28.752737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471509872020009:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:28.756754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471509639329220:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:28.965188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471509750873300:2139] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.917748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471506201514872:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.757102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471509872020094:2109] [node 6] ICS09 handshake done sender: [3:7577471509872020076:2111] self: [3:7577471509872020075:0] peer: [6:7577471506295862806:0] socket: 129 qp: -1
2025-11-27T18:19:28.761762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:28.763111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471509872020087:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.765216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471509872020094:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:28.765232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471509872020008:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20001417ns) {IncomingHandshake:261(19999216ns) {DropIncomingHandshake:442(337ns) PrepareNewSessionHandshake:477(932ns) {SwitchToState:179(310ns)} GenerateSessionVirtualId:988(367ns) IssueIncomingHandshakeReply:193(4787601ns) {SwitchToState:179(1663ns)}}}
2025-11-27T18:19:28.777820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471509872020009:2049] [node 2] ICP22 created new session: [3:7577471509872020095:2049]
2025-11-27T18:19:28.785277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:28.792786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471509872020071:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(64961122ns) {HandleHandshakeStatus:329(64958307ns) {DropIncomingHandshake:442(173ns) DropOutgoingHandshake:457(660ns) SwitchToState:179(333ns) ProcessPendingSessionEvents:495(522724ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 464692ns max# 74137ns}}}
2025-11-27T18:19:28.794380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471509872020071:2109] [node 6] ICP18 handshake (actor [3:7577471509872020087:2115]) is too old
2025-11-27T18:19:28.796054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471509872020095:2049] [node 2] ICS09 handshake done sender: [3:7577471509872020074:2110] self: [3:7577471509872020083:0] peer: [2:7577471506201514850:0] socket: 122 qp: -1
2025-11-27T18:19:28.809831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471509872020095:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:28.809906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:28.825815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471509872020079:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.825859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471509872020079:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:28.831987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471509872020099:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-11-27T18:19:28.842283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471509872020079:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:28.848041Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471509872020091:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:28.873893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471509872020008:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:28.873990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:28.874055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471509872020008:2048] [node 1] ICP22 created new session: [3:7577471509872020103:2048]
2025-11-27T18:19:28.874117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471509872020103:2048] [node 1] ICS09 handshake done sender: [3:7577471509872020091:2117] self: [3:7577471509872020093:0] peer: [1:7577471509639329226:0] socket: 159 qp: -1
2025-11-27T18:19:28.874154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471509872020103:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:28.874197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:28.882589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471509872020072:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:28.883669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471509872020101:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-11-27T18:19:28.884301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:28.884360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471509872020072:2107] [node 4] ICP22 created new session: [3:7577471509872020106:2107]
2025-11-27T18:19:28.902713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471509872020105:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-11-27T18:19:28.925464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471509872020106:2107] [node 4] ICS09 handshake done sender: [3:7577471509872020079:2112] self: [3:7577471509872020078:0] peer: [4:7577471509750873290:0] socket: 145 qp: -1
2025-11-27T18:19:28.937605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471509872020105:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.937621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471509872020106:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:28.937672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:28.949679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471509872020101:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.965452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471509872020089:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.965494Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471509872020089:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:28.766167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471506295862811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48431
2025-11-27T18:19:28.931484Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471506906569462:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31631) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471506906569533:2113] held: no
2025-11-27T18:19:28.980064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471509872020072:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(104045988ns) {HandleHandshakeStatus:329(96550185ns) {DropIncomingHandshake:442(671ns) DropOutgoingHandshake:457(566ns) SwitchToState:179(641ns) ProcessPendingSessionEvents:495(14172835ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14099901ns max# 13529956ns}}}
2025-11-27T18:19:28.980264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471509872020072:2107] [node 4] ICP18 handshake (actor [3:7577471509872020105:2121]) is too old
2025-11-27T18:19:28.917778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471506201514872:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:28.931261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471506201514872:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:28.933940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471506201514885:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-11-27T18:19:28.964851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471506201514859:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:28.969620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:28.985342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471506201514859:2109] [node 6] ICP22 created new session: [2:7577471506201514886:2109]
2025-11-27T18:19:28.761881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471509639329227:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:28.932065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471506906569462:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:28.981045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471509750873267:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(138680364ns) {HandleHandshakeStatus:329(138676180ns) {DropIncomingHandshake:442(17055ns) DropOutgoingHandshake:457(613ns) SwitchToState:179(665ns) ProcessPendingSessionEvents:495(53946116ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 53878490ns max# 42710127ns}}}
2025-11-27T18:19:28.811237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471506295862812:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48431
2025-11-27T18:19:28.761916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471509639329227:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:29.001855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471506201514886:2109] [node 6] ICS09 handshake done sender: [2:7577471506201514872:2118] self: [2:7577471506201514871:0] peer: [6:7577471506295862820:0] socket: 165 qp: -1
2025-11-27T18:19:28.932115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:28.975192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471509750873185:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:28.817841Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471506295862781:2099] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9683) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471506295862805:2116] held: no
2025-11-27T18:19:29.002788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471509750873187:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19472) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471509750873278:2134] held: no
2025-11-27T18:19:28.762877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471509639329227:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:29.009971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471506201514886:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:29.023504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471509750873299:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2889
2025-11-27T18:19:29.046029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471509750873278:2134] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:28.818604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471506295862810:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.932165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471506906569462:2048] [node 1] ICP22 created new session: [5:7577471506906569555:2048]
2025-11-27T18:19:28.932285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471506906569555:2048] [node 1] ICS09 handshake done sender: [5:7577471506906569533:2113] self: [5:7577471506906569545:0] peer: [1:7577471509639329222:0] socket: 138 qp: -1
2025-11-27T18:19:28.932332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471506906569555:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:28.932374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:28.941861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471506906569554:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54053
2025-11-27T18:19:28.942056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471506906569464:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:28.965897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471506906569554:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.983262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471506906569463:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:28.994905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471506906569463:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17286958ns) {IncomingHandshake:261(17285522ns) {DropIncomingHandshake:442(353ns) PrepareNewSessionHandshake:477(3533588ns) {SwitchToState:179(4256ns)} GenerateSessionVirtualId:988(895ns) IssueIncomingHandshakeReply:193(1847517ns) {SwitchToState:179(2185ns)}}}
2025-11-27T18:19:28.765402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471509639329231:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:29.054826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471509750873185:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31631) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471509750873300:2139] held: no
2025-11-27T18:19:29.061498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471511201536856:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54053
2025-11-27T18:19:29.021313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471506201514882:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:29.044009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471509872020089:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:29.073619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471509750873187:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:29.073694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:29.067345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471509872020073:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:29.021372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471506201514882:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:28.818653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471506295862729:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:28.835143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471509639329225:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:29.074123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471514045840598:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2889
2025-11-27T18:19:29.033656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:29.068282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471506906569527:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(161311728ns) {HandleHandshakeStatus:395(161309485ns) {LogHandshakeFail:472(9385274ns) {UpdateErrorStateLog:545(6251ns)} DropOutgoingHandshake:457(1601ns) ProcessPendingSessionEvents:495(131443568ns) {unordered DropSessionEvent:512 num# 100 sum# 131353524ns max# 20355081ns}}}
2025-11-27T18:19:28.818739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:29.067463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:29.042781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471506201514885:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.837453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471509639329220:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:29.068399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471506906569538:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:29.077526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471509750873187:2050] [node 3] ICP22 created new session: [4:7577471514045840599:2050]
2025-11-27T18:19:29.069701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471509872020073:2108] [node 5] ICP22 created new session: [3:7577471514166987406:2108]
2025-11-27T18:19:28.818793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471506295862729:2050] [node 3] ICP22 created new session: [6:7577471506295862815:2050]
2025-11-27T18:19:29.089730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471514045840599:2050] [node 3] ICS09 handshake done sender: [4:7577471509750873278:2134] self: [4:7577471509750873290:0] peer: [3:7577471509872020078:0] socket: 151 qp: -1
2025-11-27T18:19:29.046205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471506201514882:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:29.093370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471514166987406:2108] [node 5] ICS09 handshake done sender: [3:7577471509872020089:2116] self: [3:7577471509872020088:0] peer: [5:7577471506906569557:0] socket: 173 qp: -1
2025-11-27T18:19:28.839220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471509639329223:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:29.074426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471511201536857:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54053
2025-11-27T18:19:28.833670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471506295862815:2050] [node 3] ICS09 handshake done sender: [6:7577471506295862798:2113] self: [6:7577471506295862806:0] peer: [3:7577471509872020075:0] socket: 131 qp: -1
2025-11-27T18:19:29.089786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471514045840599:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:29.081105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471506906569551:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:28.848709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471509639329154:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:28.834718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471506295862815:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:28.835858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471506295862805:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:28.837374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:28.844677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471506295862728:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(208869662ns) {Configure:104(208862241ns) {ProcessConfigured:135(25851961ns) {StartInitialHandshake:157(25849094ns) {DropHandshakes:470(10413919ns) {DropIncomingHandshake:442(605ns) DropOutgoingHandshake:457(1643ns)} GenerateSessionVirtualId:988(1991ns) PrepareNewSessionHandshake:477(4008ns) {SwitchToState:179(1261ns)}}}}}
2025-11-27T18:19:28.845993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471506295862781:2099] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:28.846054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:28.846108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471506295862781:2099] [node 5] ICP22 created new session: [6:7577471506295862817:2099]
2025-11-27T18:19:28.846162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471506295862817:2099] [node 5] ICS09 handshake done sender: [6:7577471506295862805:2116] self: [6:7577471506295862809:0] peer: [5:7577471506906569531:0] socket: 150 qp: -1
2025-11-27T18:19:28.846201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471506295862817:2099] [node 5] ICS10 traffic start
2025-11-27T18:19:28.846238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:29.089843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:29.090080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471509750873187:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16383148ns) {HandleHandshakeStatus:329(16380204ns) {DropIncomingHandshake:442(978ns) DropOutgoingHandshake:457(412ns) SwitchToState:179(876ns) ProcessPendingSessionEvents:495(1820ns)}}
2025-11-27T18:19:29.142489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471509750873185:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(91098446ns) {HandleHandshakeStatus:395(91092309ns) {LogHandshakeFail:472(14490ns) {UpdateErrorStateLog:545(5917ns)} DropOutgoingHandshake:457(2063ns) ProcessPendingSessionEvents:495(86135055ns) {unordered DropSessionEvent:512 num# 100 sum# 86066360ns max# 14989276ns}}}
2025-11-27T18:19:29.142685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471509750873300:2139] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:29.103131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471514166987406:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:29.143294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471509750873185:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:29.149673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:29.149726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471509750873185:2048] [node 1] ICP22 created new session: [4:7577471514045840601:2048]
2025-11-27T18:19:29.103198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:29.149770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471514045840601:2048] [node 1] ICS09 handshake done sender: [4:7577471509750873300:2139] self: [4:7577471509750873301:0] peer: [1:7577471509639329232:0] socket: 142 qp: -1
2025-11-27T18:19:29.149814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471514045840601:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:29.073236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471506201514861:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:29.149866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:29.080428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471506201514887:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-11-27T18:19:29.097416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471506201514790:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(123614110ns) {HandleHandshakeStatus:395(123611147ns) {LogHandshakeFail:472(11596ns) {UpdateErrorStateLog:545(5044ns)} DropOutgoingHandshake:457(2062ns) ProcessPendingSessionEvents:495(117827535ns) {unordered DropSessionEvent:512 num# 100 sum# 34659617ns max# 9482137ns}}}
2025-11-27T18:19:29.097930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:29.098151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471506201514887:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:29.104959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471506201514865:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:29.109305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471506201514861:2108] [node 5] ICP22 created new session: [2:7577471510496482186:2108]
2025-11-27T18:19:29.121358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471510496482186:2108] [node 5] ICS09 handshake done sender: [2:7577471506201514882:2120] self: [2:7577471506201514881:0] peer: [5:7577471506906569559:0] socket: 182 qp: -1
2025-11-27T18:19:29.121418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471510496482186:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:29.121488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:29.129388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471506201514790:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:29.143122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:29.149536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471506201514790:2048] [node 1] ICP22 created new session: [2:7577471510496482188:2048]
2025-11-27T18:19:29.163563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471510496482188:2048] [node 1] ICS09 handshake done sender: [2:7577471506201514865:2115] self: [2:7577471506201514876:0] peer: [1:7577471509639329219:0] socket: 134 qp: -1
2025-11-27T18:19:29.165892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471510496482188:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:29.166000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:29.166167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471506201514861:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(55153820ns) {HandleHandshakeStatus:329(48934050ns) {DropIncomingHandshake:442(2930ns) DropOutgoingHandshake:457(3210ns) SwitchToState:179(886ns) ProcessPendingSessionEvents:495(524084ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 459410ns max# 65440ns}}}
2025-11-27T18:19:29.166246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471506201514790:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(36771608ns) {HandleHandshakeStatus:329(36767252ns) {DropIncomingHandshake:442(952ns) DropOutgoingHandshake:457(400ns) SwitchToState:179(750ns) ProcessPendingSessionEvents:495(1017ns)}}
2025-11-27T18:19:29.166344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471506201514861:2108] [node 5] ICP18 handshake (actor [2:7577471506201514887:2122]) is too old
2025-11-27T18:19:29.167174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471506201514857:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(245210338ns) {HandleHandshakeStatus:329(245201246ns) {DropIncomingHandshake:442(818ns) DropOutgoingHandshake:457(662ns) SwitchToState:179(822ns) ProcessPendingSessionEvents:495(192177651ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 182936045ns max# 14261291ns}}}
2025-11-27T18:19:29.167554Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471509872020073:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(103250290ns) {HandleHandshakeStatus:329(103237865ns) {DropIncomingHandshake:442(35184ns) DropOutgoingHandshake:457(537ns) SwitchToState:179(1072ns) ProcessPendingSessionEvents:495(48872672ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 48779332ns max# 12485157ns}}}
2025-11-27T18:19:28.848350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471506295862797:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:29.092007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471506906569527:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:28.849933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471509639329153:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:28.849823Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471506295862787:2104] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18699) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471506295862797:2112] held: no
2025-11-27T18:19:28.875489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471506295862728:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:28.930473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471506295862787:2104] [node 4] ICP35 event processing took too much time PendingConnection:279(79879092ns) {HandleHandshakeStatus:395(79875570ns) {LogHandshakeFail:472(13875ns) {UpdateErrorStateLog:545(5394ns)} DropOutgoingHandshake:457(5382ns) ProcessPendingSessionEvents:495(76173392ns) {unordered DropSessionEvent:512 num# 100 sum# 76122353ns max# 23565629ns}}}
2025-11-27T18:19:28.931059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471506295862787:2104] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:28.937745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471506295862821:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48431
2025-11-27T18:19:28.941373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:28.949430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471506295862787:2104] [node 4] ICP22 created new session: [6:7577471506295862824:2104]
2025-11-27T18:19:28.949489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471506295862824:2104] [node 4] ICS09 handshake done sender: [6:7577471506295862797:2112] self: [6:7577471506295862808:0] peer: [4:7577471509750873268:0] socket: 127 qp: -1
2025-11-27T18:19:28.949531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471506295862824:2104] [node 4] ICS10 traffic start
2025-11-27T18:19:28.961343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:28.985408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471506295862821:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.997799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471506295862823:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48431
2025-11-27T18:19:29.001625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471506295862787:2104] [node 4] ICP35 event processing took too much time PendingConnection:279(52284413ns) {HandleHandshakeStatus:329(52282137ns) {DropIncomingHandshake:442(7184ns) DropOutgoingHandshake:457(568ns) SwitchToState:179(1028ns) ProcessPendingSessionEvents:495(964ns)}}
2025-11-27T18:19:29.009818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471506295862727:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:29.022506Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471506295862810:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:29.034879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471506295862728:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:29.050715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:29.055515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471506295862728:2049] [node 2] ICP22 created new session: [6:7577471510590830123:2049]
2025-11-27T18:19:29.077419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471510590830123:2049] [node 2] ICS09 handshake done sender: [6:7577471506295862810:2118] self: [6:7577471506295862820:0] peer: [2:7577471506201514871:0] socket: 170 qp: -1
2025-11-27T18:19:29.080769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471510590830124:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48431
2025-11-27T18:19:29.081433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471510590830123:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:29.089350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471506295862821:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:29.097400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:29.103243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471506295862727:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:29.130120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:29.130387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471506295862727:2048] [node 1] ICP22 created new session: [6:7577471510590830126:2048]
2025-11-27T18:19:29.134644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471510590830126:2048] [node 1] ICS09 handshake done sender: [6:7577471506295862821:2122] self: [6:7577471506295862826:0] peer: [1:7577471509639329229:0] socket: 136 qp: -1
2025-11-27T18:19:29.134689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471510590830126:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:29.134766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:29.167317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471506295862728:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(131463580ns) {HandleHandshakeStatus:329(131460399ns) {DropIncomingHandshake:442(707ns) DropOutgoingHandshake:457(6290242ns) SwitchToState:179(1624ns) ProcessPendingSessionEvents:495(56449169ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 56357540ns max# 19088075ns}}}
2025-11-27T18:19:29.167329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471506295862729:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(347670727ns) {HandleHandshakeStatus:329(347668618ns) {DropIncomingHandshake:442(893ns) DropOutgoingHandshake:457(4057ns) SwitchToState:179(1167ns) ProcessPendingSessionEvents:495(307611276ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 307387293ns max# 19152883ns}}}
2025-11-27T18:19:29.167385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471506295862729:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:29.167898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471506295862727:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32194891ns) {HandleHandshakeStatus:329(32192257ns) {DropIncomingHandshake:442(1024ns) DropOutgoingHandshake:457(4737ns) SwitchToState:179(670ns) ProcessPendingSessionEvents:495(515805ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 463133ns max# 52516ns}}}
2025-11-27T18:19:29.171560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471509872020009:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(415807074ns) {HandleHandshakeStatus:329(415803618ns) {DropIncomingHandshake:442(714ns) DropOutgoingHandshake:457(5078ns) SwitchToState:179(596ns) ProcessPendingSessionEvents:495(329401198ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 313577866ns max# 22934996ns}}}
2025-11-27T18:19:29.172755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471509872020008:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(287575497ns) {HandleHandshakeStatus:329(287569102ns) {DropIncomingHandshake:442(1299ns) DropOutgoingHandshake:457(21328ns) SwitchToState:179(783ns) ProcessPendingSessionEvents:495(287127752ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 269743809ns max# 24794385ns}}}
2025-11-27T18:19:29.092081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:29.092131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471506906569527:2108] [node 4] ICP22 created new session: [5:7577471511201536858:2108]
2025-11-27T18:19:29.092178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471511201536858:2108] [node 4] ICS09 handshake done sender: [5:7577471506906569538:2116] self: [5:7577471506906569550:0] peer: [4:7577471509750873275:0] socket: 155 qp: -1
2025-11-27T18:19:29.092229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471511201536858:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:29.092269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:29.093158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471506906569554:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:29.102421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471506906569464:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19472) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471506906569551:2119] held: no
2025-11-27T18:19:29.108440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471506906569464:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:29.113414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471506906569463:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:29.113519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:29.113579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471506906569463:2049] [node 2] ICP22 created new session: [5:7577471511201536860:2049]
2025-11-27T18:19:29.113637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471511201536860:2049] [node 2] ICS09 handshake done sender: [5:7577471506906569554:2122] self: [5:7577471506906569559:0] peer: [2:7577471506201514881:0] socket: 143 qp: -1
2025-11-27T18:19:29.113677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471511201536860:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:29.113724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:29.128785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:29.130384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471506906569464:2050] [node 3] ICP22 created new session: [5:7577471511201536862:2050]
2025-11-27T18:19:29.133833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471511201536862:2050] [node 3] ICS09 handshake done sender: [5:7577471506906569551:2119] self: [5:7577471506906569557:0] peer: [3:7577471509872020088:0] socket: 174 qp: -1
2025-11-27T18:19:29.156690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471511201536862:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:29.156783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:29.169411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471506906569464:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(65014725ns) {HandleHandshakeStatus:329(65011498ns) {DropIncomingHandshake:442(3665ns) DropOutgoingHandshake:457(234ns) SwitchToState:179(9339ns) ProcessPendingSessionEvents:495(1379ns)}}
2025-11-27T18:19:28.849995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:28.850048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471509639329153:2048] [node 2] ICP22 created new session: [1:7577471509639329241:2048]
2025-11-27T18:19:28.850093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471509639329241:2048] [node 2] ICS09 handshake done sender: [1:7577471509639329220:2110] self: [1:7577471509639329219:0] peer: [2:7577471506201514876:0] socket: 133 qp: -1
2025-11-27T18:19:28.850128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471509639329241:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:28.850159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:28.866275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471509639329234:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:28.866330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471509639329217:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:28.866371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:28.866420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471509639329217:2108] [node 5] ICP22 created new session: [1:7577471509639329244:2108]
2025-11-27T18:19:28.866493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471509639329244:2108] [node 5] ICS09 handshake done sender: [1:7577471509639329223:2111] self: [1:7577471509639329222:0] peer: [5:7577471506906569545:0] socket: 135 qp: -1
2025-11-27T18:19:28.866542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471509639329244:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:28.866602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:28.868107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471509639329217:2108] [node 5] ICP18 handshake (actor [1:7577471509639329234:2117]) is too old
2025-11-27T18:19:28.869533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:28.869587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471509639329154:2049] [node 3] ICP22 created new session: [1:7577471509639329247:2049]
2025-11-27T18:19:28.869962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471509639329239:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36731
2025-11-27T18:19:28.889399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471509639329247:2049] [node 3] ICS09 handshake done sender: [1:7577471509639329227:2113] self: [1:7577471509639329226:0] peer: [3:7577471509872020093:0] socket: 158 qp: -1
2025-11-27T18:19:28.897399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471509639329247:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:28.914774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:28.924405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471509639329154:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(75771640ns) {HandleHandshakeStatus:329(75758885ns) {DropIncomingHandshake:442(955ns) DropOutgoingHandshake:457(624ns) SwitchToState:179(828ns) ProcessPendingSessionEvents:495(722726ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 656110ns max# 129487ns}}}
2025-11-27T18:19:28.924491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471509639329154:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:28.997301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471509639329249:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36731
2025-11-27T18:19:29.001906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471509639329233:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:29.001944Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471509639329233:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:29.003187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471509639329233:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:29.177738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471506201514859:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(209763031ns) {HandleHandshakeStatus:329(209758219ns) {DropIncomingHandshake:442(631ns) DropOutgoingHandshake:457(399ns) SwitchToState:179(473ns) ProcessPendingSessionEvents:495(140742079ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 140604039ns max# 13883221ns}}}
2025-11-27T18:19:29.018485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471509639329216:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:29.025666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:29.025725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471509639329216:2107] [node 4] ICP22 created new session: [1:7577471513934296547:2107]
2025-11-27T18:19:29.025776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471513934296547:2107] [node 4] ICS09 handshake done sender: [1:7577471509639329233:2116] self: [1:7577471509639329232:0] peer: [4:7577471509750873301:0] socket: 184 qp: -1
2025-11-27T18:19:29.025818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471513934296547:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:29.025861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:29.027351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471509639329216:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16483779ns) {HandleHandshakeStatus:329(16480210ns) {DropIncomingHandshake:442(29446ns) DropOutgoingHandshake:457(574ns) SwitchToState:179(881ns) ProcessPendingSessionEvents:495(517681ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 455450ns max# 48916ns}}}
2025-11-27T18:19:29.045907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471509639329230:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:29.045948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471509639329230:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:29.050800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471509639329249:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:29.081431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471509639329230:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:29.103741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471509639329218:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:29.103824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:29.103868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471509639329218:2109] [node 6] ICP22 created new session: [1:7577471513934296551:2109]
2025-11-27T18:19:29.103918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471513934296551:2109] [node 6] ICS09 handshake done sender: [1:7577471509639329230:2114] self: [1:7577471509639329229:0] peer: [6:7577471506295862826:0] socket: 123 qp: -1
2025-11-27T18:19:29.103963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471513934296551:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:29.104000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:29.172216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471509639329218:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(67598909ns) {HandleHandshakeStatus:329(67593624ns) {DropIncomingHandshake:442(19381ns) DropOutgoingHandshake:457(174ns) SwitchToState:179(243ns) ProcessPendingSessionEvents:495(67226719ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 67131187ns max# 10016016ns}}}
2025-11-27T18:19:29.178884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471506201514859:2109] [node 6] ICP18 handshake (actor [2:7577471506201514885:2121]) is too old
2025-11-27T18:19:29.178088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471509750873186:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(380737750ns) {HandleHandshakeStatus:329(380734026ns) {DropIncomingHandshake:442(1168ns) DropOutgoingHandshake:457(5335ns) SwitchToState:179(681ns) ProcessPendingSessionEvents:495(292911707ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 282406370ns max# 31665959ns}}}
2025-11-27T18:19:29.209883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:29.210125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471510496482188:2048] [node 1] ICS01 socket: 134 reason# EndOfStream
2025-11-27T18:19:29.210157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471506201514790:2048] [node 1] ICP30 unregister session Session# [2:7577471510496482188:2048] VirtualId# [2:7577471506201514876:0]
2025-11-27T18:19:29.210185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471510496482188:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.210243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:29.210272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:29.212240Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:29.212484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471509872020103:2048] [node 1] ICS07 socket disconnect 159 reason# EndOfStream
2025-11-27T18:19:29.212533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471509872020103:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.212611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:29.212658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471509872020103:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:29.213836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471509872020008:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31631) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:29.213894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471509872020103:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:29.214055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471509872020008:2048] [node 1] ICP30 unregister session Session# [3:7577471509872020103:2048] VirtualId# [3:7577471509872020093:0]
2025-11-27T18:19:29.214167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:29.214209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471509872020008:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31631) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:29.214227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31631) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:29.220673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:29.220886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471506906569555:2048] [node 1] ICS01 socket: 138 reason# EndOfStream
2025-11-27T18:19:29.220922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471506906569462:2048] [node 1] ICP30 unregister session Session# [5:7577471506906569555:2048] VirtualId# [5:7577471506906569545:0]
2025-11-27T18:19:29.220951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471506906569555:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.221018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:29.221056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:29.222274Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:29.222520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471510590830126:2048] [node 1] ICS01 socket: 136 reason# EndOfStream
2025-11-27T18:19:29.222565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471506295862727:2048] [node 1] ICP30 unregister session Session# [6:7577471510590830126:2048] VirtualId# [6:7577471506295862826:0]
2025-11-27T18:19:29.222600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471510590830126:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.222679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:29.222754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:29.225341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471509872020008:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:29.225931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:29.226427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471514045840601:2048] [node 1] ICS01 socket: 142 reason# EndOfStream
2025-11-27T18:19:29.226524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471509750873185:2048] [node 1] ICP30 unregister session Session# [4:7577471514045840601:2048] VirtualId# [4:7577471509750873301:0]
2025-11-27T18:19:29.226590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471514045840601:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.226736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:29.226840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:29.251089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:29.251316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471509872020095:2049] [node 2] ICS07 socket disconnect 122 reason# EndOfStream
2025-11-27T18:19:29.251342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471509872020095:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.251419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:29.251458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471509872020095:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:29.255276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471509872020009:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29101) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:29.255324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471509872020095:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:29.256374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471509872020009:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29101) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:29.256431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471509872020095:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:29.256465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471509872020009:2049] [node 2] ICP30 unregister session Session# [3:7577471509872020095:2049] VirtualId# [3:7577471509872020083:0]
2025-11-27T18:19:29.256576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:29.256614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471509872020009:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:29.256646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:29.261480Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:29.261712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471509750873291:2049] [node 2] ICS07 socket disconnect 140 reason# EndOfStream
2025-11-27T18:19:29.261740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471509750873291:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.261828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:29.261871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471509750873291:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:29.262407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:29.262588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471511201536860:2049] [node 2] ICS01 socket: 143 reason# EndOfStream
2025-11-27T18:19:29.262632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471506906569463:2049] [node 2] ICP30 unregister session Session# [5:7577471511201536860:2049] VirtualId# [5:7577471506906569559:0]
2025-11-27T18:19:29.262661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471511201536860:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.262722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:29.262768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:29.263854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471509750873186:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29101) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:29.263910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471509750873291:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:29.263944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471509750873186:2049] [node 2] ICP30 unregister session Session# [4:7577471509750873291:2049] VirtualId# [4:7577471509750873284:0]
2025-11-27T18:19:29.264028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:29.264064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471509750873186:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:29.264079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:29.266042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:29.266291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471510590830123:2049] [node 2] ICS01 socket: 170 reason# EndOfStream
2025-11-27T18:19:29.266348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471506295862728:2049] [node 2] ICP30 unregister session Session# [6:7577471510590830123:2049] VirtualId# [6:7577471506295862820:0]
2025-11-27T18:19:29.266392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471510590830123:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.266501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:29.266573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:29.267932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471509872020009:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:29.281143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471509750873186:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:29.369993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:29.370236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471514045840599:2050] [node 3] ICS01 socket: 151 reason# EndOfStream
2025-11-27T18:19:29.370274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471509750873187:2050] [node 3] ICP30 unregister session Session# [4:7577471514045840599:2050] VirtualId# [4:7577471509750873290:0]
2025-11-27T18:19:29.370309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471514045840599:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.370385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:29.370429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:29.373586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:29.373859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471506295862815:2050] [node 3] ICS07 socket disconnect 131 reason# EndOfStream
2025-11-27T18:19:29.373900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471506295862815:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.373992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:29.374045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471506295862815:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:29.377921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471506295862729:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19472) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:29.377990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471506295862815:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:29.378030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471506295862729:2050] [node 3] ICP30 unregister session Session# [6:7577471506295862815:2050] VirtualId# [6:7577471506295862806:0]
2025-11-27T18:19:29.378125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:29.378168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471506295862729:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19472) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:29.377615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:29.378190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19472) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:29.377871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471511201536862:2050] [node 3] ICS01 socket: 174 reason# EndOfStream
2025-11-27T18:19:29.377918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471506906569464:2050] [node 3] ICP30 unregister session Session# [5:7577471511201536862:2050] VirtualId# [5:7577471506906569557:0]
2025-11-27T18:19:29.377952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471511201536862:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.378053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:29.378118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:29.388079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471506295862729:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:29.485493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:29.485742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471506295862824:2104] [node 4] ICS01 socket: 127 reason# EndOfStream
2025-11-27T18:19:29.485798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471506295862787:2104] [node 4] ICP30 unregister session Session# [6:7577471506295862824:2104] VirtualId# [6:7577471506295862808:0]
2025-11-27T18:19:29.485836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471506295862824:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.485917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:29.485958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:29.493420Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:29.493683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471511201536858:2108] [node 4] ICS01 socket: 155 reason# EndOfStream
2025-11-27T18:19:29.493727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471506906569527:2108] [node 4] ICP30 unregister session Session# [5:7577471511201536858:2108] VirtualId# [5:7577471506906569550:0]
2025-11-27T18:19:29.493765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471511201536858:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.493851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:29.493896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:29.616265Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:29.616499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471506295862817:2099] [node 5] ICS01 socket: 150 reason# EndOfStream
2025-11-27T18:19:29.616561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471506295862781:2099] [node 5] ICP30 unregister session Session# [6:7577471506295862817:2099] VirtualId# [6:7577471506295862809:0]
2025-11-27T18:19:29.616604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471506295862817:2099] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:29.616708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:29.616756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:29.759470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471512079983735:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:29.759522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471512079983736:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:29.793457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471511072765164:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:29.793522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471511072765165:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:29.827956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471513937004121:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:29.827999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471513937004122:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:29.868875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471513950253767:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:29.868924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471513950253768:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:29.868952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471513950253769:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:29.907927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471510482643644:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:29.907983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471510482643645:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:29.908018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471510482643646:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:29.931182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471514074167653:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:29.931245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471514074167654:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:29.931288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471514074167655:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:30.187988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471518369135007:2104] [node 5] ICP01 ready to work
2025-11-27T18:19:30.299525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471518231971480:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:30.326827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471513950253768:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(75497872ns) {Configure:104(75492638ns) {ProcessConfigured:135(82351ns) {StartInitialHandshake:157(79067ns) {DropHandshakes:470(3800ns) {DropIncomingHandshake:442(1280ns) DropOutgoingHandshake:457(591ns)} GenerateSessionVirtualId:988(1334ns) PrepareNewSessionHandshake:477(2259ns) {SwitchToState:179(814ns)}}}}}
2025-11-27T18:19:30.300015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471518231971481:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:30.329533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471516374951094:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:30.329942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471516374951095:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:30.330470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471516374951096:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:30.328124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471518369135007:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(139628638ns) {Configure:104(139622737ns) {ProcessConfigured:135(12572948ns) {StartInitialHandshake:157(12569788ns) {DropHandshakes:470(3016ns) {DropIncomingHandshake:442(722ns) DropOutgoingHandshake:457(465ns)} GenerateSessionVirtualId:988(762ns) PrepareNewSessionHandshake:477(9089ns) {SwitchToState:179(2999ns)}}}}}
2025-11-27T18:19:30.311445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471518231971484:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:30.387785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471518245221131:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:30.375221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471518369135015:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:30.383458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471518231971481:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17132344ns) {Configure:104(17122379ns) {ProcessConfigured:135(77079ns) {StartInitialHandshake:157(74671ns) {DropHandshakes:470(3450ns) {DropIncomingHandshake:442(867ns) DropOutgoingHandshake:457(576ns)} GenerateSessionVirtualId:988(962ns) PrepareNewSessionHandshake:477(2559ns) {SwitchToState:179(931ns)}}}}}
2025-11-27T18:19:30.395820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471514777611003:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:30.417436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471518245221132:2111] [node 6] ICP01 ready to work
2025-11-27T18:19:30.418344Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471511072765164:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(220548778ns) {Configure:104(220539771ns) {ProcessConfigured:135(48681ns) {StartInitialHandshake:157(46927ns) {DropHandshakes:470(726ns) {DropIncomingHandshake:442(220ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(298ns) PrepareNewSessionHandshake:477(681ns) {SwitchToState:179(292ns)}}}}}
2025-11-27T18:19:30.404699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471514777611004:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:30.425613Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471514074167653:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49686584ns) {Configure:104(49680221ns) {ProcessConfigured:135(7823654ns) {StartInitialHandshake:157(7821164ns) {DropHandshakes:470(1846ns) {DropIncomingHandshake:442(452ns) DropOutgoingHandshake:457(411ns)} GenerateSessionVirtualId:988(413ns) PrepareNewSessionHandshake:477(3193ns) {SwitchToState:179(121ns)}}}}}
2025-11-27T18:19:30.433895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471518231971480:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(38023501ns) {Configure:104(38019782ns) {ProcessConfigured:135(6684636ns) {StartInitialHandshake:157(6682944ns) {DropHandshakes:470(1224ns) {DropIncomingHandshake:442(211ns) DropOutgoingHandshake:457(275ns)} GenerateSessionVirtualId:988(523ns) PrepareNewSessionHandshake:477(8102ns) {SwitchToState:179(1804ns)}}}}}
2025-11-27T18:19:30.440383Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471512079983736:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(23829437ns) {Configure:104(23825220ns) {ProcessConfigured:135(791781ns) {StartInitialHandshake:157(789322ns) {DropHandshakes:470(1755ns) {DropIncomingHandshake:442(441ns) DropOutgoingHandshake:457(377ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:477(2650ns) {SwitchToState:179(592ns)}}}}}
2025-11-27T18:19:30.441156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471515367732525:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:30.444762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471515367732526:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.451604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471516374951107:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57872
2025-11-27T18:19:30.451686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471516374951108:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57872
2025-11-27T18:19:30.451743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471516374951109:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57872
2025-11-27T18:19:30.451801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471516374951110:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57872
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.457380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471518231971496:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37370
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:19:30.461384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471518231971497:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37370
2025-11-27T18:19:30.456399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471518245221142:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14117
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.471095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471514777611018:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20306
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.478175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471518369135027:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:19:30.461557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471518231971498:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37370
2025-11-27T18:19:30.478919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471515367732527:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:30.481635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471516374951107:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.493284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471518231971498:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.492956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471514777611018:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.493090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471514777611019:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20306
2025-11-27T18:19:30.493237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471514777611020:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20306
2025-11-27T18:19:30.493731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471518369135027:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.493843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471514074167655:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.507416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471518245221142:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.507580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471513950253767:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.494312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471518231971496:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.494414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471518231971497:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.494527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471518231971488:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.494552Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471518231971488:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.496002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471514777611019:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.518719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471516374951098:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.518759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471516374951098:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.519711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471516374951109:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.521386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471511072765165:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28162292ns) {Configure:104(28154666ns) {ProcessConfigured:135(2811225ns) {StartInitialHandshake:157(2808695ns) {DropHandshakes:470(4122ns) {DropIncomingHandshake:442(1025ns) DropOutgoingHandshake:457(493ns)} GenerateSessionVirtualId:988(893ns) PrepareNewSessionHandshake:477(16525ns) {SwitchToState:179(1650ns)}}}}}
2025-11-27T18:19:30.502362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471510482643646:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.525747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471516374951098:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:30.526819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471518245221144:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14117
2025-11-27T18:19:30.531651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471518231971486:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.531687Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471518231971486:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.531130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471518245221142:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.533643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471514777611023:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20306
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.534243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471516374951108:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.534470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471516374951094:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.534566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:30.534635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471516374951094:2107] [node 4] ICP22 created new session: [1:7577471516374951117:2107]
2025-11-27T18:19:30.534691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471516374951117:2107] [node 4] ICS09 handshake done sender: [1:7577471516374951098:2110] self: [1:7577471516374951097:0] peer: [4:7577471518245221143:0] socket: 146 qp: -1
2025-11-27T18:19:30.534736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471516374951117:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:30.534794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:30.536097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471516374951110:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.533937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471518369135029:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:19:30.540690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471514777611020:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.540897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471510482643644:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.541458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471513950253767:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.545229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471518231971488:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:30.541564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:30.541622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471513950253767:2048] [node 1] ICP22 created new session: [4:7577471518245221145:2048]
2025-11-27T18:19:30.541672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471518245221145:2048] [node 1] ICS09 handshake done sender: [4:7577471518245221142:2116] self: [4:7577471518245221143:0] peer: [1:7577471516374951097:0] socket: 150 qp: -1
2025-11-27T18:19:30.541714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471518245221145:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:30.541780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:30.546005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471518369135027:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:30.546211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471514074167655:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.546262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:30.546314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471514074167655:2050] [node 3] ICP22 created new session: [6:7577471518369135030:2050]
2025-11-27T18:19:30.546360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471518369135030:2050] [node 3] ICS09 handshake done sender: [6:7577471518369135027:2115] self: [6:7577471518369135028:0] peer: [3:7577471518231971487:0] socket: 153 qp: -1
2025-11-27T18:19:30.546396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471518369135030:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:30.546455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:30.550511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471518231971486:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:30.551157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471514777611019:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:30.551434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471510482643646:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.551560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:30.551614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471510482643646:2050] [node 3] ICP22 created new session: [5:7577471514777611025:2050]
2025-11-27T18:19:30.551679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471514777611025:2050] [node 3] ICS09 handshake done sender: [5:7577471514777611019:2116] self: [5:7577471514777611022:0] peer: [3:7577471518231971485:0] socket: 155 qp: -1
2025-11-27T18:19:30.551738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471514777611025:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:30.551777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.556738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471515367732532:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21623
2025-11-27T18:19:30.557275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471515367732533:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21623
2025-11-27T18:19:30.557419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471515367732534:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21623
2025-11-27T18:19:30.557921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471515367732540:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21623
2025-11-27T18:19:30.558565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471518231971484:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.558671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:30.558723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471518231971484:2109] [node 6] ICP22 created new session: [3:7577471518231971504:2109]
2025-11-27T18:19:30.560254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471516374951102:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.560296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471516374951102:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.563532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471515367732534:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.564136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471516374951119:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57872
2025-11-27T18:19:30.561065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471518231971504:2109] [node 6] ICS09 handshake done sender: [3:7577471518231971488:2112] self: [3:7577471518231971487:0] peer: [6:7577471518369135028:0] socket: 141 qp: -1
2025-11-27T18:19:30.569490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471516374951102:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:30.565669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471515367732533:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.567298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471511072765164:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.561130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471518231971504:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:30.561177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:30.586739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471518231971484:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(27301918ns) {HandleHandshakeStatus:329(27298293ns) {DropIncomingHandshake:442(23218ns) DropOutgoingHandshake:457(757ns) SwitchToState:179(796ns) ProcessPendingSessionEvents:495(24519441ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24410459ns max# 9570194ns}}}
2025-11-27T18:19:30.586859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471518231971481:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.586957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:30.587016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471518231971481:2108] [node 5] ICP22 created new session: [3:7577471518231971506:2108]
2025-11-27T18:19:30.587060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471518231971506:2108] [node 5] ICS09 handshake done sender: [3:7577471518231971486:2111] self: [3:7577471518231971485:0] peer: [5:7577471514777611022:0] socket: 140 qp: -1
2025-11-27T18:19:30.587106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471518231971506:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:30.587147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.595747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471516374951100:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.593431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471515367732532:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.622117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471518245221148:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14117
2025-11-27T18:19:30.622330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471518245221149:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14117
2025-11-27T18:19:30.626590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471518245221148:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.628051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471513950253768:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.628925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471514777611027:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20306
2025-11-27T18:19:30.629116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471510482643644:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4322) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471514777611020:2117] held: no
2025-11-27T18:19:30.629987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471514777611028:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20306
2025-11-27T18:19:30.595790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471516374951100:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.635491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471514777611027:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.635609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471514777611020:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:30.625846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471515367732546:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21623
2025-11-27T18:19:30.637692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471510482643644:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.637753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:30.637814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471510482643644:2048] [node 1] ICP22 created new session: [5:7577471514777611031:2048]
2025-11-27T18:19:30.637862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471514777611031:2048] [node 1] ICS09 handshake done sender: [5:7577471514777611020:2117] self: [5:7577471514777611024:0] peer: [1:7577471516374951101:0] socket: 156 qp: -1
2025-11-27T18:19:30.637897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471514777611031:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:30.637934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.596353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471516374951095:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.647006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471514777611003:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.630950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471515367732531:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.651607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471518245221149:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.596454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:30.651704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471518369135033:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:19:30.662194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471518245221137:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.662257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471518245221137:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.630979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471515367732531:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.596514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471516374951095:2108] [node 5] ICP22 created new session: [1:7577471516374951124:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.666810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471518231971508:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37370
2025-11-27T18:19:30.666981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471518231971509:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37370
2025-11-27T18:19:30.596588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471516374951124:2108] [node 5] ICS09 handshake done sender: [1:7577471516374951102:2112] self: [1:7577471516374951101:0] peer: [5:7577471514777611024:0] socket: 148 qp: -1
2025-11-27T18:19:30.664116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471514777611034:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20306
2025-11-27T18:19:30.596636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471516374951124:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:30.596672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:30.624819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471516374951100:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:30.631727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471516374951119:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.631771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471512079983735:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.631849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:30.631906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471512079983735:2048] [node 2] ICP22 created new session: [1:7577471516374951128:2048]
2025-11-27T18:19:30.631964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471516374951128:2048] [node 2] ICS09 handshake done sender: [1:7577471516374951100:2111] self: [1:7577471516374951099:0] peer: [2:7577471515367732543:0] socket: 147 qp: -1
2025-11-27T18:19:30.632009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471516374951128:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:30.632045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:30.634764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471512079983735:2048] [node 2] ICP18 handshake (actor [1:7577471516374951119:2119]) is too old
2025-11-27T18:19:30.652336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471515367732540:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.670104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471518369135033:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.675024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471515367732534:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:30.670251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471518231971509:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.684386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471514777611027:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:30.667630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471518245221137:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.676960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471513937004121:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.684238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471518231971508:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.677895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471511072765164:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4322) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471515367732534:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.686568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471516374951104:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.678489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471511072765164:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.689103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471513937004122:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.689543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471518231971514:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37370
2025-11-27T18:19:30.694144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471518231971509:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:30.694460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471513937004121:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.694504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:30.694587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471513937004121:2048] [node 1] ICP22 created new session: [3:7577471518231971515:2048]
2025-11-27T18:19:30.694666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471518231971515:2048] [node 1] ICS09 handshake done sender: [3:7577471518231971509:2119] self: [3:7577471518231971512:0] peer: [1:7577471516374951103:0] socket: 176 qp: -1
2025-11-27T18:19:30.695048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471514777611003:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.694716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471518231971515:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:30.694754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:30.695102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:30.695168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471514777611003:2108] [node 4] ICP22 created new session: [5:7577471514777611036:2108]
2025-11-27T18:19:30.688835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471518245221131:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.695240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471514777611036:2108] [node 4] ICS09 handshake done sender: [5:7577471514777611027:2119] self: [5:7577471514777611033:0] peer: [4:7577471518245221136:0] socket: 170 qp: -1
2025-11-27T18:19:30.695272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471514777611036:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:30.689946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:30.695026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471518245221131:2109] [node 5] ICP22 created new session: [4:7577471518245221155:2109]
2025-11-27T18:19:30.695303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:30.695096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471518245221155:2109] [node 5] ICS09 handshake done sender: [4:7577471518245221137:2114] self: [4:7577471518245221136:0] peer: [5:7577471514777611033:0] socket: 126 qp: -1
2025-11-27T18:19:30.695136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471518245221155:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:30.678533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:30.695211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:30.678576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471511072765164:2048] [node 1] ICP22 created new session: [2:7577471515367732548:2048]
2025-11-27T18:19:30.678624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471515367732548:2048] [node 1] ICS09 handshake done sender: [2:7577471515367732534:2115] self: [2:7577471515367732543:0] peer: [1:7577471516374951099:0] socket: 161 qp: -1
2025-11-27T18:19:30.686622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471516374951104:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.678666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471515367732548:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:30.690141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471516374951104:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:30.678705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:30.685427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471515367732550:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21623
2025-11-27T18:19:30.687672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471515367732531:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:30.690306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471515367732550:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.690419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471515367732529:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.690443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471515367732529:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.692687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471515367732525:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.692764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:30.692816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471515367732525:2107] [node 4] ICP22 created new session: [2:7577471515367732553:2107]
2025-11-27T18:19:30.696488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471518369135034:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:19:30.692866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471515367732553:2107] [node 4] ICS09 handshake done sender: [2:7577471515367732531:2112] self: [2:7577471515367732530:0] peer: [4:7577471518245221150:0] socket: 169 qp: -1
2025-11-27T18:19:30.696679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471514074167653:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.692906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471515367732553:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:30.706122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471516374951106:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.716625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471518369135036:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:19:30.692939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:30.706195Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471516374951106:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.713343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471518245221153:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14117
2025-11-27T18:19:30.713562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471518245221154:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14117
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.740209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471518369135037:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:19:30.740271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471518369135038:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:19:30.717450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471515367732529:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:30.706857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471512079983736:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.706961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:30.738214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471511072765165:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.742055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471518369135033:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:30.748660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471514074167653:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.748721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:30.748782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471514074167653:2048] [node 1] ICP22 created new session: [6:7577471518369135039:2048]
2025-11-27T18:19:30.748845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471518369135039:2048] [node 1] ICS09 handshake done sender: [6:7577471518369135033:2117] self: [6:7577471518369135035:0] peer: [1:7577471516374951105:0] socket: 179 qp: -1
2025-11-27T18:19:30.748885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471518369135039:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:30.748930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:30.738294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:30.738352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471511072765165:2049] [node 3] ICP22 created new session: [2:7577471515367732558:2049]
2025-11-27T18:19:30.738399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471515367732558:2049] [node 3] ICS09 handshake done sender: [2:7577471515367732529:2111] self: [2:7577471515367732528:0] peer: [3:7577471518231971513:0] socket: 168 qp: -1
2025-11-27T18:19:30.738436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471515367732558:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:30.738479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:30.756926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471513937004122:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30548) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471518231971508:2118] held: no
2025-11-27T18:19:30.760968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471518369135036:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.707023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471512079983736:2049] [node 3] ICP22 created new session: [1:7577471516374951131:2049]
2025-11-27T18:19:30.707080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471516374951131:2049] [node 3] ICS09 handshake done sender: [1:7577471516374951104:2113] self: [1:7577471516374951103:0] peer: [3:7577471518231971512:0] socket: 171 qp: -1
2025-11-27T18:19:30.707121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471516374951131:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:30.707161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:30.742262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471516374951106:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:30.758039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471516374951096:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.758109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:30.758162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471516374951096:2109] [node 6] ICP22 created new session: [1:7577471516374951134:2109]
2025-11-27T18:19:30.758210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471516374951134:2109] [node 6] ICS09 handshake done sender: [1:7577471516374951106:2114] self: [1:7577471516374951105:0] peer: [6:7577471518369135035:0] socket: 178 qp: -1
2025-11-27T18:19:30.758256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471516374951134:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:30.758297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.762455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471518231971517:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37370
2025-11-27T18:19:30.762930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471518245221148:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.775832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471514777611035:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20306
2025-11-27T18:19:30.769555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471513950253768:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30548) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471518245221148:2118] held: no
2025-11-27T18:19:30.778066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471514777611035:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.778227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471510482643645:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.778580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471515367732536:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.778619Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471515367732536:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.780334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471518369135034:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.780606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471514074167654:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.780932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471518369135007:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.770200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471513950253768:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.770267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:30.770324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471513950253768:2049] [node 2] ICP22 created new session: [4:7577471518245221158:2049]
2025-11-27T18:19:30.770379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471518245221158:2049] [node 2] ICS09 handshake done sender: [4:7577471518245221148:2118] self: [4:7577471518245221150:0] peer: [2:7577471515367732530:0] socket: 174 qp: -1
2025-11-27T18:19:30.770417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471518245221158:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:30.770462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:30.772082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471518245221153:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.774252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471513950253769:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.776885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471518231971492:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.776929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471518231971492:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.783766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471518245221161:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14117
2025-11-27T18:19:30.784147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471518231971508:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:30.783721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471515367732536:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:30.784393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471514777611014:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.784434Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471514777611014:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.784801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471513937004122:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.784857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:30.784907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471513937004122:2049] [node 2] ICP22 created new session: [3:7577471518231971519:2049]
2025-11-27T18:19:30.784950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471518231971519:2049] [node 2] ICS09 handshake done sender: [3:7577471518231971508:2118] self: [3:7577471518231971513:0] peer: [2:7577471515367732528:0] socket: 175 qp: -1
2025-11-27T18:19:30.784990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471518231971519:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:30.785032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.787853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471514777611039:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20306
2025-11-27T18:19:30.788461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471515367732538:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.788498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471515367732538:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.789961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471518369135038:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.790984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471515367732526:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.790511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471518369135015:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:30.792926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471518245221161:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:30.791059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:30.791110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471515367732526:2108] [node 5] ICP22 created new session: [2:7577471515367732561:2108]
2025-11-27T18:19:30.791153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471515367732561:2108] [node 5] ICS09 handshake done sender: [2:7577471515367732536:2116] self: [2:7577471515367732535:0] peer: [5:7577471514777611038:0] socket: 186 qp: -1
2025-11-27T18:19:30.791188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471515367732561:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:30.791231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:30.795678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471518231971492:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:30.790609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471518369135044:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:19:30.793080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471518245221139:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.799526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471518369135046:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:19:30.800001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471518369135036:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:30.799876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471518231971480:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.799969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:30.800029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471518231971480:2107] [node 4] ICP22 created new session: [3:7577471518231971521:2107]
2025-11-27T18:19:30.800076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471518231971521:2107] [node 4] ICS09 handshake done sender: [3:7577471518231971492:2114] self: [3:7577471518231971491:0] peer: [4:7577471518245221160:0] socket: 184 qp: -1
2025-11-27T18:19:30.800116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471518231971521:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:30.799212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471515367732538:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:30.793103Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471518245221139:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:30.804655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471510482643645:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30548) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471514777611035:2122] held: no
2025-11-27T18:19:30.805531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471514777611014:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:30.804426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:30.799104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471518245221162:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14117
2025-11-27T18:19:30.800472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471515367732527:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.807129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471514074167654:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.813581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471514777611035:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:30.807199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:30.800941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471518245221153:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:30.800551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:30.800605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471515367732527:2109] [node 6] ICP22 created new session: [2:7577471515367732564:2109]
2025-11-27T18:19:30.800657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471515367732564:2109] [node 6] ICS09 handshake done sender: [2:7577471515367732538:2117] self: [2:7577471515367732537:0] peer: [6:7577471518369135042:0] socket: 139 qp: -1
2025-11-27T18:19:30.800688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471515367732564:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:30.800722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:30.801307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471518245221139:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:30.801525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471513950253769:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.801586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:30.801637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471513950253769:2050] [node 3] ICP22 created new session: [4:7577471518245221165:2050]
2025-11-27T18:19:30.801684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471518245221165:2050] [node 3] ICS09 handshake done sender: [4:7577471518245221153:2120] self: [4:7577471518245221160:0] peer: [3:7577471518231971491:0] socket: 187 qp: -1
2025-11-27T18:19:30.801724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471518245221165:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:30.801756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:30.802671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471518245221132:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.802715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:30.802745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471518245221132:2111] [node 6] ICP22 created new session: [4:7577471518245221167:2111]
2025-11-27T18:19:30.802775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471518245221167:2111] [node 6] ICS09 handshake done sender: [4:7577471518245221139:2115] self: [4:7577471518245221138:0] peer: [6:7577471518369135045:0] socket: 193 qp: -1
2025-11-27T18:19:30.802820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471518245221167:2111] [node 6] ICS10 traffic start
2025-11-27T18:19:30.802841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:30.807257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471514074167654:2049] [node 2] ICP22 created new session: [6:7577471518369135048:2049]
2025-11-27T18:19:30.807299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471518369135048:2049] [node 2] ICS09 handshake done sender: [6:7577471518369135036:2119] self: [6:7577471518369135042:0] peer: [2:7577471515367732537:0] socket: 145 qp: -1
2025-11-27T18:19:30.807338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471518369135048:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:30.807370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:30.817757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471514777611004:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:30.817875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:30.817931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471514777611004:2109] [node 6] ICP22 created new session: [5:7577471514777611041:2109]
2025-11-27T18:19:30.817981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471514777611041:2109] [node 6] ICS09 handshake done sender: [5:7577471514777611014:2114] self: [5:7577471514777611013:0] peer: [6:7577471518369135043:0] socket: 129 qp: -1
2025-11-27T18:19:30.818025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471514777611041:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:30.818063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:30.819154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471510482643645:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.819238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:30.819295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471510482643645:2049] [node 2] ICP22 created new session: [5:7577471514777611043:2049]
2025-11-27T18:19:30.819343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471514777611043:2049] [node 2] ICS09 handshake done sender: [5:7577471514777611035:2122] self: [5:7577471514777611038:0] peer: [2:7577471515367732535:0] socket: 191 qp: -1
2025-11-27T18:19:30.819374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471514777611043:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:30.819404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:30.829777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471518369135047:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34380
2025-11-27T18:19:30.830302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471518369135015:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9527) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471518369135038:2121] held: no
2025-11-27T18:19:30.831024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471518369135034:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:30.831432Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471518369135007:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21071) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471518369135034:2118] held: no
2025-11-27T18:19:30.832057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471518369135007:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.832154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:30.832218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471518369135007:2104] [node 5] ICP22 created new session: [6:7577471518369135050:2104]
2025-11-27T18:19:30.832267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471518369135050:2104] [node 5] ICS09 handshake done sender: [6:7577471518369135034:2118] self: [6:7577471518369135043:0] peer: [5:7577471514777611013:0] socket: 149 qp: -1
2025-11-27T18:19:30.832307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471518369135050:2104] [node 5] ICS10 traffic start
2025-11-27T18:19:30.832348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:30.833690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471518369135038:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:30.834730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471518369135015:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:30.834816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:30.834866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471518369135015:2110] [node 4] ICP22 created new session: [6:7577471518369135052:2110]
2025-11-27T18:19:30.834915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471518369135052:2110] [node 4] ICS09 handshake done sender: [6:7577471518369135038:2121] self: [6:7577471518369135045:0] peer: [4:7577471518245221138:0] socket: 128 qp: -1
2025-11-27T18:19:30.834962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471518369135052:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:30.835004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:30.873448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:30.873699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471515367732548:2048] [node 1] ICS01 socket: 161 reason# EndOfStream
2025-11-27T18:19:30.873753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471511072765164:2048] [node 1] ICP30 unregister session Session# [2:7577471515367732548:2048] VirtualId# [2:7577471515367732543:0]
2025-11-27T18:19:30.873789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471515367732548:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.873879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:30.873944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:30.875911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:30.876129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471518231971515:2048] [node 1] ICS07 socket disconnect 176 reason# EndOfStream
2025-11-27T18:19:30.876165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471518231971515:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.876235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:30.876273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471518231971515:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:30.880356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471513937004121:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4322) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:30.880401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471518231971515:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:30.881222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471513937004121:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4322) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:30.881302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471518231971515:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:30.881337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471513937004121:2048] [node 1] ICP30 unregister session Session# [3:7577471518231971515:2048] VirtualId# [3:7577471518231971512:0]
2025-11-27T18:19:30.881440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:30.881471Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471513937004121:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4322) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:30.881484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4322) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:30.886323Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:30.886595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471514777611031:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-27T18:19:30.886672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471510482643644:2048] [node 1] ICP30 unregister session Session# [5:7577471514777611031:2048] VirtualId# [5:7577471514777611024:0]
2025-11-27T18:19:30.886725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471514777611031:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.886821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:30.886885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:30.888579Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:30.888998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471518245221145:2048] [node 1] ICS01 socket: 150 reason# EndOfStream
2025-11-27T18:19:30.889083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471513950253767:2048] [node 1] ICP30 unregister session Session# [4:7577471518245221145:2048] VirtualId# [4:7577471518245221143:0]
2025-11-27T18:19:30.889153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471518245221145:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.889316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:30.889370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:30.889555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:30.889785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471518369135039:2048] [node 1] ICS07 socket disconnect 179 reason# EndOfStream
2025-11-27T18:19:30.889816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471518369135039:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.889899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:30.889943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471518369135039:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:30.895197Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471514074167653:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4322) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:30.895261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471518369135039:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:30.895297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471514074167653:2048] [node 1] ICP30 unregister session Session# [6:7577471518369135039:2048] VirtualId# [6:7577471518369135035:0]
2025-11-27T18:19:30.895342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:30.895371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471514074167653:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4322) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:30.895218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471513937004121:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:30.895385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4322) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:30.905507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471514074167653:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:30.949512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:30.949751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471518231971519:2049] [node 2] ICS01 socket: 175 reason# EndOfStream
2025-11-27T18:19:30.949794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471513937004122:2049] [node 2] ICP30 unregister session Session# [3:7577471518231971519:2049] VirtualId# [3:7577471518231971513:0]
2025-11-27T18:19:30.949828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471518231971519:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.949912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:30.949955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:30.959299Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:30.958741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:30.958968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471518245221158:2049] [node 2] ICS01 socket: 174 reason# EndOfStream
2025-11-27T18:19:30.959015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471513950253768:2049] [node 2] ICP30 unregister session Session# [4:7577471518245221158:2049] VirtualId# [4:7577471518245221150:0]
2025-11-27T18:19:30.959048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471518245221158:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.959133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:30.959181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:30.959776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:30.959949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471518369135048:2049] [node 2] ICS01 socket: 145 reason# EndOfStream
2025-11-27T18:19:30.959979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471514074167654:2049] [node 2] ICP30 unregister session Session# [6:7577471518369135048:2049] VirtualId# [6:7577471518369135042:0]
2025-11-27T18:19:30.960003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471518369135048:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.960058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:30.960093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:30.959492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471514777611043:2049] [node 2] ICS01 socket: 191 reason# EndOfStream
2025-11-27T18:19:30.959541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471510482643645:2049] [node 2] ICP30 unregister session Session# [5:7577471514777611043:2049] VirtualId# [5:7577471514777611038:0]
2025-11-27T18:19:30.959568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471514777611043:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.959638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:30.959674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:30.997569Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:30.997802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471518245221165:2050] [node 3] ICS01 socket: 187 reason# EndOfStream
2025-11-27T18:19:30.997849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471513950253769:2050] [node 3] ICP30 unregister session Session# [4:7577471518245221165:2050] VirtualId# [4:7577471518245221160:0]
2025-11-27T18:19:30.997885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471518245221165:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.997964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:30.998013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:30.998110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:30.998289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471514777611025:2050] [node 3] ICS07 socket disconnect 155 reason# EndOfStream
2025-11-27T18:19:30.998320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471514777611025:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.998379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:30.998418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471514777611025:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:30.998921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:30.999128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471518369135030:2050] [node 3] ICS07 socket disconnect 153 reason# EndOfStream
2025-11-27T18:19:30.999152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471518369135030:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:30.999215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:30.999259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471518369135030:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:31.000984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471514074167655:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64145) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:31.001037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471518369135030:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:31.001071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471514074167655:2050] [node 3] ICP30 unregister session Session# [6:7577471518369135030:2050] VirtualId# [6:7577471518369135028:0]
2025-11-27T18:19:31.001112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:31.001147Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471514074167655:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64145) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:31.001162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64145) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:31.001570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471510482643646:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64145) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:31.001626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471514777611025:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:31.001659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471510482643646:2050] [node 3] ICP30 unregister session Session# [5:7577471514777611025:2050] VirtualId# [5:7577471514777611022:0]
2025-11-27T18:19:31.001690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:31.001720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471510482643646:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64145) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:31.001732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64145) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:31.009379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471514074167655:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:31.012788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471510482643646:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:31.030425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:31.030648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471514777611036:2108] [node 4] ICS07 socket disconnect 170 reason# EndOfStream
2025-11-27T18:19:31.030677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471514777611036:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:31.030763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:31.030803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471514777611036:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:31.031072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:31.037277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471514777611003:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9527) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:31.037336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471514777611036:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:31.037370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471514777611003:2108] [node 4] ICP30 unregister session Session# [5:7577471514777611036:2108] VirtualId# [5:7577471514777611033:0]
2025-11-27T18:19:31.037457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:31.037480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471514777611003:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9527) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:31.037491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9527) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:31.036489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471518369135052:2110] [node 4] ICS01 socket: 128 reason# EndOfStream
2025-11-27T18:19:31.036621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471518369135015:2110] [node 4] ICP30 unregister session Session# [6:7577471518369135052:2110] VirtualId# [6:7577471518369135045:0]
2025-11-27T18:19:31.036656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471518369135052:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:31.036751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:31.036803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:31.048861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471514777611003:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:19:31.096898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:31.097130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471518369135050:2104] [node 5] ICS01 socket: 149 reason# EndOfStream
2025-11-27T18:19:31.097184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471518369135007:2104] [node 5] ICP30 unregister session Session# [6:7577471518369135050:2104] VirtualId# [6:7577471518369135043:0]
2025-11-27T18:19:31.097221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471518369135050:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:31.097673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:31.097755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:31.188126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471520143645006:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:31.188575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471520143645005:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:31.243443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471522716610569:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:31.243536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471522716610570:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:31.271806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471519872064538:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:31.271848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471519872064539:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:31.297128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471519080810978:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:31.297172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471519080810979:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:31.297199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471519080810980:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:31.318113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471519471418389:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:31.318156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471519471418390:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:31.318185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471519471418391:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:31.339570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471521136826900:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:31.339600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471521136826902:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:31.340694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471521136826901:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:31.621927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471522716610631:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:31.705788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471519872064619:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:31.718194Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471519872064538:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(74222310ns) {Configure:104(74217047ns) {ProcessConfigured:135(27902550ns) {StartInitialHandshake:157(27898372ns) {DropHandshakes:470(3180ns) {DropIncomingHandshake:442(806ns) DropOutgoingHandshake:457(337ns)} GenerateSessionVirtualId:988(1015ns) PrepareNewSessionHandshake:477(3715ns) {SwitchToState:179(1866ns)}}}}}
2025-11-27T18:19:31.718712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471519872064621:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:31.719290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471519872064622:2124] [node 6] ICP01 ready to work
2025-11-27T18:19:31.622261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471522716610632:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:31.658360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471520143645071:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:31.737464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471521136826962:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:31.761300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471520143645072:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:31.770546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471519080811041:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:31.813818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471519471418456:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:31.817912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471520143645073:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:31.830196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471519471418459:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:31.838304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471521136826964:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:31.842535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471520143645005:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(244628808ns) {Configure:104(244617625ns) {ProcessConfigured:135(58354ns) {StartInitialHandshake:157(55802ns) {DropHandshakes:470(3242ns) {DropIncomingHandshake:442(910ns) DropOutgoingHandshake:457(378ns)} GenerateSessionVirtualId:988(658ns) PrepareNewSessionHandshake:477(1056ns) {SwitchToState:179(283ns)}}}}}
2025-11-27T18:19:31.838136Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471522716610570:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(194942541ns) {Configure:104(194936374ns) {ProcessConfigured:135(28683190ns) {StartInitialHandshake:157(28679291ns) {DropHandshakes:470(2592ns) {DropIncomingHandshake:442(553ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(970ns) PrepareNewSessionHandshake:477(9883968ns) {SwitchToState:179(7437ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:31.849520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471519080811042:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:31.857639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471519872064539:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(118335819ns) {Configure:104(118330626ns) {ProcessConfigured:135(8049316ns) {StartInitialHandshake:157(8046499ns) {DropHandshakes:470(2679ns) {DropIncomingHandshake:442(723ns) DropOutgoingHandshake:457(449ns)} GenerateSessionVirtualId:988(1133ns) PrepareNewSessionHandshake:477(3139ns) {SwitchToState:179(1451ns)}}}}}
2025-11-27T18:19:31.861318Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471522716610632:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22859284ns) {Configure:104(22853919ns) {ProcessConfigured:135(75155ns) {StartInitialHandshake:157(72913ns) {DropHandshakes:470(3398ns) {DropIncomingHandshake:442(767ns) DropOutgoingHandshake:457(446ns)} GenerateSessionVirtualId:988(872ns) PrepareNewSessionHandshake:477(2225ns) {SwitchToState:179(720ns)}}}}}
2025-11-27T18:19:31.862014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471519872064621:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(121850135ns) {Configure:104(121846350ns) {ProcessConfigured:135(41855ns) {StartInitialHandshake:157(40567ns) {DropHandshakes:470(1310ns) {DropIncomingHandshake:442(422ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(368ns) PrepareNewSessionHandshake:477(783ns) {SwitchToState:179(355ns)}}}}}
2025-11-27T18:19:31.869857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471522716610638:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:31.862125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471519872064600:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:98
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:31.885328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471520143645073:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(40323076ns) {Configure:104(40319554ns) {ProcessConfigured:135(67153ns) {StartInitialHandshake:157(64812ns) {DropHandshakes:470(3641ns) {DropIncomingHandshake:442(451ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(541ns) PrepareNewSessionHandshake:477(2036ns) {SwitchToState:179(554ns)}}}}}
2025-11-27T18:19:31.863111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471519872064629:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:98
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:31.905487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471519872064629:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:31.904128Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471522716610631:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(259982337ns) {Configure:104(259974827ns) {ProcessConfigured:135(12895996ns) {StartInitialHandshake:157(12892993ns) {DropHandshakes:470(2943ns) {DropIncomingHandshake:442(635ns) DropOutgoingHandshake:457(343ns)} GenerateSessionVirtualId:988(1486ns) PrepareNewSessionHandshake:477(2799ns) {SwitchToState:179(1417ns)}}}}}
2025-11-27T18:19:31.900224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471520143645076:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30068
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:31.920350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471519080811043:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40735
2025-11-27T18:19:31.904724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471519471418459:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20601589ns) {Configure:104(20594363ns) {ProcessConfigured:135(39349ns) {StartInitialHandshake:157(35540ns) {DropHandshakes:470(1759ns) {DropIncomingHandshake:442(449ns) DropOutgoingHandshake:457(291ns)} GenerateSessionVirtualId:988(460ns) PrepareNewSessionHandshake:477(677ns) {SwitchToState:179(171ns)}}}}}
2025-11-27T18:19:31.902360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471520143645083:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30068
2025-11-27T18:19:31.905804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471520143645076:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:31.930492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471522716610640:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59184
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:31.930671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471522716610641:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59184
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:31.931784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471522716610645:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59184
2025-11-27T18:19:31.905032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471519471418468:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25101
2025-11-27T18:19:31.937476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471522716610646:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59184
2025-11-27T18:19:31.937643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471522716610640:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:31.942209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471520143645087:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30068
2025-11-27T18:19:31.951334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471522716610645:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:31.951437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471522716610569:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:31.951728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471520143645075:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:31.951764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471520143645075:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:31.952998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471522716610654:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59184
2025-11-27T18:19:31.954441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471521136826962:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(83871667ns) {Configure:104(82752395ns) {ProcessConfigured:135(52661ns) {StartInitialHandshake:157(50352ns) {DropHandshakes:470(2444ns) {DropIncomingHandshake:442(621ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(468ns) PrepareNewSessionHandshake:477(1893ns) {SwitchToState:179(542ns)}}}}}
2025-11-27T18:19:31.953646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471520143645075:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:31.982129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471519080811055:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40735
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:31.993632Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471521136826902:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(36054117ns) {Configure:104(36049961ns) {ProcessConfigured:135(15310774ns) {StartInitialHandshake:157(15305897ns) {DropHandshakes:470(2549ns) {DropIncomingHandshake:442(561ns) DropOutgoingHandshake:457(280ns)} GenerateSessionVirtualId:988(1017ns) PrepareNewSessionHandshake:477(3104ns) {SwitchToState:179(1694ns)}}}}}
2025-11-27T18:19:31.991827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471519872064622:2124] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(107705837ns) {Configure:104(107699454ns) {ProcessConfigured:135(19763538ns) {StartInitialHandshake:157(19760440ns) {DropHandshakes:470(7871543ns) {DropIncomingHandshake:442(861ns) DropOutgoingHandshake:457(2017ns)} GenerateSessionVirtualId:988(1511ns) PrepareNewSessionHandshake:477(3858ns) {SwitchToState:179(1957ns)}}}}}
2025-11-27T18:19:31.997786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471519080811043:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:31.997082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471521136826973:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58395
2025-11-27T18:19:32.005470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471519872064635:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:98
2025-11-27T18:19:32.017641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471519872064600:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.030209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471522716610645:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.030648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471522716610569:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.030740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:32.030785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471522716610569:2048] [node 1] ICP22 created new session: [2:7577471527011577952:2048]
2025-11-27T18:19:32.030834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471527011577952:2048] [node 1] ICS09 handshake done sender: [2:7577471522716610645:2116] self: [2:7577471522716610653:0] peer: [1:7577471520143645074:0] socket: 156 qp: -1
2025-11-27T18:19:32.030869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471527011577952:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:32.030917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:32.032392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471527011577951:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59184
2025-11-27T18:19:32.010557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471519471418469:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25101
2025-11-27T18:19:32.009869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471521136826900:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(137209331ns) {Configure:104(137189351ns) {ProcessConfigured:135(8084568ns) {StartInitialHandshake:157(8081436ns) {DropHandshakes:470(2945ns) {DropIncomingHandshake:442(852ns) DropOutgoingHandshake:457(625ns)} GenerateSessionVirtualId:988(1074ns) PrepareNewSessionHandshake:477(4665ns) {SwitchToState:179(2726ns)}}}}}
2025-11-27T18:19:32.017668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471520143645005:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.013322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471519471418468:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.025552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471520143645083:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.044023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:32.041499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471521136826973:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.047417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471519872064538:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.049722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471522716610646:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.049869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471522716610641:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.049461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471519872064635:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.056658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471521136826964:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.048271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471519471418389:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.050301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471519080811055:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.047506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471520143645005:2048] [node 2] ICP22 created new session: [1:7577471524438612385:2048]
2025-11-27T18:19:32.059122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471519872064539:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.057757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471525431794275:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58395
2025-11-27T18:19:32.052108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471519471418469:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.048438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471520143645087:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.059137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471519080810979:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.057898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471523766385766:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25101
2025-11-27T18:19:32.058391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471519471418390:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.059227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471523766385766:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.057574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471527011577951:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.048703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471524438612385:2048] [node 2] ICS09 handshake done sender: [1:7577471520143645075:2111] self: [1:7577471520143645074:0] peer: [2:7577471522716610653:0] socket: 149 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.059401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471519080810980:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(146163117ns) {Configure:104(145411629ns) {ProcessConfigured:135(1335675ns) {StartInitialHandshake:157(1331707ns) {DropHandshakes:470(3315ns) {DropIncomingHandshake:442(871ns) DropOutgoingHandshake:457(535ns)} GenerateSessionVirtualId:988(474ns) PrepareNewSessionHandshake:477(1724ns) {SwitchToState:179(515ns)}}}}}
2025-11-27T18:19:32.059106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471525431794276:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58395
2025-11-27T18:19:32.071127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471522716610637:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.070267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471519872064624:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.049111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471524438612385:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:32.059510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471519080810980:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.074524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471523766385770:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25101
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.069634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471525431794275:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.049163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:32.071151Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471522716610637:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.070311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471519872064624:2128] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.074199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471521136826901:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.070377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471519080811049:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.057870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471520143645078:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.072205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471522716610635:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.076657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471524167031936:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:98
2025-11-27T18:19:32.070404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471519080811049:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.057911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471520143645078:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.076824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471519872064624:2128] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:32.083027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471524167031934:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:98
2025-11-27T18:19:32.072225Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471522716610635:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.067048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471520143645005:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(55563608ns) {HandleHandshakeStatus:329(48526781ns) {DropIncomingHandshake:442(533ns) DropOutgoingHandshake:457(588ns) SwitchToState:179(914ns) ProcessPendingSessionEvents:495(7136329ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7066233ns max# 6630015ns}}}
2025-11-27T18:19:32.078887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471523375778356:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40735
2025-11-27T18:19:32.075853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471522716610642:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.080434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471519080811043:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:32.067362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471520143645066:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.090125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471519471418469:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:32.083603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471519872064619:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.087747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:32.082157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471519080810980:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.075893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471522716610642:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.067385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471520143645066:2107] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.089561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471519872064619:2107] [node 4] ICP22 created new session: [3:7577471524167031938:2107]
2025-11-27T18:19:32.095052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471519471418390:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.078330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471522716610635:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:32.085490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:32.088214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471520143645066:2107] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:32.079408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471522716610637:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:32.086584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471519080810980:2050] [node 3] ICP22 created new session: [4:7577471523375778358:2050]
2025-11-27T18:19:32.080892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471522716610648:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.086641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471523375778358:2050] [node 3] ICS09 handshake done sender: [4:7577471519080811043:2110] self: [4:7577471523375778355:0] peer: [3:7577471519872064623:0] socket: 144 qp: -1
No RDMA context found for GID: 2025-11-27T18:19:32.087788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471523375778358:2050] [node 3] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:19:32.080924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471522716610648:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.100871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471525431794279:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58395
2025-11-27T18:19:32.087896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471519080811049:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:32.082283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471522716610570:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.089511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:32.101033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471525431794280:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58395
2025-11-27T18:19:32.099942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471519080811042:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.085481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:32.100891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:32.092788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471524167031938:2107] [node 4] ICS09 handshake done sender: [3:7577471519872064624:2128] self: [3:7577471519872064623:0] peer: [4:7577471523375778355:0] socket: 141 qp: -1
2025-11-27T18:19:32.101154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:32.101085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471520143645078:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:32.085540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471522716610570:2049] [node 3] ICP22 created new session: [2:7577471527011577962:2049]
2025-11-27T18:19:32.092946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471524167031938:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:32.104337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471519080811042:2109] [node 6] ICP22 created new session: [4:7577471523375778360:2109]
2025-11-27T18:19:32.103799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471520143645006:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.100964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471519471418390:2049] [node 2] ICP22 created new session: [5:7577471523766385772:2049]
2025-11-27T18:19:32.085644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471522716610632:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.086534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:32.087825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471527011577962:2049] [node 3] ICS09 handshake done sender: [2:7577471522716610635:2110] self: [2:7577471522716610634:0] peer: [3:7577471524167031935:0] socket: 158 qp: -1
2025-11-27T18:19:32.088499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471527011577962:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:32.088698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:32.089438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471522716610632:2108] [node 5] ICP22 created new session: [2:7577471527011577963:2108]
2025-11-27T18:19:32.094528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471527011577963:2108] [node 5] ICS09 handshake done sender: [2:7577471522716610637:2111] self: [2:7577471522716610636:0] peer: [5:7577471523766385768:0] socket: 159 qp: -1
2025-11-27T18:19:32.094574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471527011577963:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:32.095270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:32.101019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471523766385772:2049] [node 2] ICS09 handshake done sender: [5:7577471519471418469:2117] self: [5:7577471523766385768:0] peer: [2:7577471522716610636:0] socket: 160 qp: -1
2025-11-27T18:19:32.108686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471525431794276:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.094530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:32.104790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:32.106037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471523375778360:2109] [node 6] ICS09 handshake done sender: [4:7577471519080811049:2113] self: [4:7577471519080811048:0] peer: [6:7577471525431794277:0] socket: 154 qp: -1
2025-11-27T18:19:32.104866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471520143645006:2049] [node 3] ICP22 created new session: [1:7577471524438612392:2049]
2025-11-27T18:19:32.095338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471524167031934:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.106085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471523375778360:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:32.112778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471521136826973:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.115572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471521136826964:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.101068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471523766385772:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:32.114827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471519872064635:2133] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:32.104918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471524438612392:2049] [node 3] ICS09 handshake done sender: [1:7577471520143645066:2107] self: [1:7577471520143645065:0] peer: [3:7577471524167031933:0] socket: 132 qp: -1
2025-11-27T18:19:32.101116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:32.116668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471521136826900:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.109178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:32.117045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471522716610570:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(18956723ns) {HandleHandshakeStatus:329(18952449ns) {DropIncomingHandshake:442(13710ns) DropOutgoingHandshake:457(2128ns) SwitchToState:179(602ns) ProcessPendingSessionEvents:495(12137248ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12040859ns max# 3463807ns}}}
2025-11-27T18:19:32.123596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471523766385771:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25101
2025-11-27T18:19:32.117085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:32.104957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471524438612392:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:32.105001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:32.123526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471521136826964:2109] [node 4] ICP22 created new session: [6:7577471525431794282:2109]
2025-11-27T18:19:32.112182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471520143645072:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.123903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471525431794282:2109] [node 4] ICS09 handshake done sender: [6:7577471521136826973:2114] self: [6:7577471525431794277:0] peer: [4:7577471519080811048:0] socket: 168 qp: -1
2025-11-27T18:19:32.112778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:32.126105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471522716610642:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:32.124322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471525431794282:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:32.112833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471520143645072:2109] [node 5] ICP22 created new session: [1:7577471524438612394:2109]
2025-11-27T18:19:32.124725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:32.128412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471522716610648:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:32.114484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471524438612394:2109] [node 5] ICS09 handshake done sender: [1:7577471520143645078:2113] self: [1:7577471520143645077:0] peer: [5:7577471523766385767:0] socket: 150 qp: -1
2025-11-27T18:19:32.114714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471524438612394:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:32.115013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:32.131062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471520143645080:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.131101Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471520143645080:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.133553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471525431794275:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.136965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471524438612397:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30068
2025-11-27T18:19:32.137426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471519471418389:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29813) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471519471418468:2116] held: no
2025-11-27T18:19:32.137123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471520143645080:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:32.139338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471520143645073:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.139389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:32.139440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471520143645073:2110] [node 6] ICP22 created new session: [1:7577471524438612399:2110]
2025-11-27T18:19:32.139498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471524438612399:2110] [node 6] ICS09 handshake done sender: [1:7577471520143645080:2114] self: [1:7577471520143645079:0] peer: [6:7577471525431794281:0] socket: 178 qp: -1
2025-11-27T18:19:32.139539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471524438612399:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:32.139579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:32.138536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471522716610631:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.139098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:32.141048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471522716610631:2107] [node 4] ICP22 created new session: [2:7577471527011577966:2107]
2025-11-27T18:19:32.141292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471527011577966:2107] [node 4] ICS09 handshake done sender: [2:7577471522716610642:2114] self: [2:7577471522716610639:0] peer: [4:7577471523375778354:0] socket: 161 qp: -1
2025-11-27T18:19:32.141335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471527011577966:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:32.141401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:32.141525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471522716610638:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.141128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471521136826901:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.141585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:32.142850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471522716610638:2109] [node 6] ICP22 created new session: [2:7577471527011577968:2109]
2025-11-27T18:19:32.142909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471527011577968:2109] [node 6] ICS09 handshake done sender: [2:7577471522716610648:2118] self: [2:7577471522716610647:0] peer: [6:7577471525431794278:0] socket: 176 qp: -1
2025-11-27T18:19:32.141595Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471519080810980:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(43683691ns) {HandleHandshakeStatus:329(43681336ns) {DropIncomingHandshake:442(905ns) DropOutgoingHandshake:457(6337ns) SwitchToState:179(735ns) ProcessPendingSessionEvents:495(31010390ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 30470199ns max# 4543518ns}}}
2025-11-27T18:19:32.142966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471527011577968:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:32.143188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471519471418468:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:32.143171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:32.143111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:32.141381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471519872064539:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.141508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:32.143321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471521136826901:2049] [node 2] ICP22 created new session: [6:7577471525431794285:2049]
2025-11-27T18:19:32.141555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471519872064539:2049] [node 2] ICP22 created new session: [3:7577471524167031942:2049]
2025-11-27T18:19:32.143598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471519080811042:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(37983383ns) {HandleHandshakeStatus:329(37978354ns) {DropIncomingHandshake:442(649ns) DropOutgoingHandshake:457(623ns) SwitchToState:179(704ns) ProcessPendingSessionEvents:495(22842644ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22497914ns max# 6706629ns}}}
2025-11-27T18:19:32.143643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471525431794285:2049] [node 2] ICS09 handshake done sender: [6:7577471525431794275:2116] self: [6:7577471525431794278:0] peer: [2:7577471522716610647:0] socket: 177 qp: -1
2025-11-27T18:19:32.146244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471519471418390:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(46400155ns) {HandleHandshakeStatus:329(46397026ns) {DropIncomingHandshake:442(2581ns) DropOutgoingHandshake:457(9255ns) SwitchToState:179(759ns) ProcessPendingSessionEvents:495(35305521ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 35170375ns max# 7548183ns}}}
2025-11-27T18:19:32.143935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471525431794285:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:32.149587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:32.146767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471519471418389:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.146856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:32.146905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471519471418389:2048] [node 1] ICP22 created new session: [5:7577471523766385775:2048]
2025-11-27T18:19:32.146946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471523766385775:2048] [node 1] ICS09 handshake done sender: [5:7577471519471418468:2116] self: [5:7577471523766385767:0] peer: [1:7577471520143645077:0] socket: 151 qp: -1
2025-11-27T18:19:32.146981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471523766385775:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:32.147011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.142853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471524167031942:2049] [node 2] ICS09 handshake done sender: [3:7577471519872064635:2133] self: [3:7577471524167031935:0] peer: [2:7577471522716610634:0] socket: 167 qp: -1
2025-11-27T18:19:32.142900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471524167031942:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:32.143093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:32.143763Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471519872064619:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(48712243ns) {HandleHandshakeStatus:329(48708284ns) {DropIncomingHandshake:442(690ns) DropOutgoingHandshake:457(460ns) SwitchToState:179(1994ns) ProcessPendingSessionEvents:495(35516288ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 35382942ns max# 6447748ns}}}
2025-11-27T18:19:32.153424Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471519872064538:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29813) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471519872064600:2109] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.153944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471520143645072:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(28861489ns) {HandleHandshakeStatus:329(28852245ns) {DropIncomingHandshake:442(20455ns) DropOutgoingHandshake:457(1455ns) SwitchToState:179(819ns) ProcessPendingSessionEvents:495(19636786ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19512588ns max# 2778249ns}}}
2025-11-27T18:19:32.153365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471523766385777:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25101
2025-11-27T18:19:32.153999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471524167031940:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:98
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.156947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471521136826964:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(27805137ns) {HandleHandshakeStatus:329(27802309ns) {DropIncomingHandshake:442(1358ns) DropOutgoingHandshake:457(5512ns) SwitchToState:179(843ns) ProcessPendingSessionEvents:495(16402971ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16279546ns max# 2028861ns}}}
2025-11-27T18:19:32.158980Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471519872064539:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:32.159054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471523766385779:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25101
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.159983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471523375778362:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40735
2025-11-27T18:19:32.161592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471519872064600:2109] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.165840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471522716610631:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(20085717ns) {HandleHandshakeStatus:329(20082433ns) {DropIncomingHandshake:442(16866ns) DropOutgoingHandshake:457(475ns) SwitchToState:179(953ns) ProcessPendingSessionEvents:495(17067475ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16705269ns max# 4518295ns}}}
2025-11-27T18:19:32.165204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471523375778363:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40735
2025-11-27T18:19:32.166816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471519080810979:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12519) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471519080811055:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.167028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471523375778363:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.166615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471519872064538:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.166747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:32.166801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471519872064538:2048] [node 1] ICP22 created new session: [3:7577471524167031945:2048]
2025-11-27T18:19:32.167153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471524167031945:2048] [node 1] ICS09 handshake done sender: [3:7577471519872064600:2109] self: [3:7577471524167031933:0] peer: [1:7577471520143645065:0] socket: 133 qp: -1
2025-11-27T18:19:32.167210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471524167031945:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:32.167411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:32.167742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471521136826901:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(18899036ns) {HandleHandshakeStatus:329(18897227ns) {DropIncomingHandshake:442(7267ns) DropOutgoingHandshake:457(20281ns) SwitchToState:179(738ns) ProcessPendingSessionEvents:495(10284217ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10199150ns max# 2202003ns}}}
2025-11-27T18:19:32.167373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471523375778365:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40735
2025-11-27T18:19:32.168809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471523375778365:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.168879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471519080811055:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.171438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471525431794288:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58395
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.171506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471523766385777:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.171969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471525431794289:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58395
2025-11-27T18:19:32.171784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471519471418391:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.172455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471524438612397:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.172783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471519872064628:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.172623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471525431794287:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58395
2025-11-27T18:19:32.172820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471519872064628:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.176152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471519080810978:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.176274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471520143645073:2110] [node 6] ICP18 handshake (actor [1:7577471524438612397:2118]) is too old
2025-11-27T18:19:32.177887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471519872064628:2130] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:32.176306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471519471418463:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.172643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471525431794288:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.176336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471519471418463:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.175875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471521136826962:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.181769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471519872064621:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.178302Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471519471418463:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:32.182311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471519471418456:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.184499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:32.184569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471519471418456:2110] [node 6] ICP22 created new session: [5:7577471523766385783:2110]
2025-11-27T18:19:32.183031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:32.185128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471523766385783:2110] [node 6] ICS09 handshake done sender: [5:7577471519471418463:2114] self: [5:7577471519471418462:0] peer: [6:7577471525431794290:0] socket: 195 qp: -1
2025-11-27T18:19:32.183093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471519872064621:2108] [node 5] ICP22 created new session: [3:7577471524167031949:2108]
2025-11-27T18:19:32.183149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471524167031949:2108] [node 5] ICS09 handshake done sender: [3:7577471519872064628:2130] self: [3:7577471519872064627:0] peer: [5:7577471523766385780:0] socket: 147 qp: -1
2025-11-27T18:19:32.183192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471524167031949:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:32.183248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:32.185277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471523766385783:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:32.185472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.186190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471523766385782:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25101
2025-11-27T18:19:32.186800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471519080810979:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.186974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:32.187033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471519080810979:2049] [node 2] ICP22 created new session: [4:7577471523375778367:2049]
2025-11-27T18:19:32.187093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471523375778367:2049] [node 2] ICS09 handshake done sender: [4:7577471519080811055:2115] self: [4:7577471523375778354:0] peer: [2:7577471522716610639:0] socket: 166 qp: -1
2025-11-27T18:19:32.187139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471523375778367:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:32.187348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.184585Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471525431794276:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:32.185551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471521136826900:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29813) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471525431794276:2117] held: no
2025-11-27T18:19:32.187049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471521136826900:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.187139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:32.187217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471521136826900:2048] [node 1] ICP22 created new session: [6:7577471525431794292:2048]
2025-11-27T18:19:32.187369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471525431794292:2048] [node 1] ICS09 handshake done sender: [6:7577471525431794276:2117] self: [6:7577471525431794281:0] peer: [1:7577471520143645079:0] socket: 179 qp: -1
2025-11-27T18:19:32.187397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471525431794292:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:32.187504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:32.188808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471525431794291:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58395
2025-11-27T18:19:32.195234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471519471418391:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25088) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471523766385777:2121] held: no
2025-11-27T18:19:32.192976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471521136826962:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3426) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471525431794288:2121] held: no
2025-11-27T18:19:32.193901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471525431794289:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.194578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471525431794288:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:32.189499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471520143645082:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.194898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471523375778370:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40735
2025-11-27T18:19:32.196498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471523766385779:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.197106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471523766385777:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:32.189547Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471520143645082:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.195948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471521136826962:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.195365Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471520143645082:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:32.197682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471520143645071:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.198143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:32.198217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471521136826962:2106] [node 5] ICP22 created new session: [6:7577471525431794294:2106]
2025-11-27T18:19:32.198225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471521136826902:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.198270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471525431794294:2106] [node 5] ICS09 handshake done sender: [6:7577471525431794288:2121] self: [6:7577471525431794290:0] peer: [5:7577471519471418462:0] socket: 200 qp: -1
2025-11-27T18:19:32.198319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471525431794294:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:32.198395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:32.198229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:32.198253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471519471418391:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.198743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471523375778363:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:32.198354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:32.198296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471520143645071:2108] [node 4] ICP22 created new session: [1:7577471524438612402:2108]
2025-11-27T18:19:32.199134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471519872064634:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.198406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471519471418391:2050] [node 3] ICP22 created new session: [5:7577471523766385785:2050]
2025-11-27T18:19:32.198359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471524438612402:2108] [node 4] ICS09 handshake done sender: [1:7577471520143645082:2115] self: [1:7577471520143645081:0] peer: [4:7577471523375778366:0] socket: 138 qp: -1
2025-11-27T18:19:32.198456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471523766385785:2050] [node 3] ICS09 handshake done sender: [5:7577471523766385777:2121] self: [5:7577471523766385780:0] peer: [3:7577471519872064627:0] socket: 194 qp: -1
2025-11-27T18:19:32.198490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471523766385785:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:32.198522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:32.198402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471524438612402:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:32.199176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471519872064634:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.200557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471519471418459:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:32.200642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471519872064634:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:32.198440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:32.200986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471519872064622:2124] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.204704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471519080810978:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.205122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:32.205141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471525431794297:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58395
2025-11-27T18:19:32.204686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:32.205199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471519080810978:2048] [node 1] ICP22 created new session: [4:7577471523375778371:2048]
2025-11-27T18:19:32.204993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471519872064622:2124] [node 6] ICP22 created new session: [3:7577471524167031952:2124]
2025-11-27T18:19:32.205052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471524167031952:2124] [node 6] ICS09 handshake done sender: [3:7577471519872064634:2132] self: [3:7577471519872064633:0] peer: [6:7577471525431794295:0] socket: 152 qp: -1
2025-11-27T18:19:32.205089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471524167031952:2124] [node 6] ICS10 traffic start
2025-11-27T18:19:32.205132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:32.205688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471523375778371:2048] [node 1] ICS09 handshake done sender: [4:7577471523375778363:2119] self: [4:7577471523375778366:0] peer: [1:7577471520143645081:0] socket: 145 qp: -1
2025-11-27T18:19:32.205847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471523375778371:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:32.205885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:32.206163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471519080811051:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:32.206187Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471519080811051:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:32.209657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471519080810978:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:32.209887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471519080811051:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:32.208871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471525431794289:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:32.210075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471521136826902:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25088) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471525431794289:2122] held: no
2025-11-27T18:19:32.210728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471521136826902:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:32.210806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:32.210859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471521136826902:2050] [node 3] ICP22 created new session: [6:7577471525431794298:2050]
2025-11-27T18:19:32.210908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471525431794298:2050] [node 3] ICS09 handshake done sender: [6:7577471525431794289:2122] self: [6:7577471525431794295:0] peer: [3:7577471519872064633:0] socket: 170 qp: -1
2025-11-27T18:19:32.211018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471525431794298:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:32.211369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:32.211384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471519080811041:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:32.211478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:32.211458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471523766385788:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25101
2025-11-27T18:19:32.211544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471519080811041:2108] [node 5] ICP22 created new session: [4:7577471523375778374:2108]
2025-11-27T18:19:32.211604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471523375778374:2108] [node 5] ICS09 handshake done sender: [4:7577471519080811051:2114] self: [4:7577471519080811050:0] peer: [5:7577471523766385787:0] socket: 197 qp: -1
2025-11-27T18:19:32.211637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471523375778374:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:32.211710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:32.211799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471523766385779:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:32.212330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471519471418459:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:32.212410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:32.212467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471519471418459:2109] [node 4] ICP22 created new session: [5:7577471523766385789:2109]
2025-11-27T18:19:32.212542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471523766385789:2109] [node 4] ICS09 handshake done sender: [5:7577471523766385779:2122] self: [5:7577471523766385787:0] peer: [4:7577471519080811050:0] socket: 157 qp: -1
2025-11-27T18:19:32.212586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471523766385789:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:32.212615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:32.255102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:32.255375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471527011577952:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-27T18:19:32.255428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471522716610569:2048] [node 1] ICP30 unregister session Session# [2:7577471527011577952:2048] VirtualId# [2:7577471522716610653:0]
2025-11-27T18:19:32.255474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471527011577952:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.255563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:32.255616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:32.256490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:32.256698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471524167031945:2048] [node 1] ICS01 socket: 133 reason# EndOfStream
2025-11-27T18:19:32.256744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471519872064538:2048] [node 1] ICP30 unregister session Session# [3:7577471524167031945:2048] VirtualId# [3:7577471524167031933:0]
2025-11-27T18:19:32.256777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471524167031945:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.256856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:32.256920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:32.260346Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:32.260653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471523766385775:2048] [node 1] ICS01 socket: 151 reason# EndOfStream
2025-11-27T18:19:32.260699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471519471418389:2048] [node 1] ICP30 unregister session Session# [5:7577471523766385775:2048] VirtualId# [5:7577471523766385767:0]
2025-11-27T18:19:32.260783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471523766385775:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.260899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:32.260959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:32.261514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:32.261715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471523375778371:2048] [node 1] ICS07 socket disconnect 145 reason# EndOfStream
2025-11-27T18:19:32.261748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471523375778371:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.261825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:32.261870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471523375778371:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:32.262670Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:32.262904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471525431794292:2048] [node 1] ICS01 socket: 179 reason# EndOfStream
2025-11-27T18:19:32.262967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471521136826900:2048] [node 1] ICP30 unregister session Session# [6:7577471525431794292:2048] VirtualId# [6:7577471525431794281:0]
2025-11-27T18:19:32.262997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471525431794292:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.263067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:32.263109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:32.262918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471519080810978:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29813) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:32.262972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471523375778371:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:32.263005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471519080810978:2048] [node 1] ICP30 unregister session Session# [4:7577471523375778371:2048] VirtualId# [4:7577471523375778366:0]
2025-11-27T18:19:32.263083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:32.263117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471519080810978:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29813) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:32.263132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29813) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:32.271422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471519080810978:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:32.336041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:32.336294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471524167031942:2049] [node 2] ICS07 socket disconnect 167 reason# EndOfStream
2025-11-27T18:19:32.336343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471524167031942:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.336447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:32.336515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471524167031942:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:32.340848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:32.341122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471523375778367:2049] [node 2] ICS01 socket: 166 reason# EndOfStream
2025-11-27T18:19:32.342509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471519080810979:2049] [node 2] ICP30 unregister session Session# [4:7577471523375778367:2049] VirtualId# [4:7577471523375778354:0]
2025-11-27T18:19:32.342564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471523375778367:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.342654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:32.342541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471519872064539:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12519) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:32.342615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471524167031942:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:32.342912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:32.342709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:32.343127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471523766385772:2049] [node 2] ICS07 socket disconnect 160 reason# EndOfStream
2025-11-27T18:19:32.343165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471523766385772:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.343273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:32.343344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471523766385772:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:32.343693Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471519872064539:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12519) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:32.343822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471524167031942:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:32.343886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471519872064539:2049] [node 2] ICP30 unregister session Session# [3:7577471524167031942:2049] VirtualId# [3:7577471524167031935:0]
2025-11-27T18:19:32.344009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:32.344055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471519872064539:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:32.344079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:32.344229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471519471418390:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12519) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:32.344225Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:32.344333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471523766385772:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:32.344368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471519471418390:2049] [node 2] ICP30 unregister session Session# [5:7577471523766385772:2049] VirtualId# [5:7577471523766385768:0]
2025-11-27T18:19:32.344475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:32.344534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471519471418390:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:32.344528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471525431794285:2049] [node 2] ICS07 socket disconnect 177 reason# EndOfStream
2025-11-27T18:19:32.344564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:32.344587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471525431794285:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.344661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:32.344708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471525431794285:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:32.345455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471521136826901:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12519) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:32.345595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471525431794285:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:32.345637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471521136826901:2049] [node 2] ICP30 unregister session Session# [6:7577471525431794285:2049] VirtualId# [6:7577471525431794278:0]
2025-11-27T18:19:32.345722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:32.345759Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471521136826901:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:32.345772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:32.354354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471519471418390:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:32.355391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471521136826901:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:32.378389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:32.378652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471523375778358:2050] [node 3] ICS01 socket: 144 reason# EndOfStream
2025-11-27T18:19:32.378714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471519080810980:2050] [node 3] ICP30 unregister session Session# [4:7577471523375778358:2050] VirtualId# [4:7577471523375778355:0]
2025-11-27T18:19:32.378779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471523375778358:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.378876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:32.378954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:32.379552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:32.379759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471523766385785:2050] [node 3] ICS01 socket: 194 reason# EndOfStream
2025-11-27T18:19:32.379803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471519471418391:2050] [node 3] ICP30 unregister session Session# [5:7577471523766385785:2050] VirtualId# [5:7577471523766385780:0]
2025-11-27T18:19:32.379854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471523766385785:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.379931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:32.379986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:32.380494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:32.380679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471525431794298:2050] [node 3] ICS01 socket: 170 reason# EndOfStream
2025-11-27T18:19:32.380711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471521136826902:2050] [node 3] ICP30 unregister session Session# [6:7577471525431794298:2050] VirtualId# [6:7577471525431794295:0]
2025-11-27T18:19:32.380730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471525431794298:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.380779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:32.380822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:32.424876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:32.425171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471525431794282:2109] [node 4] ICS07 socket disconnect 168 reason# EndOfStream
2025-11-27T18:19:32.425212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471525431794282:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.425304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:32.425372Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:32.425351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471525431794282:2109] [node 4] ICS15 start handshake
2025-11-27T18:19:32.425629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471523766385789:2109] [node 4] ICS01 socket: 157 reason# EndOfStream
2025-11-27T18:19:32.425727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471519471418459:2109] [node 4] ICP30 unregister session Session# [5:7577471523766385789:2109] VirtualId# [5:7577471523766385787:0]
2025-11-27T18:19:32.425776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471523766385789:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.425861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:32.426517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:32.427119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471521136826964:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8095) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:32.427318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471525431794282:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:32.427382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471521136826964:2109] [node 4] ICP30 unregister session Session# [6:7577471525431794282:2109] VirtualId# [6:7577471525431794277:0]
2025-11-27T18:19:32.427480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:32.427522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471521136826964:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8095) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:32.427537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8095) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:32.437355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471521136826964:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:19:32.506379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:32.506655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471525431794294:2106] [node 5] ICS01 socket: 200 reason# EndOfStream
2025-11-27T18:19:32.506697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471521136826962:2106] [node 5] ICP30 unregister session Session# [6:7577471525431794294:2106] VirtualId# [6:7577471525431794290:0]
2025-11-27T18:19:32.506735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471525431794294:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:32.506809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:32.506861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:32.601415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471526279387538:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:32.601921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471526279387539:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:32.616879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471525307399995:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:32.616953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471525307399996:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:32.644556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471526277113210:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:32.644887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471526277113211:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:32.665516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471525800624118:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:32.665574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471525800624120:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:32.665576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471525800624119:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:32.687085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471524119297701:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:32.687128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471524119297702:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:32.687170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471524119297703:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:32.711020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471523237354818:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:32.711123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471523237354819:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:32.711428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471523237354820:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:32.861686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471523237354879:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:32.882079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471526279387603:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:32.893947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471525307400056:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:32.894457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471526279387604:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:32.901635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471525307400077:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:33.025860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471530095591486:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:33.033899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471530095591489:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:33.030347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471526277113276:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:32.902104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471525307400078:2115] [node 6] ICP01 ready to work
2025-11-27T18:19:33.037829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471528414265078:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:33.054171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471525800624119:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(153683187ns) {Configure:104(153673850ns) {ProcessConfigured:135(72340ns) {StartInitialHandshake:157(68855ns) {DropHandshakes:470(2980ns) {DropIncomingHandshake:442(756ns) DropOutgoingHandshake:457(553ns)} GenerateSessionVirtualId:988(809ns) PrepareNewSessionHandshake:477(730ns) {SwitchToState:179(265ns)}}}}}
2025-11-27T18:19:33.030372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471530572080574:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:33.058507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471525307400077:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(120956782ns) {Configure:104(120951358ns) {ProcessConfigured:135(62453ns) {StartInitialHandshake:157(59068ns) {DropHandshakes:470(2375ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(386ns)} GenerateSessionVirtualId:988(1023ns) PrepareNewSessionHandshake:477(1313ns) {SwitchToState:179(700ns)}}}}}
2025-11-27T18:19:33.037351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471530574354901:2111] [node 6] ICP01 ready to work
2025-11-27T18:19:33.052412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471528414265079:2117] [node 6] ICP01 ready to work
2025-11-27T18:19:33.064383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471527532322190:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:33.037651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471530572080573:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:33.067545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471526279387539:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(204698884ns) {Configure:104(204690620ns) {ProcessConfigured:135(6630900ns) {StartInitialHandshake:157(6628149ns) {DropHandshakes:470(2842ns) {DropIncomingHandshake:442(740ns) DropOutgoingHandshake:457(390ns)} GenerateSessionVirtualId:988(1057ns) PrepareNewSessionHandshake:477(3370ns) {SwitchToState:179(1912ns)}}}}}
2025-11-27T18:19:33.064024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471525307400056:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(149451676ns) {Configure:104(149447126ns) {ProcessConfigured:135(14765621ns) {StartInitialHandshake:157(14761579ns) {DropHandshakes:470(3152ns) {DropIncomingHandshake:442(599ns) DropOutgoingHandshake:457(490ns)} GenerateSessionVirtualId:988(968ns) PrepareNewSessionHandshake:477(4832ns) {SwitchToState:179(2331ns)}}}}}
2025-11-27T18:19:33.068725Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471523237354879:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(190179124ns) {Configure:104(190167489ns) {ProcessConfigured:135(18679387ns) {StartInitialHandshake:157(18676907ns) {DropHandshakes:470(3400ns) {DropIncomingHandshake:442(908ns) DropOutgoingHandshake:457(464ns)} GenerateSessionVirtualId:988(949ns) PrepareNewSessionHandshake:477(2357ns) {SwitchToState:179(951ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.076870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471523237354818:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(147630988ns) {Configure:104(147626274ns) {ProcessConfigured:135(13034985ns) {StartInitialHandshake:157(13033113ns) {DropHandshakes:470(1424ns) {DropIncomingHandshake:442(400ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(549ns) PrepareNewSessionHandshake:477(2002ns) {SwitchToState:179(914ns)}}}}}
2025-11-27T18:19:33.079744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471525800624120:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47206160ns) {Configure:104(47195936ns) {ProcessConfigured:135(2210034ns) {StartInitialHandshake:157(2207436ns) {DropHandshakes:470(5640ns) {DropIncomingHandshake:442(3722ns) DropOutgoingHandshake:457(372ns)} GenerateSessionVirtualId:988(883ns) PrepareNewSessionHandshake:477(1517ns) {SwitchToState:179(136ns)}}}}}
2025-11-27T18:19:33.080902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471525307399995:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(176612952ns) {Configure:104(176602931ns) {ProcessConfigured:135(875474ns) {StartInitialHandshake:157(874259ns) {DropHandshakes:470(1613ns) {DropIncomingHandshake:442(338ns) DropOutgoingHandshake:457(213ns)} GenerateSessionVirtualId:988(493ns) PrepareNewSessionHandshake:477(1594ns) {SwitchToState:179(848ns)}}}}}
2025-11-27T18:19:33.080867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471526279387538:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(214283229ns) {Configure:104(214271929ns) {ProcessConfigured:135(3729274ns) {StartInitialHandshake:157(3572156ns) {DropHandshakes:470(1963ns) {DropIncomingHandshake:442(741ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(709ns) PrepareNewSessionHandshake:477(3142ns) {SwitchToState:179(1786ns)}}}}}
2025-11-27T18:19:33.113564Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471526279387603:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(45580436ns) {Configure:104(45577413ns) {ProcessConfigured:135(23873891ns) {StartInitialHandshake:157(23871611ns) {DropHandshakes:470(2864ns) {DropIncomingHandshake:442(697ns) DropOutgoingHandshake:457(451ns)} GenerateSessionVirtualId:988(779ns) PrepareNewSessionHandshake:477(3945ns) {SwitchToState:179(2208ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.122731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471527532322193:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17167
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.082803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471529602367386:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14359
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.126123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471529602367390:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14359
2025-11-27T18:19:33.126367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471527532322193:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.123548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471530095591501:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58147
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.142868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471530574354914:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38260
2025-11-27T18:19:33.143147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471530095591501:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.143346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471525800624118:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.143668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471530574354908:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.143702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471530574354908:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.144872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471530574354908:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:33.145803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471527532322190:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(65020085ns) {Configure:104(65016005ns) {ProcessConfigured:135(18856735ns) {StartInitialHandshake:157(14512091ns) {DropHandshakes:470(1864ns) {DropIncomingHandshake:442(228ns) DropOutgoingHandshake:457(342ns)} GenerateSessionVirtualId:988(1617ns) PrepareNewSessionHandshake:477(4514ns) {SwitchToState:179(1713ns)}}}}}
2025-11-27T18:19:33.149405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471523237354819:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.142296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471529602367386:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.162050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471523237354820:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(66136469ns) {Configure:104(66131235ns) {ProcessConfigured:135(7813430ns) {StartInitialHandshake:157(7811263ns) {DropHandshakes:470(1442ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(372ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(3298ns) {SwitchToState:179(1046ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.145394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471530574354906:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38260
2025-11-27T18:19:33.163516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471530572080575:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65075
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.177280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471530095591503:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58147
2025-11-27T18:19:33.145580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471526279387603:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.158739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471524119297701:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(78592482ns) {Configure:104(78586247ns) {ProcessConfigured:135(72554ns) {StartInitialHandshake:157(67836ns) {DropHandshakes:470(3055ns) {DropIncomingHandshake:442(622ns) DropOutgoingHandshake:457(497ns)} GenerateSessionVirtualId:988(650ns) PrepareNewSessionHandshake:477(1248ns) {SwitchToState:179(613ns)}}}}}
2025-11-27T18:19:33.171489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471529602367380:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.186879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471528414265080:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62591
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.164007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471530572080582:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65075
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.197214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471528414265085:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62591
2025-11-27T18:19:33.150630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:33.175026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471527532322203:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17167
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.197935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471530095591504:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58147
2025-11-27T18:19:33.150698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471526279387603:2107] [node 4] ICP22 created new session: [1:7577471530574354916:2107]
2025-11-27T18:19:33.171544Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471529602367380:2130] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.177891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471530572080582:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.195249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471527532322205:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17167
2025-11-27T18:19:33.197715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471528414265088:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62591
2025-11-27T18:19:33.196664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471530572080575:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.182126Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471529602367390:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.153648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471526279387604:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(58433802ns) {Configure:104(58428764ns) {ProcessConfigured:135(7757568ns) {StartInitialHandshake:157(7754859ns) {DropHandshakes:470(1281262ns) {DropIncomingHandshake:442(1322ns) DropOutgoingHandshake:457(576ns)} GenerateSessionVirtualId:988(1313ns) PrepareNewSessionHandshake:477(3106ns) {SwitchToState:179(1256ns)}}}}}
2025-11-27T18:19:33.196897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471527532322193:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:33.199636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471528414265091:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62591
2025-11-27T18:19:33.197765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471526277113210:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(107670895ns) {Configure:104(107663833ns) {ProcessConfigured:135(9604387ns) {StartInitialHandshake:157(9602213ns) {DropHandshakes:470(2752ns) {DropIncomingHandshake:442(566ns) DropOutgoingHandshake:457(358ns)} GenerateSessionVirtualId:988(588ns) PrepareNewSessionHandshake:477(6649ns) {SwitchToState:179(1652ns)}}}}}
2025-11-27T18:19:33.153839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471530574354906:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.198979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471527532322204:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17167
2025-11-27T18:19:33.223983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471530095591501:2124] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:33.187168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471529602367380:2130] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:33.228413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471526277113210:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.161772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471530574354916:2107] [node 4] ICS09 handshake done sender: [1:7577471530574354908:2115] self: [1:7577471530574354907:0] peer: [4:7577471530095591502:0] socket: 153 qp: -1
2025-11-27T18:19:33.197449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471529602367389:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14359
2025-11-27T18:19:33.204616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471524119297702:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(123189810ns) {Configure:104(123184198ns) {ProcessConfigured:135(5987932ns) {StartInitialHandshake:157(5984899ns) {DropHandshakes:470(327141ns) {DropIncomingHandshake:442(1159ns) DropOutgoingHandshake:457(433ns)} GenerateSessionVirtualId:988(891ns) PrepareNewSessionHandshake:477(2805ns) {SwitchToState:179(1171ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.209671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471523237354819:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.232242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471526277113211:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(136532786ns) {Configure:104(136523671ns) {ProcessConfigured:135(11856455ns) {StartInitialHandshake:157(11852734ns) {DropHandshakes:470(1177ns) {DropIncomingHandshake:442(274ns) DropOutgoingHandshake:457(315ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(2855ns) {SwitchToState:179(953ns)}}}}}
2025-11-27T18:19:33.229996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471525800624118:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.167663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471530574354916:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:33.177541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:33.203871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471530574354914:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.232454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471530574354905:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.217929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:33.204880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471528414265088:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.214061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471525307400078:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.232488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471530574354905:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.230083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:33.214144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:33.230134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471525800624118:2048] [node 1] ICP22 created new session: [4:7577471530095591506:2048]
2025-11-27T18:19:33.223830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471523237354819:2049] [node 2] ICP22 created new session: [6:7577471527532322207:2049]
2025-11-27T18:19:33.240518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471530574354903:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.240440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471526277113211:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.217966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471528414265085:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.230181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471530095591506:2048] [node 1] ICS09 handshake done sender: [4:7577471530095591501:2124] self: [4:7577471530095591502:0] peer: [1:7577471530574354907:0] socket: 155 qp: -1
2025-11-27T18:19:33.229283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471527532322207:2049] [node 2] ICS09 handshake done sender: [6:7577471527532322193:2121] self: [6:7577471527532322200:0] peer: [2:7577471529602367379:0] socket: 144 qp: -1
2025-11-27T18:19:33.227337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471528414265080:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.228333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471528414265078:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.229499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471524119297702:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.237105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471528414265091:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.240220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471524119297703:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.230218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471530095591506:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:33.214201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471525307400078:2115] [node 6] ICP22 created new session: [2:7577471529602367394:2115]
2025-11-27T18:19:33.240570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471530574354903:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.242242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471530572080581:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.214266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471529602367394:2115] [node 6] ICS09 handshake done sender: [2:7577471529602367380:2130] self: [2:7577471529602367379:0] peer: [6:7577471527532322200:0] socket: 139 qp: -1
2025-11-27T18:19:33.245435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471530574354903:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:33.251332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471528414265087:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.251368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471528414265087:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.230279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:33.229679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471527532322203:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.242277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471530572080581:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.251579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471530574354922:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38260
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.256114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471528414265103:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62591
2025-11-27T18:19:33.214313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471529602367394:2115] [node 6] ICS10 traffic start
2025-11-27T18:19:33.232807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471527532322207:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:33.248820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471530572080581:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.214370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:33.237719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471530095591505:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58147
2025-11-27T18:19:33.233562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:33.254669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471530574354905:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.240761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471530095591493:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.257534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471530574354910:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.266820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471528414265085:2131] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:33.269352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471530572080574:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.217577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471529602367389:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.237059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471527532322206:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17167
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.243397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471523237354818:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.240791Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471530095591493:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.257559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471530574354910:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.270514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:33.224667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471525307399995:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.269061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471528414265101:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62591
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.244715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471527532322204:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.258034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471526279387539:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.271111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471530572080574:2108] [node 5] ICP22 created new session: [3:7577471530572080595:2108]
2025-11-27T18:19:33.233517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471529602367378:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.269372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471528414265087:2132] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.241498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471530095591504:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.277354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471530572080579:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.249227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471527532322206:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.233550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471529602367378:2129] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.277855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471528414265078:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.250048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471523237354820:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.258074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:33.243494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471529602367382:2131] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.246583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471525800624120:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.277392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471530572080579:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.258125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471526279387539:2049] [node 3] ICP22 created new session: [1:7577471530574354925:2049]
2025-11-27T18:19:33.277932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:33.251377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471530095591493:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:33.277543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471530572080577:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.277570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471530572080577:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.283973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471530572080593:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65075
2025-11-27T18:19:33.290951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471530572080595:2108] [node 5] ICS09 handshake done sender: [3:7577471530572080581:2116] self: [3:7577471530572080580:0] peer: [5:7577471528414265100:0] socket: 172 qp: -1
2025-11-27T18:19:33.297468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471530572080595:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:33.297509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:33.311519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471530572080594:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65075
2025-11-27T18:19:33.319038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471530572080574:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(48741515ns) {HandleHandshakeStatus:329(48738459ns) {DropIncomingHandshake:442(570ns) DropOutgoingHandshake:457(553ns) SwitchToState:179(681ns) ProcessPendingSessionEvents:495(533669ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 475713ns max# 58610ns}}}
2025-11-27T18:19:33.324378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471530572080582:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:33.331488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471530572080596:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65075
2025-11-27T18:19:33.331700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471530572080579:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:33.334322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471530572080575:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:33.349642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471526277113210:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.349691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:33.349738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471526277113210:2048] [node 1] ICP22 created new session: [3:7577471530572080601:2048]
2025-11-27T18:19:33.349796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471530572080601:2048] [node 1] ICS09 handshake done sender: [3:7577471530572080582:2117] self: [3:7577471530572080590:0] peer: [1:7577471530574354902:0] socket: 165 qp: -1
2025-11-27T18:19:33.349840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471530572080601:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:33.349873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:33.352752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471526277113211:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.352799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:33.352847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471526277113211:2049] [node 2] ICP22 created new session: [3:7577471530572080604:2049]
2025-11-27T18:19:33.352894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471530572080604:2049] [node 2] ICS09 handshake done sender: [3:7577471530572080575:2113] self: [3:7577471530572080591:0] peer: [2:7577471529602367381:0] socket: 148 qp: -1
2025-11-27T18:19:33.352935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471530572080604:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:33.352965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:33.357335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471526277113276:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.357416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:33.357483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471526277113276:2109] [node 6] ICP22 created new session: [3:7577471530572080606:2109]
2025-11-27T18:19:33.357539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471530572080606:2109] [node 6] ICS09 handshake done sender: [3:7577471530572080579:2115] self: [3:7577471530572080578:0] peer: [6:7577471527532322212:0] socket: 171 qp: -1
2025-11-27T18:19:33.357579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471530572080606:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:33.357622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:33.243523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471529602367382:2131] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.277984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471528414265078:2108] [node 4] ICP22 created new session: [5:7577471528414265106:2108]
2025-11-27T18:19:33.258190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471530574354925:2049] [node 3] ICS09 handshake done sender: [1:7577471530574354903:2112] self: [1:7577471530574354902:0] peer: [3:7577471530572080590:0] socket: 151 qp: -1
2025-11-27T18:19:33.250116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471523237354879:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.251490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471530095591505:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.250188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471527532322211:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17167
2025-11-27T18:19:33.253152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471527532322211:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.256343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471527532322215:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17167
2025-11-27T18:19:33.257319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471527532322190:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.263356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471527532322216:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17167
2025-11-27T18:19:33.287285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471527532322206:2128] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:33.287777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471527532322203:2125] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:33.287963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471523237354818:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.288044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:33.288098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471523237354818:2048] [node 1] ICP22 created new session: [6:7577471527532322218:2048]
2025-11-27T18:19:33.288368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471527532322217:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17167
2025-11-27T18:19:33.300836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471527532322218:2048] [node 1] ICS09 handshake done sender: [6:7577471527532322203:2125] self: [6:7577471527532322209:0] peer: [1:7577471530574354909:0] socket: 169 qp: -1
2025-11-27T18:19:33.302262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471527532322218:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:33.321499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:33.344934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471523237354879:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.345026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:33.345078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471523237354879:2106] [node 5] ICP22 created new session: [6:7577471527532322220:2106]
2025-11-27T18:19:33.345143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471527532322220:2106] [node 5] ICS09 handshake done sender: [6:7577471527532322206:2128] self: [6:7577471527532322213:0] peer: [5:7577471528414265086:0] socket: 183 qp: -1
2025-11-27T18:19:33.345187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471527532322220:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:33.345239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:33.371105Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471523237354819:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(160518403ns) {HandleHandshakeStatus:329(160515220ns) {DropIncomingHandshake:442(1013ns) DropOutgoingHandshake:457(5930ns) SwitchToState:179(612ns) ProcessPendingSessionEvents:495(127634039ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 112607677ns max# 11674739ns}}}
2025-11-27T18:19:33.378342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471523237354818:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(89741385ns) {HandleHandshakeStatus:329(89738779ns) {DropIncomingHandshake:442(853ns) DropOutgoingHandshake:457(5196ns) SwitchToState:179(652ns) ProcessPendingSessionEvents:495(31943071ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 31878134ns max# 12198343ns}}}
2025-11-27T18:19:33.379561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471527532322211:2129] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:33.380087Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471523237354879:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(25570377ns) {HandleHandshakeStatus:329(25566652ns) {DropIncomingHandshake:442(1013ns) DropOutgoingHandshake:457(6695ns) SwitchToState:179(675ns) ProcessPendingSessionEvents:495(5015285ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4940092ns max# 4498417ns}}}
2025-11-27T18:19:33.258236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471530574354925:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:33.278030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471528414265106:2108] [node 4] ICS09 handshake done sender: [5:7577471528414265085:2131] self: [5:7577471528414265098:0] peer: [4:7577471530095591492:0] socket: 161 qp: -1
2025-11-27T18:19:33.245229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471529602367378:2129] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:33.405718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471523237354879:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:33.261698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471530095591489:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.258276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:33.278062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471528414265106:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:33.246422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471529602367399:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14359
2025-11-27T18:19:33.269340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471530095591495:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.406975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471527532322190:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.265718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471530574354922:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.278115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.407051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:33.249187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471529602367382:2131] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:33.279101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471528414265079:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.269380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471530095591495:2122] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.435022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471530572080577:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:33.407107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471527532322190:2109] [node 4] ICP22 created new session: [6:7577471527532322223:2109]
2025-11-27T18:19:33.279183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:33.250544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471525307400077:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.279222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471528414265079:2117] [node 6] ICP22 created new session: [5:7577471528414265108:2117]
2025-11-27T18:19:33.250607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:33.279262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471528414265108:2117] [node 6] ICS09 handshake done sender: [5:7577471528414265087:2132] self: [5:7577471528414265086:0] peer: [6:7577471527532322213:0] socket: 180 qp: -1
2025-11-27T18:19:33.250666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471525307400077:2108] [node 5] ICP22 created new session: [2:7577471529602367401:2108]
2025-11-27T18:19:33.279289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471528414265108:2117] [node 6] ICS10 traffic start
2025-11-27T18:19:33.250719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471529602367401:2108] [node 5] ICS09 handshake done sender: [2:7577471529602367378:2129] self: [2:7577471529602367377:0] peer: [5:7577471528414265099:0] socket: 138 qp: -1
2025-11-27T18:19:33.279322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:33.250750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471529602367401:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:33.282811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471528414265102:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62591
2025-11-27T18:19:33.250805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:33.286716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471528414265080:2128] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:33.260110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471525307399996:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.260173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:33.311914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471524119297702:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.311991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:33.260230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471525307399996:2049] [node 3] ICP22 created new session: [2:7577471529602367404:2049]
2025-11-27T18:19:33.312050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471524119297702:2049] [node 2] ICP22 created new session: [5:7577471528414265110:2049]
2025-11-27T18:19:33.260277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471529602367404:2049] [node 3] ICS09 handshake done sender: [2:7577471529602367382:2131] self: [2:7577471529602367381:0] peer: [3:7577471530572080591:0] socket: 140 qp: -1
2025-11-27T18:19:33.312098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471528414265110:2049] [node 2] ICS09 handshake done sender: [5:7577471528414265080:2128] self: [5:7577471528414265099:0] peer: [2:7577471529602367377:0] socket: 150 qp: -1
2025-11-27T18:19:33.260306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471529602367404:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:33.312134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471528414265110:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:33.312177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:33.260335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:33.313390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471528414265091:2135] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:33.264074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471529602367389:2134] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:33.341662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471528414265105:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62591
2025-11-27T18:19:33.270015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471525307399995:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.342377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471524119297703:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.342473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:33.270067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:33.270120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471525307399995:2048] [node 1] ICP22 created new session: [2:7577471529602367406:2048]
2025-11-27T18:19:33.342531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471524119297703:2050] [node 3] ICP22 created new session: [5:7577471528414265113:2050]
2025-11-27T18:19:33.270168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471529602367406:2048] [node 1] ICS09 handshake done sender: [2:7577471529602367389:2134] self: [2:7577471529602367396:0] peer: [1:7577471530574354904:0] socket: 154 qp: -1
2025-11-27T18:19:33.342622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471528414265113:2050] [node 3] ICS09 handshake done sender: [5:7577471528414265091:2135] self: [5:7577471528414265100:0] peer: [3:7577471530572080580:0] socket: 175 qp: -1
2025-11-27T18:19:33.270877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471529602367406:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:33.342661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471528414265113:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:33.270931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:33.342722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:33.371322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471529602367376:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.419218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471528414265105:2140] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.371362Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471529602367376:2128] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.423934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471524119297701:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.390249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471529602367376:2128] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:33.390568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471525307400056:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.410058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:33.419216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471525307400056:2107] [node 4] ICP22 created new session: [2:7577471529602367409:2107]
2025-11-27T18:19:33.421922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471529602367409:2107] [node 4] ICS09 handshake done sender: [2:7577471529602367376:2128] self: [2:7577471529602367375:0] peer: [4:7577471530095591518:0] socket: 160 qp: -1
2025-11-27T18:19:33.423468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471529602367409:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:33.427018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:33.444258Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471525307399995:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(152823162ns) {HandleHandshakeStatus:329(152819676ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(5426ns) SwitchToState:179(283ns) ProcessPendingSessionEvents:495(145030028ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 144894230ns max# 20557987ns}}}
2025-11-27T18:19:33.271112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:33.445195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471526277113211:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(91475215ns) {HandleHandshakeStatus:329(91472289ns) {DropIncomingHandshake:442(1066ns) DropOutgoingHandshake:457(5372ns) SwitchToState:179(647ns) ProcessPendingSessionEvents:495(91143358ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 90840895ns max# 9704600ns}}}
2025-11-27T18:19:33.272388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471530095591489:2108] [node 5] ICP22 created new session: [4:7577471530095591513:2108]
2025-11-27T18:19:33.269387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471530574354910:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:33.276604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471530095591512:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58147
2025-11-27T18:19:33.277117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471530095591495:2122] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:33.285745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471530095591508:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58147
2025-11-27T18:19:33.407173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471527532322223:2109] [node 4] ICS09 handshake done sender: [6:7577471527532322211:2129] self: [6:7577471527532322214:0] peer: [4:7577471530095591494:0] socket: 194 qp: -1
2025-11-27T18:19:33.270924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471526279387538:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.286510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471530095591513:2108] [node 5] ICS09 handshake done sender: [4:7577471530095591493:2121] self: [4:7577471530095591492:0] peer: [5:7577471528414265098:0] socket: 143 qp: -1
2025-11-27T18:19:33.407217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471527532322223:2109] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.270950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:33.407269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:33.270977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471526279387538:2048] [node 2] ICP22 created new session: [1:7577471530574354930:2048]
2025-11-27T18:19:33.447653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471527532322190:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:33.271007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471530574354930:2048] [node 2] ICS09 handshake done sender: [1:7577471530574354905:2113] self: [1:7577471530574354904:0] peer: [2:7577471529602367396:0] socket: 152 qp: -1
2025-11-27T18:19:33.277641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471530574354930:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:33.280331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:33.294596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471526279387603:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(134899462ns) {HandleHandshakeStatus:329(134892952ns) {DropIncomingHandshake:442(731ns) DropOutgoingHandshake:457(587ns) SwitchToState:179(675ns) ProcessPendingSessionEvents:495(82270220ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 82067931ns max# 7928956ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.450151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471525307400056:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(54189620ns) {HandleHandshakeStatus:329(54186110ns) {DropIncomingHandshake:442(14431ns) DropOutgoingHandshake:457(582ns) SwitchToState:179(373ns) ProcessPendingSessionEvents:495(9528636ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9468236ns max# 8983022ns}}}
2025-11-27T18:19:33.294917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471530574354901:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.294982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:33.295032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471530574354901:2111] [node 6] ICP22 created new session: [1:7577471530574354932:2111]
2025-11-27T18:19:33.295079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471530574354932:2111] [node 6] ICS09 handshake done sender: [1:7577471530574354910:2116] self: [1:7577471530574354909:0] peer: [6:7577471527532322209:0] socket: 167 qp: -1
2025-11-27T18:19:33.295111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471530574354932:2111] [node 6] ICS10 traffic start
2025-11-27T18:19:33.295146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:33.296189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471526279387603:2107] [node 4] ICP18 handshake (actor [1:7577471530574354906:2114]) is too old
2025-11-27T18:19:33.311128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471530574354929:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38260
2025-11-27T18:19:33.428752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471530574354918:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.286565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471530095591513:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:33.428771Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471530574354918:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:33.429412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471530574354935:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38260
2025-11-27T18:19:33.449370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471530574354918:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:33.450099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471527532322222:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17167
2025-11-27T18:19:33.450263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471528414265116:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62591
2025-11-27T18:19:33.300853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:33.450534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471528414265105:2140] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:33.309516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471530095591486:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.452538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471524119297701:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.449511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471530572080573:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.449578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:33.450641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471530572080573:2107] [node 4] ICP22 created new session: [3:7577471530572080608:2107]
2025-11-27T18:19:33.452034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471530572080608:2107] [node 4] ICS09 handshake done sender: [3:7577471530572080577:2114] self: [3:7577471530572080576:0] peer: [4:7577471530095591510:0] socket: 170 qp: -1
2025-11-27T18:19:33.452272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471530572080608:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:33.452563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:33.345646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:33.453018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:33.461741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471527532322204:2126] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:33.349389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471530095591486:2109] [node 6] ICP22 created new session: [4:7577471530095591517:2109]
2025-11-27T18:19:33.469630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471523237354820:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.453290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471524119297701:2048] [node 1] ICP22 created new session: [5:7577471528414265117:2048]
2025-11-27T18:19:33.482323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471526279387604:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:33.482381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:33.482415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471526279387604:2108] [node 5] ICP22 created new session: [1:7577471530574354937:2108]
2025-11-27T18:19:33.482465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471530574354937:2108] [node 5] ICS09 handshake done sender: [1:7577471530574354918:2118] self: [1:7577471530574354917:0] peer: [5:7577471528414265115:0] socket: 205 qp: -1
2025-11-27T18:19:33.482491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471530574354937:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:33.482519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.473318Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471526277113276:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(92915406ns) {HandleHandshakeStatus:329(92911898ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(578ns) SwitchToState:179(346ns) ProcessPendingSessionEvents:495(92493647ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 81442371ns max# 13691601ns}}}
2025-11-27T18:19:33.353827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471530095591512:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:33.469709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:33.453643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471528414265117:2048] [node 1] ICS09 handshake done sender: [5:7577471528414265105:2140] self: [5:7577471528414265115:0] peer: [1:7577471530574354917:0] socket: 207 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.354685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471530095591508:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.469764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471523237354820:2050] [node 3] ICP22 created new session: [6:7577471527532322226:2050]
2025-11-27T18:19:33.355086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471525800624119:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:33.369393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471530095591517:2109] [node 6] ICS09 handshake done sender: [4:7577471530095591495:2122] self: [4:7577471530095591494:0] peer: [6:7577471527532322214:0] socket: 142 qp: -1
2025-11-27T18:19:33.373494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471530095591517:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:33.375916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:33.395120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471530095591486:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(93096649ns) {HandleHandshakeStatus:329(84652095ns) {DropIncomingHandshake:442(525ns) DropOutgoingHandshake:457(562ns) SwitchToState:179(962ns) ProcessPendingSessionEvents:495(549360ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 463455ns max# 57929ns}}}
2025-11-27T18:19:33.395293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471530095591486:2109] [node 6] ICP18 handshake (actor [4:7577471530095591512:2129]) is too old
2025-11-27T18:19:33.431862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471525800624119:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471530095591508:2128] held: no
2025-11-27T18:19:33.450881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471530095591489:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(175604824ns) {HandleHandshakeStatus:329(175602238ns) {DropIncomingHandshake:442(279ns) DropOutgoingHandshake:457(246ns) SwitchToState:179(171ns) ProcessPendingSessionEvents:495(104688682ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 99337016ns max# 13231839ns}}}
2025-11-27T18:19:33.469449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471530095591489:2108] [node 5] ICP18 handshake (actor [4:7577471530095591505:2127]) is too old
2025-11-27T18:19:33.491839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471530095591520:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58147
2025-11-27T18:19:33.494073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471525800624119:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(24708314ns) {HandleHandshakeStatus:395(24705926ns) {LogHandshakeFail:472(11885ns) {UpdateErrorStateLog:545(5130ns)} DropOutgoingHandshake:457(2113ns) ProcessPendingSessionEvents:495(19710340ns) {unordered DropSessionEvent:512 num# 100 sum# 8853013ns max# 5610876ns}}}
2025-11-27T18:19:33.494197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471530095591521:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58147
2025-11-27T18:19:33.453677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471528414265117:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:33.495775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471530095591508:2128] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:33.486971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471530572080603:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65075
2025-11-27T18:19:33.498429Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471526279387538:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(181579615ns) {HandleHandshakeStatus:329(181575223ns) {DropIncomingHandshake:442(154ns) DropOutgoingHandshake:457(268ns) SwitchToState:179(359ns) ProcessPendingSessionEvents:495(151668210ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 138562067ns max# 12807687ns}}}
2025-11-27T18:19:33.453709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:33.469798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471527532322226:2050] [node 3] ICS09 handshake done sender: [6:7577471527532322204:2126] self: [6:7577471527532322212:0] peer: [3:7577471530572080578:0] socket: 174 qp: -1
2025-11-27T18:19:33.492571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471530572080596:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:33.502054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471530572080610:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65075
2025-11-27T18:19:33.505825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471530095591504:2126] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:33.505959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471525800624119:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.506053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:33.506113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471525800624119:2049] [node 2] ICP22 created new session: [4:7577471530095591522:2049]
2025-11-27T18:19:33.506172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471530095591522:2049] [node 2] ICS09 handshake done sender: [4:7577471530095591508:2128] self: [4:7577471530095591518:0] peer: [2:7577471529602367375:0] socket: 186 qp: -1
2025-11-27T18:19:33.506208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471530095591522:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:33.506275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:33.469825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471527532322226:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:33.509549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471525800624120:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:33.509610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:33.469860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:33.509657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471525800624120:2050] [node 3] ICP22 created new session: [4:7577471530095591524:2050]
2025-11-27T18:19:33.512350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471526277113276:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:33.509809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471530095591524:2050] [node 3] ICS09 handshake done sender: [4:7577471530095591504:2126] self: [4:7577471530095591510:0] peer: [3:7577471530572080576:0] socket: 173 qp: -1
2025-11-27T18:19:33.509844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471530095591524:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:33.509880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:33.514255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471523237354820:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(27348732ns) {HandleHandshakeStatus:329(27341850ns) {DropIncomingHandshake:442(938ns) DropOutgoingHandshake:457(4282ns) SwitchToState:179(615ns) ProcessPendingSessionEvents:495(26958542ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23859766ns max# 3872719ns}}}
2025-11-27T18:19:33.515419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471530572080573:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(65576559ns) {HandleHandshakeStatus:329(65572205ns) {DropIncomingHandshake:442(1489ns) DropOutgoingHandshake:457(815ns) SwitchToState:179(665ns) ProcessPendingSessionEvents:495(60849301ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60726013ns max# 5670990ns}}}
2025-11-27T18:19:33.516676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471530572080573:2107] [node 4] ICP18 handshake (actor [3:7577471530572080596:2122]) is too old
2025-11-27T18:19:33.588511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:33.589201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471530572080601:2048] [node 1] ICS07 socket disconnect 165 reason# EndOfStream
2025-11-27T18:19:33.589276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471530572080601:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.589367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:33.589424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471530572080601:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:33.589597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:33.591227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471529602367406:2048] [node 1] ICS01 socket: 154 reason# EndOfStream
2025-11-27T18:19:33.591294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471525307399995:2048] [node 1] ICP30 unregister session Session# [2:7577471529602367406:2048] VirtualId# [2:7577471529602367396:0]
2025-11-27T18:19:33.591340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471529602367406:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.591433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:33.591463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471526277113210:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29845) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:33.591523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471530572080601:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:33.591556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471526277113210:2048] [node 1] ICP30 unregister session Session# [3:7577471530572080601:2048] VirtualId# [3:7577471530572080590:0]
2025-11-27T18:19:33.591486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:33.591594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:33.591631Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471526277113210:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.591647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.594608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:33.594834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471530095591506:2048] [node 1] ICS01 socket: 155 reason# EndOfStream
2025-11-27T18:19:33.594875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471525800624118:2048] [node 1] ICP30 unregister session Session# [4:7577471530095591506:2048] VirtualId# [4:7577471530095591502:0]
2025-11-27T18:19:33.594908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471530095591506:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.594992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:33.595038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:33.595745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:33.595893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471528414265117:2048] [node 1] ICS07 socket disconnect 207 reason# EndOfStream
2025-11-27T18:19:33.595927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471528414265117:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.595998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:33.596039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471528414265117:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:33.596976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471524119297701:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29845) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:33.596888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:33.597093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471527532322218:2048] [node 1] ICS07 socket disconnect 169 reason# EndOfStream
2025-11-27T18:19:33.597111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471528414265117:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:33.597125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471527532322218:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.597160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471524119297701:2048] [node 1] ICP30 unregister session Session# [5:7577471528414265117:2048] VirtualId# [5:7577471528414265115:0]
2025-11-27T18:19:33.597200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:33.597218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:33.597251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471527532322218:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:33.597289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471524119297701:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.597310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.601625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471526277113210:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:33.601426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471523237354818:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29845) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:33.601540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471527532322218:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:33.601600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471523237354818:2048] [node 1] ICP30 unregister session Session# [6:7577471527532322218:2048] VirtualId# [6:7577471527532322209:0]
2025-11-27T18:19:33.601745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:33.601827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471523237354818:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.601897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.606747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471524119297701:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:33.611814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471523237354818:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:33.674848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:33.675151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471530572080604:2049] [node 2] ICS07 socket disconnect 148 reason# EndOfStream
2025-11-27T18:19:33.675203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471530572080604:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.675284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:33.675352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471530572080604:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:33.678122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471526277113211:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5944) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:33.678173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471530572080604:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:33.678449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:33.678721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471530095591522:2049] [node 2] ICS01 socket: 186 reason# EndOfStream
2025-11-27T18:19:33.678774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471525800624119:2049] [node 2] ICP30 unregister session Session# [4:7577471530095591522:2049] VirtualId# [4:7577471530095591518:0]
2025-11-27T18:19:33.678839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471530095591522:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.678917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:33.678990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:33.679308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471526277113211:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:33.679410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471530572080604:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:33.679559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471526277113211:2049] [node 2] ICP30 unregister session Session# [3:7577471530572080604:2049] VirtualId# [3:7577471530572080591:0]
2025-11-27T18:19:33.679698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:33.679732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:33.679776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471526277113211:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.679802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.679985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471528414265110:2049] [node 2] ICS01 socket: 150 reason# EndOfStream
2025-11-27T18:19:33.680034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471524119297702:2049] [node 2] ICP30 unregister session Session# [5:7577471528414265110:2049] VirtualId# [5:7577471528414265099:0]
2025-11-27T18:19:33.680084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471528414265110:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.680190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:33.680245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:33.681289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:33.681479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471527532322207:2049] [node 2] ICS07 socket disconnect 144 reason# EndOfStream
2025-11-27T18:19:33.681513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471527532322207:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.681609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:33.681703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471527532322207:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:33.682418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471523237354819:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:33.682476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471527532322207:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:33.682588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471523237354819:2049] [node 2] ICP30 unregister session Session# [6:7577471527532322207:2049] VirtualId# [6:7577471527532322200:0]
2025-11-27T18:19:33.682703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:33.682754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471523237354819:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.682786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.692866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471523237354819:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:33.761062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:33.761497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:33.761487Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:33.761682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471530095591524:2050] [node 3] ICS01 socket: 173 reason# EndOfStream
2025-11-27T18:19:33.761731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471525800624120:2050] [node 3] ICP30 unregister session Session# [4:7577471530095591524:2050] VirtualId# [4:7577471530095591510:0]
2025-11-27T18:19:33.761767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471530095591524:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.761841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:33.761886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:33.761695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471528414265113:2050] [node 3] ICS07 socket disconnect 175 reason# EndOfStream
2025-11-27T18:19:33.762056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471527532322226:2050] [node 3] ICS07 socket disconnect 174 reason# EndOfStream
2025-11-27T18:19:33.761760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471528414265113:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.761851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:33.761896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471528414265113:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:33.762101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471527532322226:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.762170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:33.762211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471527532322226:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:33.767805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471524119297703:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13310) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:33.767897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471528414265113:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:33.767953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471524119297703:2050] [node 3] ICP30 unregister session Session# [5:7577471528414265113:2050] VirtualId# [5:7577471528414265100:0]
2025-11-27T18:19:33.768032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:33.768145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471524119297703:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13310) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.768214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13310) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.769143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471523237354820:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13310) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:33.769199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471527532322226:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:33.769228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471523237354820:2050] [node 3] ICP30 unregister session Session# [6:7577471527532322226:2050] VirtualId# [6:7577471527532322212:0]
2025-11-27T18:19:33.769344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:33.769383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471523237354820:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13310) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.769397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13310) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.777703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471524119297703:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:33.779741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471523237354820:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:33.863271Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:33.863545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471528414265106:2108] [node 4] ICS01 socket: 161 reason# EndOfStream
2025-11-27T18:19:33.863583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471528414265078:2108] [node 4] ICP30 unregister session Session# [5:7577471528414265106:2108] VirtualId# [5:7577471528414265098:0]
2025-11-27T18:19:33.863609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471528414265106:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.863688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:33.863735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:33.863980Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:33.864260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471527532322223:2109] [node 4] ICS07 socket disconnect 194 reason# EndOfStream
2025-11-27T18:19:33.864314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471527532322223:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.864423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:33.864476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471527532322223:2109] [node 4] ICS15 start handshake
2025-11-27T18:19:33.865413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471527532322190:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9187) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:33.865539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471527532322223:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:33.865586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471527532322190:2109] [node 4] ICP30 unregister session Session# [6:7577471527532322223:2109] VirtualId# [6:7577471527532322214:0]
2025-11-27T18:19:33.865711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:33.865757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471527532322190:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9187) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.865774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9187) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.876684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471527532322190:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:19:33.914407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:33.914638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471527532322220:2106] [node 5] ICS07 socket disconnect 183 reason# EndOfStream
2025-11-27T18:19:33.914666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471527532322220:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:33.914743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:33.914780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471527532322220:2106] [node 5] ICS15 start handshake
2025-11-27T18:19:33.922355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471523237354879:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32756) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:33.922411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471527532322220:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:33.922450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471523237354879:2106] [node 5] ICP30 unregister session Session# [6:7577471527532322220:2106] VirtualId# [6:7577471527532322213:0]
2025-11-27T18:19:33.922554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:19:33.922589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471523237354879:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.922604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:33.932719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471523237354879:2106] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:34.031909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471534051529926:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:34.032013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471534051529925:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:34.053533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471531869631434:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:34.053581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471531869631435:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:34.074362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471534553405071:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:34.074437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471534553405072:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:34.098166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471533243913242:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:34.098245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471533243913244:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:34.098254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471533243913243:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:34.121425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471535993652913:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:34.121464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471535993652914:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:34.121493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471535993652915:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:34.144707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471532977625408:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:34.144746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471532977625409:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:34.144772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471532977625410:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:34.248019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471532977625468:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:34.267656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471534051529925:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21234788ns) {Configure:104(21229485ns) {ProcessConfigured:135(75802ns) {StartInitialHandshake:157(72412ns) {DropHandshakes:470(2689ns) {DropIncomingHandshake:442(853ns) DropOutgoingHandshake:457(296ns)} GenerateSessionVirtualId:988(850ns) PrepareNewSessionHandshake:477(1438ns) {SwitchToState:179(427ns)}}}}}
2025-11-27T18:19:34.276966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471534051529990:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:34.306710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471534553405142:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:34.306672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471534051529991:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:34.349425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471531869631499:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:34.348549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471534553405153:2108] [node 6] ICP01 ready to work
2025-11-27T18:19:34.381470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471535993652994:2127] [node 4] ICP01 ready to work
2025-11-27T18:19:34.380844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471533243913307:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:34.367251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471531869631498:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:34.367445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471531869631500:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:34.388684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471535993652995:2108] [node 6] ICP01 ready to work
2025-11-27T18:19:34.306770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471534051529992:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:34.385807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471532977625481:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:34.387857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471533243913308:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:34.388308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471534553405154:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:34.390140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471533243913243:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(72880124ns) {Configure:104(72872350ns) {ProcessConfigured:135(498145ns) {StartInitialHandshake:157(495788ns) {DropHandshakes:470(1047ns) {DropIncomingHandshake:442(405ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(296ns) PrepareNewSessionHandshake:477(2107ns) {SwitchToState:179(954ns)}}}}}
2025-11-27T18:19:34.392032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471534051529926:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(123300494ns) {Configure:104(123294870ns) {ProcessConfigured:135(2216527ns) {StartInitialHandshake:157(2214285ns) {DropHandshakes:470(2680ns) {DropIncomingHandshake:442(689ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(799ns) PrepareNewSessionHandshake:477(323317ns) {SwitchToState:179(1734ns)}}}}}
2025-11-27T18:19:34.415246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471535993652913:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25872176ns) {Configure:104(25867582ns) {ProcessConfigured:135(55808ns) {StartInitialHandshake:157(52856ns) {DropHandshakes:470(2415ns) {DropIncomingHandshake:442(659ns) DropOutgoingHandshake:457(347ns)} GenerateSessionVirtualId:988(907ns) PrepareNewSessionHandshake:477(1820ns) {SwitchToState:179(628ns)}}}}}
2025-11-27T18:19:34.408167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471532977625468:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(127019783ns) {Configure:104(127007803ns) {ProcessConfigured:135(7964625ns) {StartInitialHandshake:157(7962522ns) {DropHandshakes:470(1814ns) {DropIncomingHandshake:442(624ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(639ns) PrepareNewSessionHandshake:477(6318ns) {SwitchToState:179(4319ns)}}}}}
2025-11-27T18:19:34.419000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471535993652994:2127] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29030375ns) {Configure:104(29017947ns) {ProcessConfigured:135(69762ns) {StartInitialHandshake:157(67727ns) {DropHandshakes:470(2248ns) {DropIncomingHandshake:442(514ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(887ns) PrepareNewSessionHandshake:477(1844ns) {SwitchToState:179(964ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.423838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471534553405161:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11019
2025-11-27T18:19:34.435122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471532977625409:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(29687037ns) {Configure:104(29680910ns) {ProcessConfigured:135(66124ns) {StartInitialHandshake:157(62338ns) {DropHandshakes:470(2014ns) {DropIncomingHandshake:442(508ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(648ns) PrepareNewSessionHandshake:477(1343ns) {SwitchToState:179(473ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.439879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471533243913242:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50137616ns) {Configure:104(50132460ns) {ProcessConfigured:135(759929ns) {StartInitialHandshake:157(757783ns) {DropHandshakes:470(1513ns) {DropIncomingHandshake:442(361ns) DropOutgoingHandshake:457(523ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:477(1877ns) {SwitchToState:179(884ns)}}}}}
2025-11-27T18:19:34.441662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471532977625487:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59919
2025-11-27T18:19:34.442409Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471532977625410:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(50699243ns) {Configure:104(50694744ns) {ProcessConfigured:135(921933ns) {StartInitialHandshake:157(919940ns) {DropHandshakes:470(1265ns) {DropIncomingHandshake:442(332ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(535ns) PrepareNewSessionHandshake:477(1944ns) {SwitchToState:179(1015ns)}}}}}
2025-11-27T18:19:34.442526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471534051529992:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(51154587ns) {Configure:104(51144636ns) {ProcessConfigured:135(7004483ns) {StartInitialHandshake:157(7002061ns) {DropHandshakes:470(2046ns) {DropIncomingHandshake:442(617ns) DropOutgoingHandshake:457(332ns)} GenerateSessionVirtualId:988(964ns) PrepareNewSessionHandshake:477(1919ns) {SwitchToState:179(734ns)}}}}}
2025-11-27T18:19:34.444819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471531869631434:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53092840ns) {Configure:104(53088658ns) {ProcessConfigured:135(1549484ns) {StartInitialHandshake:157(1547929ns) {DropHandshakes:470(1169ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(745ns) PrepareNewSessionHandshake:477(1781ns) {SwitchToState:179(226ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.445770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471532977625408:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50579117ns) {Configure:104(50575264ns) {ProcessConfigured:135(916693ns) {StartInitialHandshake:157(915087ns) {DropHandshakes:470(943ns) {DropIncomingHandshake:442(217ns) DropOutgoingHandshake:457(154ns)} GenerateSessionVirtualId:988(485ns) PrepareNewSessionHandshake:477(916ns) {SwitchToState:179(298ns)}}}}}
2025-11-27T18:19:34.444297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471534051529990:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(51892525ns) {Configure:104(51888665ns) {ProcessConfigured:135(1536353ns) {StartInitialHandshake:157(1533882ns) {DropHandshakes:470(881ns) {DropIncomingHandshake:442(326ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(295ns) PrepareNewSessionHandshake:477(1343ns) {SwitchToState:179(111ns)}}}}}
2025-11-27T18:19:34.446559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471535993652915:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(56255559ns) {Configure:104(56250595ns) {ProcessConfigured:135(695687ns) {StartInitialHandshake:157(693449ns) {DropHandshakes:470(1171ns) {DropIncomingHandshake:442(331ns) DropOutgoingHandshake:457(182ns)} GenerateSessionVirtualId:988(360ns) PrepareNewSessionHandshake:477(1504ns) {SwitchToState:179(751ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.445640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471534553405072:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(53727520ns) {Configure:104(53721994ns) {ProcessConfigured:135(2412434ns) {StartInitialHandshake:157(2410234ns) {DropHandshakes:470(1033ns) {DropIncomingHandshake:442(223ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:477(2074ns) {SwitchToState:179(1091ns)}}}}}
2025-11-27T18:19:34.446197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471534553405071:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(51966321ns) {Configure:104(51962975ns) {ProcessConfigured:135(1931539ns) {StartInitialHandshake:157(1930053ns) {DropHandshakes:470(1195ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(483ns) PrepareNewSessionHandshake:477(1912ns) {SwitchToState:179(893ns)}}}}}
2025-11-27T18:19:34.448322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471534553405142:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(55401031ns) {Configure:104(55395516ns) {ProcessConfigured:135(1349678ns) {StartInitialHandshake:157(1347885ns) {DropHandshakes:470(1268ns) {DropIncomingHandshake:442(383ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(395ns) PrepareNewSessionHandshake:477(1532ns) {SwitchToState:179(778ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.446809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471531869631498:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(57515551ns) {Configure:104(57509978ns) {ProcessConfigured:135(3028424ns) {StartInitialHandshake:157(3026716ns) {DropHandshakes:470(1300ns) {DropIncomingHandshake:442(296ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(594ns) PrepareNewSessionHandshake:477(6037ns) {SwitchToState:179(1302ns)}}}}}
2025-11-27T18:19:34.450209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471532977625486:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59919
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.454966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471533243913308:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49731429ns) {Configure:104(49725124ns) {ProcessConfigured:135(3954303ns) {StartInitialHandshake:157(3951348ns) {DropHandshakes:470(1395ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(327ns) PrepareNewSessionHandshake:477(2577ns) {SwitchToState:179(952ns)}}}}}
2025-11-27T18:19:34.459906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471535993653008:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64314
2025-11-27T18:19:34.453556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471534051529991:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(55337090ns) {Configure:104(55331910ns) {ProcessConfigured:135(1146391ns) {StartInitialHandshake:157(1144581ns) {DropHandshakes:470(1335ns) {DropIncomingHandshake:442(357ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(375ns) PrepareNewSessionHandshake:477(1992ns) {SwitchToState:179(1007ns)}}}}}
2025-11-27T18:19:34.455553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471533243913313:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65363
2025-11-27T18:19:34.457881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471531869631500:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(57076511ns) {Configure:104(57072588ns) {ProcessConfigured:135(1037381ns) {StartInitialHandshake:157(1035426ns) {DropHandshakes:470(1169ns) {DropIncomingHandshake:442(318ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(518ns) PrepareNewSessionHandshake:477(2060ns) {SwitchToState:179(1244ns)}}}}}
2025-11-27T18:19:34.460711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471535993653007:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64314
2025-11-27T18:19:34.455724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471533243913316:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65363
2025-11-27T18:19:34.471549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471532977625487:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.455925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471533243913321:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65363
2025-11-27T18:19:34.489820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471534553405161:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.472726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471532977625486:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.475701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471532977625468:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.493389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471532977625410:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.507456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471531869631501:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28946
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.457180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471533243913316:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.459190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471533243913307:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(42417443ns) {Configure:104(42409927ns) {ProcessConfigured:135(1716669ns) {StartInitialHandshake:157(1714966ns) {DropHandshakes:470(937ns) {DropIncomingHandshake:442(222ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(326ns) PrepareNewSessionHandshake:477(2706ns) {SwitchToState:179(1219ns)}}}}}
2025-11-27T18:19:34.477777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471533243913313:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.508554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471533243913244:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.510961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471531869631509:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28946
2025-11-27T18:19:34.511266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471531869631501:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.511462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471531869631502:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28946
2025-11-27T18:19:34.515059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471531869631502:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.501504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471535993653001:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.512040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471534553405156:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.516802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471533243913244:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16830348ns) {IncomingHandshake:261(10735290ns) {DropIncomingHandshake:442(583ns) PrepareNewSessionHandshake:477(1596ns) {SwitchToState:179(703ns)} GenerateSessionVirtualId:988(783ns) IssueIncomingHandshakeReply:193(7096639ns) {SwitchToState:179(1706ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.516636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471531869631509:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.501547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471535993653007:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.517037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471534051530003:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
2025-11-27T18:19:34.523792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471532977625499:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59919
2025-11-27T18:19:34.522401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471533243913321:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.512086Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471534553405156:2128] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.517150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471531869631511:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28946
2025-11-27T18:19:34.501563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471535993653001:2130] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.524410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471534051530005:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
2025-11-27T18:19:34.516402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471534553405158:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.524328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471531869631435:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(70508459ns) {Configure:104(70500211ns) {ProcessConfigured:135(7700307ns) {StartInitialHandshake:157(7697496ns) {DropHandshakes:470(2340ns) {DropIncomingHandshake:442(503ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(702ns) PrepareNewSessionHandshake:477(2316ns) {SwitchToState:179(1095ns)}}}}}
2025-11-27T18:19:34.501858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471535993652914:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(60848658ns) {Configure:104(60842977ns) {ProcessConfigured:135(6807541ns) {StartInitialHandshake:157(6804735ns) {DropHandshakes:470(2086ns) {DropIncomingHandshake:442(652ns) DropOutgoingHandshake:457(366ns)} GenerateSessionVirtualId:988(1332ns) PrepareNewSessionHandshake:477(3391ns) {SwitchToState:179(1396ns)}}}}}
2025-11-27T18:19:34.509646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471535993653001:2130] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:34.525034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471535993653008:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.516440Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471534553405158:2129] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.530066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471534051530003:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.525223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471531869631434:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.530739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471534051529989:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.528132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471535993652995:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.525454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471531869631511:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.537867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471532977625468:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15099) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471532977625487:2122] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.540463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471533243913327:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65363
2025-11-27T18:19:34.528198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.525009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471534553405168:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11019
2025-11-27T18:19:34.530773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471534051529989:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.546268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471532977625487:2122] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.549054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471532977625468:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.549124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:34.549172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471532977625468:2106] [node 5] ICP22 created new session: [6:7577471532977625502:2106]
2025-11-27T18:19:34.549212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471532977625502:2106] [node 5] ICS09 handshake done sender: [6:7577471532977625487:2122] self: [6:7577471532977625497:0] peer: [5:7577471535993653000:0] socket: 158 qp: -1
2025-11-27T18:19:34.549239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471532977625502:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:34.549289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:34.528249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471535993652995:2108] [node 6] ICP22 created new session: [5:7577471535993653013:2108]
2025-11-27T18:19:34.527627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471534553405168:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.532218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471534051529998:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.552027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471532977625500:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59919
2025-11-27T18:19:34.528337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471535993653013:2108] [node 6] ICS09 handshake done sender: [5:7577471535993653001:2130] self: [5:7577471535993653000:0] peer: [6:7577471532977625497:0] socket: 153 qp: -1
2025-11-27T18:19:34.549880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471533243913325:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65363
2025-11-27T18:19:34.532240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471534051529998:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.530017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471534553405156:2128] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:34.533347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471534051530005:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.554778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471533243913244:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2859) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471533243913313:2112] held: no
2025-11-27T18:19:34.528378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471535993653013:2108] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.528419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:34.531191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471534553405153:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.560155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471531869631508:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.561629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471532977625500:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.539800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471534051529995:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.557657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471533243913325:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.563210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471532977625505:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59919
2025-11-27T18:19:34.531898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471534553405158:2129] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:34.560183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471531869631508:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.530212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471535993652913:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.561954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471533243913243:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.539843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471534051529995:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.538241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471535993653018:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64314
2025-11-27T18:19:34.565669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471531869631522:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28946
2025-11-27T18:19:34.532077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471534553405071:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.567555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471532977625501:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59919
2025-11-27T18:19:34.567272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471531869631508:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:34.567090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471533243913330:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65363
2025-11-27T18:19:34.549553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471535993653015:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64314
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.541841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471534051529998:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:34.533343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:34.568075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471532977625409:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.549947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471535993653008:2133] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:34.571859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471531869631513:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.576265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471532977625486:2121] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.541969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471534051529989:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:34.535474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471534553405153:2108] [node 6] ICP22 created new session: [3:7577471534553405175:2108]
2025-11-27T18:19:34.554982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471535993653015:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.571898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471531869631513:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.583892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471532977625410:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.592585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471533243913330:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.548140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471534051530007:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
2025-11-27T18:19:34.589659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471532977625508:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59919
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.555101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471535993652913:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.541732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471534553405154:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.575906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471531869631507:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.543258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471534553405175:2108] [node 6] ICS09 handshake done sender: [3:7577471534553405156:2128] self: [3:7577471534553405155:0] peer: [6:7577471532977625498:0] socket: 145 qp: -1
2025-11-27T18:19:34.551745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471534051529995:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:34.555151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.593140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471533243913242:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.552337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471534051529925:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.590992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471532977625505:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.543313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471534553405175:2108] [node 6] ICS10 traffic start
2025-11-27T18:19:34.575954Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471531869631507:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.558494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471535993652913:2048] [node 1] ICP22 created new session: [5:7577471535993653019:2048]
2025-11-27T18:19:34.593435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471533243913318:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.591137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471532977625481:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.547198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:34.554249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471534051529992:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.576892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471531869631500:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.558541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471535993653019:2048] [node 1] ICS09 handshake done sender: [5:7577471535993653008:2133] self: [5:7577471535993653016:0] peer: [1:7577471534051529996:0] socket: 175 qp: -1
2025-11-27T18:19:34.591469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:34.579246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471531869631502:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.554742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:34.593460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471533243913318:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.558571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471535993653019:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:34.547251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471534553405154:2109] [node 4] ICP22 created new session: [3:7577471534553405177:2109]
2025-11-27T18:19:34.591979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471532977625410:2050] [node 3] ICP22 created new session: [6:7577471532977625509:2050]
2025-11-27T18:19:34.554937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:34.596871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471533243913318:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:34.558619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:34.548008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:34.579892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:34.592528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471532977625509:2050] [node 3] ICS09 handshake done sender: [6:7577471532977625486:2121] self: [6:7577471532977625498:0] peer: [3:7577471534553405155:0] socket: 157 qp: -1
2025-11-27T18:19:34.584014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471531869631500:2108] [node 5] ICP22 created new session: [2:7577471531869631525:2108]
2025-11-27T18:19:34.559310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471535993652913:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:34.593196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471532977625509:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:34.557946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471534051529926:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.548589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471534553405177:2109] [node 4] ICS09 handshake done sender: [3:7577471534553405158:2129] self: [3:7577471534553405157:0] peer: [4:7577471533243913324:0] socket: 148 qp: -1
2025-11-27T18:19:34.559912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471535993652914:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.557998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471534051529925:2048] [node 2] ICP22 created new session: [1:7577471534051530012:2048]
2025-11-27T18:19:34.609118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471533243913331:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65363
2025-11-27T18:19:34.594923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471532977625507:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59919
2025-11-27T18:19:34.549916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471534553405177:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:34.589055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471531869631525:2108] [node 5] ICS09 handshake done sender: [2:7577471531869631508:2115] self: [2:7577471531869631506:0] peer: [5:7577471535993653021:0] socket: 182 qp: -1
2025-11-27T18:19:34.596291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471535993653022:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64314
2025-11-27T18:19:34.617131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471533243913307:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.552285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.594972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:34.561826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471534051529992:2108] [node 5] ICP22 created new session: [1:7577471534051530013:2108]
2025-11-27T18:19:34.601496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471535993653024:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64314
2025-11-27T18:19:34.589785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471531869631525:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:34.563605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471534051530012:2048] [node 2] ICS09 handshake done sender: [1:7577471534051529989:2110] self: [1:7577471534051529988:0] peer: [2:7577471531869631520:0] socket: 147 qp: -1
2025-11-27T18:19:34.622486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:34.616596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471532977625500:2127] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:34.567690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471534553405176:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11019
2025-11-27T18:19:34.625585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471533243913307:2109] [node 6] ICP22 created new session: [4:7577471533243913335:2109]
2025-11-27T18:19:34.601837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471535993653015:2135] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:34.563651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471534051530012:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:34.589960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471531869631513:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:34.620400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471532977625409:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.577056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471534553405176:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.614194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471535993652914:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.617027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:34.617836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471535993652914:2049] [node 2] ICP22 created new session: [5:7577471535993653025:2049]
2025-11-27T18:19:34.624848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471535993653024:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.624990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471535993652915:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.628995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471535993653026:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64314
2025-11-27T18:19:34.630007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471535993653025:2049] [node 2] ICS09 handshake done sender: [5:7577471535993653015:2135] self: [5:7577471535993653021:0] peer: [2:7577471531869631506:0] socket: 186 qp: -1
2025-11-27T18:19:34.632156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471535993653025:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:34.632243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:34.643861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471535993653029:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64314
2025-11-27T18:19:34.652166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471535993653026:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.628028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471533243913244:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(27808775ns) {HandleHandshakeStatus:395(27805340ns) {LogHandshakeFail:472(14120ns) {UpdateErrorStateLog:545(4099ns)} DropOutgoingHandshake:457(1646ns) ProcessPendingSessionEvents:495(24468257ns) {unordered DropSessionEvent:512 num# 100 sum# 24411646ns max# 3382678ns}}}
2025-11-27T18:19:34.591536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:34.563673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471534051530007:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.620460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:34.655319Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471535993653024:2138] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:34.583508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471534553405182:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11019
2025-11-27T18:19:34.628480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471533243913335:2109] [node 6] ICS09 handshake done sender: [4:7577471533243913318:2115] self: [4:7577471533243913317:0] peer: [6:7577471532977625510:0] socket: 144 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.620530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471532977625409:2049] [node 2] ICP22 created new session: [6:7577471532977625514:2049]
2025-11-27T18:19:34.592795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471531869631434:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.564019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:34.585766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471534553405182:2137] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.593054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:34.622511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471532977625514:2049] [node 2] ICS09 handshake done sender: [6:7577471532977625500:2127] self: [6:7577471532977625506:0] peer: [2:7577471531869631505:0] socket: 185 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.630254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471533243913335:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:34.662008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471535993652994:2127] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.568765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471534051529926:2049] [node 3] ICP22 created new session: [1:7577471534051530016:2049]
2025-11-27T18:19:34.627798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471532977625514:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:34.585896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471534553405181:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11019
2025-11-27T18:19:34.631174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471533243913325:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:34.593429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471531869631434:2048] [node 1] ICP22 created new session: [2:7577471531869631530:2048]
2025-11-27T18:19:34.591925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471534553405168:2134] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:34.662520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471535993652915:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.568885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471534051530013:2108] [node 5] ICS09 handshake done sender: [1:7577471534051529998:2112] self: [1:7577471534051529996:0] peer: [5:7577471535993653016:0] socket: 174 qp: -1
2025-11-27T18:19:34.627853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:34.596884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471531869631530:2048] [node 1] ICS09 handshake done sender: [2:7577471531869631502:2112] self: [2:7577471531869631520:0] peer: [1:7577471534051529988:0] socket: 156 qp: -1
2025-11-27T18:19:34.568931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471534051530013:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:34.632419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:34.662565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:34.594755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471534553405071:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.629977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471532977625513:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59919
2025-11-27T18:19:34.637041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471532977625507:2130] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.596921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471531869631530:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:34.642706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471532977625481:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21503) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471532977625505:2129] held: no
2025-11-27T18:19:34.657178Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471532977625410:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(75030078ns) {HandleHandshakeStatus:329(75026522ns) {DropIncomingHandshake:442(2314ns) DropOutgoingHandshake:457(4706ns) SwitchToState:179(804ns) ProcessPendingSessionEvents:495(59294449ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 59188786ns max# 9170718ns}}}
2025-11-27T18:19:34.597016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471531869631507:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:34.657346Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471532977625505:2129] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:34.598298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:34.568959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:34.662625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471535993652915:2050] [node 3] ICP22 created new session: [5:7577471535993653031:2050]
2025-11-27T18:19:34.595052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:34.632726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471533243913313:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:34.666447Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471532977625481:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(26869593ns) {HandleHandshakeStatus:395(26867820ns) {LogHandshakeFail:472(15935ns) {UpdateErrorStateLog:545(5641ns)} DropOutgoingHandshake:457(2166ns) ProcessPendingSessionEvents:495(22890248ns) {unordered DropSessionEvent:512 num# 100 sum# 3170494ns max# 806880ns}}}
2025-11-27T18:19:34.609352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471531869631499:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.569029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:34.595097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471534553405071:2048] [node 1] ICP22 created new session: [3:7577471534553405184:2048]
2025-11-27T18:19:34.636112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471533243913243:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.662672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471535993653031:2050] [node 3] ICS09 handshake done sender: [5:7577471535993653024:2138] self: [5:7577471535993653027:0] peer: [3:7577471534553405166:0] socket: 207 qp: -1
2025-11-27T18:19:34.616969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:34.669024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471532977625481:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.571891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471534051530016:2049] [node 3] ICS09 handshake done sender: [1:7577471534051529995:2111] self: [1:7577471534051529994:0] peer: [3:7577471534553405174:0] socket: 167 qp: -1
2025-11-27T18:19:34.617648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471531869631499:2107] [node 4] ICP22 created new session: [2:7577471531869631532:2107]
2025-11-27T18:19:34.625432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471531869631532:2107] [node 4] ICS09 handshake done sender: [2:7577471531869631513:2118] self: [2:7577471531869631512:0] peer: [4:7577471533243913329:0] socket: 183 qp: -1
2025-11-27T18:19:34.627882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471531869631532:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:34.631005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471531869631498:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.632473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:34.632547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:34.634789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471531869631498:2109] [node 6] ICP22 created new session: [2:7577471531869631533:2109]
2025-11-27T18:19:34.635723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471531869631533:2109] [node 6] ICS09 handshake done sender: [2:7577471531869631507:2114] self: [2:7577471531869631505:0] peer: [6:7577471532977625506:0] socket: 181 qp: -1
2025-11-27T18:19:34.636105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471531869631533:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:34.636147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:34.666343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471531869631500:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(88886176ns) {HandleHandshakeStatus:329(88617805ns) {DropIncomingHandshake:442(413ns) DropOutgoingHandshake:457(711ns) SwitchToState:179(395ns) ProcessPendingSessionEvents:495(70718324ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 67839919ns max# 7086200ns}}}
2025-11-27T18:19:34.666403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471531869631434:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(72766240ns) {HandleHandshakeStatus:329(72763486ns) {DropIncomingHandshake:442(507ns) DropOutgoingHandshake:457(3737ns) SwitchToState:179(237ns) ProcessPendingSessionEvents:495(55916417ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 55810731ns max# 12812185ns}}}
2025-11-27T18:19:34.666568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471531869631526:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28946
2025-11-27T18:19:34.667884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471531869631498:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(35693242ns) {HandleHandshakeStatus:329(35691332ns) {DropIncomingHandshake:442(17829ns) DropOutgoingHandshake:457(363ns) SwitchToState:179(574ns) ProcessPendingSessionEvents:495(25049223ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24947241ns max# 6162848ns}}}
2025-11-27T18:19:34.636159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:34.662706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471535993653031:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:34.597247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471534553405184:2048] [node 1] ICS09 handshake done sender: [3:7577471534553405168:2134] self: [3:7577471534553405174:0] peer: [1:7577471534051529994:0] socket: 169 qp: -1
2025-11-27T18:19:34.572589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471534051530016:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:34.669100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:34.669154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471532977625481:2109] [node 4] ICP22 created new session: [6:7577471532977625516:2109]
2025-11-27T18:19:34.637239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471533243913243:2049] [node 2] ICP22 created new session: [4:7577471533243913337:2049]
2025-11-27T18:19:34.662736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:34.601673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471534553405184:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:34.574524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:34.669209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471532977625516:2109] [node 4] ICS09 handshake done sender: [6:7577471532977625505:2129] self: [6:7577471532977625510:0] peer: [4:7577471533243913317:0] socket: 200 qp: -1
2025-11-27T18:19:34.639578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471533243913244:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.601715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:34.691124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471535993653033:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64314
2025-11-27T18:19:34.595662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471534051530001:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.739032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471531869631500:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:34.630039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471534553405167:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.640844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.640973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471533243913337:2049] [node 2] ICS09 handshake done sender: [4:7577471533243913325:2118] self: [4:7577471533243913329:0] peer: [2:7577471531869631512:0] socket: 184 qp: -1
2025-11-27T18:19:34.642219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471533243913337:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:34.642841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471533243913244:2050] [node 3] ICP22 created new session: [4:7577471533243913338:2050]
2025-11-27T18:19:34.642915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:34.647794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471533243913338:2050] [node 3] ICS09 handshake done sender: [4:7577471533243913313:2112] self: [4:7577471533243913324:0] peer: [3:7577471534553405157:0] socket: 149 qp: -1
2025-11-27T18:19:34.647847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471533243913338:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:34.647923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:34.665870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471533243913341:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65363
2025-11-27T18:19:34.666200Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471533243913243:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(25058064ns) {HandleHandshakeStatus:329(25055885ns) {DropIncomingHandshake:442(878ns) DropOutgoingHandshake:457(5847ns) SwitchToState:179(661ns) ProcessPendingSessionEvents:495(18111792ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18039905ns max# 4285598ns}}}
2025-11-27T18:19:34.696437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471535993653026:2139] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:34.666524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471533243913320:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.666552Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471533243913320:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.669243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471532977625516:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:34.684765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471533243913320:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:34.630080Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471534553405167:2133] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.595697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471534051530001:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.701009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471535993652994:2127] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.712925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:34.715246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471535993652994:2127] [node 4] ICP22 created new session: [5:7577471535993653034:2127]
2025-11-27T18:19:34.718244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471535993653034:2127] [node 4] ICS09 handshake done sender: [5:7577471535993653026:2139] self: [5:7577471535993653030:0] peer: [4:7577471533243913319:0] socket: 217 qp: -1
2025-11-27T18:19:34.669302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:34.635744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471534553405167:2133] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:34.660885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471534051530020:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
2025-11-27T18:19:34.692878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471533243913308:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.729015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471535993653034:2127] [node 4] ICS10 traffic start
2025-11-27T18:19:34.673745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471532977625408:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.680173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471534051530021:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
2025-11-27T18:19:34.730089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:34.692960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:34.683893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471534051529925:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(88230098ns) {HandleHandshakeStatus:329(88227653ns) {DropIncomingHandshake:442(602ns) DropOutgoingHandshake:457(518ns) SwitchToState:179(2889ns) ProcessPendingSessionEvents:495(65081850ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 62186578ns max# 4857295ns}}}
2025-11-27T18:19:34.695887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471533243913308:2108] [node 5] ICP22 created new session: [4:7577471533243913343:2108]
2025-11-27T18:19:34.741111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471535993652914:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(97245837ns) {HandleHandshakeStatus:329(97241741ns) {DropIncomingHandshake:442(701ns) DropOutgoingHandshake:457(17979ns) SwitchToState:179(983ns) ProcessPendingSessionEvents:495(76178800ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 76033135ns max# 7761674ns}}}
2025-11-27T18:19:34.684155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471534051530001:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:34.697490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471533243913343:2108] [node 5] ICS09 handshake done sender: [4:7577471533243913320:2116] self: [4:7577471533243913319:0] peer: [5:7577471535993653030:0] socket: 190 qp: -1
2025-11-27T18:19:34.757496Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471531869631499:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(128650251ns) {HandleHandshakeStatus:329(128647252ns) {DropIncomingHandshake:442(21167ns) DropOutgoingHandshake:457(412ns) SwitchToState:179(538ns) ProcessPendingSessionEvents:495(93560642ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 91133365ns max# 8057687ns}}}
2025-11-27T18:19:34.641899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471534553405154:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(95438799ns) {HandleHandshakeStatus:329(95433629ns) {DropIncomingHandshake:442(13580ns) DropOutgoingHandshake:457(360ns) SwitchToState:179(447ns) ProcessPendingSessionEvents:495(79192904ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 78981989ns max# 7656225ns}}}
2025-11-27T18:19:34.697847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471533243913343:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:34.699884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471532977625409:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(78548461ns) {HandleHandshakeStatus:329(78545822ns) {DropIncomingHandshake:442(1019ns) DropOutgoingHandshake:457(6238ns) SwitchToState:179(689ns) ProcessPendingSessionEvents:495(66340173ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 66222358ns max# 7786103ns}}}
2025-11-27T18:19:34.696441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471534051529999:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.741034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471532977625519:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59919
2025-11-27T18:19:34.696478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471534051529999:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.642632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471534553405142:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.644638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:34.648671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471534553405142:2107] [node 5] ICP22 created new session: [3:7577471534553405187:2107]
2025-11-27T18:19:34.657691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471534553405187:2107] [node 5] ICS09 handshake done sender: [3:7577471534553405167:2133] self: [3:7577471534553405166:0] peer: [5:7577471535993653027:0] socket: 160 qp: -1
2025-11-27T18:19:34.660556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471534553405187:2107] [node 5] ICS10 traffic start
2025-11-27T18:19:34.666968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:34.673199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471534553405153:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(105420379ns) {HandleHandshakeStatus:329(105417210ns) {DropIncomingHandshake:442(1300ns) DropOutgoingHandshake:457(937ns) SwitchToState:179(474ns) ProcessPendingSessionEvents:495(83309810ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 83128521ns max# 7731873ns}}}
2025-11-27T18:19:34.694055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471534553405153:2108] [node 6] ICP18 handshake (actor [3:7577471534553405182:2137]) is too old
2025-11-27T18:19:34.736831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471534553405142:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(60988669ns) {HandleHandshakeStatus:329(60424304ns) {DropIncomingHandshake:442(16116ns) DropOutgoingHandshake:457(415ns) SwitchToState:179(563ns) ProcessPendingSessionEvents:495(29571938ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29476841ns max# 11151491ns}}}
2025-11-27T18:19:34.764395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471534553405189:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11019
2025-11-27T18:19:34.765942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471534553405189:2138] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.766465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471534553405072:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:34.766529Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471534553405071:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(136546825ns) {HandleHandshakeStatus:329(135518872ns) {DropIncomingHandshake:442(1400ns) DropOutgoingHandshake:457(12291ns) SwitchToState:179(742ns) ProcessPendingSessionEvents:495(102240574ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 102055798ns max# 10024890ns}}}
2025-11-27T18:19:34.711195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:34.712462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471533243913330:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:34.741282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471532977625507:2130] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:34.698018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471534051529991:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.737614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471533243913242:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.738715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:34.770452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471531869631514:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.739111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471533243913242:2048] [node 1] ICP22 created new session: [4:7577471533243913345:2048]
2025-11-27T18:19:34.757523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471532977625408:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.698130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:34.740296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471533243913345:2048] [node 1] ICS09 handshake done sender: [4:7577471533243913330:2120] self: [4:7577471533243913332:0] peer: [1:7577471534051530000:0] socket: 206 qp: -1
2025-11-27T18:19:34.698172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471534051529991:2107] [node 4] ICP22 created new session: [1:7577471534051530023:2107]
2025-11-27T18:19:34.770510Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471531869631514:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:34.764890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:34.766131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471532977625408:2048] [node 1] ICP22 created new session: [6:7577471532977625520:2048]
2025-11-27T18:19:34.708887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471534051529926:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(102695867ns) {HandleHandshakeStatus:329(102692649ns) {DropIncomingHandshake:442(612ns) DropOutgoingHandshake:457(294ns) SwitchToState:179(263ns) ProcessPendingSessionEvents:495(83674645ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 81446903ns max# 12083264ns}}}
2025-11-27T18:19:34.740331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471533243913345:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:34.740390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:34.771232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471533243913307:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(124717630ns) {HandleHandshakeStatus:329(124712998ns) {DropIncomingHandshake:442(15718ns) DropOutgoingHandshake:457(1342ns) SwitchToState:179(401ns) ProcessPendingSessionEvents:495(95717288ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 89554542ns max# 7657934ns}}}
2025-11-27T18:19:34.780668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471533243913308:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(48001300ns) {HandleHandshakeStatus:329(47997452ns) {DropIncomingHandshake:442(15095ns) DropOutgoingHandshake:457(414ns) SwitchToState:179(489ns) ProcessPendingSessionEvents:495(22976998ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20758988ns max# 11235610ns}}}
2025-11-27T18:19:34.768425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471532977625520:2048] [node 1] ICS09 handshake done sender: [6:7577471532977625507:2130] self: [6:7577471532977625518:0] peer: [1:7577471534051529997:0] socket: 202 qp: -1
2025-11-27T18:19:34.787519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471535993652994:2127] [node 4] ICP35 event processing took too much time PendingConnection:279(63163267ns) {HandleHandshakeStatus:329(63160764ns) {DropIncomingHandshake:442(3187ns) DropOutgoingHandshake:457(20628ns) SwitchToState:179(716ns) ProcessPendingSessionEvents:495(26632860ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26537633ns max# 6875384ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:34.710824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471534051530021:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:34.710997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471534051529926:2049] [node 3] ICP18 handshake (actor [1:7577471534051530007:2117]) is too old
2025-11-27T18:19:34.711098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471534051530023:2107] [node 4] ICS09 handshake done sender: [1:7577471534051530001:2114] self: [1:7577471534051530000:0] peer: [4:7577471533243913332:0] socket: 205 qp: -1
2025-11-27T18:19:34.713127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471534051530023:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:34.715280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:34.730302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471534051529992:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(106440866ns) {HandleHandshakeStatus:329(106438450ns) {DropIncomingHandshake:442(14167ns) DropOutgoingHandshake:457(342ns) SwitchToState:179(138ns) ProcessPendingSessionEvents:495(84341430ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 80910005ns max# 12089204ns}}}
2025-11-27T18:19:34.736226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471534051529991:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(37455204ns) {HandleHandshakeStatus:329(37452171ns) {DropIncomingHandshake:442(24664ns) DropOutgoingHandshake:457(480ns) SwitchToState:179(399ns) ProcessPendingSessionEvents:495(5436210ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5378881ns max# 2957458ns}}}
2025-11-27T18:19:34.785053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471531869631514:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:34.787969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471534553405191:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11019
2025-11-27T18:19:34.788328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471534553405189:2138] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:34.769449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471532977625520:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:34.740128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471534051529999:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:34.770667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:34.768471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471534051529990:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.793561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471531869631435:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:34.825381Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471533243913242:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(50136797ns) {HandleHandshakeStatus:329(50134090ns) {DropIncomingHandshake:442(966ns) DropOutgoingHandshake:457(4576ns) SwitchToState:179(573ns) ProcessPendingSessionEvents:495(47191808ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 47020785ns max# 6726199ns}}}
2025-11-27T18:19:34.825495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471533243913242:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:34.810207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471534553405072:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:34.769447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:34.809724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:34.818955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:34.850434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471532977625408:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(30719897ns) {HandleHandshakeStatus:329(30716888ns) {DropIncomingHandshake:442(705ns) DropOutgoingHandshake:457(5711ns) SwitchToState:179(445ns) ProcessPendingSessionEvents:495(17029902ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16953237ns max# 4145855ns}}}
2025-11-27T18:19:34.821858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471534553405072:2049] [node 2] ICP22 created new session: [3:7577471534553405192:2049]
2025-11-27T18:19:34.770822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471534051529990:2109] [node 6] ICP22 created new session: [1:7577471534051530027:2109]
2025-11-27T18:19:34.833559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471534553405192:2049] [node 2] ICS09 handshake done sender: [3:7577471534553405189:2138] self: [3:7577471534553405190:0] peer: [2:7577471531869631510:0] socket: 173 qp: -1
2025-11-27T18:19:34.809909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471531869631435:2049] [node 3] ICP22 created new session: [2:7577471531869631538:2049]
2025-11-27T18:19:34.778263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471534051530027:2109] [node 6] ICS09 handshake done sender: [1:7577471534051529999:2113] self: [1:7577471534051529997:0] peer: [6:7577471532977625518:0] socket: 201 qp: -1
2025-11-27T18:19:34.841561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471534553405192:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:34.822276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471531869631538:2049] [node 3] ICS09 handshake done sender: [2:7577471531869631514:2119] self: [2:7577471531869631510:0] peer: [3:7577471534553405190:0] socket: 165 qp: -1
2025-11-27T18:19:34.778575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471534051530027:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:34.829070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471531869631538:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:34.845435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:34.779733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:34.803683Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471534051529990:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(19687505ns) {HandleHandshakeStatus:329(19684496ns) {DropIncomingHandshake:442(27833ns) DropOutgoingHandshake:457(670ns) SwitchToState:179(664ns) ProcessPendingSessionEvents:495(6417116ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6361521ns max# 4927576ns}}}
2025-11-27T18:19:34.855028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471531869631435:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(60700752ns) {HandleHandshakeStatus:329(60695901ns) {DropIncomingHandshake:442(18506ns) DropOutgoingHandshake:457(621ns) SwitchToState:179(770ns) ProcessPendingSessionEvents:495(1492865ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 1437221ns max# 997498ns}}}
2025-11-27T18:19:34.847806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:34.859243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471534553405072:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(48753089ns) {HandleHandshakeStatus:329(48190125ns) {DropIncomingHandshake:442(1627ns) DropOutgoingHandshake:457(24313ns) SwitchToState:179(3909ns) ProcessPendingSessionEvents:495(4982626ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4927023ns max# 4575729ns}}}
2025-11-27T18:19:34.927388Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:34.927691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471531869631530:2048] [node 1] ICS07 socket disconnect 156 reason# EndOfStream
2025-11-27T18:19:34.927738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471531869631530:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:34.927837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:34.927888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471531869631530:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:34.928876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:34.929067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471534553405184:2048] [node 1] ICS07 socket disconnect 169 reason# EndOfStream
2025-11-27T18:19:34.929112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471534553405184:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:34.929188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:34.929252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471534553405184:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:34.932473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471534553405071:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:34.932535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471534553405184:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:34.933290Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471531869631434:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:34.936679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:34.933393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471531869631530:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:34.937165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471533243913345:2048] [node 1] ICS07 socket disconnect 206 reason# EndOfStream
2025-11-27T18:19:34.937209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471533243913345:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:34.937358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:34.937418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471533243913345:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:34.937859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471534553405071:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:34.937997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471534553405184:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:34.938040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471534553405071:2048] [node 1] ICP30 unregister session Session# [3:7577471534553405184:2048] VirtualId# [3:7577471534553405174:0]
2025-11-27T18:19:34.938137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:34.938182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471534553405071:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:34.938212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471533243913242:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:34.938197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:34.938315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471533243913345:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:34.938357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471533243913242:2048] [node 1] ICP30 unregister session Session# [4:7577471533243913345:2048] VirtualId# [4:7577471533243913332:0]
2025-11-27T18:19:34.938447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:34.938586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471533243913242:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:34.938608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:34.938930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471531869631434:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:34.939052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471531869631530:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:34.939152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471531869631434:2048] [node 1] ICP30 unregister session Session# [2:7577471531869631530:2048] VirtualId# [2:7577471531869631520:0]
2025-11-27T18:19:34.939193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:34.939221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471531869631434:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:34.939233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:34.940057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:34.940253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471532977625520:2048] [node 1] ICS07 socket disconnect 202 reason# EndOfStream
2025-11-27T18:19:34.940294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471532977625520:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:34.940357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:34.940397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471532977625520:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:34.941015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471532977625408:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:34.941129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471532977625520:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:34.941171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471532977625408:2048] [node 1] ICP30 unregister session Session# [6:7577471532977625520:2048] VirtualId# [6:7577471532977625518:0]
2025-11-27T18:19:34.941280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:34.941318Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471532977625408:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:34.941330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:34.941430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:34.941634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471535993653019:2048] [node 1] ICS01 socket: 175 reason# EndOfStream
2025-11-27T18:19:34.941670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471535993652913:2048] [node 1] ICP30 unregister session Session# [5:7577471535993653019:2048] VirtualId# [5:7577471535993653016:0]
2025-11-27T18:19:34.941700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471535993653019:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:34.941782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:34.941823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:34.948158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471533243913242:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:34.948173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471534553405071:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:34.951250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471532977625408:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:35.009479Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:35.009718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471534553405192:2049] [node 2] ICS07 socket disconnect 173 reason# EndOfStream
2025-11-27T18:19:35.009763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471534553405192:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:35.009862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:35.009927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471534553405192:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:35.012117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471534553405072:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4721) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:35.012197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471534553405192:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:35.011601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:35.012580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471534553405072:2049] [node 2] ICP30 unregister session Session# [3:7577471534553405192:2049] VirtualId# [3:7577471534553405190:0]
2025-11-27T18:19:35.011817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471533243913337:2049] [node 2] ICS01 socket: 184 reason# EndOfStream
2025-11-27T18:19:35.011867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471533243913243:2049] [node 2] ICP30 unregister session Session# [4:7577471533243913337:2049] VirtualId# [4:7577471533243913329:0]
2025-11-27T18:19:35.011906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471533243913337:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:35.012704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:35.011981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:35.012045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:35.012883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471534553405072:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4721) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:35.012913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4721) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:35.012977Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:35.013179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471535993653025:2049] [node 2] ICS07 socket disconnect 186 reason# EndOfStream
2025-11-27T18:19:35.013273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471535993653025:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:35.013382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:35.013439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471535993653025:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:35.017180Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:35.017496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471532977625514:2049] [node 2] ICS01 socket: 185 reason# EndOfStream
2025-11-27T18:19:35.017592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471532977625409:2049] [node 2] ICP30 unregister session Session# [6:7577471532977625514:2049] VirtualId# [6:7577471532977625506:0]
2025-11-27T18:19:35.017677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471532977625514:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:35.017799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:35.017911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:35.018139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471535993652914:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4721) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:35.018397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471535993653025:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:35.018775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471535993652914:2049] [node 2] ICP30 unregister session Session# [5:7577471535993653025:2049] VirtualId# [5:7577471535993653021:0]
2025-11-27T18:19:35.018900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:35.018933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471535993652914:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4721) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:35.018984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4721) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:35.022942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471534553405072:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:35.029060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471535993652914:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:35.088771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:35.089062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471535993653031:2050] [node 3] ICS07 socket disconnect 207 reason# EndOfStream
2025-11-27T18:19:35.089103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471535993653031:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:35.089184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:35.089231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471535993653031:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:35.089953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:35.090136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471535993652915:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2859) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:35.090159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471532977625509:2050] [node 3] ICS01 socket: 157 reason# EndOfStream
2025-11-27T18:19:35.090286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471535993653031:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:35.090323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471535993652915:2050] [node 3] ICP30 unregister session Session# [5:7577471535993653031:2050] VirtualId# [5:7577471535993653027:0]
2025-11-27T18:19:35.090250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471532977625410:2050] [node 3] ICP30 unregister session Session# [6:7577471532977625509:2050] VirtualId# [6:7577471532977625498:0]
2025-11-27T18:19:35.093454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:35.093404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471532977625509:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:35.093584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:35.093650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471535993652915:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:35.093526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:35.093825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:35.093686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2859) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:35.093835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471533243913338:2050] [node 3] ICS01 socket: 149 reason# EndOfStream
2025-11-27T18:19:35.093894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471533243913244:2050] [node 3] ICP30 unregister session Session# [4:7577471533243913338:2050] VirtualId# [4:7577471533243913324:0]
2025-11-27T18:19:35.093933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471533243913338:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:35.094035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:35.094116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:35.103821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471535993652915:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:35.157675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:35.157933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471535993653034:2127] [node 4] ICS01 socket: 217 reason# EndOfStream
2025-11-27T18:19:35.157996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471535993652994:2127] [node 4] ICP30 unregister session Session# [5:7577471535993653034:2127] VirtualId# [5:7577471535993653030:0]
2025-11-27T18:19:35.158038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471535993653034:2127] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:35.158130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:35.158192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:35.158817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:35.159043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471532977625516:2109] [node 4] ICS01 socket: 200 reason# EndOfStream
2025-11-27T18:19:35.159086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471532977625481:2109] [node 4] ICP30 unregister session Session# [6:7577471532977625516:2109] VirtualId# [6:7577471532977625510:0]
2025-11-27T18:19:35.159119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471532977625516:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:35.159206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:35.159270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:35.221906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:35.222178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471532977625502:2106] [node 5] ICS01 socket: 158 reason# EndOfStream
2025-11-27T18:19:35.222230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471532977625468:2106] [node 5] ICP30 unregister session Session# [6:7577471532977625502:2106] VirtualId# [6:7577471532977625497:0]
2025-11-27T18:19:35.222267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471532977625502:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:35.222357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:35.222407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:35.341649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471539263583929:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:35.342040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471539263583930:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:35.362435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471536790569549:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:35.362557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471536790569550:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:35.390119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471537643673400:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:35.390155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471537643673401:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:35.410449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471537903365562:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:35.410457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471537903365563:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:35.410492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471537903365564:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:35.430854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471536343534897:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:35.430907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471536343534898:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:35.430943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471536343534899:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:35.457359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471537914997418:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:35.457661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471537914997416:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:35.457680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471537914997417:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:35.679136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471537914997477:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:35.679012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471537903365626:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:35.691265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471539263583996:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:35.696849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471536790569614:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:35.696759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471537643673473:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:35.694160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471539263583994:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:35.701055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471536790569615:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:35.701091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471537903365627:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:35.712027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471537914997480:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:35.714379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471536343534963:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:35.714214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471537643673474:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:35.701206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471539263583995:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:35.717523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471536790569616:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:35.715235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471537643673475:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:35.720288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471536343534964:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:35.775263Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471537914997416:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(62923035ns) {Configure:104(62915953ns) {ProcessConfigured:135(81167ns) {StartInitialHandshake:157(77598ns) {DropHandshakes:470(4063ns) {DropIncomingHandshake:442(798ns) DropOutgoingHandshake:457(604ns)} GenerateSessionVirtualId:988(947ns) PrepareNewSessionHandshake:477(2331ns) {SwitchToState:179(749ns)}}}}}
2025-11-27T18:19:35.796139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471536343534964:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(62278552ns) {Configure:104(62273428ns) {ProcessConfigured:135(88847ns) {StartInitialHandshake:157(85902ns) {DropHandshakes:470(6647ns) {DropIncomingHandshake:442(834ns) DropOutgoingHandshake:457(805ns)} GenerateSessionVirtualId:988(806ns) PrepareNewSessionHandshake:477(2300ns) {SwitchToState:179(1091ns)}}}}}
2025-11-27T18:19:35.806224Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471536790569549:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(207200207ns) {Configure:104(207188898ns) {ProcessConfigured:135(63494ns) {StartInitialHandshake:157(60264ns) {DropHandshakes:470(8611ns) {DropIncomingHandshake:442(497ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(638ns) PrepareNewSessionHandshake:477(1502ns) {SwitchToState:179(669ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.823374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471539263583997:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27164
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.823533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471539263584000:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27164
2025-11-27T18:19:35.857553Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471537903365562:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(178455995ns) {Configure:104(166446375ns) {ProcessConfigured:135(5518818ns) {StartInitialHandshake:157(5515873ns) {DropHandshakes:470(3097ns) {DropIncomingHandshake:442(771ns) DropOutgoingHandshake:457(649ns)} GenerateSessionVirtualId:988(754ns) PrepareNewSessionHandshake:477(3377ns) {SwitchToState:179(1602ns)}}}}}
2025-11-27T18:19:35.840639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471536343534899:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(110620802ns) {Configure:104(110614580ns) {ProcessConfigured:135(15070665ns) {StartInitialHandshake:157(15067300ns) {DropHandshakes:470(2710ns) {DropIncomingHandshake:442(515ns) DropOutgoingHandshake:457(487ns)} GenerateSessionVirtualId:988(440ns) PrepareNewSessionHandshake:477(3540ns) {SwitchToState:179(1776ns)}}}}}
2025-11-27T18:19:35.851665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471537643673400:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(72606896ns) {Configure:104(72600718ns) {ProcessConfigured:135(3482545ns) {StartInitialHandshake:157(3479596ns) {DropHandshakes:470(2085ns) {DropIncomingHandshake:442(454ns) DropOutgoingHandshake:457(514ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(4470ns) {SwitchToState:179(1817ns)}}}}}
2025-11-27T18:19:35.851080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471536790569623:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62299
2025-11-27T18:19:35.828500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471539263584001:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27164
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.869879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471536343534977:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48429
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.869955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471536790569623:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.870293Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471537914997477:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(135726174ns) {Configure:104(135719706ns) {ProcessConfigured:135(14071177ns) {StartInitialHandshake:157(14069263ns) {DropHandshakes:470(7282194ns) {DropIncomingHandshake:442(389ns) DropOutgoingHandshake:457(1558ns)} GenerateSessionVirtualId:988(2349ns) PrepareNewSessionHandshake:477(2195ns) {SwitchToState:179(911ns)}}}}}
2025-11-27T18:19:35.860024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471539263583930:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(150824906ns) {Configure:104(150813862ns) {ProcessConfigured:135(17779727ns) {StartInitialHandshake:157(17777129ns) {DropHandshakes:470(1793ns) {DropIncomingHandshake:442(376ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(803ns) PrepareNewSessionHandshake:477(3787ns) {SwitchToState:179(1779ns)}}}}}
2025-11-27T18:19:35.868629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471537643673476:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42315
2025-11-27T18:19:35.873509Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471536343534898:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(172772549ns) {Configure:104(172763097ns) {ProcessConfigured:135(31466663ns) {StartInitialHandshake:157(31461912ns) {DropHandshakes:470(4519074ns) {DropIncomingHandshake:442(613ns) DropOutgoingHandshake:457(3344ns)} GenerateSessionVirtualId:988(3895ns) PrepareNewSessionHandshake:477(4875ns) {SwitchToState:179(2788ns)}}}}}
2025-11-27T18:19:35.870962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471537914997488:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43565
2025-11-27T18:19:35.871699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471536790569624:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62299
2025-11-27T18:19:35.878036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471537903365629:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65354
2025-11-27T18:19:35.879442Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471536343534897:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(149781881ns) {Configure:104(149773605ns) {ProcessConfigured:135(8441532ns) {StartInitialHandshake:157(8439884ns) {DropHandshakes:470(2610ns) {DropIncomingHandshake:442(627ns) DropOutgoingHandshake:457(467ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(235660ns) {SwitchToState:179(382ns)}}}}}
2025-11-27T18:19:35.863648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471539263583929:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(167975313ns) {Configure:104(167963699ns) {ProcessConfigured:135(9175077ns) {StartInitialHandshake:157(9172565ns) {DropHandshakes:470(3098ns) {DropIncomingHandshake:442(787ns) DropOutgoingHandshake:457(472ns)} GenerateSessionVirtualId:988(890ns) PrepareNewSessionHandshake:477(2612ns) {SwitchToState:179(1231ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.869050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471537643673486:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42315
2025-11-27T18:19:35.880616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471536343534963:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(131377948ns) {Configure:104(131362281ns) {ProcessConfigured:135(13542718ns) {StartInitialHandshake:157(13540099ns) {DropHandshakes:470(1420ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(299ns)} GenerateSessionVirtualId:988(386ns) PrepareNewSessionHandshake:477(4786ns) {SwitchToState:179(1941ns)}}}}}
2025-11-27T18:19:35.876163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471539263584000:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.881002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471537903365563:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(148233432ns) {Configure:104(148226089ns) {ProcessConfigured:135(11870871ns) {StartInitialHandshake:157(11868883ns) {DropHandshakes:470(3518293ns) {DropIncomingHandshake:442(376ns) DropOutgoingHandshake:457(1762ns)} GenerateSessionVirtualId:988(1543ns) PrepareNewSessionHandshake:477(1851ns) {SwitchToState:179(455ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.877805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471539263583996:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(139518435ns) {Configure:104(139513790ns) {ProcessConfigured:135(17762416ns) {StartInitialHandshake:157(17759624ns) {DropHandshakes:470(3027ns) {DropIncomingHandshake:442(735ns) DropOutgoingHandshake:457(945ns)} GenerateSessionVirtualId:988(585ns) PrepareNewSessionHandshake:477(4459ns) {SwitchToState:179(2022ns)}}}}}
2025-11-27T18:19:35.879099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471536790569550:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(133231774ns) {Configure:104(133224625ns) {ProcessConfigured:135(45226ns) {StartInitialHandshake:157(43857ns) {DropHandshakes:470(1300ns) {DropIncomingHandshake:442(339ns) DropOutgoingHandshake:457(272ns)} GenerateSessionVirtualId:988(625ns) PrepareNewSessionHandshake:477(630ns) {SwitchToState:179(137ns)}}}}}
2025-11-27T18:19:35.883446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471539263583997:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.873819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471537914997417:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(130736846ns) {Configure:104(130729342ns) {ProcessConfigured:135(6494956ns) {StartInitialHandshake:157(6492174ns) {DropHandshakes:470(3004ns) {DropIncomingHandshake:442(615ns) DropOutgoingHandshake:457(564ns)} GenerateSessionVirtualId:988(703ns) PrepareNewSessionHandshake:477(11262ns) {SwitchToState:179(1924ns)}}}}}
2025-11-27T18:19:35.869856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471537643673486:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.883842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471539263584001:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.881330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471536790569624:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.878271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471537914997488:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.882918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471537903365629:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.904165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471536343534979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48429
2025-11-27T18:19:35.897464Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471536790569616:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(145363727ns) {Configure:104(145357316ns) {ProcessConfigured:135(5290099ns) {StartInitialHandshake:157(5287770ns) {DropHandshakes:470(1425ns) {DropIncomingHandshake:442(516ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(443ns) PrepareNewSessionHandshake:477(2350ns) {SwitchToState:179(120ns)}}}}}
2025-11-27T18:19:35.870855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471537643673476:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.881019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471537914997418:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:35.884292Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471537643673401:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(100330139ns) {Configure:104(100321589ns) {ProcessConfigured:135(2263150ns) {StartInitialHandshake:157(2260569ns) {DropHandshakes:470(934ns) {DropIncomingHandshake:442(280ns) DropOutgoingHandshake:457(308ns)} GenerateSessionVirtualId:988(344ns) PrepareNewSessionHandshake:477(1949ns) {SwitchToState:179(818ns)}}}}}
2025-11-27T18:19:35.884229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471537903365628:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65354
2025-11-27T18:19:35.889699Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471537643673475:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(143781870ns) {Configure:104(143776816ns) {ProcessConfigured:135(3925518ns) {StartInitialHandshake:157(3923772ns) {DropHandshakes:470(3344ns) {DropIncomingHandshake:442(717ns) DropOutgoingHandshake:457(474ns)} GenerateSessionVirtualId:988(375ns) PrepareNewSessionHandshake:477(1585ns) {SwitchToState:179(576ns)}}}}}
2025-11-27T18:19:35.884183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471537914997487:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43565
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.889613Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471537903365564:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(136294070ns) {Configure:104(136285121ns) {ProcessConfigured:135(599983ns) {StartInitialHandshake:157(597823ns) {DropHandshakes:470(1313ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(354ns)} GenerateSessionVirtualId:988(351ns) PrepareNewSessionHandshake:477(1672ns) {SwitchToState:179(776ns)}}}}}
2025-11-27T18:19:35.896605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471537643673478:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.905513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471537914997480:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(114008112ns) {Configure:104(114002072ns) {ProcessConfigured:135(5936905ns) {StartInitialHandshake:157(5934774ns) {DropHandshakes:470(754107ns) {DropIncomingHandshake:442(1156ns) DropOutgoingHandshake:457(375ns)} GenerateSessionVirtualId:988(1199ns) PrepareNewSessionHandshake:477(2266ns) {SwitchToState:179(1352ns)}}}}}
2025-11-27T18:19:35.916009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471536790569614:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(52736524ns) {Configure:104(50330730ns) {ProcessConfigured:135(116167ns) {StartInitialHandshake:157(114026ns) {DropHandshakes:470(3033ns) {DropIncomingHandshake:442(718ns) DropOutgoingHandshake:457(539ns)} GenerateSessionVirtualId:988(438ns) PrepareNewSessionHandshake:477(933ns) {SwitchToState:179(166ns)}}}}}
2025-11-27T18:19:35.908892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471537914997487:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.907158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471537903365564:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:35.896643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471537643673478:2121] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.914122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471536343534966:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.921025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471536790569618:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.914165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471536343534966:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:35.911211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471537914997477:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.908890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471537903365628:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.919559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471539263584003:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.917204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471537914997499:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43565
2025-11-27T18:19:35.899448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471537643673489:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42315
2025-11-27T18:19:35.919598Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471539263584003:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:35.921074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471536790569618:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:35.916967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471536343534966:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:35.910701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471537903365638:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65354
2025-11-27T18:19:35.912874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471537643673480:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.917521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471537914997487:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.929385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471536790569631:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62299
2025-11-27T18:19:35.921159Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471539263584003:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.931238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471536343534964:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:35.920094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471537914997477:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:35.911524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471537903365563:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:35.918874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471537903365638:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.918952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471537903365562:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:35.927171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471537903365626:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(40853092ns) {Configure:104(40843374ns) {ProcessConfigured:135(134904ns) {StartInitialHandshake:157(132533ns) {DropHandshakes:470(1408ns) {DropIncomingHandshake:442(399ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(354ns) PrepareNewSessionHandshake:477(745ns) {SwitchToState:179(219ns)}}}}}
2025-11-27T18:19:35.938363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471537903365649:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65354
2025-11-27T18:19:35.954182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471537903365650:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65354
2025-11-27T18:19:35.954238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471537903365651:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65354
2025-11-27T18:19:35.912906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471537643673480:2122] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.921168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471539263583999:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.985654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471537903365648:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65354
2025-11-27T18:19:35.931968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471536790569618:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:35.921404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.914015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471537643673489:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.998176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471537903365651:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.921199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471539263583999:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:35.932158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:35.970641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471536790569631:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.921444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471537914997477:2106] [node 5] ICP22 created new session: [6:7577471537914997500:2106]
2025-11-27T18:19:35.925728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471539263584016:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27164
2025-11-27T18:19:35.945940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471539263584016:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.947194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471539263583995:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:35.949830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:35.949921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471539263583999:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:35.964585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471539263583995:2107] [node 4] ICP22 created new session: [1:7577471539263584019:2107]
2025-11-27T18:19:35.964649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471539263584019:2107] [node 4] ICS09 handshake done sender: [1:7577471539263584003:2116] self: [1:7577471539263584002:0] peer: [4:7577471537903365644:0] socket: 177 qp: -1
2025-11-27T18:19:35.964688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471539263584019:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:35.970528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:35.970815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471539263583930:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:36.001344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471539263584006:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.001388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471539263584006:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:36.001499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:36.002770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471539263583930:2049] [node 3] ICP22 created new session: [1:7577471539263584023:2049]
2025-11-27T18:19:35.915917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471537643673400:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:36.001556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471537903365638:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:35.937722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471536343534964:2110] [node 6] ICP22 created new session: [5:7577471536343534981:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.925937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471537914997500:2106] [node 5] ICS09 handshake done sender: [6:7577471537914997487:2112] self: [6:7577471537914997497:0] peer: [5:7577471536343534965:0] socket: 158 qp: -1
2025-11-27T18:19:35.915999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471537643673478:2121] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:35.974286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471536790569615:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:36.001600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471537903365645:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65354
2025-11-27T18:19:35.937783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471536343534981:2110] [node 6] ICS09 handshake done sender: [5:7577471536343534966:2113] self: [5:7577471536343534965:0] peer: [6:7577471537914997497:0] socket: 157 qp: -1
2025-11-27T18:19:36.008611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471539263584023:2049] [node 3] ICS09 handshake done sender: [1:7577471539263583999:2113] self: [1:7577471539263583998:0] peer: [3:7577471537643673496:0] socket: 174 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.979729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:35.926743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471537914997500:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:36.001800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471537903365628:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:35.937831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471536343534981:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:36.008728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471539263584006:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:35.923171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471537643673474:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.989065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471536790569627:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.001877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471537903365633:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.925854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471537643673493:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42315
2025-11-27T18:19:35.989118Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471536790569627:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:35.925918Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471537643673480:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:35.926434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:35.989689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471536790569615:2107] [node 4] ICP22 created new session: [2:7577471536790569636:2107]
2025-11-27T18:19:35.926488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471537643673474:2110] [node 6] ICP22 created new session: [3:7577471537643673498:2110]
2025-11-27T18:19:35.994701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471536790569636:2107] [node 4] ICS09 handshake done sender: [2:7577471536790569618:2112] self: [2:7577471536790569617:0] peer: [4:7577471537903365643:0] socket: 153 qp: -1
2025-11-27T18:19:35.933144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471537643673473:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:36.001450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471536790569636:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:35.929399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:35.942335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471537643673493:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.001780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:36.005457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471536790569627:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:35.942662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471537643673401:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:35.931598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471537914997498:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43565
2025-11-27T18:19:36.020216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471541085536937:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62299
2025-11-27T18:19:36.009739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471539263584023:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:35.937890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:35.944499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471537643673498:2110] [node 6] ICS09 handshake done sender: [3:7577471537643673478:2121] self: [3:7577471537643673477:0] peer: [6:7577471537914997494:0] socket: 160 qp: -1
2025-11-27T18:19:35.949540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471537643673498:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:36.028362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471536790569550:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:35.949582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:36.030237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:35.963438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:36.034227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471541085536937:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.966324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471537643673473:2107] [node 4] ICP22 created new session: [3:7577471537643673502:2107]
2025-11-27T18:19:36.041881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471536790569550:2049] [node 3] ICP22 created new session: [2:7577471541085536939:2049]
2025-11-27T18:19:36.041884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471536790569549:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:36.001909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471537903365633:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:35.970563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471537643673503:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42315
2025-11-27T18:19:36.042814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471536790569637:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62299
2025-11-27T18:19:35.979810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471537643673502:2107] [node 4] ICS09 handshake done sender: [3:7577471537643673480:2122] self: [3:7577471537643673479:0] peer: [4:7577471537903365642:0] socket: 161 qp: -1
2025-11-27T18:19:36.043436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471541085536939:2049] [node 3] ICS09 handshake done sender: [2:7577471536790569627:2116] self: [2:7577471536790569626:0] peer: [3:7577471537643673501:0] socket: 182 qp: -1
2025-11-27T18:19:35.979872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471537643673502:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:35.979933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:36.003841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471537903365650:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.993582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471537643673400:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471537643673489:2125] held: no
2025-11-27T18:19:36.008293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471537903365563:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.008340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:35.999411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471537643673474:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(77266800ns) {HandleHandshakeStatus:329(75643140ns) {DropIncomingHandshake:442(21988ns) DropOutgoingHandshake:457(331ns) SwitchToState:179(194ns) ProcessPendingSessionEvents:495(8561384ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8520037ns max# 8189795ns}}}
2025-11-27T18:19:36.009172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471537903365562:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:35.938544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471536343534979:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.044153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471541085536939:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:36.009783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:36.001387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471537643673489:2125] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:35.941411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471537914997418:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19365) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471537914997488:2113] held: no
2025-11-27T18:19:36.009641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471537903365563:2049] [node 2] ICP22 created new session: [4:7577471542198332949:2049]
2025-11-27T18:19:36.026216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471539263583994:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:35.949732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471536343534963:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:36.008061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471537643673400:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.047694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:35.949636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471537914997488:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:35.958916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471536343534977:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.049325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471536790569550:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(25648925ns) {HandleHandshakeStatus:329(25645407ns) {DropIncomingHandshake:442(1875ns) DropOutgoingHandshake:457(617ns) SwitchToState:179(624ns) ProcessPendingSessionEvents:495(493362ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 434708ns max# 48612ns}}}
2025-11-27T18:19:36.009646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471537903365629:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:36.034207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:36.008113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:35.953093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471537914997477:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(32116095ns) {HandleHandshakeStatus:329(32112739ns) {DropIncomingHandshake:442(1682ns) DropOutgoingHandshake:457(16144ns) SwitchToState:179(334ns) ProcessPendingSessionEvents:495(7719452ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7661309ns max# 7204926ns}}}
2025-11-27T18:19:36.034284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471539263583994:2108] [node 5] ICP22 created new session: [1:7577471543558551322:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:36.019912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471542198332949:2049] [node 2] ICS09 handshake done sender: [4:7577471537903365628:2111] self: [4:7577471537903365643:0] peer: [2:7577471536790569617:0] socket: 154 qp: -1
2025-11-27T18:19:36.053307Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471536790569550:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:36.009283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471537643673400:2048] [node 1] ICP22 created new session: [3:7577471541938640802:2048]
2025-11-27T18:19:35.969471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471536343534897:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:35.980771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471537914997418:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.066078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471536790569615:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(59675181ns) {HandleHandshakeStatus:329(59673360ns) {DropIncomingHandshake:442(490ns) DropOutgoingHandshake:457(465ns) SwitchToState:179(601ns) ProcessPendingSessionEvents:495(9891800ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9820407ns max# 9384753ns}}}
2025-11-27T18:19:36.041651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471543558551322:2108] [node 5] ICS09 handshake done sender: [1:7577471539263584006:2117] self: [1:7577471539263584005:0] peer: [5:7577471536343534986:0] socket: 175 qp: -1
2025-11-27T18:19:36.022736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471542198332949:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:36.010724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471541938640802:2048] [node 1] ICS09 handshake done sender: [3:7577471537643673489:2125] self: [3:7577471537643673496:0] peer: [1:7577471539263583998:0] socket: 178 qp: -1
2025-11-27T18:19:36.001814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471536343534985:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48429
2025-11-27T18:19:36.066167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471536790569615:2107] [node 4] ICP18 handshake (actor [2:7577471536790569631:2118]) is too old
2025-11-27T18:19:35.980824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:36.042807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471543558551322:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:36.037322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:36.018717Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471536343534964:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(86536694ns) {HandleHandshakeStatus:329(86532641ns) {DropIncomingHandshake:442(1237ns) DropOutgoingHandshake:457(1804ns) SwitchToState:179(670ns) ProcessPendingSessionEvents:495(60213424ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60119835ns max# 16744662ns}}}
2025-11-27T18:19:35.987565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471537914997418:2050] [node 3] ICP22 created new session: [6:7577471537914997503:2050]
2025-11-27T18:19:36.010761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471541938640802:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:36.066821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471536790569628:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.037684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471537903365564:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:36.042934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:36.010797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:36.024000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471536343534985:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.997425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471537914997503:2050] [node 3] ICS09 handshake done sender: [6:7577471537914997488:2113] self: [6:7577471537914997494:0] peer: [3:7577471537643673477:0] socket: 169 qp: -1
2025-11-27T18:19:36.066857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471536790569628:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:36.045591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471539263584008:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:36.037745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:36.067216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471536790569633:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:35.997486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471537914997503:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:36.029711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471536343534899:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:36.011342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471541938640803:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42315
2025-11-27T18:19:36.045639Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471539263584008:2118] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:36.038836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:36.046983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471539263584012:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.067253Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471536790569633:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:35.997554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:36.011777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471537643673493:2128] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:36.043048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471540638502283:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48429
2025-11-27T18:19:36.070306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471541085536942:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62299
2025-11-27T18:19:36.043869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471540638502284:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48429
2025-11-27T18:19:36.039391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471537903365564:2050] [node 3] ICP22 created new session: [4:7577471542198332954:2050]
2025-11-27T18:19:36.078221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471536790569549:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471541085536937:2121] held: no
2025-11-27T18:19:36.050908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471540638502283:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.047017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471539263584012:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:36.028367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471537914997418:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(31081112ns) {HandleHandshakeStatus:329(31078239ns) {DropIncomingHandshake:442(2542ns) DropOutgoingHandshake:457(377ns) SwitchToState:179(909ns) ProcessPendingSessionEvents:495(966ns)}}
2025-11-27T18:19:36.011943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471537643673401:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.060116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471539263584008:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:36.039474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471542198332954:2050] [node 3] ICS09 handshake done sender: [4:7577471537903365629:2112] self: [4:7577471537903365642:0] peer: [3:7577471537643673479:0] socket: 167 qp: -1
2025-11-27T18:19:36.083534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471541085536937:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:36.011978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:36.012020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471537643673401:2049] [node 2] ICP22 created new session: [3:7577471541938640805:2049]
2025-11-27T18:19:36.012062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471541938640805:2049] [node 2] ICS09 handshake done sender: [3:7577471537643673493:2128] self: [3:7577471537643673501:0] peer: [2:7577471536790569626:0] socket: 185 qp: -1
2025-11-27T18:19:36.012677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471541938640805:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:36.054303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471536343534977:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:36.031193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471542209964801:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43565
2025-11-27T18:19:36.039518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471542198332954:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:36.062832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471539263584012:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:36.054533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471536343534898:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:36.087587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471536790569633:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:36.042467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471542209964801:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.039555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:36.023788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:36.064611Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471539263583994:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(36772556ns) {HandleHandshakeStatus:329(36761062ns) {DropIncomingHandshake:442(718ns) DropOutgoingHandshake:457(722ns) SwitchToState:179(794ns) ProcessPendingSessionEvents:495(19916710ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19822689ns max# 5985752ns}}}
2025-11-27T18:19:36.087801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471536790569628:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:36.062736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471536343534897:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.042825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471542209964802:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43565
2025-11-27T18:19:36.045828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471537643673490:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.041246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471537903365633:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:36.067400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:36.067187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471539263583929:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:36.088646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471536790569616:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:36.043594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471537914997416:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:36.043219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471537903365562:2048] [node 1] ICP22 created new session: [4:7577471542198332953:2048]
2025-11-27T18:19:36.045859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471537643673490:2126] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:36.067499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471539263583996:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:36.088961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471536790569614:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:36.059757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471537643673490:2126] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:36.064758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471537643673475:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:36.089706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:36.071060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:36.072883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471536343534963:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19199) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471536343534979:2119] held: no
2025-11-27T18:19:36.065289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:36.074702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:36.066938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471537643673475:2108] [node 5] ICP22 created new session: [3:7577471541938640808:2108]
2025-11-27T18:19:36.050908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471542209964802:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.076041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471536343534897:2048] [node 1] ICP22 created new session: [5:7577471540638502289:2048]
2025-11-27T18:19:36.075977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471539263583996:2110] [node 6] ICP22 created new session: [1:7577471543558551327:2110]
2025-11-27T18:19:36.069925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471541938640808:2108] [node 5] ICS09 handshake done sender: [3:7577471537643673490:2126] self: [3:7577471537643673487:0] peer: [5:7577471540638502285:0] socket: 204 qp: -1
2025-11-27T18:19:36.061897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471537914997417:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:36.076542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471540638502289:2048] [node 1] ICS09 handshake done sender: [5:7577471536343534977:2117] self: [5:7577471536343534986:0] peer: [1:7577471539263584005:0] socket: 176 qp: -1
2025-11-27T18:19:36.076121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471539263583929:2048] [node 2] ICP22 created new session: [1:7577471543558551326:2048]
No RDMA context found for GID: 2025-11-27T18:19:36.069970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471541938640808:2108] [node 5] ICS10 traffic start
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:19:36.066925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471542209964804:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43565
2025-11-27T18:19:36.047385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471542198332953:2048] [node 1] ICS09 handshake done sender: [4:7577471537903365638:2117] self: [4:7577471537903365644:0] peer: [1:7577471539263584002:0] socket: 181 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:36.076567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471543558551326:2048] [node 2] ICS09 handshake done sender: [1:7577471539263584008:2118] self: [1:7577471539263584007:0] peer: [2:7577471541085536940:0] socket: 166 qp: -1
2025-11-27T18:19:36.076834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471540638502289:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:36.077076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471543558551326:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:36.078034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:36.089949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471536790569549:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.053276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471542198332953:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:36.078011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:36.077444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471542209964804:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.081383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471536343534899:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19365) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471536343534985:2120] held: no
2025-11-27T18:19:36.056267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471537903365627:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:36.070274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:36.090006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:36.078195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471543558551327:2110] [node 6] ICS09 handshake done sender: [1:7577471539263584012:2119] self: [1:7577471539263584011:0] peer: [6:7577471542209964803:0] socket: 162 qp: -1
2025-11-27T18:19:36.056342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:36.085634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471540638502288:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48429
2025-11-27T18:19:36.090059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471536790569549:2048] [node 1] ICP22 created new session: [2:7577471541085536946:2048]
2025-11-27T18:19:36.079668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471537914997480:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:36.085885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471540638502286:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48429
2025-11-27T18:19:36.091108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471536343534979:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:36.094220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471536343534985:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:36.081274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471542209964806:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43565
2025-11-27T18:19:36.101689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471536343534898:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23539) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471540638502283:2121] held: no
2025-11-27T18:19:36.083476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471537914997416:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471542209964801:2119] held: no
2025-11-27T18:19:36.103037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471536343534963:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.074745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471537643673473:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(119080029ns) {HandleHandshakeStatus:329(119077618ns) {DropIncomingHandshake:442(556ns) DropOutgoingHandshake:457(311ns) SwitchToState:179(445ns) ProcessPendingSessionEvents:495(46788019ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 43482382ns max# 7754353ns}}}
2025-11-27T18:19:36.056390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471537903365627:2108] [node 5] ICP22 created new session: [4:7577471542198332957:2108]
2025-11-27T18:19:36.101580Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471542209964801:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:36.103566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471536343534899:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.102706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471537914997417:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23539) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471542209964802:2120] held: no
2025-11-27T18:19:36.104062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471540638502291:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48429
2025-11-27T18:19:36.103206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471537914997416:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.104247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:36.090107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471541085536946:2048] [node 1] ICS09 handshake done sender: [2:7577471541085536937:2121] self: [2:7577471541085536940:0] peer: [1:7577471539263584007:0] socket: 210 qp: -1
2025-11-27T18:19:36.078243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471543558551327:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:36.104541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:36.104318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471542209964808:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43565
2025-11-27T18:19:36.090140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471541085536946:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:36.107256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471542209964809:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43565
2025-11-27T18:19:36.090193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:36.105020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471536343534899:2050] [node 3] ICP22 created new session: [5:7577471540638502292:2050]
2025-11-27T18:19:36.056439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471542198332957:2108] [node 5] ICS09 handshake done sender: [4:7577471537903365633:2114] self: [4:7577471537903365632:0] peer: [5:7577471536343534984:0] socket: 187 qp: -1
2025-11-27T18:19:36.090321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:36.107391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:36.107740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471537914997416:2048] [node 1] ICP22 created new session: [6:7577471542209964810:2048]
2025-11-27T18:19:36.090368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471536790569614:2110] [node 6] ICP22 created new session: [2:7577471541085536945:2110]
2025-11-27T18:19:36.107482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471536343534963:2108] [node 4] ICP22 created new session: [5:7577471540638502293:2108]
2025-11-27T18:19:36.110534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471542209964810:2048] [node 1] ICS09 handshake done sender: [6:7577471542209964801:2119] self: [6:7577471542209964803:0] peer: [1:7577471539263584011:0] socket: 211 qp: -1
2025-11-27T18:19:36.080996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:36.090752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471541085536945:2110] [node 6] ICS09 handshake done sender: [2:7577471536790569633:2119] self: [2:7577471536790569632:0] peer: [6:7577471542209964805:0] socket: 218 qp: -1
2025-11-27T18:19:36.107668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471540638502292:2050] [node 3] ICS09 handshake done sender: [5:7577471536343534985:2120] self: [5:7577471540638502285:0] peer: [3:7577471537643673487:0] socket: 209 qp: -1
2025-11-27T18:19:36.103884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471539263583930:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(110527460ns) {HandleHandshakeStatus:329(110524416ns) {DropIncomingHandshake:442(13360ns) DropOutgoingHandshake:457(730ns) SwitchToState:179(210ns) ProcessPendingSessionEvents:495(42573893ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 40107767ns max# 5115734ns}}}
2025-11-27T18:19:36.110830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471542209964810:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:36.056474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471542198332957:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:36.107875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471540638502292:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:36.104992Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471539263583995:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(141623050ns) {HandleHandshakeStatus:329(141619541ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(872ns) SwitchToState:179(585ns) ProcessPendingSessionEvents:495(86973600ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 86800529ns max# 7564730ns}}}
2025-11-27T18:19:36.107922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:36.110518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471540638502293:2108] [node 4] ICS09 handshake done sender: [5:7577471536343534979:2119] self: [5:7577471536343534984:0] peer: [4:7577471537903365632:0] socket: 188 qp: -1
2025-11-27T18:19:36.110563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471540638502293:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:36.116159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471537643673401:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(90220629ns) {HandleHandshakeStatus:329(90218821ns) {DropIncomingHandshake:442(350ns) DropOutgoingHandshake:457(3598ns) SwitchToState:179(360ns) ProcessPendingSessionEvents:495(67905285ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 67747939ns max# 8989648ns}}}
2025-11-27T18:19:36.110604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:36.056521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:36.112321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471540638502283:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:36.056957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:36.112972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471536343534898:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.081335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471537903365647:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:36.113941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:36.081369Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471537903365647:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:36.089834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471537903365647:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:36.114015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471536343534898:2049] [node 2] ICP22 created new session: [5:7577471540638502296:2049]
2025-11-27T18:19:36.091705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471537903365626:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:36.114298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471540638502296:2049] [node 2] ICS09 handshake done sender: [5:7577471540638502283:2121] self: [5:7577471540638502287:0] peer: [2:7577471536790569625:0] socket: 213 qp: -1
2025-11-27T18:19:36.094207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:36.114636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471540638502296:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:36.101151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471537903365626:2109] [node 6] ICP22 created new session: [4:7577471542198332961:2109]
2025-11-27T18:19:36.115577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:36.101889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471542198332961:2109] [node 6] ICS09 handshake done sender: [4:7577471537903365647:2119] self: [4:7577471537903365646:0] peer: [6:7577471542209964807:0] socket: 221 qp: -1
2025-11-27T18:19:36.101939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471542198332961:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:36.102252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:36.111429Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471537903365563:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(84834585ns) {HandleHandshakeStatus:329(84832154ns) {DropIncomingHandshake:442(368ns) DropOutgoingHandshake:457(5500ns) SwitchToState:179(253ns) ProcessPendingSessionEvents:495(45934474ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 44373654ns max# 7868528ns}}}
2025-11-27T18:19:36.115602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471537903365627:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(48102641ns) {HandleHandshakeStatus:329(48099193ns) {DropIncomingHandshake:442(23464ns) DropOutgoingHandshake:457(528ns) SwitchToState:179(653ns) ProcessPendingSessionEvents:495(45226588ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 45092890ns max# 7826825ns}}}
2025-11-27T18:19:36.118399Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471537903365562:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(106968369ns) {HandleHandshakeStatus:329(106965589ns) {DropIncomingHandshake:442(2184ns) DropOutgoingHandshake:457(6571ns) SwitchToState:179(423ns) ProcessPendingSessionEvents:495(47599944ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 47416592ns max# 7420134ns}}}
2025-11-27T18:19:36.090792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471541085536945:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:36.111439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:36.091264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471536790569616:2108] [node 5] ICP22 created new session: [2:7577471541085536948:2108]
2025-11-27T18:19:36.107487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471539263583995:2107] [node 4] ICP18 handshake (actor [1:7577471539263584016:2120]) is too old
2025-11-27T18:19:36.117987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471539263583996:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(39780752ns) {HandleHandshakeStatus:329(39777557ns) {DropIncomingHandshake:442(27858ns) DropOutgoingHandshake:457(503ns) SwitchToState:179(476ns) ProcessPendingSessionEvents:495(21341247ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21249262ns max# 5776146ns}}}
2025-11-27T18:19:36.091315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471541085536948:2108] [node 5] ICS09 handshake done sender: [2:7577471536790569628:2117] self: [2:7577471536790569625:0] peer: [5:7577471540638502287:0] socket: 212 qp: -1
2025-11-27T18:19:36.112707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471537914997480:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19199) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471542209964804:2121] held: no
2025-11-27T18:19:36.091521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:36.091627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471541085536948:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:36.091661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:36.127178Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471537903365626:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(26539089ns) {HandleHandshakeStatus:329(26536434ns) {DropIncomingHandshake:442(39602ns) DropOutgoingHandshake:457(652ns) SwitchToState:179(562ns) ProcessPendingSessionEvents:495(15007208ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14900863ns max# 2901624ns}}}
2025-11-27T18:19:36.127403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471536790569616:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(27190101ns) {HandleHandshakeStatus:329(27187917ns) {DropIncomingHandshake:442(24019ns) DropOutgoingHandshake:457(144ns) SwitchToState:179(643ns) ProcessPendingSessionEvents:495(14204171ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13747908ns max# 3456175ns}}}
2025-11-27T18:19:36.114195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471542209964802:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:36.115334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471537914997417:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.115811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:36.117377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471537914997417:2049] [node 2] ICP22 created new session: [6:7577471542209964812:2049]
2025-11-27T18:19:36.117440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471542209964812:2049] [node 2] ICS09 handshake done sender: [6:7577471542209964802:2120] self: [6:7577471542209964805:0] peer: [2:7577471536790569632:0] socket: 219 qp: -1
2025-11-27T18:19:36.117489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471542209964812:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:36.118005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:36.118344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471542209964804:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:36.119128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471537914997480:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:36.120625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:36.123521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471537914997480:2107] [node 4] ICP22 created new session: [6:7577471542209964814:2107]
2025-11-27T18:19:36.126710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471542209964814:2107] [node 4] ICS09 handshake done sender: [6:7577471542209964804:2121] self: [6:7577471542209964807:0] peer: [4:7577471537903365646:0] socket: 222 qp: -1
2025-11-27T18:19:36.126771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471542209964814:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:36.126830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:36.127040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471536343534897:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(51297606ns) {HandleHandshakeStatus:329(51293935ns) {DropIncomingHandshake:442(780ns) DropOutgoingHandshake:457(7111ns) SwitchToState:179(667ns) ProcessPendingSessionEvents:495(33025809ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32904091ns max# 5377036ns}}}
2025-11-27T18:19:36.134386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471537643673475:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(47616094ns) {HandleHandshakeStatus:329(47612440ns) {DropIncomingHandshake:442(25550ns) DropOutgoingHandshake:457(3077ns) SwitchToState:179(845ns) ProcessPendingSessionEvents:495(34924924ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 31494340ns max# 7070520ns}}}
2025-11-27T18:19:36.133813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471536790569614:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(29791080ns) {HandleHandshakeStatus:329(29416548ns) {DropIncomingHandshake:442(10972ns) DropOutgoingHandshake:457(4353ns) SwitchToState:179(353ns) ProcessPendingSessionEvents:495(16518211ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16424527ns max# 2924822ns}}}
2025-11-27T18:19:36.187497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:36.187783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471541085536946:2048] [node 1] ICS01 socket: 210 reason# EndOfStream
2025-11-27T18:19:36.187837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471536790569549:2048] [node 1] ICP30 unregister session Session# [2:7577471541085536946:2048] VirtualId# [2:7577471541085536940:0]
2025-11-27T18:19:36.187870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471541085536946:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.187952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:36.188027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:36.188851Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:36.189068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471541938640802:2048] [node 1] ICS01 socket: 178 reason# EndOfStream
2025-11-27T18:19:36.189105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471537643673400:2048] [node 1] ICP30 unregister session Session# [3:7577471541938640802:2048] VirtualId# [3:7577471537643673496:0]
2025-11-27T18:19:36.189134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471541938640802:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.189214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:36.189276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:36.191659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:36.191865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471542198332953:2048] [node 1] ICS07 socket disconnect 181 reason# EndOfStream
2025-11-27T18:19:36.191904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471542198332953:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.191976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:36.192021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471542198332953:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:36.192913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471537903365562:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7274) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:36.192985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471542198332953:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:36.192951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:36.193020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471537903365562:2048] [node 1] ICP30 unregister session Session# [4:7577471542198332953:2048] VirtualId# [4:7577471537903365644:0]
2025-11-27T18:19:36.193189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471540638502289:2048] [node 1] ICS07 socket disconnect 176 reason# EndOfStream
2025-11-27T18:19:36.193238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471540638502289:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.193318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:36.193362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471540638502289:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:36.193118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:36.193180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471537903365562:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:36.193196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:36.194187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471536343534897:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7274) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:36.194238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471540638502289:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:36.194269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471536343534897:2048] [node 1] ICP30 unregister session Session# [5:7577471540638502289:2048] VirtualId# [5:7577471536343534986:0]
2025-11-27T18:19:36.194644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:36.194693Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471536343534897:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:36.194707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:36.194640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:36.194862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471542209964810:2048] [node 1] ICS01 socket: 211 reason# EndOfStream
2025-11-27T18:19:36.194903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471537914997416:2048] [node 1] ICP30 unregister session Session# [6:7577471542209964810:2048] VirtualId# [6:7577471542209964803:0]
2025-11-27T18:19:36.194931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471542209964810:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.195016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:36.195080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:36.202570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471537903365562:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:36.204644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471536343534897:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:36.306140Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:36.306385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471541938640805:2049] [node 2] ICS07 socket disconnect 185 reason# EndOfStream
2025-11-27T18:19:36.306418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471541938640805:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.306489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:36.306527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471541938640805:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:36.317493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471537643673401:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23539) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:36.317545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471541938640805:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:36.318493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471537643673401:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23539) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:36.318814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471541938640805:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:36.318970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471537643673401:2049] [node 2] ICP30 unregister session Session# [3:7577471541938640805:2049] VirtualId# [3:7577471537643673501:0]
2025-11-27T18:19:36.319175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:36.319230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471537643673401:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23539) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:36.319247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23539) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:36.321543Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:36.321771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471542198332949:2049] [node 2] ICS07 socket disconnect 154 reason# EndOfStream
2025-11-27T18:19:36.321820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471542198332949:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.321904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:36.321965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471542198332949:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:36.322963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471537903365563:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23539) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:36.323063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471542198332949:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:36.323106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471537903365563:2049] [node 2] ICP30 unregister session Session# [4:7577471542198332949:2049] VirtualId# [4:7577471537903365643:0]
2025-11-27T18:19:36.323208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:36.323251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471537903365563:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23539) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:36.323268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23539) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:36.324659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:36.324885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471540638502296:2049] [node 2] ICS01 socket: 213 reason# EndOfStream
2025-11-27T18:19:36.324937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471536343534898:2049] [node 2] ICP30 unregister session Session# [5:7577471540638502296:2049] VirtualId# [5:7577471540638502287:0]
2025-11-27T18:19:36.324973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471540638502296:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.325053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:36.325095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:36.327695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:36.327921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471542209964812:2049] [node 2] ICS01 socket: 219 reason# EndOfStream
2025-11-27T18:19:36.327957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471537914997417:2049] [node 2] ICP30 unregister session Session# [6:7577471542209964812:2049] VirtualId# [6:7577471542209964805:0]
2025-11-27T18:19:36.327992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471542209964812:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.328060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:36.328100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:36.329588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471537643673401:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:36.332955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471537903365563:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:36.374666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:36.374940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471542198332954:2050] [node 3] ICS01 socket: 167 reason# EndOfStream
2025-11-27T18:19:36.374998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471537903365564:2050] [node 3] ICP30 unregister session Session# [4:7577471542198332954:2050] VirtualId# [4:7577471537903365642:0]
2025-11-27T18:19:36.375072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471542198332954:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.375150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:36.375195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:36.376753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:36.376886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:36.376992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471537914997503:2050] [node 3] ICS01 socket: 169 reason# EndOfStream
2025-11-27T18:19:36.377043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471537914997418:2050] [node 3] ICP30 unregister session Session# [6:7577471537914997503:2050] VirtualId# [6:7577471537914997494:0]
2025-11-27T18:19:36.377077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471537914997503:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.377098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471540638502292:2050] [node 3] ICS01 socket: 209 reason# EndOfStream
2025-11-27T18:19:36.377166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471536343534899:2050] [node 3] ICP30 unregister session Session# [5:7577471540638502292:2050] VirtualId# [5:7577471540638502285:0]
2025-11-27T18:19:36.377152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:36.377560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471540638502292:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.377200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:36.377641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:36.377678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:36.460643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:36.460850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471540638502293:2108] [node 4] ICS01 socket: 188 reason# EndOfStream
2025-11-27T18:19:36.460911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471536343534963:2108] [node 4] ICP30 unregister session Session# [5:7577471540638502293:2108] VirtualId# [5:7577471536343534984:0]
2025-11-27T18:19:36.460948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471540638502293:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.461019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:36.461084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:36.461739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:36.461925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471542209964814:2107] [node 4] ICS01 socket: 222 reason# EndOfStream
2025-11-27T18:19:36.461963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471537914997480:2107] [node 4] ICP30 unregister session Session# [6:7577471542209964814:2107] VirtualId# [6:7577471542209964807:0]
2025-11-27T18:19:36.461992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471542209964814:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.462066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:36.462111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:36.554079Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:36.554327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471537914997500:2106] [node 5] ICS07 socket disconnect 158 reason# EndOfStream
2025-11-27T18:19:36.554375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471537914997500:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:36.554479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:36.554531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471537914997500:2106] [node 5] ICS15 start handshake
2025-11-27T18:19:36.555193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471537914997477:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11709) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:36.555325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471537914997500:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:36.555372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471537914997477:2106] [node 5] ICP30 unregister session Session# [6:7577471537914997500:2106] VirtualId# [6:7577471537914997497:0]
2025-11-27T18:19:36.555493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:19:36.555552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471537914997477:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11709) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:36.555576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11709) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:36.677634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471543935252205:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:36.677712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471543935252206:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:36.691432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471543707747213:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:36.691492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471543707747214:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:36.712292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471544537503136:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:36.712344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471544537503137:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:36.734002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471543970336927:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:36.734063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471543970336928:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:36.734124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471543970336929:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:36.760924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471543999264005:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:36.760993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471543999264006:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:36.761040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471543999264007:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:36.779556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471543369069506:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:36.779592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471543369069507:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:36.779641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471543369069508:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:36.894633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471543369069567:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:36.986271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471543935252268:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:36.986674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471543935252269:2108] [node 6] ICP01 ready to work
2025-11-27T18:19:36.986740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471543970336991:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:36.986279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471543707747276:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:37.027371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471548265304288:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:37.046706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471543369069567:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(151733479ns) {Configure:104(151728678ns) {ProcessConfigured:135(78980ns) {StartInitialHandshake:157(76496ns) {DropHandshakes:470(3234ns) {DropIncomingHandshake:442(806ns) DropOutgoingHandshake:457(569ns)} GenerateSessionVirtualId:988(1364ns) PrepareNewSessionHandshake:477(2045ns) {SwitchToState:179(617ns)}}}}}
2025-11-27T18:19:37.021681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471548002714573:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:37.053487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471548832470504:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:37.053873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471548832470505:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:37.054371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471548832470506:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:37.047726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471547664036869:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:37.060886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471548002714574:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:37.060641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471548230219566:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:37.069533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471543999264005:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(137031446ns) {Configure:104(137026620ns) {ProcessConfigured:135(780405ns) {StartInitialHandshake:157(777394ns) {DropHandshakes:470(697041ns) {DropIncomingHandshake:442(662ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:477(17221ns) {SwitchToState:179(15826ns)}}}}}
2025-11-27T18:19:37.079516Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471544537503137:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17838686ns) {Configure:104(17835115ns) {ProcessConfigured:135(5584002ns) {StartInitialHandshake:157(5580893ns) {DropHandshakes:470(1665ns) {DropIncomingHandshake:442(456ns) DropOutgoingHandshake:457(491ns)} GenerateSessionVirtualId:988(459ns) PrepareNewSessionHandshake:477(3230ns) {SwitchToState:179(1994ns)}}}}}
2025-11-27T18:19:37.079384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471548294231370:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:37.081207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471548294231371:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:37.089060Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471543970336929:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(46629292ns) {Configure:104(46622946ns) {ProcessConfigured:135(10307935ns) {StartInitialHandshake:157(10306060ns) {DropHandshakes:470(2615ns) {DropIncomingHandshake:442(768ns) DropOutgoingHandshake:457(557ns)} GenerateSessionVirtualId:988(931ns) PrepareNewSessionHandshake:477(7120ns) {SwitchToState:179(4590ns)}}}}}
2025-11-27T18:19:37.095875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471543935252205:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21864468ns) {Configure:104(21859393ns) {ProcessConfigured:135(51573ns) {StartInitialHandshake:157(48829ns) {DropHandshakes:470(2200ns) {DropIncomingHandshake:442(533ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(503ns) PrepareNewSessionHandshake:477(1258ns) {SwitchToState:179(534ns)}}}}}
2025-11-27T18:19:37.117640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471543369069507:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26473959ns) {Configure:104(26470217ns) {ProcessConfigured:135(1446669ns) {StartInitialHandshake:157(1441240ns) {DropHandshakes:470(1126ns) {DropIncomingHandshake:442(290ns) DropOutgoingHandshake:457(271ns)} GenerateSessionVirtualId:988(406ns) PrepareNewSessionHandshake:477(2063ns) {SwitchToState:179(964ns)}}}}}
2025-11-27T18:19:37.117901Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471548230219566:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17865140ns) {Configure:104(17862110ns) {ProcessConfigured:135(62580ns) {StartInitialHandshake:157(60731ns) {DropHandshakes:470(1204ns) {DropIncomingHandshake:442(297ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(545ns) PrepareNewSessionHandshake:477(813ns) {SwitchToState:179(379ns)}}}}}
2025-11-27T18:19:37.117666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471548832470506:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27510927ns) {Configure:104(27507737ns) {ProcessConfigured:135(3585932ns) {StartInitialHandshake:157(3581053ns) {DropHandshakes:470(1121ns) {DropIncomingHandshake:442(341ns) DropOutgoingHandshake:457(183ns)} GenerateSessionVirtualId:988(426ns) PrepareNewSessionHandshake:477(2651ns) {SwitchToState:179(1394ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.129873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471548832470517:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5704
2025-11-27T18:19:37.130171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471548832470518:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5704
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.132806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471548294231379:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59739
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.133570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471548832470518:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.131577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471548265304302:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32807
2025-11-27T18:19:37.133207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471548265304303:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32807
2025-11-27T18:19:37.133872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471548832470517:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.158670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471548230219577:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27219
2025-11-27T18:19:37.158842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471548230219578:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27219
2025-11-27T18:19:37.158955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471548230219579:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27219
2025-11-27T18:19:37.159009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471548230219580:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27219
2025-11-27T18:19:37.161977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471548230219578:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.175236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471548230219577:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.186925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471548265304304:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32807
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.200515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471548230219586:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27219
2025-11-27T18:19:37.193832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471543707747276:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(33532353ns) {Configure:104(33526904ns) {ProcessConfigured:135(85611ns) {StartInitialHandshake:157(81135ns) {DropHandshakes:470(3622ns) {DropIncomingHandshake:442(719ns) DropOutgoingHandshake:457(474ns)} GenerateSessionVirtualId:988(1150ns) PrepareNewSessionHandshake:477(5638ns) {SwitchToState:179(638ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.194485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471548002714583:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60172
2025-11-27T18:19:37.194648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471548002714584:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60172
2025-11-27T18:19:37.194764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471548002714585:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60172
2025-11-27T18:19:37.194922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471548002714588:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60172
2025-11-27T18:19:37.221844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471548294231380:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59739
2025-11-27T18:19:37.221703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471548265304302:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.223522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471548230219579:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.223673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471548230219580:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.224158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471548265304303:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.224324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471548265304304:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.224497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471543970336929:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.224729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471543970336927:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.200431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471548002714584:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.212352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471548002714583:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.222498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471548002714588:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.222609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471548002714585:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.223018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471543707747213:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.224995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471548832470512:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.225028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471548832470512:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.222133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471547664036881:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23152
2025-11-27T18:19:37.222275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471547664036882:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23152
2025-11-27T18:19:37.226469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471548230219586:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.226580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471548230219568:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.226600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471548230219568:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.227318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471548230219570:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.227342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471548230219570:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.228904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471548230219568:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:37.229361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471548294231379:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.229723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471548294231380:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.229662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471548002714596:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60172
2025-11-27T18:19:37.229842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471543999264006:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.230235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471548265304308:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32807
2025-11-27T18:19:37.230314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471548265304309:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32807
2025-11-27T18:19:37.230488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471548832470512:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:37.230941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471548832470504:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.231044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471548002714576:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.231014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:37.232231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471548230219570:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:37.232124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471548832470504:2107] [node 4] ICP22 created new session: [3:7577471548832470525:2107]
2025-11-27T18:19:37.232179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471548832470525:2107] [node 4] ICS09 handshake done sender: [3:7577471548832470512:2121] self: [3:7577471548832470511:0] peer: [4:7577471548265304305:0] socket: 168 qp: -1
2025-11-27T18:19:37.232211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471548832470525:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:37.232355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:37.231078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471548002714576:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.228903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471547664036881:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.229032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471543369069506:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.230265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471547664036882:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.230366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471543369069507:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.231525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471548002714578:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.231538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471548002714578:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.232023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471548002714588:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:37.236967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471543707747213:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.236700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471548265304302:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:37.237108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471548230219572:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.237154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471548230219572:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.237035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:37.237071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471543707747213:2048] [node 1] ICP22 created new session: [2:7577471548002714599:2048]
2025-11-27T18:19:37.237100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471548002714599:2048] [node 1] ICS09 handshake done sender: [2:7577471548002714588:2118] self: [2:7577471548002714594:0] peer: [1:7577471548230219567:0] socket: 189 qp: -1
2025-11-27T18:19:37.237125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471548002714599:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:37.238239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471543935252205:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.238308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:37.238362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471543935252205:2048] [node 2] ICP22 created new session: [1:7577471548230219592:2048]
2025-11-27T18:19:37.238519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471548230219592:2048] [node 2] ICS09 handshake done sender: [1:7577471548230219568:2110] self: [1:7577471548230219567:0] peer: [2:7577471548002714594:0] socket: 184 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.237176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:37.238706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471548230219592:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:37.241093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471547664036885:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23152
2025-11-27T18:19:37.241202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471543970336929:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.239167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:37.243644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:37.243714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471543970336929:2050] [node 3] ICP22 created new session: [4:7577471548265304310:2050]
2025-11-27T18:19:37.243758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471548265304310:2050] [node 3] ICS09 handshake done sender: [4:7577471548265304302:2116] self: [4:7577471548265304305:0] peer: [3:7577471548832470511:0] socket: 169 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.243895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471548294231383:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59739
2025-11-27T18:19:37.243974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471548294231384:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59739
2025-11-27T18:19:37.243797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471548265304310:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:37.243840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:37.245102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471543935252268:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.245202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:37.245293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471543935252268:2107] [node 4] ICP22 created new session: [1:7577471548230219594:2107]
2025-11-27T18:19:37.245361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471548230219594:2107] [node 4] ICS09 handshake done sender: [1:7577471548230219570:2111] self: [1:7577471548230219569:0] peer: [4:7577471548265304307:0] socket: 185 qp: -1
2025-11-27T18:19:37.245399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471548230219594:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:37.245379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471543970336929:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:37.245464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:37.246227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471548294231384:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.246374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471543999264007:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.246747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471548265304304:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:37.247916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471543970336927:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.247964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:37.248009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471543970336927:2048] [node 1] ICP22 created new session: [4:7577471548265304312:2048]
2025-11-27T18:19:37.248055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471548265304312:2048] [node 1] ICS09 handshake done sender: [4:7577471548265304304:2118] self: [4:7577471548265304307:0] peer: [1:7577471548230219569:0] socket: 187 qp: -1
2025-11-27T18:19:37.248090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471548265304312:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:37.248146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:37.249348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471548002714576:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:37.249800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471548832470514:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.249827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471548002714578:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:37.249837Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471548832470514:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.252842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471548230219572:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:37.253778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471548002714573:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.253738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471547664036882:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:37.254303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:37.254541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471543935252269:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.254571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471548002714573:2108] [node 5] ICP22 created new session: [2:7577471548002714601:2108]
2025-11-27T18:19:37.254604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:37.256114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471543369069507:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.254645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471543935252269:2108] [node 6] ICP22 created new session: [1:7577471548230219599:2108]
2025-11-27T18:19:37.255160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471548230219599:2108] [node 6] ICS09 handshake done sender: [1:7577471548230219572:2112] self: [1:7577471548230219571:0] peer: [6:7577471547664036883:0] socket: 186 qp: -1
2025-11-27T18:19:37.255204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471548230219599:2108] [node 6] ICS10 traffic start
2025-11-27T18:19:37.255255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:37.256473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471548832470514:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:37.256610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:37.254909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471548002714601:2108] [node 5] ICS09 handshake done sender: [2:7577471548002714576:2110] self: [2:7577471548002714575:0] peer: [5:7577471548294231382:0] socket: 190 qp: -1
2025-11-27T18:19:37.254950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471548002714601:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:37.256698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471543369069507:2049] [node 2] ICP22 created new session: [6:7577471547664036887:2049]
2025-11-27T18:19:37.256065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:37.257044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471547664036887:2049] [node 2] ICS09 handshake done sender: [6:7577471547664036882:2117] self: [6:7577471547664036884:0] peer: [2:7577471548002714577:0] socket: 197 qp: -1
2025-11-27T18:19:37.257093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471547664036887:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.257516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:37.258205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471548294231380:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:37.258233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471548832470529:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5704
2025-11-27T18:19:37.258292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471548832470505:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.258449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:37.258576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471548832470505:2108] [node 5] ICP22 created new session: [3:7577471548832470530:2108]
2025-11-27T18:19:37.258632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471548832470530:2108] [node 5] ICS09 handshake done sender: [3:7577471548832470514:2122] self: [3:7577471548832470513:0] peer: [5:7577471548294231385:0] socket: 205 qp: -1
2025-11-27T18:19:37.258690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471548832470530:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:37.258737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.258769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471543999264006:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.258869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:37.259013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471543999264006:2049] [node 2] ICP22 created new session: [5:7577471548294231388:2049]
2025-11-27T18:19:37.259080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471548294231388:2049] [node 2] ICS09 handshake done sender: [5:7577471548294231380:2116] self: [5:7577471548294231382:0] peer: [2:7577471548002714575:0] socket: 191 qp: -1
2025-11-27T18:19:37.258964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471547664036886:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23152
2025-11-27T18:19:37.259248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471548294231388:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:37.259310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:37.259245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471548002714574:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.259312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.262597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471548294231386:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59739
2025-11-27T18:19:37.262688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471548294231387:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59739
2025-11-27T18:19:37.259366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471548002714574:2109] [node 6] ICP22 created new session: [2:7577471548002714603:2109]
2025-11-27T18:19:37.259409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471548002714603:2109] [node 6] ICS09 handshake done sender: [2:7577471548002714578:2111] self: [2:7577471548002714577:0] peer: [6:7577471547664036884:0] socket: 192 qp: -1
2025-11-27T18:19:37.259438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471548002714603:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:37.259643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:37.267656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471548294231384:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:37.267715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471543369069506:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21354) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471547664036881:2116] held: no
2025-11-27T18:19:37.268125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471548832470529:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.268402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471548294231386:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.268506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471544537503136:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.268579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471548294231390:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59739
2025-11-27T18:19:37.268837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471548230219576:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.268877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471548230219576:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.270401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471543999264007:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.270550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:37.270621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471543999264007:2050] [node 3] ICP22 created new session: [5:7577471548294231391:2050]
2025-11-27T18:19:37.270673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471548294231391:2050] [node 3] ICS09 handshake done sender: [5:7577471548294231384:2118] self: [5:7577471548294231385:0] peer: [3:7577471548832470513:0] socket: 208 qp: -1
2025-11-27T18:19:37.270714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471548294231391:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:37.270781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:37.270983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471547664036881:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:37.271430Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471548230219576:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.272065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471548265304316:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32807
2025-11-27T18:19:37.271784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471543935252206:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.271841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:37.272295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471543935252206:2049] [node 3] ICP22 created new session: [1:7577471548230219602:2049]
2025-11-27T18:19:37.272771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471543999264007:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:37.276439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471548230219602:2049] [node 3] ICS09 handshake done sender: [1:7577471548230219576:2114] self: [1:7577471548230219575:0] peer: [3:7577471548832470533:0] socket: 194 qp: -1
2025-11-27T18:19:37.276540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471548230219602:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:37.276601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:37.277776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471548294231390:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.277845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471543369069506:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.277956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:37.278005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471543369069506:2048] [node 1] ICP22 created new session: [6:7577471547664036891:2048]
2025-11-27T18:19:37.278091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471547664036891:2048] [node 1] ICS09 handshake done sender: [6:7577471547664036881:2116] self: [6:7577471547664036883:0] peer: [1:7577471548230219571:0] socket: 196 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.278124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471547664036891:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:37.278180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.280499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471548832470534:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5704
2025-11-27T18:19:37.278715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471543999264005:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.279025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471548294231370:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.277079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471548265304316:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.280867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471543970336928:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.280094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471547664036889:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23152
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.282845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471548265304296:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.282895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471548265304296:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.283222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471548832470535:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5704
2025-11-27T18:19:37.282860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471547664036890:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23152
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.284653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471548265304318:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32807
2025-11-27T18:19:37.284891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471548265304296:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.286378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471548265304318:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.286438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471543970336991:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.286503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:37.286555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471543970336991:2108] [node 5] ICP22 created new session: [4:7577471548265304320:2108]
2025-11-27T18:19:37.286602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471548265304320:2108] [node 5] ICS09 handshake done sender: [4:7577471548265304296:2114] self: [4:7577471548265304295:0] peer: [5:7577471548294231396:0] socket: 212 qp: -1
2025-11-27T18:19:37.288276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471548002714587:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.286640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471548265304320:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:37.288315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471548002714587:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.286676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:37.287781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471543970336991:2108] [node 5] ICP18 handshake (actor [4:7577471548265304318:2122]) is too old
2025-11-27T18:19:37.288588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471548294231397:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59739
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.289089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471548230219574:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.289122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471548230219574:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.292129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471548230219574:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:37.288766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471547664036889:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.289080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471547664036894:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23152
2025-11-27T18:19:37.291985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471547664036890:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.292924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471548294231370:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10112) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471548294231390:2121] held: no
2025-11-27T18:19:37.292366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471547664036894:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.293021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471548002714587:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:37.293976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471548294231390:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:37.295384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471548294231370:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.295434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:37.295466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471548294231370:2108] [node 4] ICP22 created new session: [5:7577471548294231399:2108]
2025-11-27T18:19:37.295497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471548294231399:2108] [node 4] ICS09 handshake done sender: [5:7577471548294231390:2121] self: [5:7577471548294231396:0] peer: [4:7577471548265304295:0] socket: 182 qp: -1
2025-11-27T18:19:37.295525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471548294231399:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:37.295561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:37.297295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471547664036869:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.297393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471543369069508:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.297396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471548832470535:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.297586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471548832470529:2126] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:37.297987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471544537503136:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21354) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471548832470529:2126] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.298735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471548294231398:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59739
2025-11-27T18:19:37.294275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471548230219566:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.294338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:37.298861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471548230219566:2109] [node 5] ICP22 created new session: [1:7577471548230219605:2109]
2025-11-27T18:19:37.299327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471543369069567:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.298906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471548230219605:2109] [node 5] ICS09 handshake done sender: [1:7577471548230219574:2113] self: [1:7577471548230219573:0] peer: [5:7577471548294231394:0] socket: 193 qp: -1
2025-11-27T18:19:37.298949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471548230219605:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:37.298990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:37.299588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471548265304322:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32807
2025-11-27T18:19:37.298872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471543707747276:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.298954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:37.299775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471544537503136:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.300286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:37.300342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471544537503136:2048] [node 1] ICP22 created new session: [3:7577471548832470537:2048]
2025-11-27T18:19:37.298994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471543707747276:2107] [node 4] ICP22 created new session: [2:7577471548002714608:2107]
2025-11-27T18:19:37.300325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471548265304298:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.299032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471548002714608:2107] [node 4] ICS09 handshake done sender: [2:7577471548002714587:2117] self: [2:7577471548002714586:0] peer: [4:7577471548265304317:0] socket: 171 qp: -1
2025-11-27T18:19:37.300385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471548832470537:2048] [node 1] ICS09 handshake done sender: [3:7577471548832470529:2126] self: [3:7577471548832470533:0] peer: [1:7577471548230219575:0] socket: 209 qp: -1
2025-11-27T18:19:37.300358Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471548265304298:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.299061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471548002714608:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:37.300430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471548832470537:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:37.302535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471548294231375:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.302571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471548294231375:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.299092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:37.300644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:37.306086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471548265304316:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:37.303842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471548832470516:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.303178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471548294231386:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:37.304412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471543999264005:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.303883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471548832470516:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.309634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471543970336928:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.309724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:37.309934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471543970336928:2049] [node 2] ICP22 created new session: [4:7577471548265304324:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.310494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471548832470536:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5704
2025-11-27T18:19:37.310795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471544537503137:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:37.309988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471548265304324:2049] [node 2] ICS09 handshake done sender: [4:7577471548265304316:2121] self: [4:7577471548265304317:0] peer: [2:7577471548002714586:0] socket: 216 qp: -1
2025-11-27T18:19:37.304474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:37.311162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471548832470516:2123] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:37.310035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471548265304324:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:37.304526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471543999264005:2048] [node 1] ICP22 created new session: [5:7577471548294231402:2048]
2025-11-27T18:19:37.304573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471548294231402:2048] [node 1] ICS09 handshake done sender: [5:7577471548294231386:2119] self: [5:7577471548294231394:0] peer: [1:7577471548230219573:0] socket: 156 qp: -1
2025-11-27T18:19:37.304598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471548294231402:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:37.304621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:37.312425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471548832470506:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.312476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:37.312533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471548832470506:2109] [node 6] ICP22 created new session: [3:7577471548832470541:2109]
2025-11-27T18:19:37.312596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471548832470541:2109] [node 6] ICS09 handshake done sender: [3:7577471548832470516:2123] self: [3:7577471548832470515:0] peer: [6:7577471547664036896:0] socket: 161 qp: -1
2025-11-27T18:19:37.312639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471548832470541:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:37.312668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:37.310066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:37.313413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471548294231375:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:37.314202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471548002714582:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:37.314245Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471548002714582:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:37.315333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471548294231371:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.316566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471548832470536:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.315508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:37.315561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471548294231371:2109] [node 6] ICP22 created new session: [5:7577471548294231404:2109]
2025-11-27T18:19:37.315612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471548294231404:2109] [node 6] ICS09 handshake done sender: [5:7577471548294231375:2114] self: [5:7577471548294231374:0] peer: [6:7577471547664036897:0] socket: 219 qp: -1
2025-11-27T18:19:37.315655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471548294231404:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:37.316706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:37.317175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471547664036898:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23152
2025-11-27T18:19:37.316914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471548832470506:2109] [node 6] ICP18 handshake (actor [3:7577471548832470536:2129]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.324937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471548832470543:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5704
2025-11-27T18:19:37.329473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471548002714582:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:37.333320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471548832470535:2128] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:37.333949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471544537503137:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.342318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471543707747214:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.334015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:37.342389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:37.334067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471544537503137:2049] [node 2] ICP22 created new session: [3:7577471548832470544:2049]
2025-11-27T18:19:37.342609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471543707747214:2049] [node 3] ICP22 created new session: [2:7577471548002714611:2049]
2025-11-27T18:19:37.334119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471548832470544:2049] [node 2] ICS09 handshake done sender: [3:7577471548832470535:2128] self: [3:7577471548832470540:0] peer: [2:7577471548002714581:0] socket: 215 qp: -1
2025-11-27T18:19:37.342677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471548002714611:2049] [node 3] ICS09 handshake done sender: [2:7577471548002714582:2113] self: [2:7577471548002714581:0] peer: [3:7577471548832470540:0] socket: 164 qp: -1
2025-11-27T18:19:37.342721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471548002714611:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:37.334161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471548832470544:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:37.342793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:37.334199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:37.343298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471543970336928:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:37.346328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471547664036899:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23152
2025-11-27T18:19:37.346444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471547664036900:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23152
2025-11-27T18:19:37.348990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471548265304298:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:37.350927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471548265304288:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:37.351032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:37.351094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471548265304288:2109] [node 6] ICP22 created new session: [4:7577471548265304326:2109]
2025-11-27T18:19:37.351153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471548265304326:2109] [node 6] ICS09 handshake done sender: [4:7577471548265304298:2115] self: [4:7577471548265304297:0] peer: [6:7577471547664036895:0] socket: 213 qp: -1
2025-11-27T18:19:37.351198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471548265304326:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:37.351235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:37.352285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471543369069508:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18454) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471547664036890:2121] held: no
2025-11-27T18:19:37.353094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471547664036894:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:37.353402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471543369069567:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23529) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471547664036894:2122] held: no
2025-11-27T18:19:37.353996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471543369069567:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.354072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:37.354122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471543369069567:2106] [node 5] ICP22 created new session: [6:7577471547664036901:2106]
2025-11-27T18:19:37.354163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471547664036901:2106] [node 5] ICS09 handshake done sender: [6:7577471547664036894:2122] self: [6:7577471547664036897:0] peer: [5:7577471548294231374:0] socket: 220 qp: -1
2025-11-27T18:19:37.354212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471547664036901:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:37.354243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:37.356787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471547664036890:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:37.357680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471547664036869:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10112) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471547664036889:2120] held: no
2025-11-27T18:19:37.358277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471543369069508:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.358341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:37.358397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471543369069508:2050] [node 3] ICP22 created new session: [6:7577471547664036903:2050]
2025-11-27T18:19:37.358457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471547664036903:2050] [node 3] ICS09 handshake done sender: [6:7577471547664036890:2121] self: [6:7577471547664036896:0] peer: [3:7577471548832470515:0] socket: 217 qp: -1
2025-11-27T18:19:37.358498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471547664036903:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:37.358532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:37.359446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471547664036889:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:37.363237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471547664036869:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:37.363312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:37.363366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471547664036869:2108] [node 4] ICP22 created new session: [6:7577471547664036905:2108]
2025-11-27T18:19:37.363423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471547664036905:2108] [node 4] ICS09 handshake done sender: [6:7577471547664036889:2120] self: [6:7577471547664036895:0] peer: [4:7577471548265304297:0] socket: 214 qp: -1
2025-11-27T18:19:37.363460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471547664036905:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:37.363495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:37.386505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:37.386724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471548002714599:2048] [node 1] ICS07 socket disconnect 189 reason# EndOfStream
2025-11-27T18:19:37.386759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471548002714599:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.386838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:37.386880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471548002714599:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:37.387682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:37.387881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471548832470537:2048] [node 1] ICS01 socket: 209 reason# EndOfStream
2025-11-27T18:19:37.387921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471544537503136:2048] [node 1] ICP30 unregister session Session# [3:7577471548832470537:2048] VirtualId# [3:7577471548832470533:0]
2025-11-27T18:19:37.387957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471548832470537:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.388028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:37.388069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:37.390131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471543707747213:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21354) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:37.390174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471548002714599:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:37.390991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471543707747213:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21354) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:37.391042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471548002714599:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:37.391078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471543707747213:2048] [node 1] ICP30 unregister session Session# [2:7577471548002714599:2048] VirtualId# [2:7577471548002714594:0]
2025-11-27T18:19:37.390662Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:37.391148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:37.390853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471548265304312:2048] [node 1] ICS01 socket: 187 reason# EndOfStream
2025-11-27T18:19:37.391192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471543707747213:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21354) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.390900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471543970336927:2048] [node 1] ICP30 unregister session Session# [4:7577471548265304312:2048] VirtualId# [4:7577471548265304307:0]
2025-11-27T18:19:37.390944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471548265304312:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.391216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21354) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.391022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:37.391071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:37.391901Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:37.392098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471547664036891:2048] [node 1] ICS01 socket: 196 reason# EndOfStream
2025-11-27T18:19:37.392142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471543369069506:2048] [node 1] ICP30 unregister session Session# [6:7577471547664036891:2048] VirtualId# [6:7577471547664036883:0]
2025-11-27T18:19:37.392170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471547664036891:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.392236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:37.392278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:37.392895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:37.393087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471548294231402:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-27T18:19:37.393125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471543999264005:2048] [node 1] ICP30 unregister session Session# [5:7577471548294231402:2048] VirtualId# [5:7577471548294231394:0]
2025-11-27T18:19:37.393150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471548294231402:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.393229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:37.393312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:37.400861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471543707747213:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:37.458037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:37.458209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471548832470544:2049] [node 2] ICS07 socket disconnect 215 reason# EndOfStream
2025-11-27T18:19:37.458232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471548832470544:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.458291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:37.458318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471548832470544:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:37.461523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471544537503137:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3307) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:37.461566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471548832470544:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:37.461214Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:37.461464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471548265304324:2049] [node 2] ICS07 socket disconnect 216 reason# EndOfStream
2025-11-27T18:19:37.461492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471548265304324:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.461567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:37.461608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471548265304324:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:37.462568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471544537503137:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3307) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:37.462611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471548832470544:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:37.462649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471544537503137:2049] [node 2] ICP30 unregister session Session# [3:7577471548832470544:2049] VirtualId# [3:7577471548832470540:0]
2025-11-27T18:19:37.462741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:37.462770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471544537503137:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3307) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.462783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3307) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.464893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471543970336928:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3307) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:37.464943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471548265304324:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:37.464978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471543970336928:2049] [node 2] ICP30 unregister session Session# [4:7577471548265304324:2049] VirtualId# [4:7577471548265304317:0]
2025-11-27T18:19:37.465062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:37.465091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471543970336928:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3307) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.465103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3307) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.463385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:37.464056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:37.469345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471548294231388:2049] [node 2] ICS01 socket: 191 reason# EndOfStream
2025-11-27T18:19:37.469470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471547664036887:2049] [node 2] ICS07 socket disconnect 197 reason# EndOfStream
2025-11-27T18:19:37.469510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471547664036887:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.470291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471543999264006:2049] [node 2] ICP30 unregister session Session# [5:7577471548294231388:2049] VirtualId# [5:7577471548294231382:0]
2025-11-27T18:19:37.469604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:37.470343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471548294231388:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.470419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:37.470469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:37.469648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471547664036887:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:37.471878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471543369069507:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3307) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:37.471985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471547664036887:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:37.472074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471543369069507:2049] [node 2] ICP30 unregister session Session# [6:7577471547664036887:2049] VirtualId# [6:7577471547664036884:0]
2025-11-27T18:19:37.472142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:37.472192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471543369069507:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3307) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.472218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3307) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.472642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471544537503137:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:37.474296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471543970336928:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:37.485084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471543369069507:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:37.505538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:37.505764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471548265304310:2050] [node 3] ICS07 socket disconnect 169 reason# EndOfStream
2025-11-27T18:19:37.505801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471548265304310:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.505874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:37.505912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471548265304310:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:37.507010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:37.507181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471547664036903:2050] [node 3] ICS01 socket: 217 reason# EndOfStream
2025-11-27T18:19:37.507218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471543369069508:2050] [node 3] ICP30 unregister session Session# [6:7577471547664036903:2050] VirtualId# [6:7577471547664036896:0]
2025-11-27T18:19:37.507252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471547664036903:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.507304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:37.507339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:37.506376Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:37.506546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471548294231391:2050] [node 3] ICS07 socket disconnect 208 reason# EndOfStream
2025-11-27T18:19:37.506569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471548294231391:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.506620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:37.506657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471548294231391:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:37.508793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471543999264007:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18454) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:37.508846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471548294231391:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:37.508876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471543999264007:2050] [node 3] ICP30 unregister session Session# [5:7577471548294231391:2050] VirtualId# [5:7577471548294231385:0]
2025-11-27T18:19:37.508967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:37.509009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471543999264007:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18454) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.509024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18454) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.509546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471543970336929:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18454) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:37.509595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471548265304310:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:37.509625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471543970336929:2050] [node 3] ICP30 unregister session Session# [4:7577471548265304310:2050] VirtualId# [4:7577471548265304305:0]
2025-11-27T18:19:37.509901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:37.509935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471543970336929:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18454) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.509946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18454) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:37.516436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471543999264007:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:37.520461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471543970336929:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:37.567658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:37.567905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471548294231399:2108] [node 4] ICS01 socket: 182 reason# EndOfStream
2025-11-27T18:19:37.567971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471548294231370:2108] [node 4] ICP30 unregister session Session# [5:7577471548294231399:2108] VirtualId# [5:7577471548294231396:0]
2025-11-27T18:19:37.568015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471548294231399:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.568099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:37.568153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:37.573664Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:37.574038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471547664036905:2108] [node 4] ICS01 socket: 214 reason# EndOfStream
2025-11-27T18:19:37.574086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471547664036869:2108] [node 4] ICP30 unregister session Session# [6:7577471547664036905:2108] VirtualId# [6:7577471547664036895:0]
2025-11-27T18:19:37.574122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471547664036905:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.574203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:37.574245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:37.605139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:37.605408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471547664036901:2106] [node 5] ICS01 socket: 220 reason# EndOfStream
2025-11-27T18:19:37.605474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471543369069567:2106] [node 5] ICP30 unregister session Session# [6:7577471547664036901:2106] VirtualId# [6:7577471547664036897:0]
2025-11-27T18:19:37.605543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471547664036901:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:37.605698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:37.605760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:37.703668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471548108556597:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:37.703724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471548108556598:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:37.725837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471545664412433:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:37.725883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471545664412434:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:37.746928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471544962800297:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:37.746956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471544962800298:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:37.767986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471546920076436:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:37.768058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471546920076438:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:37.768073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471546920076437:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:37.789232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471548435530091:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:37.789290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471548435530093:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:37.789371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471548435530092:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:37.813791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471544898791555:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:37.813835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471544898791556:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:37.814303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471544898791554:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:37.905784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471544898791615:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:37.907041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471548108556659:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:38.010154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471549959379819:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:38.014715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471549257767656:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:38.021478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471548435530091:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(78253383ns) {Configure:104(78247920ns) {ProcessConfigured:135(76333ns) {StartInitialHandshake:157(74274ns) {DropHandshakes:470(3015ns) {DropIncomingHandshake:442(761ns) DropOutgoingHandshake:457(483ns)} GenerateSessionVirtualId:988(850ns) PrepareNewSessionHandshake:477(2031ns) {SwitchToState:179(723ns)}}}}}
2025-11-27T18:19:38.021890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471552730497470:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:38.029413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471551215043813:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:38.049418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471549959379820:2114] [node 5] ICP01 ready to work
2025-11-27T18:19:38.034033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471549257767657:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:38.037830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471552730497473:2117] [node 6] ICP01 ready to work
2025-11-27T18:19:38.054116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471549959379821:2123] [node 6] ICP01 ready to work
2025-11-27T18:19:38.078445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471548108556597:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(147636855ns) {Configure:104(147627507ns) {ProcessConfigured:135(13633993ns) {StartInitialHandshake:157(13631449ns) {DropHandshakes:470(2970ns) {DropIncomingHandshake:442(708ns) DropOutgoingHandshake:457(471ns)} GenerateSessionVirtualId:988(723ns) PrepareNewSessionHandshake:477(4239ns) {SwitchToState:179(1927ns)}}}}}
2025-11-27T18:19:38.073518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471549257767658:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:38.078466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471551215043814:2117] [node 6] ICP01 ready to work
2025-11-27T18:19:38.094195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471544898791615:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(151100576ns) {Configure:104(151093474ns) {ProcessConfigured:135(16817456ns) {StartInitialHandshake:157(16814793ns) {DropHandshakes:470(1700ns) {DropIncomingHandshake:442(536ns) DropOutgoingHandshake:457(271ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:477(3513681ns) {SwitchToState:179(6677ns)}}}}}
2025-11-27T18:19:38.095648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471548435530092:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(57046221ns) {Configure:104(57038466ns) {ProcessConfigured:135(69280ns) {StartInitialHandshake:157(67510ns) {DropHandshakes:470(3422ns) {DropIncomingHandshake:442(585ns) DropOutgoingHandshake:457(586ns)} GenerateSessionVirtualId:988(962ns) PrepareNewSessionHandshake:477(1146ns) {SwitchToState:179(553ns)}}}}}
2025-11-27T18:19:38.111544Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471544898791554:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(154252356ns) {Configure:104(154248401ns) {ProcessConfigured:135(11374169ns) {StartInitialHandshake:157(11372655ns) {DropHandshakes:470(2130ns) {DropIncomingHandshake:442(495ns) DropOutgoingHandshake:457(255ns)} GenerateSessionVirtualId:988(984ns) PrepareNewSessionHandshake:477(2253ns) {SwitchToState:179(180ns)}}}}}
2025-11-27T18:19:38.111786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471548108556598:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(187402413ns) {Configure:104(187398571ns) {ProcessConfigured:135(11078699ns) {StartInitialHandshake:157(11076177ns) {DropHandshakes:470(3152ns) {DropIncomingHandshake:442(823ns) DropOutgoingHandshake:457(348ns)} GenerateSessionVirtualId:988(853ns) PrepareNewSessionHandshake:477(4181ns) {SwitchToState:179(1382ns)}}}}}
2025-11-27T18:19:38.116676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471546920076437:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(176864998ns) {Configure:104(176859560ns) {ProcessConfigured:135(2410753ns) {StartInitialHandshake:157(2408491ns) {DropHandshakes:470(2485ns) {DropIncomingHandshake:442(680ns) DropOutgoingHandshake:457(434ns)} GenerateSessionVirtualId:988(564ns) PrepareNewSessionHandshake:477(5694ns) {SwitchToState:179(1299ns)}}}}}
2025-11-27T18:19:38.114477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471552403523963:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.120611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471549959379830:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25143
2025-11-27T18:19:38.117325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471549193758918:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:38.130176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471546920076436:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35347020ns) {Configure:104(35340466ns) {ProcessConfigured:135(83342ns) {StartInitialHandshake:157(80727ns) {DropHandshakes:470(3106ns) {DropIncomingHandshake:442(559ns) DropOutgoingHandshake:457(446ns)} GenerateSessionVirtualId:988(601ns) PrepareNewSessionHandshake:477(2248ns) {SwitchToState:179(697ns)}}}}}
2025-11-27T18:19:38.134189Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471549959379820:2114] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22446661ns) {Configure:104(22441771ns) {ProcessConfigured:135(59756ns) {StartInitialHandshake:157(56953ns) {DropHandshakes:470(3517ns) {DropIncomingHandshake:442(737ns) DropOutgoingHandshake:457(562ns)} GenerateSessionVirtualId:988(551ns) PrepareNewSessionHandshake:477(1388ns) {SwitchToState:179(546ns)}}}}}
2025-11-27T18:19:38.128763Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471552730497473:2117] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(63849423ns) {Configure:104(63844286ns) {ProcessConfigured:135(1086480ns) {StartInitialHandshake:157(1084634ns) {DropHandshakes:470(2637ns) {DropIncomingHandshake:442(697ns) DropOutgoingHandshake:457(399ns)} GenerateSessionVirtualId:988(595ns) PrepareNewSessionHandshake:477(1714ns) {SwitchToState:179(754ns)}}}}}
2025-11-27T18:19:38.132078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471552403523964:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.153469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471552403523966:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60489
2025-11-27T18:19:38.153703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471552403523967:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60489
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.160981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471552730497483:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6503
2025-11-27T18:19:38.162138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471544962800297:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50444742ns) {Configure:104(50438225ns) {ProcessConfigured:135(51401ns) {StartInitialHandshake:157(49182ns) {DropHandshakes:470(2776ns) {DropIncomingHandshake:442(760ns) DropOutgoingHandshake:457(367ns)} GenerateSessionVirtualId:988(988ns) PrepareNewSessionHandshake:477(2301ns) {SwitchToState:179(836ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.175237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471549959379836:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25143
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.178852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471551215043813:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(37890766ns) {Configure:104(37885456ns) {ProcessConfigured:135(97516ns) {StartInitialHandshake:157(95616ns) {DropHandshakes:470(3233ns) {DropIncomingHandshake:442(696ns) DropOutgoingHandshake:457(722ns)} GenerateSessionVirtualId:988(440ns) PrepareNewSessionHandshake:477(37746ns) {SwitchToState:179(714ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.176124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471549257767659:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:872
2025-11-27T18:19:38.176300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471549257767668:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:872
2025-11-27T18:19:38.181809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471549959379837:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25143
2025-11-27T18:19:38.165794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471548108556659:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(69773502ns) {Configure:104(69764166ns) {ProcessConfigured:135(13834159ns) {StartInitialHandshake:157(13830818ns) {DropHandshakes:470(3000ns) {DropIncomingHandshake:442(705ns) DropOutgoingHandshake:457(419ns)} GenerateSessionVirtualId:988(802ns) PrepareNewSessionHandshake:477(2745ns) {SwitchToState:179(1273ns)}}}}}
2025-11-27T18:19:38.181962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471552403523966:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.201882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471551215043821:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-27T18:19:38.202170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471549959379830:2141] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.216030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471552403523967:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.223286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471549257767658:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(50185217ns) {Configure:104(50180488ns) {ProcessConfigured:135(72645ns) {StartInitialHandshake:157(70131ns) {DropHandshakes:470(3967ns) {DropIncomingHandshake:442(761ns) DropOutgoingHandshake:457(482ns)} GenerateSessionVirtualId:988(716ns) PrepareNewSessionHandshake:477(2546ns) {SwitchToState:179(899ns)}}}}}
2025-11-27T18:19:38.225853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471551215043822:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-27T18:19:38.226120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471551215043821:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.226259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471546920076437:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.234955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471549257767659:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.236942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471549959379827:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.236994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471549959379827:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.239266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471552730497483:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.248676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471551215043822:2132] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.247613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471549257767668:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.246720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471552403523976:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60489
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.245856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471549959379836:2143] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.246832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471548435530091:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.247236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471552403523969:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.259949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471549193758919:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62010
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.247261Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471552403523969:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.249532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471546920076438:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.247776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471544962800297:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-27T18:19:38.246975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471545664412433:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.259783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471552403523976:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.277104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471549193758918:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29857111ns) {Configure:104(29851185ns) {ProcessConfigured:135(6172805ns) {StartInitialHandshake:157(6169850ns) {DropHandshakes:470(3446ns) {DropIncomingHandshake:442(756ns) DropOutgoingHandshake:457(729ns)} GenerateSessionVirtualId:988(682ns) PrepareNewSessionHandshake:477(2180ns) {SwitchToState:179(1018ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.257538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471551215043831:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-27T18:19:38.255194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471549257767665:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.285605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471549193758926:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62010
2025-11-27T18:19:38.247157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471549959379837:2144] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.285518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471552730497486:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6503
2025-11-27T18:19:38.293580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471552730497485:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6503
2025-11-27T18:19:38.259888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471552403523960:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.255224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471549257767665:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.273362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471549257767665:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:38.293792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471549257767656:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.293851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:38.293904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471549257767656:2107] [node 4] ICP22 created new session: [3:7577471549257767678:2107]
2025-11-27T18:19:38.293955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471549257767678:2107] [node 4] ICS09 handshake done sender: [3:7577471549257767665:2113] self: [3:7577471549257767664:0] peer: [4:7577471551215043832:0] socket: 183 qp: -1
2025-11-27T18:19:38.293998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471549257767678:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:38.294034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:38.265455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471551215043821:2131] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:38.273958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471546920076437:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.274036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:38.274085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471546920076437:2049] [node 2] ICP22 created new session: [4:7577471551215043834:2049]
2025-11-27T18:19:38.274147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471551215043834:2049] [node 2] ICS09 handshake done sender: [4:7577471551215043821:2131] self: [4:7577471551215043830:0] peer: [2:7577471549959379826:0] socket: 184 qp: -1
2025-11-27T18:19:38.274188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471551215043834:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:38.274231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:38.259925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471552403523960:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.247564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471549959379838:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25143
2025-11-27T18:19:38.305963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471549193758919:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.307396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471544898791555:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.259927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471552403523962:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.248199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471549959379827:2139] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:38.353648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471552730497483:2133] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.357449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471551215043833:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-27T18:19:38.350125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471549193758931:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62010
2025-11-27T18:19:38.375408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471549257767677:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:872
2025-11-27T18:19:38.248762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471549959379819:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.259949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471552403523962:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.350651Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471549193758919:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.248809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:38.263159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471552403523978:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60489
2025-11-27T18:19:38.365618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471551215043822:2132] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:38.386674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471548435530091:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.410670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471549257767682:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:872
2025-11-27T18:19:38.386756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:38.253842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471549959379819:2107] [node 4] ICP22 created new session: [2:7577471549959379843:2107]
2025-11-27T18:19:38.264246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471552403523980:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60489
2025-11-27T18:19:38.270593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471552403523969:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:38.275555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471552403523962:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:38.345609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471552403523978:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.345752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471552403523963:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.345811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:38.345865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471552403523963:2109] [node 5] ICP22 created new session: [1:7577471552403523985:2109]
2025-11-27T18:19:38.345916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471552403523985:2109] [node 5] ICS09 handshake done sender: [1:7577471552403523969:2116] self: [1:7577471552403523968:0] peer: [5:7577471552730497484:0] socket: 176 qp: -1
2025-11-27T18:19:38.345958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471552403523985:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:38.346005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:38.348063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471548108556598:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.348142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:38.348192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471548108556598:2049] [node 3] ICP22 created new session: [1:7577471552403523987:2049]
2025-11-27T18:19:38.348235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471552403523987:2049] [node 3] ICS09 handshake done sender: [1:7577471552403523962:2113] self: [1:7577471552403523961:0] peer: [3:7577471549257767675:0] socket: 174 qp: -1
2025-11-27T18:19:38.348270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471552403523987:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:38.348307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:38.393605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471552403523980:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.393906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471552403523960:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:38.414011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471548108556597:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.414078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:38.414142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471548108556597:2048] [node 2] ICP22 created new session: [1:7577471552403523992:2048]
2025-11-27T18:19:38.414188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471552403523992:2048] [node 2] ICS09 handshake done sender: [1:7577471552403523960:2112] self: [1:7577471552403523959:0] peer: [2:7577471549959379841:0] socket: 173 qp: -1
2025-11-27T18:19:38.414226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471552403523992:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:38.414267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.415231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471549257767682:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.367023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471546920076438:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.350833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471544898791555:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.386819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471548435530091:2048] [node 1] ICP22 created new session: [5:7577471552730497488:2048]
2025-11-27T18:19:38.386867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471552730497488:2048] [node 1] ICS09 handshake done sender: [5:7577471552730497483:2133] self: [5:7577471552730497484:0] peer: [1:7577471552403523968:0] socket: 177 qp: -1
2025-11-27T18:19:38.386909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471552730497488:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:38.386948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:38.387811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471548435530091:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:38.389193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471552730497485:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.397614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471552730497487:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6503
2025-11-27T18:19:38.413655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471552730497487:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.413857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471552730497490:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6503
2025-11-27T18:19:38.414028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471548435530093:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.367084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:38.259930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471549959379843:2107] [node 4] ICS09 handshake done sender: [2:7577471549959379827:2139] self: [2:7577471549959379826:0] peer: [4:7577471551215043830:0] socket: 169 qp: -1
2025-11-27T18:19:38.350879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.367132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471546920076438:2050] [node 3] ICP22 created new session: [4:7577471551215043837:2050]
2025-11-27T18:19:38.367183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471551215043837:2050] [node 3] ICS09 handshake done sender: [4:7577471551215043822:2132] self: [4:7577471551215043832:0] peer: [3:7577471549257767664:0] socket: 185 qp: -1
2025-11-27T18:19:38.367212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471551215043837:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:38.367248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:38.426137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471551215043824:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.426175Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471551215043824:2133] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.449814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471551215043839:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-27T18:19:38.449887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471551215043840:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-27T18:19:38.450053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471551215043824:2133] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:38.415341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471549257767668:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:38.425894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471552730497493:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6503
2025-11-27T18:19:38.259986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471549959379843:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:38.415619Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471544962800297:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18924) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471549257767668:2115] held: no
2025-11-27T18:19:38.350920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471544898791555:2049] [node 2] ICP22 created new session: [6:7577471549193758932:2049]
2025-11-27T18:19:38.456295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471551215043814:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.260039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:38.416200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471544962800297:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.350963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471549193758932:2049] [node 2] ICS09 handshake done sender: [6:7577471549193758919:2112] self: [6:7577471549193758930:0] peer: [2:7577471549959379822:0] socket: 179 qp: -1
2025-11-27T18:19:38.260183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471549959379838:2145] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.317432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471549959379823:2137] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.317471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471549959379823:2137] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.318843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471549959379823:2137] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:38.342863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471549959379819:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(93191306ns) {HandleHandshakeStatus:329(93188067ns) {DropIncomingHandshake:442(577ns) DropOutgoingHandshake:457(588ns) SwitchToState:179(344ns) ProcessPendingSessionEvents:495(60475094ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60372940ns max# 17556970ns}}}
2025-11-27T18:19:38.343019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471549959379847:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25143
2025-11-27T18:19:38.344162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471549959379819:2107] [node 4] ICP18 handshake (actor [2:7577471549959379838:2145]) is too old
2025-11-27T18:19:38.358339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471549959379821:2123] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.365494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:38.369719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471549959379847:2146] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.370688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471549959379849:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25143
2025-11-27T18:19:38.464150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471552730497487:2136] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:38.373223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471549959379821:2123] [node 6] ICP22 created new session: [2:7577471549959379848:2123]
2025-11-27T18:19:38.456353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.385344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471549959379848:2123] [node 6] ICS09 handshake done sender: [2:7577471549959379823:2137] self: [2:7577471549959379822:0] peer: [6:7577471549193758930:0] socket: 167 qp: -1
2025-11-27T18:19:38.416246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:38.393243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471549959379848:2123] [node 6] ICS10 traffic start
2025-11-27T18:19:38.350992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471549193758932:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.393356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:38.462650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471551215043814:2117] [node 6] ICP22 created new session: [4:7577471551215043843:2117]
2025-11-27T18:19:38.493010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471548435530093:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.469437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471551215043843:2117] [node 6] ICS09 handshake done sender: [4:7577471551215043824:2133] self: [4:7577471551215043823:0] peer: [6:7577471549193758935:0] socket: 192 qp: -1
2025-11-27T18:19:38.493089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:38.351020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:38.361387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471549193758926:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.376077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471549193758918:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.433945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471549193758937:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62010
2025-11-27T18:19:38.442001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471549193758936:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62010
2025-11-27T18:19:38.449622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471549193758938:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62010
2025-11-27T18:19:38.455745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471549193758926:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:38.456566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471549193758918:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.463485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:38.464092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471549193758918:2110] [node 4] ICP22 created new session: [6:7577471549193758939:2110]
2025-11-27T18:19:38.489323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471549193758939:2110] [node 4] ICS09 handshake done sender: [6:7577471549193758926:2115] self: [6:7577471549193758935:0] peer: [4:7577471551215043823:0] socket: 196 qp: -1
2025-11-27T18:19:38.489393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471549193758939:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:38.489457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:38.515000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471552403523971:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.493138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471548435530093:2050] [node 3] ICP22 created new session: [5:7577471552730497496:2050]
2025-11-27T18:19:38.397960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471549959379836:2143] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:38.408824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471545664412433:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.414518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:38.423396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471545664412433:2048] [node 1] ICP22 created new session: [2:7577471549959379852:2048]
2025-11-27T18:19:38.437430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471549959379852:2048] [node 1] ICS09 handshake done sender: [2:7577471549959379836:2143] self: [2:7577471549959379841:0] peer: [1:7577471552403523959:0] socket: 180 qp: -1
2025-11-27T18:19:38.451197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471549959379821:2123] [node 6] ICP35 event processing took too much time PendingConnection:279(55550333ns) {HandleHandshakeStatus:329(55546845ns) {DropIncomingHandshake:442(504ns) DropOutgoingHandshake:457(754ns) SwitchToState:179(1008ns) ProcessPendingSessionEvents:495(20328603ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15607856ns max# 6914713ns}}}
2025-11-27T18:19:38.451620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471549959379852:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:38.515047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471552403523971:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.472981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471551215043843:2117] [node 6] ICS10 traffic start
2025-11-27T18:19:38.451705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.493188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471552730497496:2050] [node 3] ICS09 handshake done sender: [5:7577471552730497487:2136] self: [5:7577471552730497492:0] peer: [3:7577471549257767666:0] socket: 216 qp: -1
2025-11-27T18:19:38.416276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471544962800297:2048] [node 1] ICP22 created new session: [3:7577471549257767684:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.515777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471552403523975:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.490948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471549193758918:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(33528573ns) {HandleHandshakeStatus:329(33526836ns) {DropIncomingHandshake:442(570ns) DropOutgoingHandshake:457(5793ns) SwitchToState:179(527ns) ProcessPendingSessionEvents:495(489371ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 433836ns max# 66264ns}}}
2025-11-27T18:19:38.463805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471545664412433:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(49954844ns) {HandleHandshakeStatus:329(49949935ns) {DropIncomingHandshake:442(1344ns) DropOutgoingHandshake:457(12370ns) SwitchToState:179(136ns) ProcessPendingSessionEvents:495(562190ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 511035ns max# 78404ns}}}
2025-11-27T18:19:38.466573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471549959379821:2123] [node 6] ICP18 handshake (actor [2:7577471549959379847:2146]) is too old
2025-11-27T18:19:38.517282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471549959379835:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.517306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471549959379835:2142] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.416299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471549257767684:2048] [node 1] ICS09 handshake done sender: [3:7577471549257767668:2115] self: [3:7577471549257767675:0] peer: [1:7577471552403523961:0] socket: 178 qp: -1
2025-11-27T18:19:38.473037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:38.515797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471552403523975:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.517669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471552403523971:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:38.493223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471552730497496:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:38.504507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471549193758937:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.493278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:38.494277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471552730497495:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6503
2025-11-27T18:19:38.500604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471552730497490:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.505052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471548435530092:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.498820Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471551215043814:2117] [node 6] ICP35 event processing took too much time PendingConnection:279(41711445ns) {HandleHandshakeStatus:329(41708440ns) {DropIncomingHandshake:442(867ns) DropOutgoingHandshake:457(586ns) SwitchToState:179(892ns) ProcessPendingSessionEvents:495(569787ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 506661ns max# 71518ns}}}
2025-11-27T18:19:38.547101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471552403523975:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:38.548316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471552403523964:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.416332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471549257767684:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:38.547688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471549959379829:2140] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.505844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471544898791554:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.505503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471551215043839:2137] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.540120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471552730497495:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.416369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:38.547727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471549959379829:2140] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.548374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:38.506446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471546920076436:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.506056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471549193758941:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62010
2025-11-27T18:19:38.417393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471549257767667:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.525929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471549193758941:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.526264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471549193758943:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62010
2025-11-27T18:19:38.537382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471544898791615:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.546610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471549193758937:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:38.550878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471549193758936:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.549050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471552730497470:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.550931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471544898791554:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.550996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:38.554881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471544898791554:2048] [node 1] ICP22 created new session: [6:7577471549193758945:2048]
2025-11-27T18:19:38.557520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471549193758945:2048] [node 1] ICS09 handshake done sender: [6:7577471549193758937:2119] self: [6:7577471549193758942:0] peer: [1:7577471552403523970:0] socket: 225 qp: -1
2025-11-27T18:19:38.557556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471549193758945:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:38.557601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:38.568663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471544898791556:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.548439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471552403523964:2110] [node 6] ICP22 created new session: [1:7577471552403523997:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.417416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471549257767667:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.564149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471549959379835:2142] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:38.518808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471551215043840:2138] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.549881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471552730497482:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.548512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471552403523997:2110] [node 6] ICS09 handshake done sender: [1:7577471552403523971:2117] self: [1:7577471552403523970:0] peer: [6:7577471549193758942:0] socket: 224 qp: -1
2025-11-27T18:19:38.575269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471544898791615:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26393) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471549193758941:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.549909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471552730497482:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.422395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471549257767681:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:872
2025-11-27T18:19:38.597568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471549959379829:2140] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:38.522549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471551215043846:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-27T18:19:38.548549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471552403523997:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:38.556796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471552730497482:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:38.444519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471549257767683:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:872
2025-11-27T18:19:38.548584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:38.551795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471548108556659:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.551841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:38.551884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471548108556659:2107] [node 4] ICP22 created new session: [1:7577471552403523999:2107]
2025-11-27T18:19:38.551923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471552403523999:2107] [node 4] ICS09 handshake done sender: [1:7577471552403523975:2118] self: [1:7577471552403523965:0] peer: [4:7577471551215043845:0] socket: 181 qp: -1
2025-11-27T18:19:38.551958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471552403523999:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:38.551992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:38.561605Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471548108556597:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(71484086ns) {HandleHandshakeStatus:329(71480181ns) {DropIncomingHandshake:442(13976ns) DropOutgoingHandshake:457(745ns) SwitchToState:179(470ns) ProcessPendingSessionEvents:495(71078093ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 69797446ns max# 14845146ns}}}
2025-11-27T18:19:38.575930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471549193758947:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62010
2025-11-27T18:19:38.550513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471551215043839:2137] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:38.601659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471549959379820:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.444675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471549257767667:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:38.463995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471549257767656:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:38.481661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471549257767657:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.495333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:38.506397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471549257767681:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.506714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471549257767657:2108] [node 5] ICP22 created new session: [3:7577471549257767689:2108]
2025-11-27T18:19:38.521649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471549257767689:2108] [node 5] ICS09 handshake done sender: [3:7577471549257767667:2114] self: [3:7577471549257767666:0] peer: [5:7577471552730497492:0] socket: 211 qp: -1
2025-11-27T18:19:38.521700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471549257767689:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:38.522838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:38.539215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471544962800298:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:38.572974Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471549257767657:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(74671121ns) {HandleHandshakeStatus:329(74666184ns) {DropIncomingHandshake:442(25027ns) DropOutgoingHandshake:457(1005ns) SwitchToState:179(4635ns) ProcessPendingSessionEvents:495(8669708ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8614237ns max# 6935326ns}}}
2025-11-27T18:19:38.576257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471549257767673:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.576285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471549257767673:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.577279Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471549257767673:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:38.577652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471549257767658:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.577701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:38.577753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471549257767658:2109] [node 6] ICP22 created new session: [3:7577471549257767694:2109]
2025-11-27T18:19:38.577798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471549257767694:2109] [node 6] ICS09 handshake done sender: [3:7577471549257767673:2116] self: [3:7577471549257767672:0] peer: [6:7577471549193758948:0] socket: 218 qp: -1
2025-11-27T18:19:38.577830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471549257767694:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:38.577865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:38.595119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471549257767692:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:872
2025-11-27T18:19:38.604491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471549257767681:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:38.604645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471544962800298:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.604685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:38.604719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471544962800298:2049] [node 2] ICP22 created new session: [3:7577471549257767696:2049]
2025-11-27T18:19:38.604767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471549257767696:2049] [node 2] ICS09 handshake done sender: [3:7577471549257767681:2118] self: [3:7577471549257767691:0] peer: [2:7577471549959379828:0] socket: 217 qp: -1
2025-11-27T18:19:38.604798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471549257767696:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:38.604820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:38.553285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471546920076436:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.560777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:38.563951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471546920076436:2048] [node 1] ICP22 created new session: [4:7577471551215043848:2048]
2025-11-27T18:19:38.570240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471551215043820:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:38.570285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471551215043820:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:38.571368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471551215043848:2048] [node 1] ICS09 handshake done sender: [4:7577471551215043839:2137] self: [4:7577471551215043845:0] peer: [1:7577471552403523965:0] socket: 226 qp: -1
2025-11-27T18:19:38.571959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471551215043848:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:38.572013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:38.616321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471551215043820:2130] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:38.622987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471551215043813:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.623046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:38.623100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471551215043813:2108] [node 5] ICP22 created new session: [4:7577471551215043851:2108]
2025-11-27T18:19:38.623145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471551215043851:2108] [node 5] ICS09 handshake done sender: [4:7577471551215043820:2130] self: [4:7577471551215043819:0] peer: [5:7577471552730497500:0] socket: 230 qp: -1
2025-11-27T18:19:38.623198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471551215043851:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:38.623230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:38.601731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:38.601784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471549959379820:2114] [node 5] ICP22 created new session: [2:7577471549959379856:2114]
2025-11-27T18:19:38.601824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471549959379856:2114] [node 5] ICS09 handshake done sender: [2:7577471549959379835:2142] self: [2:7577471549959379834:0] peer: [5:7577471552730497499:0] socket: 220 qp: -1
2025-11-27T18:19:38.601851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471549959379856:2114] [node 5] ICS10 traffic start
2025-11-27T18:19:38.601875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:38.602808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471545664412434:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.602851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:38.602898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471545664412434:2049] [node 3] ICP22 created new session: [2:7577471549959379858:2049]
2025-11-27T18:19:38.602944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471549959379858:2049] [node 3] ICS09 handshake done sender: [2:7577471549959379829:2140] self: [2:7577471549959379828:0] peer: [3:7577471549257767691:0] socket: 213 qp: -1
2025-11-27T18:19:38.602970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471549959379858:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:38.602993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:38.557008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471552730497473:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:38.599398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471549193758941:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:38.599664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471544898791556:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26627) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471549193758936:2118] held: no
2025-11-27T18:19:38.600312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471544898791615:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.600356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:38.600397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471544898791615:2106] [node 5] ICP22 created new session: [6:7577471549193758950:2106]
2025-11-27T18:19:38.600448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471549193758950:2106] [node 5] ICS09 handshake done sender: [6:7577471549193758941:2121] self: [6:7577471549193758944:0] peer: [5:7577471552730497481:0] socket: 168 qp: -1
2025-11-27T18:19:38.600493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471549193758950:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:38.600536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:38.607076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471549193758949:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62010
2025-11-27T18:19:38.613267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471549193758936:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:38.613438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471544898791556:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.613482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:38.613527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471544898791556:2050] [node 3] ICP22 created new session: [6:7577471549193758952:2050]
2025-11-27T18:19:38.613574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471549193758952:2050] [node 3] ICS09 handshake done sender: [6:7577471549193758936:2118] self: [6:7577471549193758948:0] peer: [3:7577471549257767672:0] socket: 221 qp: -1
2025-11-27T18:19:38.613609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471549193758952:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:38.613638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:38.557067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:38.560927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471552730497473:2117] [node 6] ICP22 created new session: [5:7577471552730497503:2117]
2025-11-27T18:19:38.564529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471552730497503:2117] [node 6] ICS09 handshake done sender: [5:7577471552730497482:2132] self: [5:7577471552730497481:0] peer: [6:7577471549193758944:0] socket: 162 qp: -1
2025-11-27T18:19:38.564589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471552730497503:2117] [node 6] ICS10 traffic start
2025-11-27T18:19:38.564646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:38.594596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471552730497501:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6503
2025-11-27T18:19:38.615395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471552730497490:2137] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:38.634941Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471546920076436:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(82743765ns) {HandleHandshakeStatus:329(80764044ns) {DropIncomingHandshake:442(371ns) DropOutgoingHandshake:457(5669ns) SwitchToState:179(2492ns) ProcessPendingSessionEvents:495(36205393ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36100799ns max# 16757707ns}}}
2025-11-27T18:19:38.637014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471548435530092:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.637082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:38.637130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471548435530092:2049] [node 2] ICP22 created new session: [5:7577471552730497506:2049]
2025-11-27T18:19:38.637176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471552730497506:2049] [node 2] ICS09 handshake done sender: [5:7577471552730497490:2137] self: [5:7577471552730497499:0] peer: [2:7577471549959379834:0] socket: 163 qp: -1
2025-11-27T18:19:38.637216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471552730497506:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:38.637360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:38.638397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471552730497470:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18488) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471552730497495:2139] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:38.639133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471552730497505:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6503
2025-11-27T18:19:38.639866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471548435530092:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:38.642231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471552730497495:2139] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:38.642445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471552730497470:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:38.642489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:38.642535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471552730497470:2108] [node 4] ICP22 created new session: [5:7577471552730497508:2108]
2025-11-27T18:19:38.642583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471552730497508:2108] [node 4] ICS09 handshake done sender: [5:7577471552730497495:2139] self: [5:7577471552730497500:0] peer: [4:7577471551215043819:0] socket: 172 qp: -1
2025-11-27T18:19:38.642620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471552730497508:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:38.642651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:38.683153Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:38.683393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471549959379852:2048] [node 1] ICS01 socket: 180 reason# EndOfStream
2025-11-27T18:19:38.683435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471545664412433:2048] [node 1] ICP30 unregister session Session# [2:7577471549959379852:2048] VirtualId# [2:7577471549959379841:0]
2025-11-27T18:19:38.683471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471549959379852:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.683571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:38.683620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:38.685814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:38.686157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471549257767684:2048] [node 1] ICS01 socket: 178 reason# EndOfStream
2025-11-27T18:19:38.686203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471544962800297:2048] [node 1] ICP30 unregister session Session# [3:7577471549257767684:2048] VirtualId# [3:7577471549257767675:0]
2025-11-27T18:19:38.686242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471549257767684:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.686343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:38.686401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:38.688847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:38.689070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471551215043848:2048] [node 1] ICS07 socket disconnect 226 reason# EndOfStream
2025-11-27T18:19:38.689100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471551215043848:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.689170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:38.689211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471551215043848:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:38.693360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471546920076436:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18924) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:38.693461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471551215043848:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:38.693510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471546920076436:2048] [node 1] ICP30 unregister session Session# [4:7577471551215043848:2048] VirtualId# [4:7577471551215043845:0]
2025-11-27T18:19:38.693639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:38.693677Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471546920076436:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18924) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:38.693703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18924) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:38.696249Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:38.696592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471552730497488:2048] [node 1] ICS01 socket: 177 reason# EndOfStream
2025-11-27T18:19:38.696642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471548435530091:2048] [node 1] ICP30 unregister session Session# [5:7577471552730497488:2048] VirtualId# [5:7577471552730497484:0]
2025-11-27T18:19:38.696682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471552730497488:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.696768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:38.696836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:38.696559Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:38.696781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471549193758945:2048] [node 1] ICS01 socket: 225 reason# EndOfStream
2025-11-27T18:19:38.696829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471544898791554:2048] [node 1] ICP30 unregister session Session# [6:7577471549193758945:2048] VirtualId# [6:7577471549193758942:0]
2025-11-27T18:19:38.696859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471549193758945:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.696930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:38.696965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:38.703295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471546920076436:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:38.787602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:38.787824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471549257767696:2049] [node 2] ICS01 socket: 217 reason# EndOfStream
2025-11-27T18:19:38.787862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471544962800298:2049] [node 2] ICP30 unregister session Session# [3:7577471549257767696:2049] VirtualId# [3:7577471549257767691:0]
2025-11-27T18:19:38.787894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471549257767696:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.787967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:38.788018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:38.790643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:38.790864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471551215043834:2049] [node 2] ICS07 socket disconnect 184 reason# EndOfStream
2025-11-27T18:19:38.790901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471551215043834:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.790965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:38.791009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471551215043834:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:38.791938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471546920076437:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14178) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:38.792004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471551215043834:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:38.792039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471546920076437:2049] [node 2] ICP30 unregister session Session# [4:7577471551215043834:2049] VirtualId# [4:7577471551215043830:0]
2025-11-27T18:19:38.792079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:38.792128Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471546920076437:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14178) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:38.792177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14178) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:38.796905Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:38.797251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471549193758932:2049] [node 2] ICS01 socket: 179 reason# EndOfStream
2025-11-27T18:19:38.797351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471544898791555:2049] [node 2] ICP30 unregister session Session# [6:7577471549193758932:2049] VirtualId# [6:7577471549193758930:0]
2025-11-27T18:19:38.797391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471549193758932:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.797481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:38.797537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:38.797406Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:38.797606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471552730497506:2049] [node 2] ICS07 socket disconnect 163 reason# EndOfStream
2025-11-27T18:19:38.797635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471552730497506:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.802684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471546920076437:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:38.797693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:38.797734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471552730497506:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:38.809522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471548435530092:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14178) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:38.809605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471552730497506:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:38.809641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471548435530092:2049] [node 2] ICP30 unregister session Session# [5:7577471552730497506:2049] VirtualId# [5:7577471552730497499:0]
2025-11-27T18:19:38.809767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:38.809837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471548435530092:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14178) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:38.809853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14178) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:38.819914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471548435530092:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:38.833339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:38.833531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471549193758952:2050] [node 3] ICS01 socket: 221 reason# EndOfStream
2025-11-27T18:19:38.833579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471544898791556:2050] [node 3] ICP30 unregister session Session# [6:7577471549193758952:2050] VirtualId# [6:7577471549193758948:0]
2025-11-27T18:19:38.833615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471549193758952:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.833670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:38.833701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:38.833285Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:38.833531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471551215043837:2050] [node 3] ICS07 socket disconnect 185 reason# EndOfStream
2025-11-27T18:19:38.833561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471551215043837:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.833633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:38.833670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471551215043837:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:38.834197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:38.834373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471552730497496:2050] [node 3] ICS01 socket: 216 reason# EndOfStream
2025-11-27T18:19:38.834406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471548435530093:2050] [node 3] ICP30 unregister session Session# [5:7577471552730497496:2050] VirtualId# [5:7577471552730497492:0]
2025-11-27T18:19:38.834433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471552730497496:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.834493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:38.834530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:38.835556Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471546920076438:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26627) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:38.835617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471551215043837:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:38.835651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471546920076438:2050] [node 3] ICP30 unregister session Session# [4:7577471551215043837:2050] VirtualId# [4:7577471551215043832:0]
2025-11-27T18:19:38.835751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:38.835783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471546920076438:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26627) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:38.835810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26627) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:38.845491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471546920076438:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:38.920625Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:38.920879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471552730497508:2108] [node 4] ICS01 socket: 172 reason# EndOfStream
2025-11-27T18:19:38.920926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471552730497470:2108] [node 4] ICP30 unregister session Session# [5:7577471552730497508:2108] VirtualId# [5:7577471552730497500:0]
2025-11-27T18:19:38.920963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471552730497508:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.921044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:38.921090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:38.921189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:38.921380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471549193758939:2110] [node 4] ICS01 socket: 196 reason# EndOfStream
2025-11-27T18:19:38.921412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471549193758918:2110] [node 4] ICP30 unregister session Session# [6:7577471549193758939:2110] VirtualId# [6:7577471549193758935:0]
2025-11-27T18:19:38.921442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471549193758939:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:38.921502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:38.921538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:39.017791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:39.018074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471549193758950:2106] [node 5] ICS01 socket: 168 reason# EndOfStream
2025-11-27T18:19:39.018119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471544898791615:2106] [node 5] ICP30 unregister session Session# [6:7577471549193758950:2106] VirtualId# [6:7577471549193758944:0]
2025-11-27T18:19:39.018157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471549193758950:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:39.018233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:39.018274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:39.144781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471557220859218:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:39.145401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471557220859219:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:39.166707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471556443906558:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:39.166751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471556443906559:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:39.195786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471557262557364:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:39.195819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471557262557365:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:39.217639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471555481258015:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:39.217685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471555481258016:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:39.217717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471555481258017:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:39.243408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471556001403529:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:39.243458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471556001403530:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:39.243491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471556001403531:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:39.279294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471555030009177:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:39.279346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471555030009178:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:39.279432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471555030009179:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:39.681808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471555030009238:2105] [node 5] ICP01 ready to work
2025-11-27T18:19:39.690231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471557220859279:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:39.694121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471556001403590:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50460
2025-11-27T18:19:39.707092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471557220859219:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18957047ns) {Configure:104(18952065ns) {ProcessConfigured:135(81222ns) {StartInitialHandshake:157(77992ns) {DropHandshakes:470(5603ns) {DropIncomingHandshake:442(776ns) DropOutgoingHandshake:457(337ns)} GenerateSessionVirtualId:988(1049ns) PrepareNewSessionHandshake:477(1569ns) {SwitchToState:179(527ns)}}}}}
2025-11-27T18:19:39.749887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471557220859218:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(25226127ns) {Configure:104(25220463ns) {ProcessConfigured:135(88667ns) {StartInitialHandshake:157(86576ns) {DropHandshakes:470(3312ns) {DropIncomingHandshake:442(668ns) DropOutgoingHandshake:457(376ns)} GenerateSessionVirtualId:988(856ns) PrepareNewSessionHandshake:477(2117ns) {SwitchToState:179(871ns)}}}}}
2025-11-27T18:19:39.721399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471556001403590:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:39.857310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471556443906633:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:39.857552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471557220859287:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:39.937447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471556443906640:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:39.965026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471556001403602:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:39.965649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471555030009244:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:39.974826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471556443906641:2113] [node 6] ICP01 ready to work
2025-11-27T18:19:39.961746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471557262557445:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:39.945693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471557220859279:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(172144025ns) {Configure:104(172139358ns) {ProcessConfigured:135(136880609ns) {StartInitialHandshake:157(136876701ns) {DropHandshakes:470(3277ns) {DropIncomingHandshake:442(764ns) DropOutgoingHandshake:457(468ns)} GenerateSessionVirtualId:988(828ns) PrepareNewSessionHandshake:477(6374ns) {SwitchToState:179(3626ns)}}}}}
2025-11-27T18:19:39.989165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471557262557446:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:39.962325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471557220859294:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:40.005780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471557262557447:2114] [node 6] ICP01 ready to work
2025-11-27T18:19:39.985411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471556001403603:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:40.013761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471556001403529:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(254821540ns) {Configure:104(254816301ns) {ProcessConfigured:135(59333ns) {StartInitialHandshake:157(56772ns) {DropHandshakes:470(3670ns) {DropIncomingHandshake:442(790ns) DropOutgoingHandshake:457(793ns)} GenerateSessionVirtualId:988(944ns) PrepareNewSessionHandshake:477(1333ns) {SwitchToState:179(658ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.053238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471561557524744:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13593
2025-11-27T18:19:40.052957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471555481258016:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(321662192ns) {Configure:104(321647903ns) {ProcessConfigured:135(8961517ns) {StartInitialHandshake:157(8959215ns) {DropHandshakes:470(2111ns) {DropIncomingHandshake:442(478ns) DropOutgoingHandshake:457(450ns)} GenerateSessionVirtualId:988(525ns) PrepareNewSessionHandshake:477(4219ns) {SwitchToState:179(2394ns)}}}}}
2025-11-27T18:19:40.053452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471559776225380:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.064187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471561557524753:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13593
2025-11-27T18:19:40.055690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471556443906641:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(70852815ns) {Configure:104(70848087ns) {ProcessConfigured:135(51994ns) {StartInitialHandshake:157(50606ns) {DropHandshakes:470(1437ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(703ns) PrepareNewSessionHandshake:477(829ns) {SwitchToState:179(307ns)}}}}}
2025-11-27T18:19:40.061201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471555481258017:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(263575489ns) {Configure:104(263569870ns) {ProcessConfigured:135(24226903ns) {StartInitialHandshake:157(24225611ns) {DropHandshakes:470(1796ns) {DropIncomingHandshake:442(505ns) DropOutgoingHandshake:457(442ns)} GenerateSessionVirtualId:988(320ns) PrepareNewSessionHandshake:477(2766ns) {SwitchToState:179(1566ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.084753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471555481258015:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(312750071ns) {Configure:104(312744841ns) {ProcessConfigured:135(32823611ns) {StartInitialHandshake:157(32821490ns) {DropHandshakes:470(8279570ns) {DropIncomingHandshake:442(2720ns) DropOutgoingHandshake:457(425ns)} GenerateSessionVirtualId:988(3144ns) PrepareNewSessionHandshake:477(3774ns) {SwitchToState:179(2096ns)}}}}}
2025-11-27T18:19:40.088418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471557262557445:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(47672007ns) {Configure:104(47666418ns) {ProcessConfigured:135(13678557ns) {StartInitialHandshake:157(13674556ns) {DropHandshakes:470(1410ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(561ns) PrepareNewSessionHandshake:477(3770ns) {SwitchToState:179(1648ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.102362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471560296370911:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50460
2025-11-27T18:19:40.088010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471559776225386:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:40.105297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471561515826594:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
2025-11-27T18:19:40.096251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471561557524744:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.116164Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471556001403602:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(84395468ns) {Configure:104(84393046ns) {ProcessConfigured:135(14424409ns) {StartInitialHandshake:157(10460269ns) {DropHandshakes:470(3933ns) {DropIncomingHandshake:442(1059ns) DropOutgoingHandshake:457(607ns)} GenerateSessionVirtualId:988(6445ns) PrepareNewSessionHandshake:477(4101ns) {SwitchToState:179(1761ns)}} IncomingHandshake:261(3920822ns) {DropIncomingHandshake:442(762ns) PrepareNewSessionHandshake:477(380ns) {SwitchToState:179(132ns)} GenerateSessionVirtualId:988(911ns) IssueIncomingHandshakeReply:193(3500ns)}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.128390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471555030009245:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61967
2025-11-27T18:19:40.150716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471555030009245:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.151058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471555030009177:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.161153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471559776225387:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-11-27T18:19:40.161734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471559776225390:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-11-27T18:19:40.129664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471561515826593:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
2025-11-27T18:19:40.151634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471561557524753:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.162197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471559776225390:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.195402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471560296370911:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.169512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471557220859288:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.181656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471556443906559:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(162711706ns) {Configure:104(162704728ns) {ProcessConfigured:135(9481205ns) {StartInitialHandshake:157(9323828ns) {DropHandshakes:470(4237ns) {DropIncomingHandshake:442(1045ns) DropOutgoingHandshake:457(715ns)} GenerateSessionVirtualId:988(1670ns) PrepareNewSessionHandshake:477(6134ns) {SwitchToState:179(2359ns)}}}}}
2025-11-27T18:19:40.169741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471559776225387:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.157675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471557262557364:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.169559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471557220859288:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.196232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471559324976551:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61967
2025-11-27T18:19:40.157820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471561557524759:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13593
2025-11-27T18:19:40.226074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471555030009178:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(97858130ns) {Configure:104(97848822ns) {ProcessConfigured:135(61070ns) {StartInitialHandshake:157(56689ns) {DropHandshakes:470(2292ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(474ns)} GenerateSessionVirtualId:988(549ns) PrepareNewSessionHandshake:477(1356ns) {SwitchToState:179(368ns)}}}}}
2025-11-27T18:19:40.219430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471556443906642:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21581
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.220273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471556001403531:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.187239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471557220859283:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.224483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471560738873948:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21581
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.224220Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471556001403531:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(22843308ns) {IncomingHandshake:261(22840796ns) {DropIncomingHandshake:442(687ns) PrepareNewSessionHandshake:477(2256ns) {SwitchToState:179(873ns)} GenerateSessionVirtualId:988(1210ns) IssueIncomingHandshakeReply:193(22810297ns) {SwitchToState:179(4317ns)}}}
2025-11-27T18:19:40.165723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471561557524759:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.187285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471557220859283:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.233382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471555030009245:2111] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:40.228452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471560738873947:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21581
2025-11-27T18:19:40.228714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471556443906642:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.231128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471556443906558:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.233031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471560738873948:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.233344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471560738873949:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21581
2025-11-27T18:19:40.242052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471560738873947:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.233537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471559776225393:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-11-27T18:19:40.224096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471561557524750:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.253921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471559776225393:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.269362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471560738873949:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.189346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471561515826593:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.241778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471559324976554:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61967
2025-11-27T18:19:40.268417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471560296370918:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50460
2025-11-27T18:19:40.195825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471557220859288:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:40.224125Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471561557524750:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.254053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471555481258015:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.269899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471560738873958:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21581
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.254338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471555030009177:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.206093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471561515826596:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
2025-11-27T18:19:40.317919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471556001403531:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6453) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471560296370911:2124] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.233977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471561557524764:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13593
2025-11-27T18:19:40.254394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:40.258998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471555030009177:2048] [node 1] ICP22 created new session: [6:7577471559324976555:2048]
2025-11-27T18:19:40.281453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471559324976555:2048] [node 1] ICS09 handshake done sender: [6:7577471555030009245:2111] self: [6:7577471559324976550:0] peer: [1:7577471557220859286:0] socket: 174 qp: -1
2025-11-27T18:19:40.281521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471559324976555:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:40.281573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:40.283310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471555030009177:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(39611326ns) {HandleHandshakeStatus:329(35305571ns) {DropIncomingHandshake:442(2140ns) DropOutgoingHandshake:457(10513ns) SwitchToState:179(116ns) ProcessPendingSessionEvents:495(635994ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 581816ns max# 102537ns}}}
2025-11-27T18:19:40.321879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471559324976558:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61967
2025-11-27T18:19:40.270079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471560738873959:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21581
2025-11-27T18:19:40.326442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471560296370916:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50460
2025-11-27T18:19:40.329822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471559776225395:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-11-27T18:19:40.221624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471557220859279:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.235228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471561557524744:2128] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:40.236604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471557262557364:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.341646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471560296370916:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.341802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471556001403530:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.335290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471559324976554:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.221776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:40.278268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471556443906642:2128] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:40.369401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471560296370917:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50460
2025-11-27T18:19:40.352712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471555030009178:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.236661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:40.349463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471559776225397:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-11-27T18:19:40.222493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471561515826594:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.380880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471555481258015:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471559776225393:2117] held: no
2025-11-27T18:19:40.278441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471556443906558:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.369953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471559324976558:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.236718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471557262557364:2048] [node 1] ICP22 created new session: [3:7577471561557524765:2048]
2025-11-27T18:19:40.381776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471560296370918:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.227415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471557220859283:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:40.370118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471555030009179:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.278504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:40.236792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471561557524765:2048] [node 1] ICS09 handshake done sender: [3:7577471561557524744:2128] self: [3:7577471561557524760:0] peer: [1:7577471557220859282:0] socket: 177 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.236846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471561557524765:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:40.227672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471557220859279:2107] [node 6] ICP22 created new session: [1:7577471561515826598:2107]
2025-11-27T18:19:40.278559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471556443906558:2048] [node 1] ICP22 created new session: [2:7577471560738873961:2048]
2025-11-27T18:19:40.381865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471560296370911:2124] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.381933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471556001403529:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.400941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471559776225393:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:40.229586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471561515826598:2107] [node 6] ICS09 handshake done sender: [1:7577471557220859288:2112] self: [1:7577471557220859286:0] peer: [6:7577471559324976550:0] socket: 173 qp: -1
2025-11-27T18:19:40.236888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:40.278604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471560738873961:2048] [node 1] ICS09 handshake done sender: [2:7577471556443906642:2128] self: [2:7577471560738873955:0] peer: [1:7577471557220859284:0] socket: 175 qp: -1
2025-11-27T18:19:40.404730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471559776225396:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-11-27T18:19:40.230477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471561515826598:2107] [node 6] ICS10 traffic start
2025-11-27T18:19:40.403923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471559324976561:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61967
2025-11-27T18:19:40.246492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471561557524762:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13593
2025-11-27T18:19:40.278640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471560738873961:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:40.230524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:40.246898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471561557524750:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:40.406225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471559776225396:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.395615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471560296370922:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50460
2025-11-27T18:19:40.278671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:40.406122Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471559324976554:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:40.406412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471555481258015:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.250846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471557220859279:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(28029009ns) {HandleHandshakeStatus:329(28025796ns) {DropIncomingHandshake:442(689ns) DropOutgoingHandshake:457(977ns) SwitchToState:179(456ns) ProcessPendingSessionEvents:495(4099605ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4027039ns max# 3353196ns}}}
2025-11-27T18:19:40.284351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471561557524762:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.396821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471556001403531:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.330076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471560738873959:2138] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.250906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471557220859219:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.251073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:40.251126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471557220859219:2049] [node 3] ICP22 created new session: [1:7577471561515826603:2049]
2025-11-27T18:19:40.251164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471561515826603:2049] [node 3] ICS09 handshake done sender: [1:7577471557220859283:2110] self: [1:7577471557220859282:0] peer: [3:7577471561557524760:0] socket: 171 qp: -1
2025-11-27T18:19:40.251202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471561515826603:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:40.251244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:40.254507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471557220859285:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.254542Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471557220859285:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.265865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471557220859290:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.265895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471557220859290:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.273840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471557220859285:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:40.289522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471557220859290:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:40.326891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471557220859218:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.326949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:40.327002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471557220859218:2048] [node 2] ICP22 created new session: [1:7577471561515826608:2048]
2025-11-27T18:19:40.327049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471561515826608:2048] [node 2] ICS09 handshake done sender: [1:7577471557220859285:2111] self: [1:7577471557220859284:0] peer: [2:7577471560738873955:0] socket: 172 qp: -1
2025-11-27T18:19:40.327086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471561515826608:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:40.327123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:40.327963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471557220859218:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:40.329594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471557220859287:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.329693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:40.329737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471557220859287:2108] [node 4] ICP22 created new session: [1:7577471561515826610:2108]
2025-11-27T18:19:40.330134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471561515826610:2108] [node 4] ICS09 handshake done sender: [1:7577471557220859290:2113] self: [1:7577471557220859289:0] peer: [4:7577471559776225394:0] socket: 210 qp: -1
2025-11-27T18:19:40.332930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471561515826610:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:40.335514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:40.385676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471561515826612:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
2025-11-27T18:19:40.387405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471557220859296:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.387435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471557220859296:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.391043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471557220859296:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:40.392925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471561515826612:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.397661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471557220859294:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.397696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:40.397738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471557220859294:2109] [node 5] ICP22 created new session: [1:7577471561515826614:2109]
2025-11-27T18:19:40.397780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471561515826614:2109] [node 5] ICS09 handshake done sender: [1:7577471557220859296:2114] self: [1:7577471557220859295:0] peer: [5:7577471560296370921:0] socket: 219 qp: -1
2025-11-27T18:19:40.397815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471561515826614:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:40.397846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:40.400199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471557220859294:2109] [node 5] ICP18 handshake (actor [1:7577471561515826612:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.410689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471555030009179:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6453) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471559324976558:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.396871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:40.284411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471557262557446:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.409148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:40.411299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471555030009178:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.386625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471556443906646:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.409844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471555481258015:2048] [node 1] ICP22 created new session: [4:7577471559776225399:2048]
2025-11-27T18:19:40.284486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:40.399438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471556001403531:2050] [node 3] ICP22 created new session: [5:7577471560296370924:2050]
2025-11-27T18:19:40.411365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:40.386671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471556443906646:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.412708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471559776225397:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.411419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471555030009178:2049] [node 2] ICP22 created new session: [6:7577471559324976563:2049]
2025-11-27T18:19:40.411463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471559324976563:2049] [node 2] ICS09 handshake done sender: [6:7577471559324976554:2117] self: [6:7577471559324976559:0] peer: [2:7577471560738873943:0] socket: 213 qp: -1
2025-11-27T18:19:40.411498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471559324976563:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:40.411531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:40.423429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471559324976562:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61967
2025-11-27T18:19:40.432291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471559324976558:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:40.437540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471555030009179:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.437571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:40.437620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471555030009179:2050] [node 3] ICP22 created new session: [6:7577471559324976566:2050]
2025-11-27T18:19:40.437664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471559324976566:2050] [node 3] ICS09 handshake done sender: [6:7577471559324976558:2118] self: [6:7577471559324976560:0] peer: [3:7577471561557524747:0] socket: 181 qp: -1
2025-11-27T18:19:40.437690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471559324976566:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:40.437717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:40.454237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471559324976565:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61967
2025-11-27T18:19:40.465519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471559324976565:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.473343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471555030009238:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.400877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471560296370923:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50460
2025-11-27T18:19:40.284538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471557262557446:2108] [node 5] ICP22 created new session: [3:7577471561557524768:2108]
2025-11-27T18:19:40.404707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471560296370924:2050] [node 3] ICS09 handshake done sender: [5:7577471560296370911:2124] self: [5:7577471560296370915:0] peer: [3:7577471561557524749:0] socket: 186 qp: -1
2025-11-27T18:19:40.405827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471560296370924:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:40.406194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:40.417019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471556001403530:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19796) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471560296370916:2126] held: no
2025-11-27T18:19:40.418000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471560296370918:2128] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:40.425573Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471556001403529:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471560296370918:2128] held: no
2025-11-27T18:19:40.433749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471560296370916:2126] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:40.434708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471560296370926:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50460
2025-11-27T18:19:40.436652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471560296370926:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.436884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471556001403529:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.436921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:40.436956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471556001403529:2048] [node 1] ICP22 created new session: [5:7577471560296370928:2048]
2025-11-27T18:19:40.436990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471560296370928:2048] [node 1] ICS09 handshake done sender: [5:7577471560296370918:2128] self: [5:7577471560296370921:0] peer: [1:7577471557220859295:0] socket: 222 qp: -1
2025-11-27T18:19:40.437016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471560296370928:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:40.437046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:40.438012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471556001403530:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.438044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:40.438083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471556001403530:2049] [node 2] ICP22 created new session: [5:7577471560296370930:2049]
2025-11-27T18:19:40.438146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471560296370930:2049] [node 2] ICS09 handshake done sender: [5:7577471560296370916:2126] self: [5:7577471560296370920:0] peer: [2:7577471556443906645:0] socket: 208 qp: -1
2025-11-27T18:19:40.438173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471560296370930:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:40.438213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:40.441108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471556001403603:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.476342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471560296370913:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.476374Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471560296370913:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.477225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471556001403529:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23051734ns) {HandleHandshakeStatus:329(23049269ns) {DropIncomingHandshake:442(1232ns) DropOutgoingHandshake:457(248ns) SwitchToState:179(276ns) ProcessPendingSessionEvents:495(1180ns)}}
2025-11-27T18:19:40.389803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471560738873944:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.284590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471561557524768:2108] [node 5] ICS09 handshake done sender: [3:7577471561557524750:2131] self: [3:7577471561557524749:0] peer: [5:7577471560296370915:0] socket: 183 qp: -1
2025-11-27T18:19:40.412839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471555481258017:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.389821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471560738873944:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.284623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471561557524768:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:40.394008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471556443906646:2130] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:40.284660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:40.394234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471556443906633:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.298625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471557262557364:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(61118950ns) {HandleHandshakeStatus:329(61115739ns) {DropIncomingHandshake:442(911ns) DropOutgoingHandshake:457(5918ns) SwitchToState:179(832ns) ProcessPendingSessionEvents:495(60673296ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 56258430ns max# 16144734ns}}}
2025-11-27T18:19:40.394290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:40.392931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471561557524748:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.491114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471560296370913:2125] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:40.501413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471556001403602:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.413080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471555481258016:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.394324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471556443906633:2107] [node 5] ICP22 created new session: [2:7577471560738873967:2107]
2025-11-27T18:19:40.501505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:40.392969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471561557524748:2130] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.415238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471559776225399:2048] [node 1] ICS09 handshake done sender: [4:7577471559776225393:2117] self: [4:7577471559776225394:0] peer: [1:7577471557220859289:0] socket: 211 qp: -1
2025-11-27T18:19:40.394356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471560738873967:2107] [node 5] ICS09 handshake done sender: [2:7577471556443906646:2130] self: [2:7577471556443906645:0] peer: [5:7577471560296370920:0] socket: 207 qp: -1
2025-11-27T18:19:40.405495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471561557524748:2130] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.394382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471560738873967:2107] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.504615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471556001403602:2109] [node 6] ICP22 created new session: [5:7577471560296370934:2109]
2025-11-27T18:19:40.405700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471557262557447:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.418255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471559776225399:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:40.406195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:40.421338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.394406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:40.509723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471560296370934:2109] [node 6] ICS09 handshake done sender: [5:7577471560296370913:2125] self: [5:7577471560296370912:0] peer: [6:7577471559324976568:0] socket: 226 qp: -1
2025-11-27T18:19:40.429196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471559776225404:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:40.443999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471559776225403:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-11-27T18:19:40.523895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471555030009238:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7365) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471559324976565:2121] held: no
2025-11-27T18:19:40.525115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471559324976569:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61967
2025-11-27T18:19:40.395851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471560738873944:2131] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:40.515467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471560296370935:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50460
2025-11-27T18:19:40.449382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471559776225382:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.406513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471557262557447:2114] [node 6] ICP22 created new session: [3:7577471561557524774:2114]
2025-11-27T18:19:40.410125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471561557524774:2114] [node 6] ICS09 handshake done sender: [3:7577471561557524748:2130] self: [3:7577471561557524747:0] peer: [6:7577471559324976560:0] socket: 185 qp: -1
2025-11-27T18:19:40.414021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471561557524758:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.414047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471561557524758:2134] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.527084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471559324976570:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61967
2025-11-27T18:19:40.533872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471559324976570:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.516045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471560296370934:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:40.418252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471556443906641:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.415901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471561557524774:2114] [node 6] ICS10 traffic start
2025-11-27T18:19:40.534151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471559324976565:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:40.449435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471559776225382:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.420226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:40.518068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:40.545521Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471557220859287:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(186398478ns) {HandleHandshakeStatus:329(186395338ns) {DropIncomingHandshake:442(16890ns) DropOutgoingHandshake:457(434ns) SwitchToState:179(617ns) ProcessPendingSessionEvents:495(150257576ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 143227320ns max# 20873076ns}}}
2025-11-27T18:19:40.418575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471561557524758:2134] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:40.459899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471555481258015:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(38488501ns) {HandleHandshakeStatus:329(38115184ns) {DropIncomingHandshake:442(1125ns) DropOutgoingHandshake:457(119ns) SwitchToState:179(365ns) ProcessPendingSessionEvents:495(1062ns)}}
2025-11-27T18:19:40.421554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471556443906641:2113] [node 6] ICP22 created new session: [2:7577471560738873969:2113]
2025-11-27T18:19:40.534244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471555030009244:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.523459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471560296370926:2131] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:40.483628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471559776225396:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:40.535633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471555030009238:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.535670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:40.536205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471555030009238:2105] [node 5] ICP22 created new session: [6:7577471559324976572:2105]
2025-11-27T18:19:40.540216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471559324976572:2105] [node 5] ICS09 handshake done sender: [6:7577471559324976565:2121] self: [6:7577471559324976568:0] peer: [5:7577471560296370912:0] socket: 235 qp: -1
2025-11-27T18:19:40.544710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471559324976572:2105] [node 5] ICS10 traffic start
2025-11-27T18:19:40.546439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:40.420408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:40.421610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471560738873969:2113] [node 6] ICS09 handshake done sender: [2:7577471560738873944:2131] self: [2:7577471560738873943:0] peer: [6:7577471559324976559:0] socket: 209 qp: -1
2025-11-27T18:19:40.435637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471561557524777:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13593
2025-11-27T18:19:40.421647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471560738873969:2113] [node 6] ICS10 traffic start
2025-11-27T18:19:40.441791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471557262557445:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.421684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:40.441867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:40.423589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471560738873946:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.441915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471557262557445:2107] [node 4] ICP22 created new session: [3:7577471561557524778:2107]
2025-11-27T18:19:40.423623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471560738873946:2132] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.443810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471557262557447:2114] [node 6] ICP35 event processing took too much time PendingConnection:279(37131077ns) {HandleHandshakeStatus:329(37128909ns) {DropIncomingHandshake:442(421288ns) DropOutgoingHandshake:457(1281ns) SwitchToState:179(654ns) ProcessPendingSessionEvents:495(6909902ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6850622ns max# 6392774ns}}}
2025-11-27T18:19:40.445474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471560738873946:2132] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:40.459978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471561557524778:2107] [node 4] ICS09 handshake done sender: [3:7577471561557524758:2134] self: [3:7577471561557524757:0] peer: [4:7577471559776225400:0] socket: 205 qp: -1
2025-11-27T18:19:40.470362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471556443906640:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.524869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471556001403603:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.472267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471561557524778:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:40.470446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:40.524942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:40.475319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:40.470514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471556443906640:2108] [node 4] ICP22 created new session: [2:7577471560738873973:2108]
2025-11-27T18:19:40.528283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471556001403603:2110] [node 4] ICP22 created new session: [5:7577471560296370937:2110]
2025-11-27T18:19:40.477564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471561557524777:2138] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.470568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471560738873973:2108] [node 4] ICS09 handshake done sender: [2:7577471560738873946:2132] self: [2:7577471560738873945:0] peer: [4:7577471559776225401:0] socket: 179 qp: -1
2025-11-27T18:19:40.528327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471560296370937:2110] [node 4] ICS09 handshake done sender: [5:7577471560296370926:2131] self: [5:7577471560296370932:0] peer: [4:7577471559776225381:0] socket: 233 qp: -1
2025-11-27T18:19:40.470613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471560738873973:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:40.481178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471557262557365:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:40.528364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471560296370937:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:40.489645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471557262557445:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(55817659ns) {HandleHandshakeStatus:329(55813367ns) {DropIncomingHandshake:442(28406ns) DropOutgoingHandshake:457(484ns) SwitchToState:179(263ns) ProcessPendingSessionEvents:495(13201914ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13137215ns max# 6985679ns}}}
2025-11-27T18:19:40.470654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:40.531397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:40.508103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471561557524781:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13593
2025-11-27T18:19:40.532842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471556001403602:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(30417399ns) {HandleHandshakeStatus:329(30413108ns) {DropIncomingHandshake:442(23778ns) DropOutgoingHandshake:457(313ns) SwitchToState:179(1084ns) ProcessPendingSessionEvents:495(6269607ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6228744ns max# 2775055ns}}}
2025-11-27T18:19:40.500007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471560738873954:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.520946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471561557524777:2138] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:40.500053Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471560738873954:2136] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.529553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471557262557365:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.542525Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471556001403603:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(17962596ns) {HandleHandshakeStatus:329(17960228ns) {DropIncomingHandshake:442(2800ns) DropOutgoingHandshake:457(7916ns) SwitchToState:179(1316ns) ProcessPendingSessionEvents:495(3538246ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3472154ns max# 3063301ns}}}
2025-11-27T18:19:40.532452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:40.511485Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471560738873954:2136] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:40.532500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471557262557365:2049] [node 2] ICP22 created new session: [3:7577471561557524782:2049]
2025-11-27T18:19:40.535627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471561557524782:2049] [node 2] ICS09 handshake done sender: [3:7577471561557524777:2138] self: [3:7577471561557524780:0] peer: [2:7577471560738873953:0] socket: 234 qp: -1
2025-11-27T18:19:40.519618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471556443906559:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.535815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471561557524782:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:40.519687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:40.535870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:40.519735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471556443906559:2049] [node 3] ICP22 created new session: [2:7577471560738873976:2049]
2025-11-27T18:19:40.519779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471560738873976:2049] [node 3] ICS09 handshake done sender: [2:7577471560738873954:2136] self: [2:7577471560738873953:0] peer: [3:7577471561557524780:0] socket: 199 qp: -1
2025-11-27T18:19:40.519811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471560738873976:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:40.519860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:40.500457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471559776225397:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:40.500688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471555481258016:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19796) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471559776225397:2120] held: no
2025-11-27T18:19:40.502316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471555481258017:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6453) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471559776225396:2119] held: no
2025-11-27T18:19:40.502936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471555481258017:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:19:40.502980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
::ffff:127.1.0.0
2025-11-27T18:19:40.503034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471555481258017:2050] [node 3] ICP22 created new session: [4:7577471559776225406:2050]
2025-11-27T18:19:40.503078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471559776225406:2050] [node 3] ICS09 handshake done sender: [4:7577471559776225396:2119] self: [4:7577471559776225400:0] peer: [3:7577471561557524757:0] socket: 225 qp: -1
2025-11-27T18:19:40.503117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471559776225406:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:40.553312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471559324976574:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61967
2025-11-27T18:19:40.503152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:40.505850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471555481258016:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.505910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:40.505960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471555481258016:2049] [node 2] ICP22 created new session: [4:7577471559776225408:2049]
2025-11-27T18:19:40.505997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471559776225408:2049] [node 2] ICS09 handshake done sender: [4:7577471559776225397:2120] self: [4:7577471559776225401:0] peer: [2:7577471560738873945:0] socket: 206 qp: -1
2025-11-27T18:19:40.506030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471559776225408:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:40.506080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:40.510671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471559776225382:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:40.531338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471559776225380:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.535758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:40.535815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471559776225380:2108] [node 5] ICP22 created new session: [4:7577471559776225411:2108]
2025-11-27T18:19:40.535896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471559776225411:2108] [node 5] ICS09 handshake done sender: [4:7577471559776225382:2113] self: [4:7577471559776225381:0] peer: [5:7577471560296370932:0] socket: 193 qp: -1
2025-11-27T18:19:40.536077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471559776225411:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:40.538177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471559776225389:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:40.538214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471559776225389:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:40.538430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:40.553006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471559776225389:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:40.553840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471559324976570:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:40.555155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471559776225386:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:40.555241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:40.555298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471559776225386:2109] [node 6] ICP22 created new session: [4:7577471559776225414:2109]
2025-11-27T18:19:40.555343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471559776225414:2109] [node 6] ICS09 handshake done sender: [4:7577471559776225389:2115] self: [4:7577471559776225388:0] peer: [6:7577471559324976571:0] socket: 170 qp: -1
2025-11-27T18:19:40.555400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471559776225414:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:40.555433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:40.555712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471555030009244:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:40.555788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:40.555835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471555030009244:2110] [node 4] ICP22 created new session: [6:7577471559324976575:2110]
2025-11-27T18:19:40.555875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471559324976575:2110] [node 4] ICS09 handshake done sender: [6:7577471559324976570:2123] self: [6:7577471559324976571:0] peer: [4:7577471559776225388:0] socket: 237 qp: -1
2025-11-27T18:19:40.555904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471559324976575:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:40.555930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:40.557441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471555030009244:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:40.632393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:40.632645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471560738873961:2048] [node 1] ICS07 socket disconnect 175 reason# EndOfStream
2025-11-27T18:19:40.632680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471560738873961:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.632764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:40.632800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471560738873961:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:40.635354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:40.635578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471561557524765:2048] [node 1] ICS07 socket disconnect 177 reason# EndOfStream
2025-11-27T18:19:40.635613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471561557524765:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.635694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:40.635742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471561557524765:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:40.639291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471557262557364:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:40.642754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:40.645430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:40.645354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471556443906558:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:40.645416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471560738873961:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:40.641569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471561557524765:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:40.645671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471560296370928:2048] [node 1] ICS01 socket: 222 reason# EndOfStream
2025-11-27T18:19:40.646322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471559324976555:2048] [node 1] ICS07 socket disconnect 174 reason# EndOfStream
2025-11-27T18:19:40.646239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471556001403529:2048] [node 1] ICP30 unregister session Session# [5:7577471560296370928:2048] VirtualId# [5:7577471560296370921:0]
2025-11-27T18:19:40.646297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471560296370928:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.646374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:40.646433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:40.646357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471559324976555:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.647094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:40.647144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471559324976555:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:40.645383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:40.646842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471559776225399:2048] [node 1] ICS01 socket: 211 reason# EndOfStream
2025-11-27T18:19:40.653101Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471556443906558:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:40.646889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471555481258015:2048] [node 1] ICP30 unregister session Session# [4:7577471559776225399:2048] VirtualId# [4:7577471559776225394:0]
2025-11-27T18:19:40.646917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471559776225399:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.646991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:40.657233Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471557262557364:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:40.647035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:40.657359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471560738873961:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:40.658078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471561557524765:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:40.657441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471556443906558:2048] [node 1] ICP30 unregister session Session# [2:7577471560738873961:2048] VirtualId# [2:7577471560738873955:0]
2025-11-27T18:19:40.657494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:40.658121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471557262557364:2048] [node 1] ICP30 unregister session Session# [3:7577471561557524765:2048] VirtualId# [3:7577471561557524760:0]
2025-11-27T18:19:40.657534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471556443906558:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.657552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.658233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:40.663563Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471555030009177:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:40.658292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471557262557364:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.658306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.663648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471559324976555:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:40.663682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471555030009177:2048] [node 1] ICP30 unregister session Session# [6:7577471559324976555:2048] VirtualId# [6:7577471559324976550:0]
2025-11-27T18:19:40.663748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:40.667174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471555030009177:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.667196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.668326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471557262557364:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:40.677495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471555030009177:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:40.691164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:40.691405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471561557524782:2049] [node 2] ICS07 socket disconnect 234 reason# EndOfStream
2025-11-27T18:19:40.691435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471561557524782:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.691514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:40.691549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471561557524782:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:40.695057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471557262557365:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19796) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:40.695101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471561557524782:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:40.695943Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471557262557365:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19796) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:40.696006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471561557524782:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:40.698399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:40.698473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:40.698703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471560296370930:2049] [node 2] ICS01 socket: 208 reason# EndOfStream
2025-11-27T18:19:40.698745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471556001403530:2049] [node 2] ICP30 unregister session Session# [5:7577471560296370930:2049] VirtualId# [5:7577471560296370920:0]
2025-11-27T18:19:40.698775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471560296370930:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.698836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:40.698873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:40.698652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471559776225408:2049] [node 2] ICS01 socket: 206 reason# EndOfStream
2025-11-27T18:19:40.698698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471555481258016:2049] [node 2] ICP30 unregister session Session# [4:7577471559776225408:2049] VirtualId# [4:7577471559776225401:0]
2025-11-27T18:19:40.698731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471559776225408:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.698812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:40.698850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:40.696128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471557262557365:2049] [node 2] ICP30 unregister session Session# [3:7577471561557524782:2049] VirtualId# [3:7577471561557524780:0]
2025-11-27T18:19:40.700038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:40.700085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471557262557365:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19796) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.700101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19796) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.704584Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:40.704822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471559324976563:2049] [node 2] ICS07 socket disconnect 213 reason# EndOfStream
2025-11-27T18:19:40.704868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471559324976563:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.705117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:40.705170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471559324976563:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:40.709578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471555030009178:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19796) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:40.709718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471559324976563:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:40.709766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471555030009178:2049] [node 2] ICP30 unregister session Session# [6:7577471559324976563:2049] VirtualId# [6:7577471559324976559:0]
2025-11-27T18:19:40.709875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:40.709929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471555030009178:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19796) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.709947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19796) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.713740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471557262557365:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:40.725600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471555030009178:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:40.791964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:40.792201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471559776225406:2050] [node 3] ICS01 socket: 225 reason# EndOfStream
2025-11-27T18:19:40.792254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471555481258017:2050] [node 3] ICP30 unregister session Session# [4:7577471559776225406:2050] VirtualId# [4:7577471559776225400:0]
2025-11-27T18:19:40.792295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471559776225406:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.792350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:40.792384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:40.793839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:40.793903Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:40.794044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471559324976566:2050] [node 3] ICS01 socket: 181 reason# EndOfStream
2025-11-27T18:19:40.794105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471555030009179:2050] [node 3] ICP30 unregister session Session# [6:7577471559324976566:2050] VirtualId# [6:7577471559324976560:0]
2025-11-27T18:19:40.794135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471559324976566:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.794372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:40.794465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:40.794396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471560296370924:2050] [node 3] ICS01 socket: 186 reason# EndOfStream
2025-11-27T18:19:40.794444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471556001403531:2050] [node 3] ICP30 unregister session Session# [5:7577471560296370924:2050] VirtualId# [5:7577471560296370915:0]
2025-11-27T18:19:40.794478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471560296370924:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.794564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:40.794607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:40.863585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:40.865476Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:40.865666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471560296370937:2110] [node 4] ICS07 socket disconnect 233 reason# EndOfStream
2025-11-27T18:19:40.863838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471559324976575:2110] [node 4] ICS07 socket disconnect 237 reason# EndOfStream
2025-11-27T18:19:40.863872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471559324976575:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.865694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471560296370937:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.865763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:40.865800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471560296370937:2110] [node 4] ICS15 start handshake
2025-11-27T18:19:40.863949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:40.863986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471559324976575:2110] [node 4] ICS15 start handshake
2025-11-27T18:19:40.865320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471555030009244:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26801) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:40.865374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471559324976575:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:40.865417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471555030009244:2110] [node 4] ICP30 unregister session Session# [6:7577471559324976575:2110] VirtualId# [6:7577471559324976571:0]
2025-11-27T18:19:40.865525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:40.865557Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471555030009244:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26801) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.865574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26801) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:40.895825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471555030009244:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:19:40.942840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:40.943281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471559324976572:2105] [node 5] ICS01 socket: 235 reason# EndOfStream
2025-11-27T18:19:40.943347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471555030009238:2105] [node 5] ICP30 unregister session Session# [6:7577471559324976572:2105] VirtualId# [6:7577471559324976568:0]
2025-11-27T18:19:40.943396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471559324976572:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:40.943510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:40.943562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:41.084162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471565706596370:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:41.084222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471565706596371:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:41.132261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471563211226583:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:41.132302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471563211226584:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:41.163270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471564210605087:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:41.163316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471564210605088:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:41.187126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471562107027088:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:41.187170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471562107027089:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:41.187199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471562107027090:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:41.210123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471564201745548:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:41.210165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471564201745549:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:41.211740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471564201745547:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:41.233476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471563516102573:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:41.233552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471563516102574:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:41.233684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471563516102575:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:41.416704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471563516102633:2105] [node 5] ICP01 ready to work
2025-11-27T18:19:41.429476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471565706596439:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:41.443922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471563211226645:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:41.437520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471565706596438:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:41.482029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471563211226647:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:41.501694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471565706596442:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:41.605774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471564201745611:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:41.606005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471564210605177:2135] [node 5] ICP01 ready to work
2025-11-27T18:19:41.605906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471562107027151:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:41.620250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471563516102637:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:41.617473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471562107027152:2108] [node 6] ICP01 ready to work
2025-11-27T18:19:41.621882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471563211226648:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:41.612418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471564201745612:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:41.630045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471565706596439:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(177707814ns) {Configure:104(177705121ns) {ProcessConfigured:135(135576509ns) {StartInitialHandshake:157(135573363ns) {DropHandshakes:470(3641ns) {DropIncomingHandshake:442(1149ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(1489ns) PrepareNewSessionHandshake:477(3317ns) {SwitchToState:179(1834ns)}}}}}
2025-11-27T18:19:41.637549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471564210605178:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:41.667405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471563211226583:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(208793500ns) {Configure:104(208788048ns) {ProcessConfigured:135(6877647ns) {StartInitialHandshake:157(6873282ns) {DropHandshakes:470(3831ns) {DropIncomingHandshake:442(1267ns) DropOutgoingHandshake:457(491ns)} GenerateSessionVirtualId:988(1425ns) PrepareNewSessionHandshake:477(5444ns) {SwitchToState:179(2762ns)}}}}}
2025-11-27T18:19:41.642167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471564210605179:2125] [node 6] ICP01 ready to work
2025-11-27T18:19:41.670900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471563211226645:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(205328171ns) {Configure:104(205323529ns) {ProcessConfigured:135(37250317ns) {StartInitialHandshake:157(37247066ns) {DropHandshakes:470(3682ns) {DropIncomingHandshake:442(762ns) DropOutgoingHandshake:457(456ns)} GenerateSessionVirtualId:988(943ns) PrepareNewSessionHandshake:477(3361ns) {SwitchToState:179(1733ns)}}}}}
2025-11-27T18:19:41.679811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471563516102573:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(67438145ns) {Configure:104(67429208ns) {ProcessConfigured:135(49006ns) {StartInitialHandshake:157(47568ns) {DropHandshakes:470(2862ns) {DropIncomingHandshake:442(746ns) DropOutgoingHandshake:457(357ns)} GenerateSessionVirtualId:988(761ns) PrepareNewSessionHandshake:477(1410ns) {SwitchToState:179(817ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.709730Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471564201745611:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(39804757ns) {Configure:104(39798433ns) {ProcessConfigured:135(56249ns) {StartInitialHandshake:157(53855ns) {DropHandshakes:470(3078ns) {DropIncomingHandshake:442(729ns) DropOutgoingHandshake:457(444ns)} GenerateSessionVirtualId:988(575ns) PrepareNewSessionHandshake:477(15631ns) {SwitchToState:179(683ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.725901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471564210605186:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21278
2025-11-27T18:19:41.682336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471563516102575:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28199208ns) {Configure:104(28194713ns) {ProcessConfigured:135(49332ns) {StartInitialHandshake:157(46944ns) {DropHandshakes:470(1419ns) {DropIncomingHandshake:442(429ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(1519ns) {SwitchToState:179(729ns)}}}}}
2025-11-27T18:19:41.710200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471562107027159:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-27T18:19:41.704412Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471563516102633:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(173881414ns) {Configure:104(142990725ns) {ProcessConfigured:135(6143485ns) {StartInitialHandshake:157(6140924ns) {DropHandshakes:470(3540ns) {DropIncomingHandshake:442(940ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(768ns) PrepareNewSessionHandshake:477(5133ns) {SwitchToState:179(2412ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.749731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471562107027090:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(78471762ns) {Configure:104(78465634ns) {ProcessConfigured:135(64551ns) {StartInitialHandshake:157(62608ns) {DropHandshakes:470(5416ns) {DropIncomingHandshake:442(777ns) DropOutgoingHandshake:457(393ns)} GenerateSessionVirtualId:988(1190ns) PrepareNewSessionHandshake:477(2601ns) {SwitchToState:179(1108ns)}}}}}
2025-11-27T18:19:41.753873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471564201745547:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53257013ns) {Configure:104(53247644ns) {ProcessConfigured:135(33258386ns) {StartInitialHandshake:157(33255779ns) {DropHandshakes:470(1764ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(726ns) PrepareNewSessionHandshake:477(4532ns) {SwitchToState:179(2340ns)}}}}}
2025-11-27T18:19:41.705687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471563516102649:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23607
2025-11-27T18:19:41.763436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471565706596447:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.741805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471563516102649:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.788036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471565706596448:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.793735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471562107027159:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.761811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471563516102574:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(52459102ns) {Configure:104(52452879ns) {ProcessConfigured:135(29328231ns) {StartInitialHandshake:157(29324659ns) {DropHandshakes:470(13484240ns) {DropIncomingHandshake:442(2238ns) DropOutgoingHandshake:457(730ns)} GenerateSessionVirtualId:988(1768ns) PrepareNewSessionHandshake:477(5917ns) {SwitchToState:179(2216ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.801066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471563211226653:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7984
2025-11-27T18:19:41.795293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471564201745614:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-11-27T18:19:41.800814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471565706596449:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
2025-11-27T18:19:41.810396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471563211226584:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47020875ns) {Configure:104(47009790ns) {ProcessConfigured:135(60066ns) {StartInitialHandshake:157(57738ns) {DropHandshakes:470(2035ns) {DropIncomingHandshake:442(540ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(485ns) PrepareNewSessionHandshake:477(6549ns) {SwitchToState:179(493ns)}}}}}
2025-11-27T18:19:41.810548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471563211226653:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.810802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471563211226583:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:41.764533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471563516102575:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.809337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471564210605087:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(89942518ns) {Configure:104(81552312ns) {ProcessConfigured:135(7418015ns) {StartInitialHandshake:157(7414985ns) {DropHandshakes:470(3787ns) {DropIncomingHandshake:442(753ns) DropOutgoingHandshake:457(822ns)} GenerateSessionVirtualId:988(923ns) PrepareNewSessionHandshake:477(5104ns) {SwitchToState:179(2884ns)}}}}}
2025-11-27T18:19:41.814415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471564210605178:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(84850979ns) {Configure:104(84843953ns) {ProcessConfigured:135(19407587ns) {StartInitialHandshake:157(16804556ns) {DropHandshakes:470(1845ns) {DropIncomingHandshake:442(412ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(855ns) PrepareNewSessionHandshake:477(2848ns) {SwitchToState:179(1210ns)}}}}}
2025-11-27T18:19:41.798076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471564201745621:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.817946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471564210605183:2138] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.817997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471564210605183:2138] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:41.821024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471564210605186:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.821360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471564210605187:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21278
2025-11-27T18:19:41.821863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471564210605194:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21278
2025-11-27T18:19:41.803390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471562107027088:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.802942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471565706596448:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.832261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471562107027169:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-27T18:19:41.836813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471563211226648:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(42697014ns) {Configure:104(42688834ns) {ProcessConfigured:135(63579ns) {StartInitialHandshake:157(59960ns) {DropHandshakes:470(3308ns) {DropIncomingHandshake:442(884ns) DropOutgoingHandshake:457(316ns)} GenerateSessionVirtualId:988(988ns) PrepareNewSessionHandshake:477(1398ns) {SwitchToState:179(512ns)}}}}}
2025-11-27T18:19:41.839441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471562107027089:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(133775608ns) {Configure:104(133767924ns) {ProcessConfigured:135(4785113ns) {StartInitialHandshake:157(4781893ns) {DropHandshakes:470(2848ns) {DropIncomingHandshake:442(675ns) DropOutgoingHandshake:457(359ns)} GenerateSessionVirtualId:988(1054ns) PrepareNewSessionHandshake:477(3271ns) {SwitchToState:179(1903ns)}}}}}
2025-11-27T18:19:41.789314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471563516102650:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23607
2025-11-27T18:19:41.849941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471562107027159:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.803838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471565706596371:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(131978897ns) {Configure:104(130949776ns) {ProcessConfigured:135(3325400ns) {StartInitialHandshake:157(3321048ns) {DropHandshakes:470(3311ns) {DropIncomingHandshake:442(704ns) DropOutgoingHandshake:457(325ns)} GenerateSessionVirtualId:988(1260ns) PrepareNewSessionHandshake:477(1826ns) {SwitchToState:179(816ns)}}}}}
2025-11-27T18:19:41.830504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471564210605183:2138] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:41.814493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471563516102650:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.840719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471564201745621:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.804060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471565706596447:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.851075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471562107027088:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:41.822569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471563516102654:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23607
2025-11-27T18:19:41.846000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471564201745618:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-11-27T18:19:41.858017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471563211226652:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.841579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471564210605187:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.854364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471564201745614:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.814829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471565706596444:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.855146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471562107027158:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.858049Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471563211226652:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:41.847606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471564210605179:2125] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.850301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471563516102655:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23607
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.814864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471565706596444:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.856875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471564201745548:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(86428101ns) {Configure:104(86419676ns) {ProcessConfigured:135(55388ns) {StartInitialHandshake:157(53185ns) {DropHandshakes:470(2367ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(654ns)} GenerateSessionVirtualId:988(505ns) PrepareNewSessionHandshake:477(1125ns) {SwitchToState:179(433ns)}}}}}
2025-11-27T18:19:41.855170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471562107027158:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:41.850707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471564210605088:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(137811456ns) {Configure:104(137803544ns) {ProcessConfigured:135(11040754ns) {StartInitialHandshake:157(9064006ns) {DropHandshakes:470(4437ns) {DropIncomingHandshake:442(857ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(1889ns) PrepareNewSessionHandshake:477(2126ns) {SwitchToState:179(812ns)}}}}}
2025-11-27T18:19:41.850858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471563516102637:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:41.873777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471563211226657:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.822847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471565706596441:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.855965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:41.857286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471564201745548:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:41.873804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471563211226657:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:41.863387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471563516102637:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(17762839ns) {IncomingHandshake:261(4821516ns) {DropIncomingHandshake:442(584ns) PrepareNewSessionHandshake:477(2213ns) {SwitchToState:179(1046ns)} GenerateSessionVirtualId:988(1239ns) IssueIncomingHandshakeReply:193(55205ns) {SwitchToState:179(715ns)}}}
2025-11-27T18:19:41.857381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471564201745625:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-11-27T18:19:41.854222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:41.822893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471565706596441:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:41.859538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471562107027088:2048] [node 1] ICP22 created new session: [4:7577471562107027170:2048]
2025-11-27T18:19:41.863641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471564201745618:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.859747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471564210605179:2125] [node 6] ICP22 created new session: [3:7577471564210605200:2125]
2025-11-27T18:19:41.877369Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471563211226652:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:41.831763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471565706596454:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
2025-11-27T18:19:41.880194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471563516102654:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.866901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471562107027170:2048] [node 1] ICS09 handshake done sender: [4:7577471562107027159:2113] self: [4:7577471562107027165:0] peer: [1:7577471565706596443:0] socket: 187 qp: -1
2025-11-27T18:19:41.861328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471564210605194:2144] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.883606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471564201745630:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-11-27T18:19:41.922015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471563211226662:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7984
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.867123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471564210605088:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:41.845484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471565706596441:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:41.866958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471562107027170:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:41.880762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471563516102633:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:41.924265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471563211226657:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:41.874206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471564210605200:2125] [node 6] ICS09 handshake done sender: [3:7577471564210605183:2138] self: [3:7577471564210605182:0] peer: [6:7577471563516102653:0] socket: 175 qp: -1
2025-11-27T18:19:41.893238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471564201745625:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.873363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:41.849244Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471565706596444:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.926889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471563211226645:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:41.866548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471565706596459:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
2025-11-27T18:19:41.897356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471563516102649:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.905627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471564201745617:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.878461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471564210605200:2125] [node 6] ICS10 traffic start
2025-11-27T18:19:41.875795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471562107027166:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-27T18:19:41.897473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471563516102575:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7763) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471563516102649:2114] held: no
2025-11-27T18:19:41.926928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:41.905672Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471564201745617:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:41.876616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471565706596449:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.880677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:41.893162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471562107027173:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-27T18:19:41.927486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471563211226653:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:41.878034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471565706596454:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.933240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471563516102658:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23607
2025-11-27T18:19:41.931324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471564210605192:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.919967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471564201745631:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.944972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471563516102650:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:41.931275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471563211226645:2108] [node 5] ICP22 created new session: [2:7577471563211226665:2108]
2025-11-27T18:19:41.925473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471564201745547:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:41.897305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471562107027166:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.901602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471565706596442:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(124765649ns) {Configure:104(124756708ns) {ProcessConfigured:135(10239335ns) {StartInitialHandshake:157(10236357ns) {DropHandshakes:470(2219ns) {DropIncomingHandshake:442(573ns) DropOutgoingHandshake:457(590ns)} GenerateSessionVirtualId:988(510ns) PrepareNewSessionHandshake:477(1818003ns) {SwitchToState:179(3884ns)}}}}}
2025-11-27T18:19:41.931354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471564210605192:2142] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:41.950645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471563516102633:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471563516102654:2117] held: no
2025-11-27T18:19:41.931988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471563211226647:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(109379477ns) {Configure:104(109371841ns) {ProcessConfigured:135(46116ns) {StartInitialHandshake:157(43809ns) {DropHandshakes:470(1272ns) {DropIncomingHandshake:442(274ns) DropOutgoingHandshake:457(213ns)} GenerateSessionVirtualId:988(412ns) PrepareNewSessionHandshake:477(637ns) {SwitchToState:179(139ns)}}}}}
2025-11-27T18:19:41.918743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471562107027088:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(66838986ns) {HandleHandshakeStatus:329(66836557ns) {DropIncomingHandshake:442(1211ns) DropOutgoingHandshake:457(10518ns) SwitchToState:179(296ns) ProcessPendingSessionEvents:495(16513060ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16446980ns max# 16014456ns}}}
2025-11-27T18:19:41.920035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471562107027158:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:41.925892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471562107027173:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.926512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471562107027090:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:41.931054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471562107027152:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:41.931345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:41.932079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471562107027152:2108] [node 6] ICP22 created new session: [4:7577471562107027177:2108]
2025-11-27T18:19:41.936861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471562107027177:2108] [node 6] ICS09 handshake done sender: [4:7577471562107027158:2112] self: [4:7577471562107027157:0] peer: [6:7577471563516102656:0] socket: 192 qp: -1
2025-11-27T18:19:41.939481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471562107027177:2108] [node 6] ICS10 traffic start
2025-11-27T18:19:41.942955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471562107027156:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.942987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471562107027156:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:41.948275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:41.950140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471562107027180:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-27T18:19:41.950332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471562107027156:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:41.977369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471562107027151:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:41.984644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:41.987614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471562107027151:2109] [node 5] ICP22 created new session: [4:7577471562107027182:2109]
2025-11-27T18:19:41.988545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471562107027182:2109] [node 5] ICS09 handshake done sender: [4:7577471562107027156:2111] self: [4:7577471562107027155:0] peer: [5:7577471564201745640:0] socket: 223 qp: -1
2025-11-27T18:19:41.989453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471562107027182:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:41.992999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:41.901762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471565706596459:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.943249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471564210605203:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21278
2025-11-27T18:19:41.925669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471564201745621:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:41.958115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471564210605181:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.958155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471564210605181:2137] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:41.972877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471564210605194:2144] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:41.985938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471564210605088:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:41.985998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:41.988107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471564210605204:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21278
2025-11-27T18:19:41.989055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471564210605088:2049] [node 2] ICP22 created new session: [3:7577471564210605208:2049]
2025-11-27T18:19:41.989506Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471564210605192:2142] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:41.993671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471564210605208:2049] [node 2] ICS09 handshake done sender: [3:7577471564210605194:2144] self: [3:7577471564210605201:0] peer: [2:7577471563211226656:0] socket: 211 qp: -1
2025-11-27T18:19:41.993718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471564210605208:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:42.001796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:41.901957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471565706596439:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:41.932665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471563211226583:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471563211226653:2112] held: no
2025-11-27T18:19:41.927854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471564201745631:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.933148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471563211226584:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:41.935273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:41.935331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471563211226584:2049] [node 3] ICP22 created new session: [2:7577471563211226668:2049]
2025-11-27T18:19:41.939338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471563211226665:2108] [node 5] ICS09 handshake done sender: [2:7577471563211226652:2111] self: [2:7577471563211226649:0] peer: [5:7577471564201745628:0] socket: 200 qp: -1
2025-11-27T18:19:41.939377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471563211226665:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:41.942810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:41.944967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471563211226668:2049] [node 3] ICS09 handshake done sender: [2:7577471563211226657:2113] self: [2:7577471563211226656:0] peer: [3:7577471564210605201:0] socket: 210 qp: -1
2025-11-27T18:19:41.948058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471563211226668:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:41.948116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:41.963557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471563516102637:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:41.963608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:41.963662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471563516102637:2108] [node 4] ICP22 created new session: [6:7577471563516102662:2108]
2025-11-27T18:19:41.963712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471563516102662:2108] [node 4] ICS09 handshake done sender: [6:7577471563516102650:2115] self: [6:7577471563516102656:0] peer: [4:7577471562107027157:0] socket: 193 qp: -1
2025-11-27T18:19:41.963753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471563516102662:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:41.963780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:41.965856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471563516102659:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23607
2025-11-27T18:19:41.966318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471563516102661:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23607
2025-11-27T18:19:41.975013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471563516102661:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.982794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471563516102573:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:41.992895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471563516102666:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23607
2025-11-27T18:19:42.038236Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471563516102575:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(44095939ns) {HandleHandshakeStatus:395(44092481ns) {LogHandshakeFail:472(16970ns) {UpdateErrorStateLog:545(5433ns)} DropOutgoingHandshake:457(1238ns) ProcessPendingSessionEvents:495(27706924ns) {unordered DropSessionEvent:512 num# 100 sum# 27662536ns max# 8173816ns}}}
2025-11-27T18:19:42.041998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471563516102633:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(77776111ns) {HandleHandshakeStatus:395(77772805ns) {LogHandshakeFail:472(12931ns) {UpdateErrorStateLog:545(5201ns)} DropOutgoingHandshake:457(2160ns) ProcessPendingSessionEvents:495(72195309ns) {unordered DropSessionEvent:512 num# 100 sum# 21043187ns max# 5235753ns}}}
2025-11-27T18:19:42.042100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471563516102573:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471563516102661:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.902015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:41.927945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471564201745548:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:42.042784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471563516102575:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:42.042260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471564210605088:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(55418215ns) {HandleHandshakeStatus:329(55414099ns) {DropIncomingHandshake:442(944ns) DropOutgoingHandshake:457(5800ns) SwitchToState:179(978ns) ProcessPendingSessionEvents:495(18331634ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18252929ns max# 14466114ns}}}
2025-11-27T18:19:41.929707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471564201745549:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(143355436ns) {Configure:104(143348149ns) {ProcessConfigured:135(27620370ns) {StartInitialHandshake:157(27617380ns) {DropHandshakes:470(9648143ns) {DropIncomingHandshake:442(1534ns) DropOutgoingHandshake:457(1026ns)} GenerateSessionVirtualId:988(1038ns) PrepareNewSessionHandshake:477(3702ns) {SwitchToState:179(2408ns)}}}}}
2025-11-27T18:19:41.902064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471565706596439:2107] [node 4] ICP22 created new session: [1:7577471565706596462:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:42.044941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:42.045673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471562107027151:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(67441879ns) {HandleHandshakeStatus:329(67438720ns) {DropIncomingHandshake:442(19845ns) DropOutgoingHandshake:457(553ns) SwitchToState:179(891ns) ProcessPendingSessionEvents:495(29798124ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29714606ns max# 11481769ns}}}
2025-11-27T18:19:42.042328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471564210605178:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:42.044992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471563516102575:2050] [node 3] ICP22 created new session: [6:7577471567811069964:2050]
2025-11-27T18:19:42.042390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:41.931136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471564201745617:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:41.902110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471565706596462:2107] [node 4] ICS09 handshake done sender: [1:7577471565706596444:2116] self: [1:7577471565706596443:0] peer: [4:7577471562107027165:0] socket: 176 qp: -1
2025-11-27T18:19:42.046448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471563516102654:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:41.931245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:42.042447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471564210605178:2107] [node 4] ICP22 created new session: [3:7577471568505572506:2107]
2025-11-27T18:19:42.065141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471563211226583:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(53043273ns) {HandleHandshakeStatus:395(53040913ns) {LogHandshakeFail:472(11642ns) {UpdateErrorStateLog:545(4172ns)} DropOutgoingHandshake:457(370ns) ProcessPendingSessionEvents:495(52975146ns) {unordered DropSessionEvent:512 num# 100 sum# 52910612ns max# 9942872ns}}}
2025-11-27T18:19:41.902146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471565706596462:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:42.050241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471567811069964:2050] [node 3] ICS09 handshake done sender: [6:7577471563516102649:2114] self: [6:7577471563516102653:0] peer: [3:7577471564210605182:0] socket: 179 qp: -1
2025-11-27T18:19:42.042531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471568505572506:2107] [node 4] ICS09 handshake done sender: [3:7577471564210605192:2142] self: [3:7577471564210605190:0] peer: [4:7577471562107027176:0] socket: 222 qp: -1
2025-11-27T18:19:41.934947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471564201745548:2049] [node 2] ICP22 created new session: [5:7577471564201745636:2049]
2025-11-27T18:19:41.902206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:42.042574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471568505572506:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:42.072102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471563211226671:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7984
2025-11-27T18:19:42.054346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471567811069964:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:41.935118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471564201745549:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:41.903482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471565706596370:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:42.061614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471563516102667:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23607
2025-11-27T18:19:41.938080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471564201745611:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:42.073233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471563211226583:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:42.042610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:41.903515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:41.938137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:42.042630Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471564210605179:2125] [node 6] ICP35 event processing took too much time PendingConnection:279(200059315ns) {HandleHandshakeStatus:329(200055629ns) {DropIncomingHandshake:442(20068ns) DropOutgoingHandshake:457(610ns) SwitchToState:179(756ns) ProcessPendingSessionEvents:495(122283756ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 116606007ns max# 11400983ns}}}
2025-11-27T18:19:42.073813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:41.938181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471564201745611:2109] [node 6] ICP22 created new session: [5:7577471564201745638:2109]
2025-11-27T18:19:42.042971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471564210605204:2146] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.938229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471564201745638:2109] [node 6] ICS09 handshake done sender: [5:7577471564201745617:2113] self: [5:7577471564201745616:0] peer: [6:7577471563516102657:0] socket: 202 qp: -1
2025-11-27T18:19:42.044173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471564210605181:2137] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:42.073865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471563211226583:2048] [node 1] ICP22 created new session: [2:7577471567506193971:2048]
2025-11-27T18:19:41.938261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471564201745638:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:42.049308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471564210605207:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21278
2025-11-27T18:19:42.073906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471567506193971:2048] [node 1] ICS09 handshake done sender: [2:7577471563211226653:2112] self: [2:7577471563211226658:0] peer: [1:7577471565706596440:0] socket: 184 qp: -1
2025-11-27T18:19:41.938294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:42.058128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471564210605177:2135] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:41.939366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471564201745636:2049] [node 2] ICS09 handshake done sender: [5:7577471564201745621:2116] self: [5:7577471564201745628:0] peer: [2:7577471563211226649:0] socket: 201 qp: -1
2025-11-27T18:19:42.058163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:42.058217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471564210605177:2135] [node 5] ICP22 created new session: [3:7577471568505572508:2135]
2025-11-27T18:19:41.940181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471564201745612:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:42.073937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471567506193971:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:42.078810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471562107027090:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7763) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471562107027173:2118] held: no
2025-11-27T18:19:41.942599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471564201745636:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:42.058247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471568505572508:2135] [node 5] ICS09 handshake done sender: [3:7577471564210605181:2137] self: [3:7577471564210605180:0] peer: [5:7577471564201745637:0] socket: 174 qp: -1
2025-11-27T18:19:41.942640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:42.074134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:41.957779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471564201745643:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-11-27T18:19:41.973117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471564201745625:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:42.058274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471568505572508:2135] [node 5] ICS10 traffic start
2025-11-27T18:19:42.079893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471563211226645:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(147348037ns) {HandleHandshakeStatus:329(147345354ns) {DropIncomingHandshake:442(190ns) DropOutgoingHandshake:457(568ns) SwitchToState:179(329ns) ProcessPendingSessionEvents:495(115945133ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 115787060ns max# 16809498ns}}}
2025-11-27T18:19:41.979744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471564201745548:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(50906315ns) {HandleHandshakeStatus:329(50904079ns) {DropIncomingHandshake:442(521ns) DropOutgoingHandshake:457(4560ns) SwitchToState:179(151ns) ProcessPendingSessionEvents:495(29793698ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29714900ns max# 21317045ns}}}
2025-11-27T18:19:42.058298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:41.979807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471564201745547:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:42.084849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471567506193969:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7984
2025-11-27T18:19:41.979851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:42.059643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471564210605087:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:41.979899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471564201745547:2048] [node 1] ICP22 created new session: [5:7577471564201745645:2048]
2025-11-27T18:19:42.066181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471564210605207:2147] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:42.066810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471564210605177:2135] [node 5] ICP18 handshake (actor [3:7577471564210605207:2147]) is too old
2025-11-27T18:19:41.979956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471564201745645:2048] [node 1] ICS09 handshake done sender: [5:7577471564201745625:2117] self: [5:7577471564201745635:0] peer: [1:7577471565706596450:0] socket: 220 qp: -1
2025-11-27T18:19:41.979993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471564201745645:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:42.085122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471568505572511:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21278
2025-11-27T18:19:41.980031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:41.991594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471564201745644:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-11-27T18:19:42.063504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471563516102633:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:42.063543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:41.903560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471565706596370:2048] [node 2] ICP22 created new session: [1:7577471565706596464:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:42.092944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471566401994480:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-27T18:19:42.085151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471563211226661:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:42.013700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471568496712943:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-11-27T18:19:41.903603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471565706596464:2048] [node 2] ICS09 handshake done sender: [1:7577471565706596441:2115] self: [1:7577471565706596440:0] peer: [2:7577471563211226658:0] socket: 177 qp: -1
2025-11-27T18:19:42.063774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471563516102575:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(20906290ns) {HandleHandshakeStatus:329(20904048ns) {DropIncomingHandshake:442(525ns) DropOutgoingHandshake:457(111ns) SwitchToState:179(272ns) ProcessPendingSessionEvents:495(843ns)}}
2025-11-27T18:19:42.085184Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471563211226661:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:42.013977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471564201745612:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26245) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471564201745631:2120] held: no
2025-11-27T18:19:42.093211Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471564210605204:2146] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:42.065208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:42.065792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471563516102633:2105] [node 5] ICP22 created new session: [6:7577471567811069966:2105]
2025-11-27T18:19:42.066664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471563516102661:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:42.067263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471567811069966:2105] [node 5] ICS09 handshake done sender: [6:7577471563516102654:2117] self: [6:7577471563516102657:0] peer: [5:7577471564201745616:0] socket: 203 qp: -1
2025-11-27T18:19:42.067460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471563516102573:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:42.068276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:42.068724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471567811069966:2105] [node 5] ICS10 traffic start
2025-11-27T18:19:42.069127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471563516102573:2048] [node 1] ICP22 created new session: [6:7577471567811069967:2048]
2025-11-27T18:19:41.903684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471565706596464:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:42.094352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471567506193969:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.903711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:42.100187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471564210605087:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471564210605204:2146] held: no
2025-11-27T18:19:42.041706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471564201745611:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(48646904ns) {HandleHandshakeStatus:329(48643329ns) {DropIncomingHandshake:442(13278ns) DropOutgoingHandshake:457(919ns) SwitchToState:179(787ns) ProcessPendingSessionEvents:495(48225568ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 48134129ns max# 18801380ns}}}
2025-11-27T18:19:42.095079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471563211226671:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:42.069174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:41.906875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471565706596370:2048] [node 2] ICP18 handshake (actor [1:7577471565706596449:2119]) is too old
2025-11-27T18:19:42.042606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471564201745612:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(29140807ns) {HandleHandshakeStatus:395(29137314ns) {LogHandshakeFail:472(1365324ns) {UpdateErrorStateLog:545(5079ns)} DropOutgoingHandshake:457(413ns) ProcessPendingSessionEvents:495(27699700ns) {unordered DropSessionEvent:512 num# 100 sum# 5318072ns max# 4685052ns}}}
2025-11-27T18:19:42.073015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471567811069967:2048] [node 1] ICS09 handshake done sender: [6:7577471563516102661:2121] self: [6:7577471563516102665:0] peer: [1:7577471565706596460:0] socket: 232 qp: -1
2025-11-27T18:19:42.100390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471563211226661:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:42.073052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471567811069967:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:42.073115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:42.083129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471563516102666:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:42.083239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471563516102574:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:42.104910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471562107027173:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:42.106728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471562107027152:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(167283672ns) {HandleHandshakeStatus:329(167280644ns) {DropIncomingHandshake:442(1349ns) DropOutgoingHandshake:457(635ns) SwitchToState:179(198ns) ProcessPendingSessionEvents:495(131092344ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 128303823ns max# 10804459ns}}}
2025-11-27T18:19:42.106940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471562107027090:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:42.106972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:42.106553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471564210605087:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:42.043659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471564201745631:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:19:42.045036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471564201745612:2108] [node 4] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:19:42.106614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:42.045667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:41.934260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471565706596451:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:42.106661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471564210605087:2048] [node 1] ICP22 created new session: [3:7577471568505572512:2048]
2025-11-27T18:19:42.106731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471568505572512:2048] [node 1] ICS09 handshake done sender: [3:7577471564210605204:2146] self: [3:7577471568505572510:0] peer: [1:7577471565706596452:0] socket: 231 qp: -1
2025-11-27T18:19:42.106785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471568505572512:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:42.106821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:42.107336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471563211226648:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:42.107015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471562107027090:2050] [node 3] ICP22 created new session: [4:7577471566401994482:2050]
2025-11-27T18:19:42.107086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471566401994482:2050] [node 3] ICS09 handshake done sender: [4:7577471562107027173:2118] self: [4:7577471562107027176:0] peer: [3:7577471564210605190:0] socket: 191 qp: -1
2025-11-27T18:19:42.109772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471566401994482:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:42.109820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:42.118948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471567811069971:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23607
2025-11-27T18:19:42.051546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471564201745612:2108] [node 4] ICP22 created new session: [5:7577471568496712944:2108]
2025-11-27T18:19:42.059991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471568496712944:2108] [node 4] ICS09 handshake done sender: [5:7577471564201745631:2120] self: [5:7577471564201745640:0] peer: [4:7577471562107027155:0] socket: 225 qp: -1
2025-11-27T18:19:42.061614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471564201745614:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:42.061784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471568496712944:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:42.064752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:42.066612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471564201745549:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:42.107405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:42.109986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471563211226648:2109] [node 6] ICP22 created new session: [2:7577471567506193976:2109]
2025-11-27T18:19:42.066687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:41.934294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471565706596451:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:42.124620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471563516102666:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:42.112567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471567506193976:2109] [node 6] ICS09 handshake done sender: [2:7577471563211226661:2114] self: [2:7577471563211226660:0] peer: [6:7577471567811069970:0] socket: 241 qp: -1
2025-11-27T18:19:42.118938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471566401994481:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-27T18:19:42.066743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471564201745549:2050] [node 3] ICP22 created new session: [5:7577471568496712946:2050]
2025-11-27T18:19:41.943254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471565706596451:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:41.974866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471565706596438:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:41.974948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:41.983300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471565706596461:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:41.983326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471565706596461:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:41.984132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471565706596438:2108] [node 5] ICP22 created new session: [1:7577471565706596472:2108]
2025-11-27T18:19:41.988271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471565706596472:2108] [node 5] ICS09 handshake done sender: [1:7577471565706596451:2120] self: [1:7577471565706596450:0] peer: [5:7577471564201745635:0] socket: 215 qp: -1
2025-11-27T18:19:41.988322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471565706596461:2124] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:41.989118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471565706596472:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:41.989176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:42.003578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471565706596442:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:42.003656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:42.003708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471565706596442:2109] [node 6] ICP22 created new session: [1:7577471570001563771:2109]
2025-11-27T18:19:42.003767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471570001563771:2109] [node 6] ICS09 handshake done sender: [1:7577471565706596461:2124] self: [1:7577471565706596460:0] peer: [6:7577471563516102665:0] socket: 195 qp: -1
2025-11-27T18:19:42.003809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471570001563771:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:42.003873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:42.042655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471565706596438:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(80134933ns) {HandleHandshakeStatus:329(72134597ns) {DropIncomingHandshake:442(31741ns) DropOutgoingHandshake:457(356ns) SwitchToState:179(420ns) ProcessPendingSessionEvents:495(47989850ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 47892906ns max# 18251654ns}}}
2025-11-27T18:19:42.065868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471565706596453:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:42.065921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471565706596453:2121] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:42.071986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471565706596453:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:42.074275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471565706596371:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:42.074331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:42.076468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471565706596371:2049] [node 3] ICP22 created new session: [1:7577471570001563774:2049]
2025-11-27T18:19:42.077937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471570001563774:2049] [node 3] ICS09 handshake done sender: [1:7577471565706596453:2121] self: [1:7577471565706596452:0] peer: [3:7577471568505572510:0] socket: 230 qp: -1
2025-11-27T18:19:42.078661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471570001563774:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:42.078726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:42.091723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471565706596371:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17114912ns) {HandleHandshakeStatus:329(17112420ns) {DropIncomingHandshake:442(16668ns) DropOutgoingHandshake:457(207ns) SwitchToState:179(233ns) ProcessPendingSessionEvents:495(11918711ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11840630ns max# 6061702ns}}}
2025-11-27T18:19:42.119276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471565706596442:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(89911982ns) {HandleHandshakeStatus:329(89907939ns) {DropIncomingHandshake:442(26237ns) DropOutgoingHandshake:457(424ns) SwitchToState:179(605ns) ProcessPendingSessionEvents:495(89471170ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 89337103ns max# 15127194ns}}}
2025-11-27T18:19:42.112622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471567506193976:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:42.124808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471566401994481:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:42.067270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471568496712946:2050] [node 3] ICS09 handshake done sender: [5:7577471564201745614:2111] self: [5:7577471564201745637:0] peer: [3:7577471564210605180:0] socket: 180 qp: -1
2025-11-27T18:19:42.129310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471563516102574:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471563516102666:2122] held: no
2025-11-27T18:19:42.129906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471563516102574:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:42.129958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:42.130007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471563516102574:2049] [node 2] ICP22 created new session: [6:7577471567811069972:2049]
2025-11-27T18:19:42.130053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471567811069972:2049] [node 2] ICS09 handshake done sender: [6:7577471563516102666:2122] self: [6:7577471567811069970:0] peer: [2:7577471563211226660:0] socket: 204 qp: -1
2025-11-27T18:19:42.130089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471567811069972:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:42.130126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:42.068363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471568496712946:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:42.112662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:42.132749Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471563211226584:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(163588938ns) {HandleHandshakeStatus:329(162493020ns) {DropIncomingHandshake:442(405ns) DropOutgoingHandshake:457(424ns) SwitchToState:179(620ns) ProcessPendingSessionEvents:495(111186321ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 104272387ns max# 10672840ns}}}
2025-11-27T18:19:42.127011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471562107027089:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:42.068365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471564201745612:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(22500151ns) {HandleHandshakeStatus:329(22241078ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(543ns) SwitchToState:179(408ns) ProcessPendingSessionEvents:495(1511ns)}}
2025-11-27T18:19:42.136872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471563211226667:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:42.069636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:42.136922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471563211226667:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:42.144951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471563211226667:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:42.147917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471566401994485:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34150
2025-11-27T18:19:42.153832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471566401994481:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:42.158084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471563211226647:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:42.158182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:42.158362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471562107027089:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:42.161373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471563211226647:2107] [node 4] ICP22 created new session: [2:7577471567506193979:2107]
2025-11-27T18:19:42.158441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:42.161511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471562107027089:2049] [node 2] ICP22 created new session: [4:7577471566401994486:2049]
2025-11-27T18:19:42.161575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471566401994486:2049] [node 2] ICS09 handshake done sender: [4:7577471566401994481:2121] self: [4:7577471566401994484:0] peer: [2:7577471563211226666:0] socket: 197 qp: -1
2025-11-27T18:19:42.161615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471566401994486:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:42.162280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:42.173529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471567506193979:2107] [node 4] ICS09 handshake done sender: [2:7577471563211226667:2116] self: [2:7577471563211226666:0] peer: [4:7577471566401994484:0] socket: 188 qp: -1
2025-11-27T18:19:42.173613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471567506193979:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:42.173680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:42.175041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471563211226647:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16302798ns) {HandleHandshakeStatus:329(16299145ns) {DropIncomingHandshake:442(24101ns) DropOutgoingHandshake:457(513ns) SwitchToState:179(447ns) ProcessPendingSessionEvents:495(542647ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 481582ns max# 70320ns}}}
2025-11-27T18:19:42.245154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:42.245433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471567506193971:2048] [node 1] ICS01 socket: 184 reason# EndOfStream
2025-11-27T18:19:42.245478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471563211226583:2048] [node 1] ICP30 unregister session Session# [2:7577471567506193971:2048] VirtualId# [2:7577471563211226658:0]
2025-11-27T18:19:42.245513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471567506193971:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.245590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:42.245632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:42.247549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:42.247806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471568505572512:2048] [node 1] ICS01 socket: 231 reason# EndOfStream
2025-11-27T18:19:42.247886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471564210605087:2048] [node 1] ICP30 unregister session Session# [3:7577471568505572512:2048] VirtualId# [3:7577471568505572510:0]
2025-11-27T18:19:42.247927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471568505572512:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.248030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:42.248081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:42.249397Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:42.249639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471562107027170:2048] [node 1] ICS07 socket disconnect 187 reason# EndOfStream
2025-11-27T18:19:42.249672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471562107027170:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.249762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:42.249829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471562107027170:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:42.250693Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471562107027088:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:42.260028Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:42.260110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:42.250759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471562107027170:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:42.250802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471562107027088:2048] [node 1] ICP30 unregister session Session# [4:7577471562107027170:2048] VirtualId# [4:7577471562107027165:0]
2025-11-27T18:19:42.261493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:42.262206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471567811069967:2048] [node 1] ICS01 socket: 232 reason# EndOfStream
2025-11-27T18:19:42.262156Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471562107027088:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:42.262181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:42.262258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471563516102573:2048] [node 1] ICP30 unregister session Session# [6:7577471567811069967:2048] VirtualId# [6:7577471563516102665:0]
2025-11-27T18:19:42.262816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471564201745645:2048] [node 1] ICS07 socket disconnect 220 reason# EndOfStream
2025-11-27T18:19:42.262892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471564201745645:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.262295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471567811069967:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.262974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:42.263032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471564201745645:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:42.262377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:42.262422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:42.264509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471564201745547:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:42.264962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471564201745645:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:42.265036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471564201745547:2048] [node 1] ICP30 unregister session Session# [5:7577471564201745645:2048] VirtualId# [5:7577471564201745635:0]
2025-11-27T18:19:42.265082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:42.265133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471564201745547:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:42.265149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:42.272054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471562107027088:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:42.274888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471564201745547:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:42.283778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:42.284060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471564210605208:2049] [node 2] ICS01 socket: 211 reason# EndOfStream
2025-11-27T18:19:42.284117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471564210605088:2049] [node 2] ICP30 unregister session Session# [3:7577471564210605208:2049] VirtualId# [3:7577471564210605201:0]
2025-11-27T18:19:42.284157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471564210605208:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.284238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:42.284286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:42.286424Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:42.286667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471566401994486:2049] [node 2] ICS01 socket: 197 reason# EndOfStream
2025-11-27T18:19:42.286724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471562107027089:2049] [node 2] ICP30 unregister session Session# [4:7577471566401994486:2049] VirtualId# [4:7577471566401994484:0]
2025-11-27T18:19:42.286769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471566401994486:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.286842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:42.286889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:42.287559Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:42.287732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471564201745636:2049] [node 2] ICS01 socket: 201 reason# EndOfStream
2025-11-27T18:19:42.287774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471564201745548:2049] [node 2] ICP30 unregister session Session# [5:7577471564201745636:2049] VirtualId# [5:7577471564201745628:0]
2025-11-27T18:19:42.287798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471564201745636:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.287865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:42.287907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:42.288584Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:42.288780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471567811069972:2049] [node 2] ICS01 socket: 204 reason# EndOfStream
2025-11-27T18:19:42.288822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471563516102574:2049] [node 2] ICP30 unregister session Session# [6:7577471567811069972:2049] VirtualId# [6:7577471567811069970:0]
2025-11-27T18:19:42.288850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471567811069972:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.288919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:42.288967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:42.403022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:42.405028Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:42.403271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471566401994482:2050] [node 3] ICS01 socket: 191 reason# EndOfStream
2025-11-27T18:19:42.405229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471568496712946:2050] [node 3] ICS07 socket disconnect 180 reason# EndOfStream
2025-11-27T18:19:42.403315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471562107027090:2050] [node 3] ICP30 unregister session Session# [4:7577471566401994482:2050] VirtualId# [4:7577471562107027176:0]
2025-11-27T18:19:42.403354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471566401994482:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.403424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:42.403470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:42.405267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471568496712946:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.405339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:42.405383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471568496712946:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:42.405506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:42.406692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471567811069964:2050] [node 3] ICS01 socket: 179 reason# EndOfStream
2025-11-27T18:19:42.406734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471563516102575:2050] [node 3] ICP30 unregister session Session# [6:7577471567811069964:2050] VirtualId# [6:7577471563516102653:0]
2025-11-27T18:19:42.406768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471567811069964:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.406838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:42.406881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:42.407351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471564201745549:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7763) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:42.407426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471568496712946:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:42.407527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471564201745549:2050] [node 3] ICP30 unregister session Session# [5:7577471568496712946:2050] VirtualId# [5:7577471564201745637:0]
2025-11-27T18:19:42.407633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:42.407672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471564201745549:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7763) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:42.407696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7763) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:42.417205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471564201745549:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:42.459743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:42.459974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471563516102662:2108] [node 4] ICS01 socket: 193 reason# EndOfStream
2025-11-27T18:19:42.460023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471563516102637:2108] [node 4] ICP30 unregister session Session# [6:7577471563516102662:2108] VirtualId# [6:7577471563516102656:0]
2025-11-27T18:19:42.460059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471563516102662:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.460154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:42.460226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:42.460755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:42.460992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471568496712944:2108] [node 4] ICS01 socket: 225 reason# EndOfStream
2025-11-27T18:19:42.461046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471564201745612:2108] [node 4] ICP30 unregister session Session# [5:7577471568496712944:2108] VirtualId# [5:7577471564201745640:0]
2025-11-27T18:19:42.461092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471568496712944:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.461183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:42.461229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:42.547325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:42.547592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471567811069966:2105] [node 5] ICS01 socket: 203 reason# EndOfStream
2025-11-27T18:19:42.547643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471563516102633:2105] [node 5] ICP30 unregister session Session# [6:7577471567811069966:2105] VirtualId# [6:7577471563516102657:0]
2025-11-27T18:19:42.547673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471567811069966:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:42.547782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:42.547839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:42.676468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471567052786229:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:42.678556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471567052786230:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:42.702200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471567565391454:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:42.702216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471567565391453:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:42.726918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471566584149639:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:42.726988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471566584149640:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:42.749828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471567669876858:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:42.750036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471567669876859:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:42.750067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471567669876860:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:42.773400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471568923408558:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:42.773447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471568923408559:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:42.773472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471568923408560:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:42.796083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471568042547780:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:42.796142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471568042547782:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:42.797190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471568042547781:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:42.871865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471568042547842:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:42.880668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471567052786310:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:42.885501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471567052786322:2113] [node 5] ICP01 ready to work
2025-11-27T18:19:42.889562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471567052786323:2134] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:42.942522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471566584149718:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:42.948443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471567669876931:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:42.957770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471567565391525:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:42.962717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471566584149719:2125] [node 6] ICP01 ready to work
2025-11-27T18:19:42.957890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471568923408623:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:42.957764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471567565391526:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:42.962812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471566584149720:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:42.965572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471568042547847:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:42.957883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471567565391527:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:42.962556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471567669876933:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:42.965468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471568923408619:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15876
2025-11-27T18:19:42.962743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471567052786229:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(58973562ns) {Configure:104(58966998ns) {ProcessConfigured:135(33282496ns) {StartInitialHandshake:157(33279929ns) {DropHandshakes:470(3779ns) {DropIncomingHandshake:442(842ns) DropOutgoingHandshake:457(406ns)} GenerateSessionVirtualId:988(1753ns) PrepareNewSessionHandshake:477(3809ns) {SwitchToState:179(1920ns)}}}}}
2025-11-27T18:19:42.963166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471568923408624:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:42.975648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471568923408619:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:42.993578Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471568042547781:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(35175722ns) {Configure:104(35169732ns) {ProcessConfigured:135(9489249ns) {StartInitialHandshake:157(9107640ns) {DropHandshakes:470(2042ns) {DropIncomingHandshake:442(522ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(1253ns) PrepareNewSessionHandshake:477(4278ns) {SwitchToState:179(2223ns)}}}}}
2025-11-27T18:19:43.015533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471567565391454:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(30908061ns) {Configure:104(30897290ns) {ProcessConfigured:135(1795643ns) {StartInitialHandshake:157(1792927ns) {DropHandshakes:470(1330ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(489ns) PrepareNewSessionHandshake:477(2037ns) {SwitchToState:179(720ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.018041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471566584149640:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40329426ns) {Configure:104(40322092ns) {ProcessConfigured:135(11087988ns) {StartInitialHandshake:157(11086010ns) {DropHandshakes:470(1357ns) {DropIncomingHandshake:442(807ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(915ns) PrepareNewSessionHandshake:477(4037ns) {SwitchToState:179(1494ns)}}}}}
2025-11-27T18:19:43.017944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471567669876858:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28793193ns) {Configure:104(28787642ns) {ProcessConfigured:135(2397448ns) {StartInitialHandshake:157(2394621ns) {DropHandshakes:470(1044ns) {DropIncomingHandshake:442(220ns) DropOutgoingHandshake:457(112ns)} GenerateSessionVirtualId:988(443ns) PrepareNewSessionHandshake:477(4416ns) {SwitchToState:179(2104ns)}}}}}
2025-11-27T18:19:43.015640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471567565391526:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29107423ns) {Configure:104(29095067ns) {ProcessConfigured:135(1823366ns) {StartInitialHandshake:157(1820943ns) {DropHandshakes:470(1322ns) {DropIncomingHandshake:442(271ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(398ns) PrepareNewSessionHandshake:477(2005ns) {SwitchToState:179(991ns)}}}}}
2025-11-27T18:19:43.021733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471566584149718:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(40355524ns) {Configure:104(40347544ns) {ProcessConfigured:135(13071909ns) {StartInitialHandshake:157(13069807ns) {DropHandshakes:470(1228ns) {DropIncomingHandshake:442(319ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(507ns) PrepareNewSessionHandshake:477(2854ns) {SwitchToState:179(604ns)}}}}}
2025-11-27T18:19:43.022712Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471568042547847:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27023142ns) {Configure:104(27017009ns) {ProcessConfigured:135(3000063ns) {StartInitialHandshake:157(2993554ns) {DropHandshakes:470(3385ns) {DropIncomingHandshake:442(1129ns) DropOutgoingHandshake:457(479ns)} GenerateSessionVirtualId:988(1108ns) PrepareNewSessionHandshake:477(3178ns) {SwitchToState:179(1231ns)}}}}}
2025-11-27T18:19:43.026173Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471568923408558:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44208657ns) {Configure:104(44197847ns) {ProcessConfigured:135(3573607ns) {StartInitialHandshake:157(3571426ns) {DropHandshakes:470(915ns) {DropIncomingHandshake:442(272ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(404ns) PrepareNewSessionHandshake:477(2550ns) {SwitchToState:179(971ns)}}}}}
2025-11-27T18:19:43.021650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471567669876860:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29173742ns) {Configure:104(29167599ns) {ProcessConfigured:135(2024345ns) {StartInitialHandshake:157(2022891ns) {DropHandshakes:470(1545ns) {DropIncomingHandshake:442(412ns) DropOutgoingHandshake:457(308ns)} GenerateSessionVirtualId:988(431ns) PrepareNewSessionHandshake:477(2175ns) {SwitchToState:179(771ns)}}}}}
2025-11-27T18:19:43.026215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471566584149639:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50427724ns) {Configure:104(50422639ns) {ProcessConfigured:135(12342942ns) {StartInitialHandshake:157(6664012ns) {DropHandshakes:470(2475ns) {DropIncomingHandshake:442(442ns) DropOutgoingHandshake:457(178ns)} GenerateSessionVirtualId:988(1816ns) PrepareNewSessionHandshake:477(2458ns) {SwitchToState:179(1173ns)}}}}}
2025-11-27T18:19:43.022761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471567565391453:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(90368547ns) {Configure:104(90362126ns) {ProcessConfigured:135(16364446ns) {StartInitialHandshake:157(16362578ns) {DropHandshakes:470(1023ns) {DropIncomingHandshake:442(229ns) DropOutgoingHandshake:457(181ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:477(2349ns) {SwitchToState:179(96ns)}}}}}
2025-11-27T18:19:43.026098Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471567669876859:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45882888ns) {Configure:104(45863338ns) {ProcessConfigured:135(5702468ns) {StartInitialHandshake:157(5700373ns) {DropHandshakes:470(1088ns) {DropIncomingHandshake:442(322ns) DropOutgoingHandshake:457(263ns)} GenerateSessionVirtualId:988(372ns) PrepareNewSessionHandshake:477(3645ns) {SwitchToState:179(2137ns)}}}}}
2025-11-27T18:19:43.026735Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471566584149719:2125] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(47347045ns) {Configure:104(47342244ns) {ProcessConfigured:135(7117565ns) {StartInitialHandshake:157(7115131ns) {DropHandshakes:470(1377ns) {DropIncomingHandshake:442(430ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(490ns) PrepareNewSessionHandshake:477(3150235ns) {SwitchToState:179(5098ns)}}}}}
No RDMA context found for GID: 2025-11-27T18:19:43.026716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471567669876932:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25465
2025-11-27T18:19:43.024837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471567565391527:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(40240558ns) {Configure:104(40235043ns) {ProcessConfigured:135(12041757ns) {StartInitialHandshake:157(12039365ns) {DropHandshakes:470(1019ns) {DropIncomingHandshake:442(260ns) DropOutgoingHandshake:457(159ns)} GenerateSessionVirtualId:988(547ns) PrepareNewSessionHandshake:477(3494ns) {SwitchToState:179(1998ns)}}}}}
2025-11-27T18:19:43.027610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471567052786323:2134] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48748292ns) {Configure:104(48726040ns) {ProcessConfigured:135(12738596ns) {StartInitialHandshake:157(12736446ns) {DropHandshakes:470(3902863ns) {DropIncomingHandshake:442(2416ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(1633ns) PrepareNewSessionHandshake:477(3011ns) {SwitchToState:179(1658ns)}}}}}
::ffff:127.1.0.0
2025-11-27T18:19:43.026573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471568923408559:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47561772ns) {Configure:104(47557777ns) {ProcessConfigured:135(7278696ns) {StartInitialHandshake:157(7276358ns) {DropHandshakes:470(3299ns) {DropIncomingHandshake:442(928ns) DropOutgoingHandshake:457(474ns)} GenerateSessionVirtualId:988(1186ns) PrepareNewSessionHandshake:477(2289ns) {SwitchToState:179(1048ns)}}}}}
2025-11-27T18:19:43.028209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471567669876933:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50421273ns) {Configure:104(50415793ns) {ProcessConfigured:135(1401603ns) {StartInitialHandshake:157(1399722ns) {DropHandshakes:470(2888ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(545ns) PrepareNewSessionHandshake:477(1783ns) {SwitchToState:179(687ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.028312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471568923408624:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(50901702ns) {Configure:104(50895435ns) {ProcessConfigured:135(8365327ns) {StartInitialHandshake:157(8361862ns) {DropHandshakes:470(1267ns) {DropIncomingHandshake:442(372ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:477(3612661ns) {SwitchToState:179(3302ns)}}}}}
2025-11-27T18:19:43.028363Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471568923408560:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52015746ns) {Configure:104(52010960ns) {ProcessConfigured:135(16076555ns) {StartInitialHandshake:157(12201462ns) {DropHandshakes:470(2637ns) {DropIncomingHandshake:442(302ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(2125ns) PrepareNewSessionHandshake:477(4234ns) {SwitchToState:179(2178ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.028989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471566584149720:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50124143ns) {Configure:104(50118667ns) {ProcessConfigured:135(1671673ns) {StartInitialHandshake:157(1289968ns) {DropHandshakes:470(2171ns) {DropIncomingHandshake:442(757ns) DropOutgoingHandshake:457(280ns)} GenerateSessionVirtualId:988(1043ns) PrepareNewSessionHandshake:477(1531ns) {SwitchToState:179(622ns)}}}}}
2025-11-27T18:19:43.028153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471567052786322:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49412514ns) {Configure:104(49406288ns) {ProcessConfigured:135(1768644ns) {StartInitialHandshake:157(1765950ns) {DropHandshakes:470(3350ns) {DropIncomingHandshake:442(985ns) DropOutgoingHandshake:457(637ns)} GenerateSessionVirtualId:988(9319ns) PrepareNewSessionHandshake:477(3906ns) {SwitchToState:179(2193ns)}}}}}
2025-11-27T18:19:43.028925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471567565391528:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45920
2025-11-27T18:19:43.029033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471568923408623:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(51195137ns) {Configure:104(51189092ns) {ProcessConfigured:135(1516225ns) {StartInitialHandshake:157(1514917ns) {DropHandshakes:470(1338ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(432ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(1797ns) {SwitchToState:179(720ns)}}}}}
2025-11-27T18:19:43.029078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471567669876932:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.029539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471566584149699:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3848
2025-11-27T18:19:43.029118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471571347753624:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18740
2025-11-27T18:19:43.029740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471567669876931:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(50510068ns) {Configure:104(50505941ns) {ProcessConfigured:135(1238053ns) {StartInitialHandshake:157(1236633ns) {DropHandshakes:470(629546ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(1364ns)} GenerateSessionVirtualId:988(1530ns) PrepareNewSessionHandshake:477(1299ns) {SwitchToState:179(490ns)}}}}}
2025-11-27T18:19:43.030675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471567565391528:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.030243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471567669876858:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.032657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471567565391453:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.033441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471566584149699:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.033347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471571347753624:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.035507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471571964844244:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25465
2025-11-27T18:19:43.035414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471571860358836:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45920
2025-11-27T18:19:43.034413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471570879117025:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3848
2025-11-27T18:19:43.033692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471571347753628:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18740
2025-11-27T18:19:43.036429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471570879117028:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3848
2025-11-27T18:19:43.037030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471567565391525:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(38127419ns) {Configure:104(38119450ns) {ProcessConfigured:135(1361551ns) {StartInitialHandshake:157(1359556ns) {DropHandshakes:470(421317ns) {DropIncomingHandshake:442(316ns) DropOutgoingHandshake:457(1590ns)} GenerateSessionVirtualId:988(1267ns) PrepareNewSessionHandshake:477(1836ns) {SwitchToState:179(923ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.033825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471567052786319:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.041143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471573218375935:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15876
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-27T18:19:43.036241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471566584149639:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.033861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471567052786319:2133] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.039085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471570879117025:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.034890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471571347753629:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18740
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.035015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471567052786325:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.045404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471571964844245:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25465
2025-11-27T18:19:43.035044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471567052786325:2135] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.046016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471573218375935:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.042586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471570879117029:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3848
2025-11-27T18:19:43.045405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471571860358838:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45920
2025-11-27T18:19:43.042356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471567052786319:2133] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.045855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471571964844244:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.046072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471570879117029:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:19:43.042395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471567052786325:2135] [node 2] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:19:43.046894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471568923408559:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.046704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471571860358836:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.049025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471570879117028:2132] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.042439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471567052786293:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.048990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471571964844245:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.049432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471566584149640:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.042473Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471567052786293:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.048374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471571860358841:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45920
2025-11-27T18:19:43.049050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471571964844246:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25465
2025-11-27T18:19:43.048962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471571860358838:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.042991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471571347753629:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.049149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471567669876859:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.046019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471571347753628:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.049371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471567565391534:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.051187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471571964844247:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25465
2025-11-27T18:19:43.049402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471567565391534:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.048424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471567052786229:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.051850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471571964844246:2128] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.048934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471567052786310:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.050040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471571860358842:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45920
2025-11-27T18:19:43.053908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471570879117036:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3848
2025-11-27T18:19:43.052063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471567669876860:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.048945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:43.051192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471571860358841:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.055231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471570879117021:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.049117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:43.055274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471567669876858:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471567669876932:2120] held: no
2025-11-27T18:19:43.049171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471567052786293:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:43.055276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471570879117021:2127] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.055546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471567669876932:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.049311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471567052786229:2048] [node 2] ICP22 created new session: [1:7577471571347753636:2048]
2025-11-27T18:19:43.059136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471566584149699:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:43.059894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471573218375937:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15876
2025-11-27T18:19:43.051195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471571860358831:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.051237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471571860358831:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.049347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471571347753631:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18740
2025-11-27T18:19:43.059292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471570879117021:2127] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:43.060125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471568923408559:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24755) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471573218375935:2117] held: no
2025-11-27T18:19:43.049371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471571347753636:2048] [node 2] ICS09 handshake done sender: [1:7577471567052786325:2135] self: [1:7577471567052786324:0] peer: [2:7577471571860358837:0] socket: 188 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.051514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471567565391533:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-27T18:19:43.060195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471566584149639:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.063584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471567669876858:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.051543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471567565391533:2120] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.060498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:43.049419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471571347753636:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:43.065044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471572337515154:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3910
2025-11-27T18:19:43.063668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:43.054014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471567565391534:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:43.065041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471573218375938:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15876
2025-11-27T18:19:43.049456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:43.061253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471566584149639:2048] [node 1] ICP22 created new session: [3:7577471570879117040:2048]
2025-11-27T18:19:43.054523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471567565391528:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:43.066475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471572337515154:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.063709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471567669876858:2048] [node 1] ICP22 created new session: [4:7577471571964844254:2048]
2025-11-27T18:19:43.066786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471573218375938:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.049793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471567052786310:2107] [node 4] ICP22 created new session: [1:7577471571347753638:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.061537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471566584149718:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.064177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471567669876860:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2063) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471571964844246:2128] held: no
2025-11-27T18:19:43.054694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471567565391526:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.068912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471568042547780:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.051208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471571347753638:2107] [node 4] ICS09 handshake done sender: [1:7577471567052786319:2133] self: [1:7577471567052786318:0] peer: [4:7577471571964844243:0] socket: 184 qp: -1
2025-11-27T18:19:43.061590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:43.068565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471568923408558:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.064522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471571964844252:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25465
2025-11-27T18:19:43.055476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471567565391453:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.051514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471571347753638:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:43.065034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471571964844254:2048] [node 1] ICS09 handshake done sender: [4:7577471567669876932:2120] self: [4:7577471571964844243:0] peer: [1:7577471567052786318:0] socket: 187 qp: -1
2025-11-27T18:19:43.062929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471566584149718:2108] [node 4] ICP22 created new session: [3:7577471570879117041:2108]
2025-11-27T18:19:43.051526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471567052786230:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.055561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:43.062943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471570879117040:2048] [node 1] ICS09 handshake done sender: [3:7577471566584149699:2107] self: [3:7577471570879117033:0] peer: [1:7577471567052786292:0] socket: 185 qp: -1
2025-11-27T18:19:43.065901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471571964844254:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:43.068873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471573218375935:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:43.055970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471567565391526:2107] [node 5] ICP22 created new session: [2:7577471571860358849:2107]
2025-11-27T18:19:43.074082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471572337515156:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3910
2025-11-27T18:19:43.066671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471571964844251:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25465
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.066740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:43.062986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471570879117040:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:43.057316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:43.073794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471573218375939:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15876
2025-11-27T18:19:43.051578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:43.063023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:43.075150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471572337515155:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3910
2025-11-27T18:19:43.063526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471570879117041:2108] [node 4] ICS09 handshake done sender: [3:7577471570879117021:2127] self: [3:7577471566584149721:0] peer: [4:7577471571964844250:0] socket: 217 qp: -1
2025-11-27T18:19:43.075882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471572337515156:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.073819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471568923408559:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.064159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471570879117041:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:43.073554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471571964844245:2127] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:43.051780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:43.064200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:43.076948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471572337515155:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.057343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471571860358849:2107] [node 5] ICS09 handshake done sender: [2:7577471567565391534:2121] self: [2:7577471567565391530:0] peer: [5:7577471573218375936:0] socket: 198 qp: -1
2025-11-27T18:19:43.053785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471567052786230:2049] [node 3] ICP22 created new session: [1:7577471571347753641:2049]
2025-11-27T18:19:43.068930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471566584149640:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24755) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471570879117028:2132] held: no
2025-11-27T18:19:43.076973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471568042547842:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.074127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:43.057367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471567565391453:2048] [node 1] ICP22 created new session: [2:7577471571860358850:2048]
2025-11-27T18:19:43.073863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471567669876859:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24755) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471571964844245:2127] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.054159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471571347753631:2141] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.077458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471572337515158:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3910
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.074178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471568923408559:2049] [node 2] ICP22 created new session: [5:7577471573218375943:2049]
2025-11-27T18:19:43.074159Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471571964844246:2128] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:43.069532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471570879117039:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3848
2025-11-27T18:19:43.057514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471571860358849:2107] [node 5] ICS10 traffic start
2025-11-27T18:19:43.054622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471571347753641:2049] [node 3] ICS09 handshake done sender: [1:7577471567052786293:2110] self: [1:7577471567052786292:0] peer: [3:7577471570879117033:0] socket: 182 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.078003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471568042547781:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.076975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471567669876860:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.077552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471570879117028:2132] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:43.075372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471573218375943:2049] [node 2] ICS09 handshake done sender: [5:7577471573218375935:2117] self: [5:7577471573218375936:0] peer: [2:7577471567565391530:0] socket: 199 qp: -1
2025-11-27T18:19:43.057533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471567565391533:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:43.055284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471571347753641:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:43.057561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:43.077427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:43.078973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471572337515159:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3910
2025-11-27T18:19:43.076002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471573218375943:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:43.079072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471566584149640:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.077470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471567669876860:2050] [node 3] ICP22 created new session: [4:7577471571964844257:2050]
2025-11-27T18:19:43.078285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471571964844257:2050] [node 3] ICS09 handshake done sender: [4:7577471571964844246:2128] self: [4:7577471571964844250:0] peer: [3:7577471566584149721:0] socket: 218 qp: -1
2025-11-27T18:19:43.076436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:43.057916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471571860358850:2048] [node 1] ICS09 handshake done sender: [2:7577471567565391528:2119] self: [2:7577471571860358837:0] peer: [1:7577471567052786324:0] socket: 189 qp: -1
2025-11-27T18:19:43.055557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:43.080885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:43.078327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471571964844257:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:43.079026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471572337515158:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.070052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471571347753625:2137] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.076725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471573218375942:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15876
2025-11-27T18:19:43.057973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471571860358850:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:43.081314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471570879117023:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.078695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:43.077144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471573218375939:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.059131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:43.081338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471570879117023:2128] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.078888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471567669876859:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.070089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471571347753625:2137] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.077186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471573218375930:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.060053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471567565391454:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.079914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471568042547782:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.081520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471566584149640:2049] [node 2] ICP22 created new session: [3:7577471570879117046:2049]
2025-11-27T18:19:43.077207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471573218375930:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.079408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.060844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471571860358831:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:43.081943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471570879117046:2049] [node 2] ICS09 handshake done sender: [3:7577471570879117028:2132] self: [3:7577471570879117037:0] peer: [2:7577471567565391529:0] socket: 201 qp: -1
2025-11-27T18:19:43.061510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:43.080163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471572337515159:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.079455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471567669876859:2049] [node 2] ICP22 created new session: [4:7577471571964844259:2049]
2025-11-27T18:19:43.061991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471567565391454:2049] [node 3] ICP22 created new session: [2:7577471571860358853:2049]
2025-11-27T18:19:43.070905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471571347753627:2138] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.082099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471570879117046:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:43.081279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471568042547847:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.080292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471571964844259:2049] [node 2] ICS09 handshake done sender: [4:7577471571964844245:2127] self: [4:7577471571964844249:0] peer: [2:7577471567565391532:0] socket: 210 qp: -1
No RDMA context found for GID: 2025-11-27T18:19:43.077972Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471568923408558:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471573218375938:2119] held: no
::ffff:127.1.0.0
2025-11-27T18:19:43.084098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:43.063060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471567565391527:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.080871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471571964844259:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:43.070930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471571347753627:2138] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.078079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471573218375942:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.073839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471571347753627:2138] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:43.063538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:43.081052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:43.081462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471572337515161:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3910
2025-11-27T18:19:43.084613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471570879117027:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.076097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471567052786322:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.082044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471571964844239:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.080136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471573218375944:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15876
2025-11-27T18:19:43.064343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471571860358853:2049] [node 3] ICS09 handshake done sender: [2:7577471567565391533:2120] self: [2:7577471567565391529:0] peer: [3:7577471570879117037:0] socket: 195 qp: -1
2025-11-27T18:19:43.084639Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471570879117027:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.087608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471572337515154:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.080186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471568923408624:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.076155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:43.082072Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471571964844239:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.064386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471571860358853:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:43.088334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471572337515165:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3910
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.084776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471571964844237:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.088293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471570879117023:2128] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:43.076747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471567052786322:2113] [node 5] ICP22 created new session: [1:7577471571347753647:2113]
2025-11-27T18:19:43.064417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:43.081455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471568923408560:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.084806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471571964844237:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.077151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471571347753647:2113] [node 5] ICS09 handshake done sender: [1:7577471571347753627:2138] self: [1:7577471571347753626:0] peer: [5:7577471573218375941:0] socket: 233 qp: -1
2025-11-27T18:19:43.084127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471573218375930:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:43.090096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471572337515166:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3910
2025-11-27T18:19:43.085881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471573218375938:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:43.064148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471567565391527:2108] [node 4] ICP22 created new session: [2:7577471571860358855:2108]
2025-11-27T18:19:43.087325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471571964844239:2125] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:43.090548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471568042547780:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471572337515154:2115] held: no
2025-11-27T18:19:43.087058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471568923408623:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.066349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471571860358855:2108] [node 4] ICS09 handshake done sender: [2:7577471571860358831:2122] self: [2:7577471567565391532:0] peer: [4:7577471571964844249:0] socket: 203 qp: -1
2025-11-27T18:19:43.077401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471571347753647:2113] [node 5] ICS10 traffic start
2025-11-27T18:19:43.093000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471572337515156:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.087660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471571964844237:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:43.087244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471568923408558:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.089782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471570879117027:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:43.077447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:43.094826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471572337515155:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:43.066496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471571860358855:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:43.087294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:43.089872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471566584149719:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.066543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:43.094817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471568042547842:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1086) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471572337515156:2117] held: no
2025-11-27T18:19:43.089321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471567669876931:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.087323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: 2025-11-27T18:19:43.091023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
2025-11-27T18:19:43.089480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471567669876933:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.098361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471568042547780:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.087342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471568923408558:2048] [node 1] ICP22 created new session: [5:7577471573218375950:2048]
2025-11-27T18:19:43.080280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471571860358840:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.090141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:43.091081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471566584149719:2125] [node 6] ICP22 created new session: [3:7577471570879117051:2125]
2025-11-27T18:19:43.080306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471571860358840:2126] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.092987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471566584149718:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(20619446ns) {HandleHandshakeStatus:329(20616511ns) {DropIncomingHandshake:442(12954ns) DropOutgoingHandshake:457(465ns) SwitchToState:179(361ns) ProcessPendingSessionEvents:495(15555482ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15461159ns max# 2629671ns}}}
2025-11-27T18:19:43.090380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471567669876931:2109] [node 6] ICP22 created new session: [4:7577471571964844263:2109]
2025-11-27T18:19:43.087672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471568923408623:2110] [node 6] ICP22 created new session: [5:7577471573218375951:2110]
2025-11-27T18:19:43.087732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471573218375951:2110] [node 6] ICS09 handshake done sender: [5:7577471573218375930:2116] self: [5:7577471573218375929:0] peer: [6:7577471572337515160:0] socket: 219 qp: -1
2025-11-27T18:19:43.090715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:43.085898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471571860358840:2126] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:43.093046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471566584149720:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.087944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471573218375951:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:43.091029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471571964844263:2109] [node 6] ICS09 handshake done sender: [4:7577471571964844239:2125] self: [4:7577471571964844238:0] peer: [6:7577471572337515164:0] socket: 241 qp: -1
2025-11-27T18:19:43.088255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471567565391525:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.088342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471573218375950:2048] [node 1] ICS09 handshake done sender: [5:7577471573218375938:2119] self: [5:7577471573218375941:0] peer: [1:7577471571347753626:0] socket: 234 qp: -1
2025-11-27T18:19:43.091429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471567669876933:2108] [node 5] ICP22 created new session: [4:7577471571964844264:2108]
2025-11-27T18:19:43.093079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:43.088421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:43.088384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471573218375950:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:43.093130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471566584149720:2109] [node 5] ICP22 created new session: [3:7577471570879117052:2109]
2025-11-27T18:19:43.091471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471571964844263:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:43.099184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471572337515168:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3910
2025-11-27T18:19:43.092152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471571964844264:2108] [node 5] ICS09 handshake done sender: [4:7577471571964844237:2124] self: [4:7577471571964844236:0] peer: [5:7577471573218375946:0] socket: 237 qp: -1
2025-11-27T18:19:43.093182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471570879117051:2125] [node 6] ICS09 handshake done sender: [3:7577471570879117023:2128] self: [3:7577471570879117020:0] peer: [6:7577471572337515163:0] socket: 200 qp: -1
2025-11-27T18:19:43.088429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:43.099198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:43.092987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:43.088457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:43.093410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471570879117051:2125] [node 6] ICS10 traffic start
2025-11-27T18:19:43.089169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471567565391525:2109] [node 6] ICP22 created new session: [2:7577471571860358859:2109]
2025-11-27T18:19:43.093005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471571964844264:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:43.099258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471568042547780:2048] [node 1] ICP22 created new session: [6:7577471572337515170:2048]
2025-11-27T18:19:43.093526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471566584149639:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23894025ns) {HandleHandshakeStatus:329(23892356ns) {DropIncomingHandshake:442(2126ns) DropOutgoingHandshake:457(16161ns) SwitchToState:179(410ns) ProcessPendingSessionEvents:495(18989001ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18194093ns max# 1994377ns}}}
2025-11-27T18:19:43.093064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:43.089987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471567565391453:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23818631ns) {HandleHandshakeStatus:329(23313249ns) {DropIncomingHandshake:442(2994ns) DropOutgoingHandshake:457(9435ns) SwitchToState:179(465ns) ProcessPendingSessionEvents:495(17256649ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17148573ns max# 1435999ns}}}
2025-11-27T18:19:43.093905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471568923408624:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31075) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471573218375939:2120] held: no
2025-11-27T18:19:43.099334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471572337515170:2048] [node 1] ICS09 handshake done sender: [6:7577471572337515154:2115] self: [6:7577471572337515157:0] peer: [1:7577471571347753623:0] socket: 235 qp: -1
2025-11-27T18:19:43.093757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471570879117052:2109] [node 5] ICS09 handshake done sender: [3:7577471570879117027:2131] self: [3:7577471570879117026:0] peer: [5:7577471573218375947:0] socket: 214 qp: -1
2025-11-27T18:19:43.090088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471571860358859:2109] [node 6] ICS09 handshake done sender: [2:7577471571860358840:2126] self: [2:7577471571860358839:0] peer: [6:7577471572337515162:0] socket: 211 qp: -1
2025-11-27T18:19:43.099371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471572337515170:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:43.095286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471573218375954:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15876
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.079524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471571347753625:2137] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:43.093888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471570879117048:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3848
2025-11-27T18:19:43.098163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471573218375955:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15876
2025-11-27T18:19:43.090325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471571860358859:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:43.099416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:43.093937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:43.090361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:43.099920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471573218375939:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:43.100038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471572337515167:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3910
2025-11-27T18:19:43.082241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471567052786323:2134] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.093954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471570879117052:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:43.091810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471567565391526:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(27370560ns) {HandleHandshakeStatus:329(27368677ns) {DropIncomingHandshake:442(554066ns) DropOutgoingHandshake:457(810ns) SwitchToState:179(6504ns) ProcessPendingSessionEvents:495(20789636ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20636734ns max# 1387657ns}}}
2025-11-27T18:19:43.100528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471573218375942:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:43.095076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:43.082285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:43.093871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471567565391454:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(23622844ns) {HandleHandshakeStatus:329(23620271ns) {DropIncomingHandshake:442(11689ns) DropOutgoingHandshake:457(627ns) SwitchToState:179(978ns) ProcessPendingSessionEvents:495(17236270ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17134524ns max# 2253148ns}}}
2025-11-27T18:19:43.084633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471567052786323:2134] [node 6] ICP22 created new session: [1:7577471571347753649:2134]
2025-11-27T18:19:43.098747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471567565391527:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(23609464ns) {HandleHandshakeStatus:329(23606952ns) {DropIncomingHandshake:442(20258ns) DropOutgoingHandshake:457(249ns) SwitchToState:179(519ns) ProcessPendingSessionEvents:495(16224549ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16104607ns max# 1772025ns}}}
2025-11-27T18:19:43.103021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471568042547782:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2063) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471572337515158:2118] held: no
2025-11-27T18:19:43.085019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471571347753649:2134] [node 6] ICS09 handshake done sender: [1:7577471571347753625:2137] self: [1:7577471571347753623:0] peer: [6:7577471572337515157:0] socket: 232 qp: -1
2025-11-27T18:19:43.103863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471568923408624:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.099204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471570879117048:2136] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.104168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:43.103386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471568042547781:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.104238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471568923408624:2109] [node 4] ICP22 created new session: [5:7577471573218375956:2109]
2025-11-27T18:19:43.085897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471571347753649:2134] [node 6] ICS10 traffic start
2025-11-27T18:19:43.104211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:43.086314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:43.104297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471573218375956:2109] [node 4] ICS09 handshake done sender: [5:7577471573218375939:2120] self: [5:7577471573218375946:0] peer: [4:7577471571964844236:0] socket: 206 qp: -1
2025-11-27T18:19:43.104275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471568042547781:2049] [node 2] ICP22 created new session: [6:7577471572337515172:2049]
2025-11-27T18:19:43.104537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471572337515172:2049] [node 2] ICS09 handshake done sender: [6:7577471572337515155:2116] self: [6:7577471572337515162:0] peer: [2:7577471571860358839:0] socket: 216 qp: -1
2025-11-27T18:19:43.104346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471568923408560:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.104583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471572337515172:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:43.104633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:43.086629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471567052786230:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(25784796ns) {HandleHandshakeStatus:329(25782376ns) {DropIncomingHandshake:442(2233ns) DropOutgoingHandshake:457(984ns) SwitchToState:179(429ns) ProcessPendingSessionEvents:495(19183655ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19069320ns max# 1853800ns}}}
2025-11-27T18:19:43.104566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471573218375956:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:43.086759Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471567052786229:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(28025207ns) {HandleHandshakeStatus:329(28021922ns) {DropIncomingHandshake:442(499ns) DropOutgoingHandshake:457(12449ns) SwitchToState:179(713ns) ProcessPendingSessionEvents:495(24864650ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24332476ns max# 2602118ns}}}
2025-11-27T18:19:43.087109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471567052786230:2049] [node 3] ICP18 handshake (actor [1:7577471571347753631:2141]) is too old
2025-11-27T18:19:43.104615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.088281Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471567052786310:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(29679380ns) {HandleHandshakeStatus:329(29677641ns) {DropIncomingHandshake:442(17381ns) DropOutgoingHandshake:457(415ns) SwitchToState:179(347ns) ProcessPendingSessionEvents:495(23359192ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22217663ns max# 2367695ns}}}
2025-11-27T18:19:43.105245Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471567052786322:2113] [node 5] ICP35 event processing took too much time PendingConnection:279(19876377ns) {HandleHandshakeStatus:329(19874044ns) {DropIncomingHandshake:442(14669ns) DropOutgoingHandshake:457(464ns) SwitchToState:179(893ns) ProcessPendingSessionEvents:495(16573516ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16467643ns max# 1778825ns}}}
2025-11-27T18:19:43.105290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:43.105246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471566584149720:2109] [node 5] ICP18 handshake (actor [3:7577471570879117048:2136]) is too old
2025-11-27T18:19:43.105223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471571860358861:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45920
2025-11-27T18:19:43.106370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471567052786323:2134] [node 6] ICP35 event processing took too much time PendingConnection:279(17154777ns) {HandleHandshakeStatus:329(17152996ns) {DropIncomingHandshake:442(13375ns) DropOutgoingHandshake:457(343ns) SwitchToState:179(238ns) ProcessPendingSessionEvents:495(11476280ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11376836ns max# 2188031ns}}}
2025-11-27T18:19:43.106312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471568042547842:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.106971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:43.105531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471568923408560:2050] [node 3] ICP22 created new session: [5:7577471573218375958:2050]
2025-11-27T18:19:43.107281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471568042547842:2106] [node 5] ICP22 created new session: [6:7577471572337515174:2106]
2025-11-27T18:19:43.106380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471573218375958:2050] [node 3] ICS09 handshake done sender: [5:7577471573218375942:2121] self: [5:7577471573218375947:0] peer: [3:7577471570879117026:0] socket: 239 qp: -1
2025-11-27T18:19:43.107697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471572337515174:2106] [node 5] ICS09 handshake done sender: [6:7577471572337515156:2117] self: [6:7577471572337515160:0] peer: [5:7577471573218375929:0] socket: 238 qp: -1
2025-11-27T18:19:43.106407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471573218375958:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:43.107992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471572337515159:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:43.106947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:43.108224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471572337515174:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:43.108650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:43.108691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471568042547847:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.108719Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471572337515158:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:43.108955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:43.110097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471567669876933:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17512105ns) {HandleHandshakeStatus:329(17509987ns) {DropIncomingHandshake:442(347151ns) DropOutgoingHandshake:457(574ns) SwitchToState:179(415ns) ProcessPendingSessionEvents:495(13044077ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12790958ns max# 2004642ns}}}
2025-11-27T18:19:43.109029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471568042547847:2110] [node 4] ICP22 created new session: [6:7577471572337515177:2110]
2025-11-27T18:19:43.109070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471572337515177:2110] [node 4] ICS09 handshake done sender: [6:7577471572337515159:2119] self: [6:7577471572337515164:0] peer: [4:7577471571964844238:0] socket: 242 qp: -1
2025-11-27T18:19:43.109282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471572337515177:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:43.109709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.114053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471571964844267:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25465
2025-11-27T18:19:43.114906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471568042547782:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.114987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:43.115178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471568042547782:2050] [node 3] ICP22 created new session: [6:7577471572337515179:2050]
2025-11-27T18:19:43.115407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471572337515179:2050] [node 3] ICS09 handshake done sender: [6:7577471572337515158:2118] self: [6:7577471572337515163:0] peer: [3:7577471570879117020:0] socket: 205 qp: -1
2025-11-27T18:19:43.115445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471572337515179:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:43.115643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:43.184951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:43.185211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471571860358850:2048] [node 1] ICS07 socket disconnect 189 reason# EndOfStream
2025-11-27T18:19:43.185253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471571860358850:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.185343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:43.185393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471571860358850:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:43.186281Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:43.186475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471570879117040:2048] [node 1] ICS07 socket disconnect 185 reason# EndOfStream
2025-11-27T18:19:43.186506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471570879117040:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.186594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:43.186639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471570879117040:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:43.187589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471567565391453:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13385) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:43.187626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471571860358850:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:43.188861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471566584149639:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13385) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:43.192273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471570879117040:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:43.194317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471567565391453:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13385) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:43.194319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:43.194398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471571860358850:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:43.194440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471567565391453:2048] [node 1] ICP30 unregister session Session# [2:7577471571860358850:2048] VirtualId# [2:7577471571860358837:0]
2025-11-27T18:19:43.194494Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471566584149639:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13385) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:43.194561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471570879117040:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:43.194581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:43.194583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471566584149639:2048] [node 1] ICP30 unregister session Session# [3:7577471570879117040:2048] VirtualId# [3:7577471570879117033:0]
2025-11-27T18:19:43.194703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471571964844254:2048] [node 1] ICS01 socket: 187 reason# EndOfStream
2025-11-27T18:19:43.194722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471567565391453:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:43.194666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:43.194757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:43.194706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471566584149639:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:43.194722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:43.194813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471567669876858:2048] [node 1] ICP30 unregister session Session# [4:7577471571964844254:2048] VirtualId# [4:7577471571964844243:0]
2025-11-27T18:19:43.194863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471571964844254:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.194921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:43.194955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:43.195780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:43.195950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471573218375950:2048] [node 1] ICS01 socket: 234 reason# EndOfStream
2025-11-27T18:19:43.195992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471568923408558:2048] [node 1] ICP30 unregister session Session# [5:7577471573218375950:2048] VirtualId# [5:7577471573218375941:0]
2025-11-27T18:19:43.196020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471573218375950:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.196087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:43.196133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:43.196728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:43.196847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471572337515170:2048] [node 1] ICS01 socket: 235 reason# EndOfStream
2025-11-27T18:19:43.196876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471568042547780:2048] [node 1] ICP30 unregister session Session# [6:7577471572337515170:2048] VirtualId# [6:7577471572337515157:0]
2025-11-27T18:19:43.196896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471572337515170:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.196957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:43.196990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:43.204651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471566584149639:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:43.256426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:43.256661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471570879117046:2049] [node 2] ICS01 socket: 201 reason# EndOfStream
2025-11-27T18:19:43.256729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471566584149640:2049] [node 2] ICP30 unregister session Session# [3:7577471570879117046:2049] VirtualId# [3:7577471570879117037:0]
2025-11-27T18:19:43.256785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471570879117046:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.256887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:43.256946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:43.259128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:43.259350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471573218375943:2049] [node 2] ICS01 socket: 199 reason# EndOfStream
2025-11-27T18:19:43.259393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471568923408559:2049] [node 2] ICP30 unregister session Session# [5:7577471573218375943:2049] VirtualId# [5:7577471573218375936:0]
2025-11-27T18:19:43.259430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471573218375943:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.259503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:43.259554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:43.260209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:43.260382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471571964844259:2049] [node 2] ICS01 socket: 210 reason# EndOfStream
2025-11-27T18:19:43.260428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471567669876859:2049] [node 2] ICP30 unregister session Session# [4:7577471571964844259:2049] VirtualId# [4:7577471571964844249:0]
2025-11-27T18:19:43.260461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471571964844259:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.260533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:43.260573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:43.261294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:43.261478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471572337515172:2049] [node 2] ICS07 socket disconnect 216 reason# EndOfStream
2025-11-27T18:19:43.261515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471572337515172:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.261607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:43.261660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471572337515172:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:43.262366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471568042547781:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24755) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:43.262509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471572337515172:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:43.262560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471568042547781:2049] [node 2] ICP30 unregister session Session# [6:7577471572337515172:2049] VirtualId# [6:7577471572337515162:0]
2025-11-27T18:19:43.262666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:43.262737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471568042547781:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24755) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:43.262763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24755) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:43.272249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471568042547781:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:43.324745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:43.324963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471571964844257:2050] [node 3] ICS01 socket: 218 reason# EndOfStream
2025-11-27T18:19:43.325007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471567669876860:2050] [node 3] ICP30 unregister session Session# [4:7577471571964844257:2050] VirtualId# [4:7577471571964844250:0]
2025-11-27T18:19:43.325031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471571964844257:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.325100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:43.325135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:43.325971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:43.326191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471573218375958:2050] [node 3] ICS07 socket disconnect 239 reason# EndOfStream
2025-11-27T18:19:43.326226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471573218375958:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.326302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:43.326339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471573218375958:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:43.327078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:43.327204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471568923408560:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2063) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:43.327266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471572337515179:2050] [node 3] ICS01 socket: 205 reason# EndOfStream
2025-11-27T18:19:43.327382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471573218375958:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:43.327320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471568042547782:2050] [node 3] ICP30 unregister session Session# [6:7577471572337515179:2050] VirtualId# [6:7577471572337515163:0]
2025-11-27T18:19:43.327437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471568923408560:2050] [node 3] ICP30 unregister session Session# [5:7577471573218375958:2050] VirtualId# [5:7577471573218375947:0]
2025-11-27T18:19:43.327360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471572337515179:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.327421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:43.327470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:43.327543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:43.327607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471568923408560:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2063) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:43.327629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2063) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:43.337804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471568923408560:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:43.421897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:43.422101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471573218375956:2109] [node 4] ICS01 socket: 206 reason# EndOfStream
2025-11-27T18:19:43.422148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471568923408624:2109] [node 4] ICP30 unregister session Session# [5:7577471573218375956:2109] VirtualId# [5:7577471573218375946:0]
2025-11-27T18:19:43.422183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471573218375956:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.422249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:43.422338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:43.422844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:43.422997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471572337515177:2110] [node 4] ICS07 socket disconnect 242 reason# EndOfStream
2025-11-27T18:19:43.423023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471572337515177:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.423079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:43.423123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471572337515177:2110] [node 4] ICS15 start handshake
2025-11-27T18:19:43.423772Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471568042547847:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31075) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:43.423814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471572337515177:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:43.423907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471568042547847:2110] [node 4] ICP30 unregister session Session# [6:7577471572337515177:2110] VirtualId# [6:7577471572337515164:0]
2025-11-27T18:19:43.424004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:43.424034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471568042547847:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31075) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:43.424044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31075) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:43.434037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471568042547847:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:19:43.479854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:43.480041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471572337515174:2106] [node 5] ICS01 socket: 238 reason# EndOfStream
2025-11-27T18:19:43.480116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471568042547842:2106] [node 5] ICP30 unregister session Session# [6:7577471572337515174:2106] VirtualId# [6:7577471572337515160:0]
2025-11-27T18:19:43.480147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471572337515174:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:43.480215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:43.480266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:43.580819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471573333546677:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:43.580876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471573333546678:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:43.601045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471572593591452:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:43.601112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471572593591453:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:43.624554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471571286919749:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:43.624623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471571286919750:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:43.647020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471573924385745:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:43.647088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471573924385746:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:43.647134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471573924385747:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:43.667625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471572451164847:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:43.667689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471572451164848:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:43.667745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471572451164849:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:43.688530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471573365213828:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:43.688590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471573365213829:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:43.688615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471573365213830:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:43.842986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471573333546743:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:43.849402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471572451164910:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:43.849315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471573365213891:2106] [node 4] ICP01 ready to work
2025-11-27T18:19:43.849419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471571286919839:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:43.849493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471573924385808:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:43.849621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471572593591542:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:43.850373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471571286919840:2111] [node 5] ICP01 ready to work
2025-11-27T18:19:43.850384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471573333546744:2110] [node 5] ICP01 ready to work
2025-11-27T18:19:43.850477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471573365213893:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:43.850437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471572593591543:2130] [node 5] ICP01 ready to work
2025-11-27T18:19:43.850498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471573924385809:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:43.850619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471572451164911:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:43.851728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471571286919841:2120] [node 6] ICP01 ready to work
2025-11-27T18:19:43.852549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471573333546745:2111] [node 6] ICP01 ready to work
2025-11-27T18:19:43.853094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471572593591544:2132] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.868394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471571286919855:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29466
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.868474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471572593591558:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31524
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.868579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471572593591559:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31524
2025-11-27T18:19:43.868671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471572593591560:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31524
2025-11-27T18:19:43.868606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471571286919856:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29466
2025-11-27T18:19:43.869095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471573333546756:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8215
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.870837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471573333546757:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8215
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.876947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471571286919857:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29466
2025-11-27T18:19:43.877039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471571286919858:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29466
2025-11-27T18:19:43.877180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471571286919859:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29466
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.877929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471573333546758:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8215
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.880206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471572593591561:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31524
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.881123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471573333546762:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8215
2025-11-27T18:19:43.882086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471572593591562:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31524
2025-11-27T18:19:43.882483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471573333546763:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8215
2025-11-27T18:19:43.890919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471573333546757:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.891411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471571286919856:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.894138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471571286919857:2144] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.894264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471572593591559:2143] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.894359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471573333546758:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.894485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471573333546762:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.901436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471573924385823:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26706
2025-11-27T18:19:43.902223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471573924385823:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.902299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471573924385824:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26706
2025-11-27T18:19:43.902359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471573924385825:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26706
2025-11-27T18:19:43.902523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471573924385747:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.900841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471571286919859:2146] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.901402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471571286919749:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.903663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471573333546749:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.903699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471573333546749:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.908493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471572593591561:2145] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.908654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471572593591560:2144] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.905673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471573924385827:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26706
2025-11-27T18:19:43.903794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471571286919847:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.903831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471571286919847:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.904528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471571286919855:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.904959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471571286919847:2139] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:43.906113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471571286919839:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.906219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:43.906268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471571286919839:2107] [node 4] ICP22 created new session: [3:7577471571286919865:2107]
2025-11-27T18:19:43.906327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471571286919865:2107] [node 4] ICS09 handshake done sender: [3:7577471571286919847:2139] self: [3:7577471571286919846:0] peer: [4:7577471573924385826:0] socket: 194 qp: -1
2025-11-27T18:19:43.906362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471571286919865:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:43.908928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:43.910893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471572593591558:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.910507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471573333546756:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.910865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471573924385825:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.910928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471573924385823:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:43.911138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471572593591562:2146] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.911232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471573333546763:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.911321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471572593591452:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.911631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471573333546749:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:43.911739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471573924385824:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.912050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471573333546747:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.912079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471573333546747:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.911834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471573924385747:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.911915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:43.911961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471573924385747:2050] [node 3] ICP22 created new session: [4:7577471573924385828:2050]
2025-11-27T18:19:43.912027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471573924385828:2050] [node 3] ICS09 handshake done sender: [4:7577471573924385823:2115] self: [4:7577471573924385826:0] peer: [3:7577471571286919846:0] socket: 200 qp: -1
2025-11-27T18:19:43.912063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471573924385828:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:43.912120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.922037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471571286919858:2145] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.922186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471571286919867:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29466
2025-11-27T18:19:43.923031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471571286919750:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.924621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471571286919859:2146] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:43.922984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471573333546678:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.923078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:43.923138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471573333546678:2049] [node 3] ICP22 created new session: [1:7577471573333546770:2049]
2025-11-27T18:19:43.923182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471573333546770:2049] [node 3] ICS09 handshake done sender: [1:7577471573333546749:2114] self: [1:7577471573333546748:0] peer: [3:7577471571286919862:0] socket: 215 qp: -1
2025-11-27T18:19:43.923204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471573333546770:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:43.923243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.928154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471572593591568:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31524
2025-11-27T18:19:43.927740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471572451164925:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38924
2025-11-27T18:19:43.928202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471572451164926:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38924
2025-11-27T18:19:43.929248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471573333546747:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:43.929677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471573333546677:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.928183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471573924385747:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:43.932935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471571286919749:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.938663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:43.939109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471572593591548:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.933027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:43.938869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471573333546677:2048] [node 2] ICP22 created new session: [1:7577471573333546774:2048]
2025-11-27T18:19:43.939150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471572593591548:2138] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.933083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471571286919749:2048] [node 1] ICP22 created new session: [3:7577471571286919870:2048]
2025-11-27T18:19:43.938954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471573333546774:2048] [node 2] ICS09 handshake done sender: [1:7577471573333546747:2113] self: [1:7577471573333546746:0] peer: [2:7577471572593591567:0] socket: 212 qp: -1
2025-11-27T18:19:43.933137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471571286919870:2048] [node 1] ICS09 handshake done sender: [3:7577471571286919859:2146] self: [3:7577471571286919862:0] peer: [1:7577471573333546748:0] socket: 222 qp: -1
2025-11-27T18:19:43.933176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471571286919870:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:43.938999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471573333546774:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:43.939051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471573924385745:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.939164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471573924385746:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.941488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471572593591562:2146] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:43.939043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:43.933209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:43.933966Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471571286919749:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:43.944474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471572593591452:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.944534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:43.944576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471572593591452:2048] [node 1] ICP22 created new session: [2:7577471572593591570:2048]
2025-11-27T18:19:43.944616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471572593591570:2048] [node 1] ICS09 handshake done sender: [2:7577471572593591562:2146] self: [2:7577471572593591567:0] peer: [1:7577471573333546746:0] socket: 221 qp: -1
2025-11-27T18:19:43.944651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471572593591570:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:43.944689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:43.945882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471572451164925:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.945932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471572593591452:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.946620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471572451164927:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38924
2025-11-27T18:19:43.946935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471572451164849:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.947196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471572451164927:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.947309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471572451164910:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.947220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471573365213908:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37432
2025-11-27T18:19:43.947542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471573924385817:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.947576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471573924385817:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.947439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471573333546751:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.947488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471573333546751:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.949869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471573924385817:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:43.949917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471572593591550:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.949968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471571286919849:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.949955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471572593591550:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.950013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471571286919849:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.952388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471573924385808:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.951704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471572593591548:2138] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:43.954253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471572451164930:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38924
2025-11-27T18:19:43.952484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:43.952594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471573924385808:2108] [node 5] ICP22 created new session: [4:7577471573924385834:2108]
2025-11-27T18:19:43.954330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471573924385834:2108] [node 5] ICS09 handshake done sender: [4:7577471573924385817:2113] self: [4:7577471573924385816:0] peer: [5:7577471572451164929:0] socket: 189 qp: -1
2025-11-27T18:19:43.954379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471573924385834:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:43.954873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:43.955322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471573365213908:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.955316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471572593591453:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.955408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:43.955564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471573365213830:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.955558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471572593591453:2049] [node 3] ICP22 created new session: [2:7577471572593591574:2049]
2025-11-27T18:19:43.955681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471572451164927:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:43.956124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471572451164910:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.956198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:43.956246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471572451164910:2108] [node 4] ICP22 created new session: [5:7577471572451164931:2108]
2025-11-27T18:19:43.956298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471572451164931:2108] [node 4] ICS09 handshake done sender: [5:7577471572451164927:2117] self: [5:7577471572451164929:0] peer: [4:7577471573924385816:0] socket: 205 qp: -1
2025-11-27T18:19:43.956271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471572593591574:2049] [node 3] ICS09 handshake done sender: [2:7577471572593591548:2138] self: [2:7577471572593591547:0] peer: [3:7577471571286919869:0] socket: 207 qp: -1
2025-11-27T18:19:43.956331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471572451164931:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:43.956320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471572593591574:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:43.956496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:43.956010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471571286919851:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.956044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471571286919851:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.956366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.956993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471573365213910:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37432
2025-11-27T18:19:43.957347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471571286919873:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29466
2025-11-27T18:19:43.957248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471573333546751:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:43.957891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471573333546743:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.958175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:43.958246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471573333546743:2107] [node 4] ICP22 created new session: [1:7577471573333546778:2107]
2025-11-27T18:19:43.958287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471573333546778:2107] [node 4] ICS09 handshake done sender: [1:7577471573333546751:2115] self: [1:7577471573333546750:0] peer: [4:7577471573924385830:0] socket: 217 qp: -1
2025-11-27T18:19:43.958324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471573333546778:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:43.958350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:43.958346Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471571286919849:2140] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.958743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471571286919840:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.958832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:43.958882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471571286919840:2111] [node 5] ICP22 created new session: [3:7577471571286919876:2111]
2025-11-27T18:19:43.958833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471573365213911:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37432
2025-11-27T18:19:43.958919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471571286919876:2111] [node 5] ICS09 handshake done sender: [3:7577471571286919849:2140] self: [3:7577471571286919848:0] peer: [5:7577471572451164928:0] socket: 231 qp: -1
2025-11-27T18:19:43.958952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471571286919876:2111] [node 5] ICS10 traffic start
2025-11-27T18:19:43.958985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.959450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471573924385836:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26706
2025-11-27T18:19:43.959623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471573924385837:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26706
2025-11-27T18:19:43.959823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471572451164926:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.959823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471572593591550:2139] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:43.960301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471572593591542:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.960353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:43.960418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471572593591542:2107] [node 4] ICP22 created new session: [2:7577471572593591577:2107]
2025-11-27T18:19:43.960460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471572593591577:2107] [node 4] ICS09 handshake done sender: [2:7577471572593591550:2139] self: [2:7577471572593591549:0] peer: [4:7577471573924385831:0] socket: 210 qp: -1
2025-11-27T18:19:43.960479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471572593591577:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:43.960504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:43.960527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471571286919858:2145] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.960843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471573924385825:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:43.961363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471572451164933:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38924
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.961605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471573924385824:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:43.961600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471573365213912:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37432
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.962172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471572451164934:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38924
2025-11-27T18:19:43.962251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471573924385745:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.962282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471572451164847:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.962334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471573365213910:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.962370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:43.962424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471573924385745:2048] [node 1] ICP22 created new session: [4:7577471573924385839:2048]
2025-11-27T18:19:43.962483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471573365213828:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.962499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471573924385839:2048] [node 1] ICS09 handshake done sender: [4:7577471573924385825:2117] self: [4:7577471573924385830:0] peer: [1:7577471573333546750:0] socket: 225 qp: -1
2025-11-27T18:19:43.962541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471573924385839:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:43.962994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471573333546755:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.962602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:43.963027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471573333546755:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.964962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471571286919750:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.965284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:43.965407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471572451164925:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:43.965346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471571286919750:2049] [node 2] ICP22 created new session: [3:7577471571286919878:2049]
2025-11-27T18:19:43.965487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471573333546753:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.965516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471573333546753:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.965424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471571286919878:2049] [node 2] ICS09 handshake done sender: [3:7577471571286919858:2145] self: [3:7577471571286919869:0] peer: [2:7577471572593591547:0] socket: 216 qp: -1
2025-11-27T18:19:43.966985Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471572451164849:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6771) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471572451164925:2115] held: no
2025-11-27T18:19:43.967240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471573333546755:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:43.965470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471571286919878:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:43.965505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.967657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471571286919750:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:43.967690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471573365213914:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37432
2025-11-27T18:19:43.967704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471572451164849:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.967782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:43.967830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471572451164849:2050] [node 3] ICP22 created new session: [5:7577471572451164937:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.967879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471572451164937:2050] [node 3] ICS09 handshake done sender: [5:7577471572451164925:2115] self: [5:7577471572451164928:0] peer: [3:7577471571286919848:0] socket: 232 qp: -1
2025-11-27T18:19:43.967868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471573365213915:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37432
2025-11-27T18:19:43.967913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471572451164937:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:43.967947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471573365213916:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37432
2025-11-27T18:19:43.967944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:43.968565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471573365213910:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:43.969188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471573365213828:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.969238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:43.969313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471573365213828:2048] [node 1] ICP22 created new session: [6:7577471573365213917:2048]
2025-11-27T18:19:43.969396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471573365213917:2048] [node 1] ICS09 handshake done sender: [6:7577471573365213910:2118] self: [6:7577471573365213913:0] peer: [1:7577471573333546754:0] socket: 243 qp: -1
2025-11-27T18:19:43.969438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471573365213917:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:43.969472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:43.969826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471573333546745:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.969923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:43.969968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471573333546745:2111] [node 6] ICP22 created new session: [1:7577471573333546782:2111]
2025-11-27T18:19:43.970021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471573333546782:2111] [node 6] ICS09 handshake done sender: [1:7577471573333546755:2117] self: [1:7577471573333546754:0] peer: [6:7577471573365213913:0] socket: 242 qp: -1
2025-11-27T18:19:43.970060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471573333546782:2111] [node 6] ICS10 traffic start
2025-11-27T18:19:43.970093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:43.970179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471571286919851:2141] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:43.974650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471571286919841:2120] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.974951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:43.974999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471571286919841:2120] [node 6] ICP22 created new session: [3:7577471571286919880:2120]
2025-11-27T18:19:43.975052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471571286919880:2120] [node 6] ICS09 handshake done sender: [3:7577471571286919851:2141] self: [3:7577471571286919850:0] peer: [6:7577471573365213909:0] socket: 219 qp: -1
2025-11-27T18:19:43.975226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471571286919880:2120] [node 6] ICS10 traffic start
2025-11-27T18:19:43.975296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:43.975749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471573924385746:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.975890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:43.975936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471572451164934:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.975969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471573924385746:2049] [node 2] ICP22 created new session: [4:7577471573924385841:2049]
2025-11-27T18:19:43.976106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471573924385841:2049] [node 2] ICS09 handshake done sender: [4:7577471573924385824:2116] self: [4:7577471573924385831:0] peer: [2:7577471572593591549:0] socket: 224 qp: -1
2025-11-27T18:19:43.976267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471573365213914:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.976152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471573924385841:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:43.976191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:43.977749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471573333546753:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.978292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471572451164939:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38924
2025-11-27T18:19:43.979751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471573333546744:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.979839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:43.979794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471572451164848:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.979892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471573333546744:2110] [node 5] ICP22 created new session: [1:7577471573333546784:2110]
2025-11-27T18:19:43.979946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471573333546784:2110] [node 5] ICS09 handshake done sender: [1:7577471573333546753:2116] self: [1:7577471573333546752:0] peer: [5:7577471572451164936:0] socket: 233 qp: -1
2025-11-27T18:19:43.979993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471572451164926:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:43.980016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471573333546784:2110] [node 5] ICS10 traffic start
2025-11-27T18:19:43.980047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:43.980042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471573365213908:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:43.980249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471572451164847:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.980361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:43.980385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471572451164847:2048] [node 1] ICP22 created new session: [5:7577471572451164941:2048]
2025-11-27T18:19:43.980946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471572451164941:2048] [node 1] ICS09 handshake done sender: [5:7577471572451164926:2116] self: [5:7577471572451164936:0] peer: [1:7577471573333546752:0] socket: 234 qp: -1
2025-11-27T18:19:43.980984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471572451164941:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:43.981015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:43.981131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471573365213830:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6771) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471573365213908:2117] held: no
2025-11-27T18:19:43.981220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471572593591552:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.981268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471572593591552:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.982218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471573365213830:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.982279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:43.982258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471572593591552:2140] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:43.982316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471573365213830:2050] [node 3] ICP22 created new session: [6:7577471573365213920:2050]
2025-11-27T18:19:43.982364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471573365213920:2050] [node 3] ICS09 handshake done sender: [6:7577471573365213908:2117] self: [6:7577471573365213909:0] peer: [3:7577471571286919850:0] socket: 235 qp: -1
2025-11-27T18:19:43.982296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471573924385745:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:43.982417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471573365213920:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:43.982452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:43.982690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471573924385746:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:43.982816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471572593591543:2130] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.982886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:43.982931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471572593591543:2130] [node 5] ICP22 created new session: [2:7577471572593591580:2130]
2025-11-27T18:19:43.983009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471572593591580:2130] [node 5] ICS09 handshake done sender: [2:7577471572593591552:2140] self: [2:7577471572593591551:0] peer: [5:7577471572451164940:0] socket: 249 qp: -1
2025-11-27T18:19:43.983041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471572593591580:2130] [node 5] ICS10 traffic start
2025-11-27T18:19:43.983071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:43.983585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471573365213829:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.984093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471573924385843:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26706
2025-11-27T18:19:43.985375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471573365213911:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.985992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471572593591554:2141] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.986045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471572593591554:2141] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.986047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471573365213915:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.987347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471572593591554:2141] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:43.987727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471572593591544:2132] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.987796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:43.987881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471572593591544:2132] [node 6] ICP22 created new session: [2:7577471572593591583:2132]
2025-11-27T18:19:43.987968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471572593591583:2132] [node 6] ICS09 handshake done sender: [2:7577471572593591554:2141] self: [2:7577471572593591553:0] peer: [6:7577471573365213922:0] socket: 190 qp: -1
2025-11-27T18:19:43.988006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471572593591583:2132] [node 6] ICS10 traffic start
2025-11-27T18:19:43.988048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:43.988685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471572451164848:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471572451164934:2120] held: no
2025-11-27T18:19:43.988726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471573365213891:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.989189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471573365213893:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:43.989111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471573924385819:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.989145Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471573924385819:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.990334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471573924385843:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.990750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471573924385819:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.991002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471572451164943:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38924
2025-11-27T18:19:43.991022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471573924385809:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.991098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:43.991149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471573924385809:2109] [node 6] ICP22 created new session: [4:7577471573924385846:2109]
2025-11-27T18:19:43.991206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471573924385846:2109] [node 6] ICS09 handshake done sender: [4:7577471573924385819:2114] self: [4:7577471573924385818:0] peer: [6:7577471573365213924:0] socket: 247 qp: -1
2025-11-27T18:19:43.991240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471573924385846:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:43.991271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.992110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471572451164921:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.992140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471572451164921:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:43.992221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471573365213925:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37432
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.993314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471572451164944:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38924
2025-11-27T18:19:43.993449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471572451164934:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:43.994296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471572451164848:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.994343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:43.994389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471572451164848:2049] [node 2] ICP22 created new session: [5:7577471572451164946:2049]
2025-11-27T18:19:43.994427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471573365213829:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471573365213914:2121] held: no
2025-11-27T18:19:43.994456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471572451164946:2049] [node 2] ICS09 handshake done sender: [5:7577471572451164934:2120] self: [5:7577471572451164940:0] peer: [2:7577471572593591551:0] socket: 251 qp: -1
2025-11-27T18:19:43.994494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471572451164946:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:43.994542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:43.995835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471572451164944:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:43.995959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471572451164921:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:43.996367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471572451164911:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:43.996405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:43.996459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471572451164911:2109] [node 6] ICP22 created new session: [5:7577471572451164948:2109]
2025-11-27T18:19:43.996562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471573365213914:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:43.996528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471572451164948:2109] [node 6] ICS09 handshake done sender: [5:7577471572451164921:2114] self: [5:7577471572451164920:0] peer: [6:7577471573365213926:0] socket: 195 qp: -1
2025-11-27T18:19:43.996578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471572451164948:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:43.996607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:43.997189Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471573365213891:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21096) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471573365213911:2119] held: no
2025-11-27T18:19:43.997668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577471572451164911:2109] [node 6] ICP18 handshake (actor [5:7577471572451164944:2123]) is too old
2025-11-27T18:19:43.997843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471573365213829:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:43.997889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:43.997928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471573365213829:2049] [node 2] ICP22 created new session: [6:7577471573365213929:2049]
2025-11-27T18:19:43.997980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471573365213929:2049] [node 2] ICS09 handshake done sender: [6:7577471573365213914:2121] self: [6:7577471573365213922:0] peer: [2:7577471572593591553:0] socket: 193 qp: -1
2025-11-27T18:19:43.998013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471573365213929:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:43.998039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:43.999626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471573365213927:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37432
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.000018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471573365213928:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37432
2025-11-27T18:19:44.001900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471573365213893:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3224) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471573365213915:2122] held: no
2025-11-27T18:19:44.002576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471573365213911:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:44.002802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471573365213915:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:44.003306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471573365213891:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.003350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:44.003390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471573365213891:2106] [node 4] ICP22 created new session: [6:7577471577660181227:2106]
2025-11-27T18:19:44.003445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471577660181227:2106] [node 4] ICS09 handshake done sender: [6:7577471573365213911:2119] self: [6:7577471573365213924:0] peer: [4:7577471573924385818:0] socket: 248 qp: -1
2025-11-27T18:19:44.003480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471577660181227:2106] [node 4] ICS10 traffic start
2025-11-27T18:19:44.003506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:44.004408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471573365213893:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.004461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:44.004500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471573365213893:2108] [node 5] ICP22 created new session: [6:7577471577660181229:2108]
2025-11-27T18:19:44.004538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471577660181229:2108] [node 5] ICS09 handshake done sender: [6:7577471573365213915:2122] self: [6:7577471573365213926:0] peer: [5:7577471572451164920:0] socket: 213 qp: -1
2025-11-27T18:19:44.004567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471577660181229:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:44.004592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:44.025734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:44.025986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471572593591570:2048] [node 1] ICS07 socket disconnect 221 reason# EndOfStream
2025-11-27T18:19:44.026021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471572593591570:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.026111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:44.026152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471572593591570:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.027186Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:44.027400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471571286919870:2048] [node 1] ICS07 socket disconnect 222 reason# EndOfStream
2025-11-27T18:19:44.027432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471571286919870:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.027507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:44.027703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471571286919870:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.030387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471572593591452:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:44.030433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471572593591570:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.031659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:44.032032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471573924385839:2048] [node 1] ICS07 socket disconnect 225 reason# EndOfStream
2025-11-27T18:19:44.032078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471573924385839:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.032169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:44.032224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471573924385839:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.032557Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471571286919749:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:44.032627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471571286919870:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.032896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471572593591452:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.032983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471572593591570:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.033024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471572593591452:2048] [node 1] ICP30 unregister session Session# [2:7577471572593591570:2048] VirtualId# [2:7577471572593591567:0]
2025-11-27T18:19:44.033078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:44.033116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471572593591452:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.033136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.033449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471573924385745:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.033525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471573924385839:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.033566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471573924385745:2048] [node 1] ICP30 unregister session Session# [4:7577471573924385839:2048] VirtualId# [4:7577471573924385830:0]
2025-11-27T18:19:44.033571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:44.033659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:44.033782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471572451164941:2048] [node 1] ICS07 socket disconnect 234 reason# EndOfStream
2025-11-27T18:19:44.033703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471573924385745:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.033717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.033823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471572451164941:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.033895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:44.033948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471572451164941:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.034017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471571286919749:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.034213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471571286919870:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.034302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471571286919749:2048] [node 1] ICP30 unregister session Session# [3:7577471571286919870:2048] VirtualId# [3:7577471571286919862:0]
2025-11-27T18:19:44.034357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:44.034495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471571286919749:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.034514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.035110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:44.035210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471572451164847:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.035257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471572451164941:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.035293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471572451164847:2048] [node 1] ICP30 unregister session Session# [5:7577471572451164941:2048] VirtualId# [5:7577471572451164936:0]
2025-11-27T18:19:44.035346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471573365213917:2048] [node 1] ICS07 socket disconnect 243 reason# EndOfStream
2025-11-27T18:19:44.035372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:44.035378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471573365213917:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.035429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471572451164847:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.035461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:44.035492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.035532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471573365213917:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.036395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471573365213828:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.036466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471573365213917:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.036494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471573365213828:2048] [node 1] ICP30 unregister session Session# [6:7577471573365213917:2048] VirtualId# [6:7577471573365213913:0]
2025-11-27T18:19:44.036531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:44.036557Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471573365213828:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.036590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.043312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471573924385745:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:44.043310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471572593591452:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:44.044332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471571286919749:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:44.045375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471572451164847:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:44.047392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471573365213828:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:44.076086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:44.076333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471571286919878:2049] [node 2] ICS07 socket disconnect 216 reason# EndOfStream
2025-11-27T18:19:44.076370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471571286919878:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.076503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:44.076565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471571286919878:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:44.078994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:44.079174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471573924385841:2049] [node 2] ICS07 socket disconnect 224 reason# EndOfStream
2025-11-27T18:19:44.079206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471573924385841:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.079280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:44.079336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471573924385841:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:44.079618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471571286919750:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9339) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:44.079678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471571286919878:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:44.080239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:44.080317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471571286919750:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9339) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.080363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471571286919878:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.080412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471571286919750:2049] [node 2] ICP30 unregister session Session# [3:7577471571286919878:2049] VirtualId# [3:7577471571286919869:0]
2025-11-27T18:19:44.080435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471572451164946:2049] [node 2] ICS01 socket: 251 reason# EndOfStream
2025-11-27T18:19:44.080516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:44.080549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471571286919750:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9339) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.080514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471572451164848:2049] [node 2] ICP30 unregister session Session# [5:7577471572451164946:2049] VirtualId# [5:7577471572451164940:0]
2025-11-27T18:19:44.080514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471573924385746:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9339) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.080584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471573924385841:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.080586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9339) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.080551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471572451164946:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.080646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471573924385746:2049] [node 2] ICP30 unregister session Session# [4:7577471573924385841:2049] VirtualId# [4:7577471573924385831:0]
2025-11-27T18:19:44.080623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:44.080676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:44.080803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:44.081007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471573924385746:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9339) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.081030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9339) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.081340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:44.081527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471573365213929:2049] [node 2] ICS01 socket: 193 reason# EndOfStream
2025-11-27T18:19:44.081577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471573365213829:2049] [node 2] ICP30 unregister session Session# [6:7577471573365213929:2049] VirtualId# [6:7577471573365213922:0]
2025-11-27T18:19:44.081600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471573365213929:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.081667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:44.081720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:44.090431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471573924385746:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:44.107907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:44.108091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471573924385828:2050] [node 3] ICS07 socket disconnect 200 reason# EndOfStream
2025-11-27T18:19:44.108127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471573924385828:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.108200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:44.108236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471573924385828:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:44.109094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:44.109140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471573924385747:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6771) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.109190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471573924385828:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.109223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471573924385747:2050] [node 3] ICP30 unregister session Session# [4:7577471573924385828:2050] VirtualId# [4:7577471573924385826:0]
2025-11-27T18:19:44.109323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471572451164937:2050] [node 3] ICS01 socket: 232 reason# EndOfStream
2025-11-27T18:19:44.109364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471572451164849:2050] [node 3] ICP30 unregister session Session# [5:7577471572451164937:2050] VirtualId# [5:7577471572451164928:0]
2025-11-27T18:19:44.109358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:44.109394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471573924385747:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6771) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.109407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471572451164937:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.109413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6771) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.109477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:44.109561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:44.110086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:44.110230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471573365213920:2050] [node 3] ICS01 socket: 235 reason# EndOfStream
2025-11-27T18:19:44.110265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471573365213830:2050] [node 3] ICP30 unregister session Session# [6:7577471573365213920:2050] VirtualId# [6:7577471573365213909:0]
2025-11-27T18:19:44.110303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471573365213920:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.110375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:44.110450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:44.119090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471573924385747:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:44.160899Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:44.161135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471572451164931:2108] [node 4] ICS07 socket disconnect 205 reason# EndOfStream
2025-11-27T18:19:44.161170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471572451164931:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.161283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:44.161337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471572451164931:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:44.162193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:44.162193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471572451164910:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21096) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.162251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471572451164931:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.162287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471572451164910:2108] [node 4] ICP30 unregister session Session# [5:7577471572451164931:2108] VirtualId# [5:7577471572451164929:0]
2025-11-27T18:19:44.162369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:44.162379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471577660181227:2106] [node 4] ICS01 socket: 248 reason# EndOfStream
2025-11-27T18:19:44.162401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471572451164910:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21096) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.162418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471573365213891:2106] [node 4] ICP30 unregister session Session# [6:7577471577660181227:2106] VirtualId# [6:7577471573365213924:0]
2025-11-27T18:19:44.162422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21096) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.162444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471577660181227:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.162516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:44.162580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:44.191746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:44.192024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471577660181229:2108] [node 5] ICS01 socket: 213 reason# EndOfStream
2025-11-27T18:19:44.192086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471573365213893:2108] [node 5] ICP30 unregister session Session# [6:7577471577660181229:2108] VirtualId# [6:7577471573365213926:0]
2025-11-27T18:19:44.192119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471577660181229:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.192194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:44.192307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:44.263030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471577075132656:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:44.263153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471577075132657:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:44.283049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471578032274435:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:44.283194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471578032274436:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:44.302934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471577145958896:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:44.303013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471577145958897:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:44.323481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471575890857485:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:44.323562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471575890857487:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:44.323625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471575890857486:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:44.344515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471578180402734:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:44.344565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471578180402735:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:44.344603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471578180402736:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:44.365679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471575446580187:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:44.365751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471575446580188:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:44.365903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471575446580189:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:44.436065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471575446580246:2105] [node 5] ICP01 ready to work
2025-11-27T18:19:44.477376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471577145958986:2136] [node 5] ICP01 ready to work
2025-11-27T18:19:44.485144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471577075132720:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:44.493691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471575890857548:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:44.493491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471578032274516:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:44.493596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471577075132721:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:44.493240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471577145958987:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:44.493488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471578180402797:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:44.493561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471575446580251:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:44.493633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471578180402798:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:44.494080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471575890857549:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:44.494297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471577145958988:2125] [node 6] ICP01 ready to work
2025-11-27T18:19:44.494392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471578032274517:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:44.494830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471577075132722:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:44.495105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471578032274518:2117] [node 6] ICP01 ready to work
2025-11-27T18:19:44.495237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471577075132657:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(60918425ns) {Configure:104(60489018ns) {ProcessConfigured:135(52541ns) {StartInitialHandshake:157(50761ns) {DropHandshakes:470(2098ns) {DropIncomingHandshake:442(491ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(488ns) PrepareNewSessionHandshake:477(1037ns) {SwitchToState:179(330ns)}}}}}
2025-11-27T18:19:44.495783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471578032274435:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49889186ns) {Configure:104(49881284ns) {ProcessConfigured:135(92558ns) {StartInitialHandshake:157(90533ns) {DropHandshakes:470(1559ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(244ns)} GenerateSessionVirtualId:988(554ns) PrepareNewSessionHandshake:477(1249ns) {SwitchToState:179(469ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.503443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471578032274532:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5140
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.504466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471577075132736:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56056
2025-11-27T18:19:44.504875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471575446580261:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11291
2025-11-27T18:19:44.504888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471575890857558:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24169
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.505510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471578180402812:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28695
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.507871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471577075132737:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56056
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:19:44.508004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471578032274533:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5140
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.508370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471577145959000:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-27T18:19:44.508447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471578032274534:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5140
2025-11-27T18:19:44.508669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471577145958997:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-27T18:19:44.511633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471577145959001:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.513282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471575446580261:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.513781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471578032274535:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5140
2025-11-27T18:19:44.514135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471575446580188:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.514119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471578032274532:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.514266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471578032274534:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.514317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471577145959000:2143] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.514338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471578032274435:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.514398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471578032274533:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.514421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471577145959001:2144] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.514300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471578180402812:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.514451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471578180402813:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28695
2025-11-27T18:19:44.514436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471577075132736:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.514478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471577145959005:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-27T18:19:44.514478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471578032274526:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.514541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471577145959006:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-27T18:19:44.514517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471578032274526:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.514872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471575890857558:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.515120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471577075132738:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56056
2025-11-27T18:19:44.515174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471577145958997:2141] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.515267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471575890857485:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.515334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471577075132739:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56056
2025-11-27T18:19:44.515342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471577145959005:2145] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.515352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471577075132737:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.515491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471577145958897:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.515507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471578032274526:2131] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:44.515518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471577075132728:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.515559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471577075132728:2113] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.515642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471577075132738:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.515777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471577145959006:2146] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.515856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471575446580267:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11291
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.517162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471578032274518:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.517446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471577075132726:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.517429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471577145958896:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.517228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:44.517476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471577075132726:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.517789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471575446580261:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:44.517305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471578032274518:2117] [node 6] ICP22 created new session: [2:7577471578032274540:2117]
2025-11-27T18:19:44.517877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471577075132740:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56056
2025-11-27T18:19:44.518025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471575446580188:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.519159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471577075132739:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.519153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:44.517473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471578032274540:2117] [node 6] ICS09 handshake done sender: [2:7577471578032274526:2131] self: [2:7577471578032274525:0] peer: [6:7577471575446580266:0] socket: 197 qp: -1
2025-11-27T18:19:44.517520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471578032274540:2117] [node 6] ICS10 traffic start
2025-11-27T18:19:44.520095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471577075132724:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.519512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471575446580188:2049] [node 2] ICP22 created new session: [6:7577471575446580268:2049]
2025-11-27T18:19:44.519885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471575446580268:2049] [node 2] ICS09 handshake done sender: [6:7577471575446580261:2114] self: [6:7577471575446580266:0] peer: [2:7577471578032274525:0] socket: 202 qp: -1
2025-11-27T18:19:44.517557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:44.520125Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471577075132724:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.517885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471578032274522:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.519934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471575446580268:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:44.520114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:44.523384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471577075132728:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:44.523489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471577075132726:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:44.517915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471578032274522:2129] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.524420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471575890857565:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24169
2025-11-27T18:19:44.524544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471578180402813:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.525134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471578180402736:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.525678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471577075132656:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.525805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471577145958994:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.525754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471577075132720:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.525984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471578032274535:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.526682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471578032274542:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5140
2025-11-27T18:19:44.525822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:44.525844Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471577145958994:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.525955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:44.526256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471577075132656:2048] [node 2] ICP22 created new session: [1:7577471577075132749:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.526578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471577075132720:2107] [node 4] ICP22 created new session: [1:7577471577075132748:2107]
2025-11-27T18:19:44.526855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471575890857558:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:44.526839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471577075132749:2048] [node 2] ICS09 handshake done sender: [1:7577471577075132728:2113] self: [1:7577471577075132727:0] peer: [2:7577471578032274536:0] socket: 194 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.527061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471578032274522:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:44.526993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471575890857566:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24169
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.526844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471577075132748:2107] [node 4] ICS09 handshake done sender: [1:7577471577075132726:2112] self: [1:7577471577075132725:0] peer: [4:7577471575890857564:0] socket: 196 qp: -1
2025-11-27T18:19:44.526887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471577075132749:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:44.527324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471575890857485:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.527315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471577145959012:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-27T18:19:44.526890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471577075132748:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:44.527370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.527529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471575890857485:2048] [node 1] ICP22 created new session: [4:7577471575890857568:2048]
2025-11-27T18:19:44.526940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:44.527611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471577145959013:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13172
2025-11-27T18:19:44.527686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471575890857568:2048] [node 1] ICS09 handshake done sender: [4:7577471575890857558:2114] self: [4:7577471575890857564:0] peer: [1:7577471577075132725:0] socket: 201 qp: -1
2025-11-27T18:19:44.526988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.527720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471575890857568:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:44.527778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471577145958994:2139] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:44.528076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471577145958986:2136] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.527841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471575890857567:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24169
2025-11-27T18:19:44.527855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:44.528184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:44.528350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471577145958986:2136] [node 5] ICP22 created new session: [3:7577471577145959015:2136]
2025-11-27T18:19:44.528291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471578180402816:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28695
2025-11-27T18:19:44.528087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471575890857566:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.528400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471578032274436:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.528453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:44.528513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471578032274436:2049] [node 3] ICP22 created new session: [2:7577471578032274545:2049]
2025-11-27T18:19:44.529732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471577145959015:2136] [node 5] ICS09 handshake done sender: [3:7577471577145958994:2139] self: [3:7577471577145958993:0] peer: [5:7577471578180402815:0] socket: 230 qp: -1
2025-11-27T18:19:44.529790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471578032274545:2049] [node 3] ICS09 handshake done sender: [2:7577471578032274522:2129] self: [2:7577471578032274521:0] peer: [3:7577471577145959007:0] socket: 195 qp: -1
2025-11-27T18:19:44.529771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471577145959000:2143] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:44.529889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471578032274545:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:44.529773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471577145959015:2136] [node 5] ICS10 traffic start
2025-11-27T18:19:44.530130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:44.529895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471578180402813:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:44.529812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:44.530513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471578180402736:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.530676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:44.530723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471578180402736:2050] [node 3] ICP22 created new session: [5:7577471578180402817:2050]
2025-11-27T18:19:44.530850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471578180402817:2050] [node 3] ICS09 handshake done sender: [5:7577471578180402813:2116] self: [5:7577471578180402815:0] peer: [3:7577471577145958993:0] socket: 231 qp: -1
2025-11-27T18:19:44.530883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471578180402817:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:44.530945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:44.535425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471577145958897:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.536019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:44.536153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471577145958897:2049] [node 2] ICP22 created new session: [3:7577471577145959017:2049]
2025-11-27T18:19:44.536219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471577145959017:2049] [node 2] ICS09 handshake done sender: [3:7577471577145959000:2143] self: [3:7577471577145959007:0] peer: [2:7577471578032274521:0] socket: 207 qp: -1
2025-11-27T18:19:44.536260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471577145959017:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:44.536337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:44.536802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471577075132740:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.536940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471578032274436:2049] [node 3] ICP18 handshake (actor [2:7577471578032274535:2136]) is too old
2025-11-27T18:19:44.536972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471578032274532:2133] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.537229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471578180402819:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28695
2025-11-27T18:19:44.537380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471577075132724:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:44.538192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471578180402819:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.539191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471578180402797:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.539433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471577075132657:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.539441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471575890857557:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.539481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.540167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471577075132657:2049] [node 3] ICP22 created new session: [1:7577471577075132754:2049]
2025-11-27T18:19:44.539488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471575890857557:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.540850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471578032274435:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.541064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:44.541033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471577075132754:2049] [node 3] ICS09 handshake done sender: [1:7577471577075132724:2111] self: [1:7577471577075132723:0] peer: [3:7577471577145959009:0] socket: 192 qp: -1
2025-11-27T18:19:44.541473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471575890857557:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:44.541727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471578032274435:2048] [node 1] ICP22 created new session: [2:7577471578032274547:2048]
2025-11-27T18:19:44.542719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471578032274547:2048] [node 1] ICS09 handshake done sender: [2:7577471578032274532:2133] self: [2:7577471578032274536:0] peer: [1:7577471577075132727:0] socket: 199 qp: -1
2025-11-27T18:19:44.541686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471575890857572:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24169
2025-11-27T18:19:44.543120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471578032274547:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:44.543247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471575890857548:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.541395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471577075132754:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:44.543171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.543671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471578180402822:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28695
2025-11-27T18:19:44.543303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:44.543344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471575890857548:2108] [node 5] ICP22 created new session: [4:7577471575890857574:2108]
2025-11-27T18:19:44.543679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471575890857574:2108] [node 5] ICS09 handshake done sender: [4:7577471575890857557:2113] self: [4:7577471575890857556:0] peer: [5:7577471578180402821:0] socket: 216 qp: -1
2025-11-27T18:19:44.543726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471575890857574:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:44.543766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:44.541783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:44.543905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471578180402819:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:44.544609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471578180402797:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.544172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471575890857572:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.544922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:44.549374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471578180402797:2108] [node 4] ICP22 created new session: [5:7577471578180402823:2108]
2025-11-27T18:19:44.550642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471578180402823:2108] [node 4] ICS09 handshake done sender: [5:7577471578180402819:2118] self: [5:7577471578180402821:0] peer: [4:7577471575890857556:0] socket: 198 qp: -1
2025-11-27T18:19:44.552099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471577145958997:2141] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:44.550693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471578180402823:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.550741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:44.553783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471575890857576:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24169
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.556526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471575890857567:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.556619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471577145958896:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.556684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:44.558502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471578032274435:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:44.556663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471578180402824:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28695
2025-11-27T18:19:44.564634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471575446580272:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11291
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.559632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471577145958896:2048] [node 1] ICP22 created new session: [3:7577471577145959021:2048]
2025-11-27T18:19:44.565545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471575446580273:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11291
2025-11-27T18:19:44.565047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471575890857486:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.559709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471577145959021:2048] [node 1] ICS09 handshake done sender: [3:7577471577145958997:2141] self: [3:7577471577145959009:0] peer: [1:7577471577075132723:0] socket: 204 qp: -1
2025-11-27T18:19:44.565636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471575446580274:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11291
2025-11-27T18:19:44.559759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471577145959021:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:44.565730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471575446580271:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11291
2025-11-27T18:19:44.566957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471577075132657:2049] [node 3] ICP18 handshake (actor [1:7577471577075132740:2120]) is too old
2025-11-27T18:19:44.567638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471578032274524:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.559807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:44.566363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471575446580271:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.568090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471575890857563:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.567665Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471578032274524:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.567030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471575446580272:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.568574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471578180402824:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.568115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471575890857563:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.567199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471575446580187:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.567850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471575446580251:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.569089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471575890857563:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:44.569509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471578180402734:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.569498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471575890857548:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(22625304ns) {HandleHandshakeStatus:329(22622868ns) {DropIncomingHandshake:442(545ns) DropOutgoingHandshake:457(665ns) SwitchToState:179(676ns) ProcessPendingSessionEvents:495(21832007ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21727181ns max# 3439419ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.569783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471575890857576:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.569765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471575446580274:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.569887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471575890857549:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.569900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471578180402827:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28695
2025-11-27T18:19:44.569973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471575446580246:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.570105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471575446580277:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11291
2025-11-27T18:19:44.569991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471577075132730:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.569952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:44.570035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471577075132730:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.570004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471575890857549:2109] [node 6] ICP22 created new session: [4:7577471575890857581:2109]
2025-11-27T18:19:44.570703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471578180402808:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.570059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471575890857581:2109] [node 6] ICS09 handshake done sender: [4:7577471575890857563:2115] self: [4:7577471575890857562:0] peer: [6:7577471575446580276:0] socket: 209 qp: -1
2025-11-27T18:19:44.570737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471578180402808:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.570092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471575890857581:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:44.570125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:44.571510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471575446580272:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:44.571569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471578032274524:2130] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:44.571870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471575890857548:2108] [node 5] ICP18 handshake (actor [4:7577471575890857572:2119]) is too old
2025-11-27T18:19:44.572248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471575446580251:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.572722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471578180402808:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:44.572698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:44.572289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471577075132732:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.572319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471577075132732:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.572782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471578180402797:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(24596407ns) {HandleHandshakeStatus:329(24593851ns) {DropIncomingHandshake:442(435ns) DropOutgoingHandshake:457(19326ns) SwitchToState:179(323ns) ProcessPendingSessionEvents:495(14330812ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14213749ns max# 2940273ns}}}
2025-11-27T18:19:44.575704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471577075132730:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:44.573118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471578032274516:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.573789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471578180402798:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.572899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471575446580251:2108] [node 4] ICP22 created new session: [6:7577471575446580280:2108]
2025-11-27T18:19:44.573604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:44.573849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:44.573784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471578032274516:2107] [node 4] ICP22 created new session: [2:7577471578032274551:2107]
2025-11-27T18:19:44.573071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471575446580280:2108] [node 4] ICS09 handshake done sender: [6:7577471575446580272:2118] self: [6:7577471575446580276:0] peer: [4:7577471575890857562:0] socket: 254 qp: -1
2025-11-27T18:19:44.575913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471578032274551:2107] [node 4] ICS09 handshake done sender: [2:7577471578032274524:2130] self: [2:7577471578032274523:0] peer: [4:7577471575890857578:0] socket: 243 qp: -1
2025-11-27T18:19:44.573100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471575446580280:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:44.573487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:44.576164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471578180402798:2109] [node 6] ICP22 created new session: [5:7577471578180402830:2109]
2025-11-27T18:19:44.575950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471578032274551:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:44.575996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.579031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471578180402830:2109] [node 6] ICS09 handshake done sender: [5:7577471578180402808:2114] self: [5:7577471578180402807:0] peer: [6:7577471575446580278:0] socket: 257 qp: -1
2025-11-27T18:19:44.582293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471578180402830:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:44.582775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:44.582813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471575890857580:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24169
2025-11-27T18:19:44.583067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471577075132722:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.583079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471577075132732:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:44.583212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471575446580273:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.583205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:44.583249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471577075132722:2109] [node 6] ICP22 created new session: [1:7577471577075132758:2109]
2025-11-27T18:19:44.583252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471575890857487:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.583299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471577075132758:2109] [node 6] ICS09 handshake done sender: [1:7577471577075132730:2114] self: [1:7577471577075132729:0] peer: [6:7577471575446580275:0] socket: 251 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.583326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471577075132758:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:44.583794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471575890857486:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5140) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471575890857567:2118] held: no
2025-11-27T18:19:44.583360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:44.583464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471578180402827:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.583787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471575446580279:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11291
2025-11-27T18:19:44.584379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471577145958996:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.584419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471577145958996:2140] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.584696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471575890857567:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:44.585067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471575446580189:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.585631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471577075132721:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.585705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:44.585699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471575890857486:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.585870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471577145958996:2140] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:44.585750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471577075132721:2108] [node 5] ICP22 created new session: [1:7577471577075132760:2108]
2025-11-27T18:19:44.585748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:44.585791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471575890857486:2049] [node 2] ICP22 created new session: [4:7577471575890857584:2049]
2025-11-27T18:19:44.585820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471577075132760:2108] [node 5] ICS09 handshake done sender: [1:7577471577075132732:2115] self: [1:7577471577075132731:0] peer: [5:7577471578180402828:0] socket: 227 qp: -1
2025-11-27T18:19:44.585856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471577075132760:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:44.585966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471575890857584:2049] [node 2] ICS09 handshake done sender: [4:7577471575890857567:2118] self: [4:7577471575890857578:0] peer: [2:7577471578032274523:0] socket: 247 qp: -1
2025-11-27T18:19:44.586307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:44.586028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471575890857584:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:44.586121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471577145958987:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.586062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:44.586168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:44.586573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471577145958987:2107] [node 4] ICP22 created new session: [3:7577471577145959024:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.586762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471577145959024:2107] [node 4] ICS09 handshake done sender: [3:7577471577145958996:2140] self: [3:7577471577145958995:0] peer: [4:7577471575890857583:0] socket: 226 qp: -1
2025-11-27T18:19:44.586798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471577145959024:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:44.587135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471578180402735:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:44.587154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471578180402832:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28695
2025-11-27T18:19:44.586840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:44.587575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471578032274528:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.587598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471578180402824:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:44.587598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471575446580246:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6000) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471575446580274:2120] held: no
2025-11-27T18:19:44.587618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471578032274528:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.588464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471578180402734:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.588515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:44.588571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471578180402734:2048] [node 1] ICP22 created new session: [5:7577471578180402834:2048]
2025-11-27T18:19:44.588608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471578180402834:2048] [node 1] ICS09 handshake done sender: [5:7577471578180402824:2120] self: [5:7577471578180402828:0] peer: [1:7577471577075132731:0] socket: 253 qp: -1
2025-11-27T18:19:44.588645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471578180402834:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:44.589071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471578032274528:2132] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:44.588684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.590653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471575446580282:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11291
2025-11-27T18:19:44.592545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471578032274517:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.592618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471578180402734:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:44.592677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:44.592737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471578032274517:2108] [node 5] ICP22 created new session: [2:7577471578032274554:2108]
2025-11-27T18:19:44.593269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471578032274554:2108] [node 5] ICS09 handshake done sender: [2:7577471578032274528:2132] self: [2:7577471578032274527:0] peer: [5:7577471578180402833:0] socket: 259 qp: -1
2025-11-27T18:19:44.593442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471578032274554:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:44.593489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:44.593504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471575446580274:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:44.593555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471577145958999:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:44.593591Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471577145958999:2142] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:44.594425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471575890857487:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29747) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471575890857576:2120] held: no
2025-11-27T18:19:44.595587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471575446580246:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.595731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471575446580187:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471575446580271:2117] held: no
2025-11-27T18:19:44.595856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:44.595917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471575446580246:2105] [node 5] ICP22 created new session: [6:7577471575446580284:2105]
2025-11-27T18:19:44.595794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471577145958999:2142] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:44.596015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471575446580284:2105] [node 5] ICS09 handshake done sender: [6:7577471575446580274:2120] self: [6:7577471575446580278:0] peer: [5:7577471578180402807:0] socket: 258 qp: -1
2025-11-27T18:19:44.596136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471577145958988:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:44.596148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471575446580284:2105] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.596578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:44.596631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:44.596695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471577145958988:2125] [node 6] ICP22 created new session: [3:7577471577145959027:2125]
2025-11-27T18:19:44.597353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471575890857586:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24169
2025-11-27T18:19:44.596957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471577145959027:2125] [node 6] ICS09 handshake done sender: [3:7577471577145958999:2142] self: [3:7577471577145958998:0] peer: [6:7577471575446580283:0] socket: 255 qp: -1
2025-11-27T18:19:44.597041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471577145959027:2125] [node 6] ICS10 traffic start
2025-11-27T18:19:44.597102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:44.597456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471578180402735:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5140) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471578180402827:2121] held: no
2025-11-27T18:19:44.597717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471575890857576:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.598270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471578180402836:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28695
2025-11-27T18:19:44.598133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471575890857487:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.598183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:44.598227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471575890857487:2050] [node 3] ICP22 created new session: [4:7577471575890857587:2050]
2025-11-27T18:19:44.598267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471575890857587:2050] [node 3] ICS09 handshake done sender: [4:7577471575890857576:2120] self: [4:7577471575890857583:0] peer: [3:7577471577145958995:0] socket: 250 qp: -1
2025-11-27T18:19:44.598298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471575890857587:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:44.598340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:44.599609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471578180402827:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:44.599710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471575446580271:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:44.599771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471578180402735:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.599880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:44.600008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471578180402735:2049] [node 2] ICP22 created new session: [5:7577471578180402837:2049]
2025-11-27T18:19:44.600089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471578180402837:2049] [node 2] ICS09 handshake done sender: [5:7577471578180402827:2121] self: [5:7577471578180402833:0] peer: [2:7577471578032274527:0] socket: 229 qp: -1
2025-11-27T18:19:44.600164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471578180402837:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:44.600208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:44.600186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471575446580187:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.600223Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471575446580189:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29747) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471575446580273:2119] held: no
2025-11-27T18:19:44.600233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:44.600280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471575446580187:2048] [node 1] ICP22 created new session: [6:7577471575446580287:2048]
2025-11-27T18:19:44.600322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471575446580287:2048] [node 1] ICS09 handshake done sender: [6:7577471575446580271:2117] self: [6:7577471575446580275:0] peer: [1:7577471577075132729:0] socket: 252 qp: -1
2025-11-27T18:19:44.600347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471575446580287:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:44.600380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:44.601284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471575446580286:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11291
2025-11-27T18:19:44.601483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471575446580273:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:44.601611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471575446580189:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:44.601636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:44.601686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471575446580189:2050] [node 3] ICP22 created new session: [6:7577471575446580289:2050]
2025-11-27T18:19:44.601727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471575446580289:2050] [node 3] ICS09 handshake done sender: [6:7577471575446580273:2119] self: [6:7577471575446580283:0] peer: [3:7577471577145958998:0] socket: 256 qp: -1
2025-11-27T18:19:44.601746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471575446580289:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:44.601772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:44.643579Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:44.643839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471578032274547:2048] [node 1] ICS07 socket disconnect 199 reason# EndOfStream
2025-11-27T18:19:44.643872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471578032274547:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.643945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:44.643998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471578032274547:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.644925Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:44.645138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471577145959021:2048] [node 1] ICS07 socket disconnect 204 reason# EndOfStream
2025-11-27T18:19:44.645175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471577145959021:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.645277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:44.645323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471577145959021:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.646454Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471578032274435:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:44.646507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471578032274547:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.647948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471577145958896:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:44.648022Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471578032274435:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.648004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471577145959021:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.648094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471578032274547:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.649062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471578032274435:2048] [node 1] ICP30 unregister session Session# [2:7577471578032274547:2048] VirtualId# [2:7577471578032274536:0]
2025-11-27T18:19:44.649332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:44.649399Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471578032274435:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.649360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:44.649423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.649592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471575890857568:2048] [node 1] ICS07 socket disconnect 201 reason# EndOfStream
2025-11-27T18:19:44.649629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471575890857568:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.649722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:44.649789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471575890857568:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.650048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471577145958896:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.650109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471577145959021:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.650133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471577145958896:2048] [node 1] ICP30 unregister session Session# [3:7577471577145959021:2048] VirtualId# [3:7577471577145959009:0]
2025-11-27T18:19:44.650221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:44.650248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471577145958896:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.650281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.650429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:44.650578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471575890857485:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.650632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471575890857568:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.650672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471575890857485:2048] [node 1] ICP30 unregister session Session# [4:7577471575890857568:2048] VirtualId# [4:7577471575890857564:0]
2025-11-27T18:19:44.650657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471578180402834:2048] [node 1] ICS07 socket disconnect 253 reason# EndOfStream
2025-11-27T18:19:44.650689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471578180402834:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.650748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:44.650789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:44.650809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471578180402834:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:44.650830Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471575890857485:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.650877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.651525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471578180402734:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.651570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471578180402834:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.651628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471578180402734:2048] [node 1] ICP30 unregister session Session# [5:7577471578180402834:2048] VirtualId# [5:7577471578180402828:0]
2025-11-27T18:19:44.651696Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:44.651751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:44.651788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471578180402734:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.651805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.651831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471575446580287:2048] [node 1] ICS01 socket: 252 reason# EndOfStream
2025-11-27T18:19:44.651861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471575446580187:2048] [node 1] ICP30 unregister session Session# [6:7577471575446580287:2048] VirtualId# [6:7577471575446580275:0]
2025-11-27T18:19:44.651882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471575446580287:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.651925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:44.651958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:44.659720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471578032274435:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:44.659739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471577145958896:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:44.660779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471575890857485:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:44.661776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471578180402734:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:44.727904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:44.728149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471577145959017:2049] [node 2] ICS07 socket disconnect 207 reason# EndOfStream
2025-11-27T18:19:44.728186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471577145959017:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.728265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:44.728322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471577145959017:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:44.731577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:44.731641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471577145958897:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5140) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:44.731692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471577145959017:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:44.731790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471575890857584:2049] [node 2] ICS01 socket: 247 reason# EndOfStream
2025-11-27T18:19:44.732151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471575890857486:2049] [node 2] ICP30 unregister session Session# [4:7577471575890857584:2049] VirtualId# [4:7577471575890857578:0]
2025-11-27T18:19:44.732207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471575890857584:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.732305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:44.732516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:44.733046Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471577145958897:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5140) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.733212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471577145959017:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.733280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471577145958897:2049] [node 2] ICP30 unregister session Session# [3:7577471577145959017:2049] VirtualId# [3:7577471577145959007:0]
2025-11-27T18:19:44.733400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:44.733438Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471577145958897:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.733436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:44.733463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.733635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471578180402837:2049] [node 2] ICS01 socket: 229 reason# EndOfStream
2025-11-27T18:19:44.733773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471578180402735:2049] [node 2] ICP30 unregister session Session# [5:7577471578180402837:2049] VirtualId# [5:7577471578180402833:0]
2025-11-27T18:19:44.733829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471578180402837:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.733921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:44.733974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:44.734544Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:44.734734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471575446580268:2049] [node 2] ICS01 socket: 202 reason# EndOfStream
2025-11-27T18:19:44.734777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471575446580188:2049] [node 2] ICP30 unregister session Session# [6:7577471575446580268:2049] VirtualId# [6:7577471575446580266:0]
2025-11-27T18:19:44.734804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471575446580268:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.734881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:44.734924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:44.782499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:44.782757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471575890857587:2050] [node 3] ICS01 socket: 250 reason# EndOfStream
2025-11-27T18:19:44.782813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471575890857487:2050] [node 3] ICP30 unregister session Session# [4:7577471575890857587:2050] VirtualId# [4:7577471575890857583:0]
2025-11-27T18:19:44.782863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471575890857587:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.782942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:44.782989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:44.783658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:44.783853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471575446580289:2050] [node 3] ICS01 socket: 256 reason# EndOfStream
2025-11-27T18:19:44.783903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471575446580189:2050] [node 3] ICP30 unregister session Session# [6:7577471575446580289:2050] VirtualId# [6:7577471575446580283:0]
2025-11-27T18:19:44.783929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471575446580289:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.783995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:44.784037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:44.784676Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:44.784865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471578180402817:2050] [node 3] ICS07 socket disconnect 231 reason# EndOfStream
2025-11-27T18:19:44.784901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471578180402817:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.784968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:44.785010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471578180402817:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:44.786182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471578180402736:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29747) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.786237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471578180402817:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.786279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471578180402736:2050] [node 3] ICP30 unregister session Session# [5:7577471578180402817:2050] VirtualId# [5:7577471578180402815:0]
2025-11-27T18:19:44.786371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:44.786421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471578180402736:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29747) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.786441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29747) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.796024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471578180402736:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:44.847999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:44.848267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471578180402823:2108] [node 4] ICS07 socket disconnect 198 reason# EndOfStream
2025-11-27T18:19:44.848303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471578180402823:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.848371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:44.848426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471578180402823:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:44.849032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:44.849162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471575446580280:2108] [node 4] ICS07 socket disconnect 254 reason# EndOfStream
2025-11-27T18:19:44.849187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471575446580280:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.849219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471578180402797:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26974) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.849394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471578180402823:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.849468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471578180402797:2108] [node 4] ICP30 unregister session Session# [5:7577471578180402823:2108] VirtualId# [5:7577471578180402821:0]
2025-11-27T18:19:44.849594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:44.849639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471578180402797:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26974) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.849654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26974) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.849252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:44.849321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471575446580280:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:44.850147Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471575446580251:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26974) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:44.850194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471575446580280:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:44.850274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471575446580251:2108] [node 4] ICP30 unregister session Session# [6:7577471575446580280:2108] VirtualId# [6:7577471575446580276:0]
2025-11-27T18:19:44.850387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:44.850432Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471575446580251:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26974) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.850447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26974) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:44.859479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471578180402797:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:19:44.860466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471575446580251:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:19:44.902051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:44.902260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471575446580284:2105] [node 5] ICS01 socket: 258 reason# EndOfStream
2025-11-27T18:19:44.902306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471575446580246:2105] [node 5] ICP30 unregister session Session# [6:7577471575446580284:2105] VirtualId# [6:7577471575446580278:0]
2025-11-27T18:19:44.902373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471575446580284:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:44.902472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:44.902520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:45.001931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471581306261481:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:45.001945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471581306261480:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:45.024122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471582868269815:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:45.024206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471582868269816:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:45.044067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471581559177415:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:45.044228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471581559177416:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:45.064936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471579090449455:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:45.065005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471579090449456:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:45.065145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471579090449454:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:45.086475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471582130788572:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:45.086617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471582130788573:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:45.086745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471582130788574:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:45.106528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471580307682254:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:45.106591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471580307682253:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:45.106768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471580307682255:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:45.192389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471580307682316:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:45.204187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471581306261542:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:45.208802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471581306261545:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:45.247515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471582868269887:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:45.255564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471581559177496:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:45.261653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471579090449520:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:45.262970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471582130788635:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:45.247959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471581306261546:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:45.262848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471579090449519:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:45.263140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471580307682319:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:45.261640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471582868269888:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:45.264191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471581559177497:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:45.264938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471581559177498:2118] [node 6] ICP01 ready to work
2025-11-27T18:19:45.263800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471582868269889:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:45.264424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471582130788636:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:45.271853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471581559177415:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(72695973ns) {Configure:104(72690066ns) {ProcessConfigured:135(76191ns) {StartInitialHandshake:157(73248ns) {DropHandshakes:470(2386ns) {DropIncomingHandshake:442(818ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(1809ns) PrepareNewSessionHandshake:477(3968ns) {SwitchToState:179(505ns)}}}}}
2025-11-27T18:19:45.271868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471581306261480:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(80048525ns) {Configure:104(80038110ns) {ProcessConfigured:135(115663ns) {StartInitialHandshake:157(114022ns) {DropHandshakes:470(1121ns) {DropIncomingHandshake:442(238ns) DropOutgoingHandshake:457(168ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:477(658ns) {SwitchToState:179(291ns)}}}}}
2025-11-27T18:19:45.272120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471581306261481:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(77672696ns) {Configure:104(77657206ns) {ProcessConfigured:135(55859ns) {StartInitialHandshake:157(51866ns) {DropHandshakes:470(1481ns) {DropIncomingHandshake:442(471ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(862ns) PrepareNewSessionHandshake:477(1194ns) {SwitchToState:179(534ns)}}}}}
2025-11-27T18:19:45.276340Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471582868269815:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(76683471ns) {Configure:104(76675603ns) {ProcessConfigured:135(296289ns) {StartInitialHandshake:157(293753ns) {DropHandshakes:470(1284ns) {DropIncomingHandshake:442(390ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(5680ns) {SwitchToState:179(1021ns)}}}}}
2025-11-27T18:19:45.276307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471579090449455:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(70460078ns) {Configure:104(70452718ns) {ProcessConfigured:135(1701143ns) {StartInitialHandshake:157(1699119ns) {DropHandshakes:470(1485ns) {DropIncomingHandshake:442(574ns) DropOutgoingHandshake:457(315ns)} GenerateSessionVirtualId:988(432ns) PrepareNewSessionHandshake:477(2184ns) {SwitchToState:179(808ns)}}}}}
2025-11-27T18:19:45.279523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471582130788572:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53254987ns) {Configure:104(53245153ns) {ProcessConfigured:135(1655192ns) {StartInitialHandshake:157(1653207ns) {DropHandshakes:470(170543ns) {DropIncomingHandshake:442(398ns) DropOutgoingHandshake:457(46ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:477(1406ns) {SwitchToState:179(136ns)}}}}}
2025-11-27T18:19:45.279559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471580307682316:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22280329ns) {Configure:104(22272725ns) {ProcessConfigured:135(1372380ns) {StartInitialHandshake:157(1370534ns) {DropHandshakes:470(1174ns) {DropIncomingHandshake:442(362ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(370ns) PrepareNewSessionHandshake:477(728045ns) {SwitchToState:179(2988ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.304403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471581306261560:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1304
2025-11-27T18:19:45.306931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471579090449520:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23583099ns) {Configure:104(23577299ns) {ProcessConfigured:135(236584ns) {StartInitialHandshake:157(233166ns) {DropHandshakes:470(2443ns) {DropIncomingHandshake:442(419ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(1132ns) PrepareNewSessionHandshake:477(1852ns) {SwitchToState:179(642ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.305918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471580307682254:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24407512ns) {Configure:104(24402919ns) {ProcessConfigured:135(1630878ns) {StartInitialHandshake:157(1628110ns) {DropHandshakes:470(2272ns) {DropIncomingHandshake:442(593ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(571ns) PrepareNewSessionHandshake:477(2847ns) {SwitchToState:179(849ns)}}}}}
2025-11-27T18:19:45.307643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471581306261560:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.307651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471581559177512:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20337
2025-11-27T18:19:45.308142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471579090449519:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(26290679ns) {Configure:104(26285789ns) {ProcessConfigured:135(552338ns) {StartInitialHandshake:157(549866ns) {DropHandshakes:470(2375ns) {DropIncomingHandshake:442(737ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(677ns) PrepareNewSessionHandshake:477(3094ns) {SwitchToState:179(1478ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.308511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471582868269889:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21974757ns) {Configure:104(21967967ns) {ProcessConfigured:135(667132ns) {StartInitialHandshake:157(664771ns) {DropHandshakes:470(1950ns) {DropIncomingHandshake:442(412ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(687ns) PrepareNewSessionHandshake:477(298929ns) {SwitchToState:179(1922ns)}}}}}
2025-11-27T18:19:45.308628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471581559177512:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.309429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471581306261561:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1304
2025-11-27T18:19:45.309451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471581559177415:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.309662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471580307682330:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59720
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.310021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471581306261549:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.310031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471581559177513:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20337
2025-11-27T18:19:45.310069Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471581306261549:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.311759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471579090449530:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3898
No RDMA context found for GID: 2025-11-27T18:19:45.312320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471580307682330:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.312503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471581306261564:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1304
2025-11-27T18:19:45.312481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471581559177515:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20337
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.312509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471581559177516:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20337
2025-11-27T18:19:45.312823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471580307682255:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.312750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471579090449530:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.313234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471581306261563:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1304
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.314622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471582868269896:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42549
2025-11-27T18:19:45.313490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471579090449454:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.313859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471581306261549:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:45.315649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471581559177506:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.320805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471581306261552:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.320928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471582868269897:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42549
2025-11-27T18:19:45.315688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471581559177506:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.320791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471582130788646:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39452
2025-11-27T18:19:45.320848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471581306261552:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.323270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471581306261481:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.327493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471581559177515:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.327499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471582868269896:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.328590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471581559177516:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.325333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.328647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471582130788646:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.327883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471582868269897:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.328618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471581559177517:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20337
2025-11-27T18:19:45.325437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471581306261481:2049] [node 3] ICP22 created new session: [1:7577471581306261567:2049]
2025-11-27T18:19:45.328619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471582868269903:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42549
2025-11-27T18:19:45.326642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471581306261567:2049] [node 3] ICS09 handshake done sender: [1:7577471581306261549:2111] self: [1:7577471581306261547:0] peer: [3:7577471581559177514:0] socket: 199 qp: -1
No RDMA context found for GID: 2025-11-27T18:19:45.329161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471581559177506:2131] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:19:45.327474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471581306261567:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:45.331634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471579090449536:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3898
2025-11-27T18:19:45.329042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471582868269905:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42549
2025-11-27T18:19:45.327543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:45.330042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471582868269903:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.329090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471581306261563:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.331841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471581559177518:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20337
2025-11-27T18:19:45.331457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471582868269815:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.330010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471581306261564:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.331567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471581306261566:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1304
2025-11-27T18:19:45.332659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471581559177415:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6149) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471581559177512:2133] held: no
2025-11-27T18:19:45.331761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471582868269905:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.334194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471582868269895:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.335901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471580307682335:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59720
2025-11-27T18:19:45.332614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471579090449537:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3898
2025-11-27T18:19:45.336742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471581559177498:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.331732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471581306261561:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.334217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471582868269895:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.333811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471579090449536:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.336242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471580307682336:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59720
2025-11-27T18:19:45.332201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471581306261552:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:45.337584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:45.337224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471582868269895:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:45.333938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471579090449455:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.332272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471581306261550:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.342875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471581559177498:2118] [node 6] ICP22 created new session: [3:7577471581559177521:2118]
2025-11-27T18:19:45.343671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471579090449530:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:45.332307Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471581306261550:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.344250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471582868269887:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.343369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471581559177521:2118] [node 6] ICS09 handshake done sender: [3:7577471581559177506:2131] self: [3:7577471581559177505:0] peer: [6:7577471580307682334:0] socket: 206 qp: -1
2025-11-27T18:19:45.343684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471580307682330:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:45.337515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471581306261542:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.343424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471581559177521:2118] [node 6] ICS10 traffic start
2025-11-27T18:19:45.344029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471580307682255:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.344338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:45.345453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471579090449454:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.341549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:45.343326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471581306261542:2107] [node 4] ICP22 created new session: [1:7577471581306261574:2107]
2025-11-27T18:19:45.345608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471582868269887:2107] [node 4] ICP22 created new session: [2:7577471582868269912:2107]
2025-11-27T18:19:45.343459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:45.344752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:45.345808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:45.343700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471581306261574:2107] [node 4] ICS09 handshake done sender: [1:7577471581306261552:2113] self: [1:7577471581306261551:0] peer: [4:7577471579090449535:0] socket: 200 qp: -1
2025-11-27T18:19:45.343825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471581306261574:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:45.343957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.348242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471581559177512:2133] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:45.349048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471582868269912:2107] [node 4] ICS09 handshake done sender: [2:7577471582868269895:2121] self: [2:7577471582868269894:0] peer: [4:7577471579090449538:0] socket: 237 qp: -1
2025-11-27T18:19:45.347147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471579090449454:2048] [node 1] ICP22 created new session: [4:7577471579090449540:2048]
2025-11-27T18:19:45.345539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471580307682255:2050] [node 3] ICP22 created new session: [6:7577471580307682337:2050]
2025-11-27T18:19:45.348502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471581559177513:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.347098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471580307682337:2050] [node 3] ICS09 handshake done sender: [6:7577471580307682330:2117] self: [6:7577471580307682334:0] peer: [3:7577471581559177505:0] socket: 210 qp: -1
2025-11-27T18:19:45.349529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471579090449540:2048] [node 1] ICS09 handshake done sender: [4:7577471579090449530:2116] self: [4:7577471579090449535:0] peer: [1:7577471581306261551:0] socket: 211 qp: -1
2025-11-27T18:19:45.349835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471579090449540:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.350800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471581559177517:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.349873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:45.349093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471582868269912:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:45.349133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:45.351940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471581559177415:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.354007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471582130788652:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39452
2025-11-27T18:19:45.348492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471580307682337:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.351982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:45.357848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471582868269915:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42549
2025-11-27T18:19:45.357577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471581306261550:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:45.348537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:45.358029Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471579090449455:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13734) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471579090449536:2118] held: no
2025-11-27T18:19:45.352968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471581559177415:2048] [node 1] ICP22 created new session: [3:7577471581559177526:2048]
2025-11-27T18:19:45.354459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471580307682336:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.358086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471580307682339:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59720
2025-11-27T18:19:45.358548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471580307682253:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.358612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471579090449539:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3898
2025-11-27T18:19:45.359339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471580307682339:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.361119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471582868269892:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.353650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471581559177526:2048] [node 1] ICS09 handshake done sender: [3:7577471581559177512:2133] self: [3:7577471581559177514:0] peer: [1:7577471581306261547:0] socket: 205 qp: -1
2025-11-27T18:19:45.361163Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471582868269892:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.361324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471580307682254:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.363263Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471582868269896:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:45.363609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471581306261566:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.354408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471581559177526:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:45.366973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471581306261480:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.354454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:45.367034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:45.357311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471581559177416:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.367257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471581306261480:2048] [node 2] ICP22 created new session: [1:7577471581306261578:2048]
2025-11-27T18:19:45.367468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471581306261578:2048] [node 2] ICS09 handshake done sender: [1:7577471581306261550:2112] self: [1:7577471581306261548:0] peer: [2:7577471582868269908:0] socket: 198 qp: -1
2025-11-27T18:19:45.367047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471579090449536:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:19:45.368244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471581306261578:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:45.368249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471582130788652:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:19:45.368288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:45.367555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471582868269899:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.369300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471582130788574:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.370877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471579090449542:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3898
2025-11-27T18:19:45.371155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471581306261481:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(34612852ns) {HandleHandshakeStatus:329(34609850ns) {DropIncomingHandshake:442(14436ns) DropOutgoingHandshake:457(413ns) SwitchToState:179(924ns) ProcessPendingSessionEvents:495(29169919ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29031997ns max# 5168792ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.367594Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471582868269899:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.371462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471579090449455:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.371957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471581306261554:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.371514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:45.369033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471582868269815:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.372001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471581306261554:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.371959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471581559177508:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.369079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:45.373103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471580307682343:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59720
2025-11-27T18:19:45.371999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471581559177508:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.371565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471579090449455:2049] [node 2] ICP22 created new session: [4:7577471579090449544:2049]
2025-11-27T18:19:45.373498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471582130788653:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39452
2025-11-27T18:19:45.372051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471579090449544:2049] [node 2] ICS09 handshake done sender: [4:7577471579090449536:2118] self: [4:7577471579090449538:0] peer: [2:7577471582868269894:0] socket: 239 qp: -1
2025-11-27T18:19:45.369297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471582868269815:2048] [node 1] ICP22 created new session: [2:7577471582868269918:2048]
2025-11-27T18:19:45.373506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471581559177530:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20337
2025-11-27T18:19:45.371674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471582868269918:2048] [node 1] ICS09 handshake done sender: [2:7577471582868269896:2122] self: [2:7577471582868269908:0] peer: [1:7577471581306261548:0] socket: 208 qp: -1
2025-11-27T18:19:45.375095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471581306261542:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(29462576ns) {HandleHandshakeStatus:329(29461200ns) {DropIncomingHandshake:442(9715ns) DropOutgoingHandshake:457(209ns) SwitchToState:179(130ns) ProcessPendingSessionEvents:495(22444336ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22147011ns max# 2787160ns}}}
2025-11-27T18:19:45.373855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471581559177498:2118] [node 6] ICP35 event processing took too much time PendingConnection:279(29055817ns) {HandleHandshakeStatus:329(29052274ns) {DropIncomingHandshake:442(14040ns) DropOutgoingHandshake:457(615ns) SwitchToState:179(398ns) ProcessPendingSessionEvents:495(20912203ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19191761ns max# 2780619ns}}}
2025-11-27T18:19:45.373107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471579090449544:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:45.371803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471582868269918:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.373147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:45.373901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471581559177508:2132] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:45.372533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:45.376622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471580307682255:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(27010917ns) {HandleHandshakeStatus:329(27008548ns) {DropIncomingHandshake:442(3978ns) DropOutgoingHandshake:457(9913ns) SwitchToState:179(351ns) ProcessPendingSessionEvents:495(20047398ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19682059ns max# 2392693ns}}}
2025-11-27T18:19:45.373277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471582868269892:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:45.375400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471581559177497:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.373326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471579090449543:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3898
2025-11-27T18:19:45.376947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471580307682255:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:45.375428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471582868269887:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22833899ns) {HandleHandshakeStatus:329(22830944ns) {DropIncomingHandshake:442(15507ns) DropOutgoingHandshake:457(342ns) SwitchToState:179(410ns) ProcessPendingSessionEvents:495(15987907ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15870206ns max# 2276914ns}}}
2025-11-27T18:19:45.375651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:45.379092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471580307682339:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:45.375605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471582868269899:2124] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:45.375707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471581559177497:2108] [node 5] ICP22 created new session: [3:7577471581559177532:2108]
2025-11-27T18:19:45.377888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471582868269816:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.375708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471581559177517:2137] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:45.377948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:45.376319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471581559177532:2108] [node 5] ICS09 handshake done sender: [3:7577471581559177508:2132] self: [3:7577471581559177507:0] peer: [5:7577471582130788654:0] socket: 228 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.378085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471582868269816:2049] [node 3] ICP22 created new session: [2:7577471582868269920:2049]
2025-11-27T18:19:45.376519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471581559177532:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:45.379120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471582868269920:2049] [node 3] ICS09 handshake done sender: [2:7577471582868269892:2119] self: [2:7577471582868269891:0] peer: [3:7577471581559177529:0] socket: 232 qp: -1
2025-11-27T18:19:45.376928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:45.379159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471582868269920:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:45.379244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471581559177416:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.379297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:45.379346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471581559177416:2049] [node 2] ICP22 created new session: [3:7577471581559177534:2049]
2025-11-27T18:19:45.379385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471581559177534:2049] [node 2] ICS09 handshake done sender: [3:7577471581559177517:2137] self: [3:7577471581559177529:0] peer: [2:7577471582868269891:0] socket: 236 qp: -1
2025-11-27T18:19:45.376899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471582130788655:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39452
2025-11-27T18:19:45.379427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471581559177534:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:45.374397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471579090449542:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.379645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:45.379450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471582130788656:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39452
2025-11-27T18:19:45.379503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:45.379564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471580307682344:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59720
2025-11-27T18:19:45.375006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471579090449543:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.378910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471579090449456:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.378914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471579090449454:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27032049ns) {HandleHandshakeStatus:329(27029733ns) {DropIncomingHandshake:442(1731ns) DropOutgoingHandshake:457(4773ns) SwitchToState:179(399ns) ProcessPendingSessionEvents:495(19411709ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19311130ns max# 2831126ns}}}
2025-11-27T18:19:45.379578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471582130788574:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29007) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471582130788652:2116] held: no
2025-11-27T18:19:45.382069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471580307682254:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.385091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471582130788656:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.382111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:45.382436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471580307682254:2049] [node 2] ICP22 created new session: [6:7577471580307682345:2049]
2025-11-27T18:19:45.385569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471582130788652:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:45.385864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471581559177504:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.383432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471580307682345:2049] [node 2] ICS09 handshake done sender: [6:7577471580307682339:2120] self: [6:7577471580307682342:0] peer: [2:7577471582868269898:0] socket: 230 qp: -1
2025-11-27T18:19:45.385967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471581306261480:2048] [node 2] ICP18 handshake (actor [1:7577471581306261566:2120]) is too old
2025-11-27T18:19:45.385712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471582130788635:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.385910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471581559177504:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.390156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471582130788574:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.390623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471579090449531:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.383469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471580307682345:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:45.390203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:45.390830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471581306261554:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:45.390632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471582130788574:2050] [node 3] ICP22 created new session: [5:7577471582130788659:2050]
2025-11-27T18:19:45.391750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471582868269889:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.391495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471581559177504:2130] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:45.392346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471582130788659:2050] [node 3] ICS09 handshake done sender: [5:7577471582130788652:2116] self: [5:7577471582130788654:0] peer: [3:7577471581559177507:0] socket: 250 qp: -1
2025-11-27T18:19:45.390673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471579090449531:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.383593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:45.392796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:45.392258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471579090449550:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3898
2025-11-27T18:19:45.395069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471581306261546:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.393277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471582130788659:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:45.390029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471580307682344:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.394522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471582868269889:2109] [node 6] ICP22 created new session: [2:7577471582868269922:2109]
2025-11-27T18:19:45.401368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471582868269922:2109] [node 6] ICS09 handshake done sender: [2:7577471582868269899:2124] self: [2:7577471582868269898:0] peer: [6:7577471580307682342:0] socket: 221 qp: -1
2025-11-27T18:19:45.395132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:45.409979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471581559177496:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.410047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:45.393330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:45.405876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471582868269922:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:45.395160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471581306261546:2109] [node 6] ICP22 created new session: [1:7577471581306261581:2109]
2025-11-27T18:19:45.405923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:45.395204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471582130788653:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.395194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471581306261581:2109] [node 6] ICS09 handshake done sender: [1:7577471581306261554:2114] self: [1:7577471581306261553:0] peer: [6:7577471580307682341:0] socket: 244 qp: -1
2025-11-27T18:19:45.395385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471582130788572:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.410103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471581559177496:2107] [node 4] ICP22 created new session: [3:7577471581559177537:2107]
2025-11-27T18:19:45.395215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471581306261581:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:45.408513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471582130788574:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(18869005ns) {HandleHandshakeStatus:329(18866018ns) {DropIncomingHandshake:442(1733ns) DropOutgoingHandshake:457(215ns) SwitchToState:179(316ns) ProcessPendingSessionEvents:495(1181ns)}}
2025-11-27T18:19:45.395237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:45.410150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471581559177537:2107] [node 4] ICS09 handshake done sender: [3:7577471581559177504:2130] self: [3:7577471581559177503:0] peer: [4:7577471579090449548:0] socket: 249 qp: -1
2025-11-27T18:19:45.410185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471581559177537:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:45.410148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471581306261559:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.410175Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471581306261559:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.410222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:45.412155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471582868269815:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18221338ns) {HandleHandshakeStatus:329(18218956ns) {DropIncomingHandshake:442(1110ns) DropOutgoingHandshake:457(5994ns) SwitchToState:179(311ns) ProcessPendingSessionEvents:495(13819046ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13573339ns max# 1574236ns}}}
2025-11-27T18:19:45.412784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471581306261559:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.413840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471580307682319:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.417329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471579090449550:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.423872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471581306261545:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.421402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471580307682253:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6149) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471580307682336:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.424766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471579090449551:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3898
2025-11-27T18:19:45.423941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:45.425359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471582130788662:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39452
2025-11-27T18:19:45.424377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471581306261545:2108] [node 5] ICP22 created new session: [1:7577471581306261584:2108]
2025-11-27T18:19:45.427390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471582868269816:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(47540669ns) {HandleHandshakeStatus:329(47538250ns) {DropIncomingHandshake:442(13759ns) DropOutgoingHandshake:457(213ns) SwitchToState:179(365ns) ProcessPendingSessionEvents:495(42859221ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 42708760ns max# 4248002ns}}}
2025-11-27T18:19:45.425967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471581306261584:2108] [node 5] ICS09 handshake done sender: [1:7577471581306261559:2115] self: [1:7577471581306261558:0] peer: [5:7577471582130788661:0] socket: 252 qp: -1
2025-11-27T18:19:45.426005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471581306261584:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:45.430253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471579090449531:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.433619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471580307682347:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59720
2025-11-27T18:19:45.432834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471582130788572:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6149) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471582130788653:2117] held: no
2025-11-27T18:19:45.426580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:45.434175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471582868269889:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(41290285ns) {HandleHandshakeStatus:329(41288058ns) {DropIncomingHandshake:442(900ns) DropOutgoingHandshake:457(588ns) SwitchToState:179(409ns) ProcessPendingSessionEvents:495(23065977ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21836370ns max# 4077488ns}}}
2025-11-27T18:19:45.435154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471579090449519:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.435190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471581559177416:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(44537711ns) {HandleHandshakeStatus:329(44535705ns) {DropIncomingHandshake:442(410ns) DropOutgoingHandshake:457(4378ns) SwitchToState:179(352ns) ProcessPendingSessionEvents:495(44143849ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 41109502ns max# 7665615ns}}}
2025-11-27T18:19:45.435389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471582130788653:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.436280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:45.436360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471579090449528:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.436751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471582130788572:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.436386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471579090449528:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.436779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471582130788663:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39452
2025-11-27T18:19:45.436796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471579090449519:2108] [node 5] ICP22 created new session: [4:7577471579090449553:2108]
2025-11-27T18:19:45.437506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471579090449553:2108] [node 5] ICS09 handshake done sender: [4:7577471579090449531:2117] self: [4:7577471579090449529:0] peer: [5:7577471582130788658:0] socket: 261 qp: -1
2025-11-27T18:19:45.437868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471580307682254:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(50714287ns) {HandleHandshakeStatus:329(50712576ns) {DropIncomingHandshake:442(453ns) DropOutgoingHandshake:457(4279ns) SwitchToState:179(253ns) ProcessPendingSessionEvents:495(47458700ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 47202855ns max# 4194840ns}}}
2025-11-27T18:19:45.436903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:45.437544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471579090449553:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:45.437920Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471580307682253:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18941883ns) {HandleHandshakeStatus:395(18940325ns) {LogHandshakeFail:472(10519ns) {UpdateErrorStateLog:545(3410ns)} DropOutgoingHandshake:457(1733ns) ProcessPendingSessionEvents:495(11407195ns) {unordered DropSessionEvent:512 num# 100 sum# 3089924ns max# 1268604ns}}}
2025-11-27T18:19:45.437870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471579090449542:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:45.438089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:45.438765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471580307682336:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:45.437010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471582130788572:2048] [node 1] ICP22 created new session: [5:7577471582130788665:2048]
2025-11-27T18:19:45.437639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471582130788665:2048] [node 1] ICS09 handshake done sender: [5:7577471582130788653:2117] self: [5:7577471582130788661:0] peer: [1:7577471581306261558:0] socket: 253 qp: -1
2025-11-27T18:19:45.439773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471579090449456:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.437845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471582130788665:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:45.439640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471580307682253:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.440238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:45.439762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:45.440396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471579090449456:2050] [node 3] ICP22 created new session: [4:7577471579090449556:2050]
2025-11-27T18:19:45.438037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:45.441033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471582130788656:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:45.441072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471579090449556:2050] [node 3] ICS09 handshake done sender: [4:7577471579090449542:2121] self: [4:7577471579090449548:0] peer: [3:7577471581559177503:0] socket: 223 qp: -1
2025-11-27T18:19:45.439803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471580307682253:2048] [node 1] ICP22 created new session: [6:7577471580307682349:2048]
2025-11-27T18:19:45.440394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471580307682349:2048] [node 1] ICS09 handshake done sender: [6:7577471580307682336:2119] self: [6:7577471580307682341:0] peer: [1:7577471581306261553:0] socket: 246 qp: -1
2025-11-27T18:19:45.441109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471579090449556:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:45.441331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:45.440830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471580307682349:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:45.441058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.442389Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471581559177496:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(26906372ns) {HandleHandshakeStatus:329(26899505ns) {DropIncomingHandshake:442(13711ns) DropOutgoingHandshake:457(401ns) SwitchToState:179(557ns) ProcessPendingSessionEvents:495(24762761ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24642277ns max# 3916672ns}}}
2025-11-27T18:19:45.442556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471579090449528:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:45.442883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471582130788635:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.442952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471582130788664:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39452
2025-11-27T18:19:45.443175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:45.444360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471579090449520:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.443381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471582130788635:2108] [node 4] ICP22 created new session: [5:7577471582130788668:2108]
2025-11-27T18:19:45.447242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471581306261545:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(16969287ns) {HandleHandshakeStatus:329(16966418ns) {DropIncomingHandshake:442(15297ns) DropOutgoingHandshake:457(297ns) SwitchToState:179(423ns) ProcessPendingSessionEvents:495(9135633ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8821916ns max# 1147594ns}}}
2025-11-27T18:19:45.444413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:45.445738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471579090449520:2110] [node 6] ICP22 created new session: [4:7577471579090449558:2110]
2025-11-27T18:19:45.447551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471579090449558:2110] [node 6] ICS09 handshake done sender: [4:7577471579090449528:2115] self: [4:7577471579090449527:0] peer: [6:7577471580307682348:0] socket: 259 qp: -1
2025-11-27T18:19:45.448499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471579090449558:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:45.448544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.443666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471582130788668:2108] [node 4] ICS09 handshake done sender: [5:7577471582130788656:2119] self: [5:7577471582130788658:0] peer: [4:7577471579090449529:0] socket: 262 qp: -1
2025-11-27T18:19:45.450625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471579090449519:2108] [node 5] ICP18 handshake (actor [4:7577471579090449550:2123]) is too old
2025-11-27T18:19:45.443824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471582130788668:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:45.450351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471580307682351:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59720
2025-11-27T18:19:45.451333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471579090449456:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:45.451401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471582868269906:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.444358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:45.451442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471582868269906:2127] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.448983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471582130788664:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.450441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471582130788573:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.453878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471580307682319:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14863) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471580307682344:2122] held: no
2025-11-27T18:19:45.453878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471580307682352:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59720
2025-11-27T18:19:45.454264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471582868269906:2127] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:45.454458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471580307682352:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.455152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471580307682316:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:45.456072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471582868269888:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.456143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:45.456192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471582868269888:2108] [node 5] ICP22 created new session: [2:7577471582868269926:2108]
2025-11-27T18:19:45.456241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471582868269926:2108] [node 5] ICS09 handshake done sender: [2:7577471582868269906:2127] self: [2:7577471582868269904:0] peer: [5:7577471582130788670:0] socket: 209 qp: -1
2025-11-27T18:19:45.456279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471582868269926:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:45.456308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:45.456359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471582130788650:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:45.457784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471580307682344:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:45.456393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471582130788650:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:45.458354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471580307682319:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.458793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:45.459082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471580307682319:2110] [node 4] ICP22 created new session: [6:7577471580307682354:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.459998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471580307682354:2110] [node 4] ICS09 handshake done sender: [6:7577471580307682344:2122] self: [6:7577471580307682348:0] peer: [4:7577471579090449527:0] socket: 260 qp: -1
2025-11-27T18:19:45.460024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471582130788671:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39452
2025-11-27T18:19:45.460054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471580307682354:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:45.460106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:45.461130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471582130788573:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13734) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471582130788664:2122] held: no
2025-11-27T18:19:45.461542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471582130788664:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:45.461995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471582130788573:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.462059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:45.462125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471582130788573:2049] [node 2] ICP22 created new session: [5:7577471582130788673:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:45.462163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471582130788673:2049] [node 2] ICS09 handshake done sender: [5:7577471582130788664:2122] self: [5:7577471582130788670:0] peer: [2:7577471582868269904:0] socket: 212 qp: -1
2025-11-27T18:19:45.462207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471582130788673:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:45.462725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471580307682356:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59720
2025-11-27T18:19:45.462235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:45.462518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471582130788650:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:45.462669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471582130788636:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:45.462770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:45.462818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471582130788636:2109] [node 6] ICP22 created new session: [5:7577471582130788675:2109]
2025-11-27T18:19:45.462892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471582130788675:2109] [node 6] ICS09 handshake done sender: [5:7577471582130788650:2115] self: [5:7577471582130788649:0] peer: [6:7577471580307682353:0] socket: 224 qp: -1
2025-11-27T18:19:45.462925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471582130788675:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:45.462953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:45.462985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471580307682352:2125] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:45.463147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471580307682316:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:45.463188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:45.463518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471580307682316:2106] [node 5] ICP22 created new session: [6:7577471580307682357:2106]
2025-11-27T18:19:45.463820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471580307682357:2106] [node 5] ICS09 handshake done sender: [6:7577471580307682352:2125] self: [6:7577471580307682353:0] peer: [5:7577471582130788649:0] socket: 225 qp: -1
2025-11-27T18:19:45.463851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471580307682357:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:45.464086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:45.508708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:45.508975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471582868269918:2048] [node 1] ICS07 socket disconnect 208 reason# EndOfStream
2025-11-27T18:19:45.509014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471582868269918:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.509122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:45.509189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471582868269918:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:45.510262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:45.510474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471581559177526:2048] [node 1] ICS01 socket: 205 reason# EndOfStream
2025-11-27T18:19:45.510522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471581559177415:2048] [node 1] ICP30 unregister session Session# [3:7577471581559177526:2048] VirtualId# [3:7577471581559177514:0]
2025-11-27T18:19:45.510554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471581559177526:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.510636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:45.510687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:45.511702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471582868269815:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6149) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:45.511749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471582868269918:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:45.516337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:45.516577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471579090449540:2048] [node 1] ICS07 socket disconnect 211 reason# EndOfStream
2025-11-27T18:19:45.516615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471579090449540:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.516693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:45.516744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471579090449540:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:45.516746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471582868269815:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6149) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:45.516919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471582868269918:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:45.517014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471582868269815:2048] [node 1] ICP30 unregister session Session# [2:7577471582868269918:2048] VirtualId# [2:7577471582868269908:0]
2025-11-27T18:19:45.517153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:45.517188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471582868269815:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.517204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.517731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:45.518001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471582130788665:2048] [node 1] ICS01 socket: 253 reason# EndOfStream
2025-11-27T18:19:45.518050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471582130788572:2048] [node 1] ICP30 unregister session Session# [5:7577471582130788665:2048] VirtualId# [5:7577471582130788661:0]
2025-11-27T18:19:45.518106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471582130788665:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.518194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:45.518242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:45.517654Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471579090449454:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6149) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:45.517826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471579090449540:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:45.517869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471579090449454:2048] [node 1] ICP30 unregister session Session# [4:7577471579090449540:2048] VirtualId# [4:7577471579090449535:0]
2025-11-27T18:19:45.517970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:45.518026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471579090449454:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.518043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.519174Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:45.519400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471580307682349:2048] [node 1] ICS01 socket: 246 reason# EndOfStream
2025-11-27T18:19:45.519432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471580307682253:2048] [node 1] ICP30 unregister session Session# [6:7577471580307682349:2048] VirtualId# [6:7577471580307682341:0]
2025-11-27T18:19:45.519450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471580307682349:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.519499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:45.519533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:45.528128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471579090449454:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:45.594718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:45.595052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471581559177534:2049] [node 2] ICS07 socket disconnect 236 reason# EndOfStream
2025-11-27T18:19:45.595089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471581559177534:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.595160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:45.595208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471581559177534:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:45.597797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:45.598011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471579090449544:2049] [node 2] ICS01 socket: 239 reason# EndOfStream
2025-11-27T18:19:45.598058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471579090449455:2049] [node 2] ICP30 unregister session Session# [4:7577471579090449544:2049] VirtualId# [4:7577471579090449538:0]
2025-11-27T18:19:45.598109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471579090449544:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.598175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:45.598219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:45.598267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471581559177416:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13734) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:45.598316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471581559177534:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:45.599267Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:45.599275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471581559177416:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13734) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:45.599340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471581559177534:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:45.599415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471581559177416:2049] [node 2] ICP30 unregister session Session# [3:7577471581559177534:2049] VirtualId# [3:7577471581559177529:0]
2025-11-27T18:19:45.599583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:45.599538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471582130788673:2049] [node 2] ICS01 socket: 212 reason# EndOfStream
2025-11-27T18:19:45.599643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471581559177416:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13734) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.599593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471582130788573:2049] [node 2] ICP30 unregister session Session# [5:7577471582130788673:2049] VirtualId# [5:7577471582130788670:0]
2025-11-27T18:19:45.599619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471582130788673:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.599688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13734) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.599703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:45.599748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:45.600518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:45.600692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471580307682345:2049] [node 2] ICS07 socket disconnect 230 reason# EndOfStream
2025-11-27T18:19:45.600722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471580307682345:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.600800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:45.600863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471580307682345:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:45.601634Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471580307682254:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13734) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:45.601750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471580307682345:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:45.601810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471580307682254:2049] [node 2] ICP30 unregister session Session# [6:7577471580307682345:2049] VirtualId# [6:7577471580307682342:0]
2025-11-27T18:19:45.601924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:45.601976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471580307682254:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13734) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.601990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13734) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.612067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471580307682254:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:45.658169Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:45.658411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471579090449556:2050] [node 3] ICS07 socket disconnect 223 reason# EndOfStream
2025-11-27T18:19:45.658451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471579090449556:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.658532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:45.658573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471579090449556:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:45.659364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:45.659316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471579090449456:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29007) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:45.659429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471579090449556:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:45.659467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471579090449456:2050] [node 3] ICP30 unregister session Session# [4:7577471579090449556:2050] VirtualId# [4:7577471579090449548:0]
2025-11-27T18:19:45.659584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:45.659590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471582130788659:2050] [node 3] ICS01 socket: 250 reason# EndOfStream
2025-11-27T18:19:45.659616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471579090449456:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29007) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.659633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471582130788574:2050] [node 3] ICP30 unregister session Session# [5:7577471582130788659:2050] VirtualId# [5:7577471582130788654:0]
2025-11-27T18:19:45.659630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29007) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.659668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471582130788659:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.659734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:45.659790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:45.660555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:45.660744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471580307682337:2050] [node 3] ICS07 socket disconnect 210 reason# EndOfStream
2025-11-27T18:19:45.660785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471580307682337:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.660857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:45.660903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471580307682337:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:45.661646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471580307682255:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29007) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:45.661786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471580307682337:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:45.661841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471580307682255:2050] [node 3] ICP30 unregister session Session# [6:7577471580307682337:2050] VirtualId# [6:7577471580307682334:0]
2025-11-27T18:19:45.661937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:45.661993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471580307682255:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29007) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.662022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29007) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.669453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471579090449456:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:45.673221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471580307682255:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:45.726804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:45.727054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471582130788668:2108] [node 4] ICS07 socket disconnect 262 reason# EndOfStream
2025-11-27T18:19:45.727102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471582130788668:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.727179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:45.727249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471582130788668:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:45.727780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:45.727924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471580307682354:2110] [node 4] ICS01 socket: 260 reason# EndOfStream
2025-11-27T18:19:45.727957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471580307682319:2110] [node 4] ICP30 unregister session Session# [6:7577471580307682354:2110] VirtualId# [6:7577471580307682348:0]
2025-11-27T18:19:45.727980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471580307682354:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.727984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471582130788635:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14863) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:45.728035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471582130788668:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:45.728049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:45.728098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471582130788635:2108] [node 4] ICP30 unregister session Session# [5:7577471582130788668:2108] VirtualId# [5:7577471582130788658:0]
2025-11-27T18:19:45.728103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:45.728193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:45.728264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471582130788635:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14863) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.728301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14863) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.793194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:45.793471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471580307682357:2106] [node 5] ICS07 socket disconnect 225 reason# EndOfStream
2025-11-27T18:19:45.793510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471580307682357:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:45.793594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:45.793640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471580307682357:2106] [node 5] ICS15 start handshake
2025-11-27T18:19:45.794552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471580307682316:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7322) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:45.794652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471580307682357:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:45.794761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471580307682316:2106] [node 5] ICP30 unregister session Session# [6:7577471580307682357:2106] VirtualId# [6:7577471580307682353:0]
2025-11-27T18:19:45.794875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:19:45.794921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471580307682316:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7322) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.794936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7322) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:45.804622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471580307682316:2106] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:45.899152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471580119601574:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:45.899259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471580119601575:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:45.919839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471579302078049:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:45.919877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471579302078050:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:45.941844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471582568152917:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:45.941899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471582568152918:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:45.966420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471581953607623:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:45.966463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471581953607624:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:45.966627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471581953607622:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:45.991313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471580609101644:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:45.991376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471580609101645:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:45.992396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471580609101646:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:46.015483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471587128166972:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:46.015529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471587128166973:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:46.015889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471587128166971:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:46.206571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471587128167030:2105] [node 5] ICP01 ready to work
2025-11-27T18:19:46.206717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471584414568936:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:46.222896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471586248574981:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:46.209514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471586863120276:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:46.236759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471586248574982:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:46.236784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471583597045409:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:46.236793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471583597045408:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:46.207203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471584414568934:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:46.241756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471584904069006:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:46.248441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471583597045410:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:46.243298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471584904069007:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:46.239791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471587128167035:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:46.242275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471586863120278:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:46.242678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471586863120277:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:46.222847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471584414568935:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:46.273485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471584414568935:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23852091ns) {Configure:104(23848363ns) {ProcessConfigured:135(7740791ns) {StartInitialHandshake:157(7737187ns) {DropHandshakes:470(3238ns) {DropIncomingHandshake:442(774ns) DropOutgoingHandshake:457(404ns)} GenerateSessionVirtualId:988(1295ns) PrepareNewSessionHandshake:477(6889ns) {SwitchToState:179(3187ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.275658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471584414568941:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14324
2025-11-27T18:19:46.276617Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471580119601575:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(131991757ns) {Configure:104(131984268ns) {ProcessConfigured:135(564413ns) {StartInitialHandshake:157(560150ns) {DropHandshakes:470(3690ns) {DropIncomingHandshake:442(1214ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(2690ns) PrepareNewSessionHandshake:477(2743ns) {SwitchToState:179(937ns)}}}}}
2025-11-27T18:19:46.282933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471581953607624:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34390706ns) {Configure:104(34386179ns) {ProcessConfigured:135(51888ns) {StartInitialHandshake:157(50255ns) {DropHandshakes:470(2922ns) {DropIncomingHandshake:442(712ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(857ns) PrepareNewSessionHandshake:477(1605ns) {SwitchToState:179(617ns)}}}}}
2025-11-27T18:19:46.287572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471583597045410:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25530800ns) {Configure:104(25527101ns) {ProcessConfigured:135(62083ns) {StartInitialHandshake:157(59635ns) {DropHandshakes:470(1828ns) {DropIncomingHandshake:442(361ns) DropOutgoingHandshake:457(420ns)} GenerateSessionVirtualId:988(679ns) PrepareNewSessionHandshake:477(4681ns) {SwitchToState:179(605ns)}}}}}
2025-11-27T18:19:46.292447Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471580119601574:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36705158ns) {Configure:104(36697508ns) {ProcessConfigured:135(4477412ns) {StartInitialHandshake:157(4474880ns) {DropHandshakes:470(2220ns) {DropIncomingHandshake:442(513ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(1077ns) PrepareNewSessionHandshake:477(4122ns) {SwitchToState:179(2344ns)}}}}}
2025-11-27T18:19:46.301177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471587128166971:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38267361ns) {Configure:104(38263305ns) {ProcessConfigured:135(76150ns) {StartInitialHandshake:157(73751ns) {DropHandshakes:470(3159ns) {DropIncomingHandshake:442(789ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(1093ns) PrepareNewSessionHandshake:477(1457ns) {SwitchToState:179(743ns)}}}}}
2025-11-27T18:19:46.292941Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471580609101645:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21961246ns) {Configure:104(21952849ns) {ProcessConfigured:135(1259201ns) {StartInitialHandshake:157(1256189ns) {DropHandshakes:470(2144ns) {DropIncomingHandshake:442(488ns) DropOutgoingHandshake:457(266ns)} GenerateSessionVirtualId:988(422ns) PrepareNewSessionHandshake:477(2087ns) {SwitchToState:179(964ns)}}}}}
2025-11-27T18:19:46.293228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471584414568941:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.291496Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471579302078050:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34981259ns) {Configure:104(34977248ns) {ProcessConfigured:135(1006685ns) {StartInitialHandshake:157(1004482ns) {DropHandshakes:470(1192ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(591ns) PrepareNewSessionHandshake:477(1764ns) {SwitchToState:179(704ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.307863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471587128167037:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55559
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.291552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471579302078049:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(39393960ns) {Configure:104(39386886ns) {ProcessConfigured:135(4611283ns) {StartInitialHandshake:157(4607767ns) {DropHandshakes:470(1258ns) {DropIncomingHandshake:442(380ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(380ns) PrepareNewSessionHandshake:477(1927ns) {SwitchToState:179(963ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.327479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471584414568949:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14324
2025-11-27T18:19:46.309430Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471587128166973:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(43903156ns) {Configure:104(43896508ns) {ProcessConfigured:135(3893899ns) {StartInitialHandshake:157(3890743ns) {DropHandshakes:470(2117ns) {DropIncomingHandshake:442(597ns) DropOutgoingHandshake:457(524ns)} GenerateSessionVirtualId:988(481ns) PrepareNewSessionHandshake:477(3164ns) {SwitchToState:179(1609ns)}}}}}
2025-11-27T18:19:46.313517Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471586863120276:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(47934421ns) {Configure:104(47926175ns) {ProcessConfigured:135(8421221ns) {StartInitialHandshake:157(8419061ns) {DropHandshakes:470(1699ns) {DropIncomingHandshake:442(453ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(762ns) PrepareNewSessionHandshake:477(3610ns) {SwitchToState:179(1428ns)}}}}}
2025-11-27T18:19:46.309833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471581953607623:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27817023ns) {Configure:104(27811202ns) {ProcessConfigured:135(5520534ns) {StartInitialHandshake:157(5517606ns) {DropHandshakes:470(1255ns) {DropIncomingHandshake:442(372ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(484ns) PrepareNewSessionHandshake:477(3152ns) {SwitchToState:179(1909ns)}}}}}
2025-11-27T18:19:46.318982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471583597045409:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(43675698ns) {Configure:104(43670452ns) {ProcessConfigured:135(8711652ns) {StartInitialHandshake:157(8708698ns) {DropHandshakes:470(11716ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(449ns) PrepareNewSessionHandshake:477(1723ns) {SwitchToState:179(482ns)}}}}}
2025-11-27T18:19:46.332147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471587128167037:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.318932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471584904069018:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11759
2025-11-27T18:19:46.318149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471586248574983:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32057
2025-11-27T18:19:46.313575Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471582568152917:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(48576132ns) {Configure:104(48571655ns) {ProcessConfigured:135(5935982ns) {StartInitialHandshake:157(5933665ns) {DropHandshakes:470(797ns) {DropIncomingHandshake:442(300ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(498ns) PrepareNewSessionHandshake:477(2747ns) {SwitchToState:179(997ns)}}}}}
2025-11-27T18:19:46.349383Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471587128166972:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(69358232ns) {Configure:104(69350099ns) {ProcessConfigured:135(20066143ns) {StartInitialHandshake:157(16747589ns) {DropHandshakes:470(2674ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(1327ns) PrepareNewSessionHandshake:477(3972ns) {SwitchToState:179(3054ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.337944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471584904069018:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.343331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471586863120278:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(52627654ns) {Configure:104(52623300ns) {ProcessConfigured:135(13095870ns) {StartInitialHandshake:157(13093771ns) {DropHandshakes:470(2544994ns) {DropIncomingHandshake:442(2600ns) DropOutgoingHandshake:457(462ns)} GenerateSessionVirtualId:988(2261ns) PrepareNewSessionHandshake:477(7191ns) {SwitchToState:179(2468ns)}}}}}
2025-11-27T18:19:46.343040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471583597045408:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39147375ns) {Configure:104(39137773ns) {ProcessConfigured:135(78152ns) {StartInitialHandshake:157(75015ns) {DropHandshakes:470(3126ns) {DropIncomingHandshake:442(697ns) DropOutgoingHandshake:457(372ns)} GenerateSessionVirtualId:988(1587ns) PrepareNewSessionHandshake:477(4883ns) {SwitchToState:179(689ns)}}}}}
2025-11-27T18:19:46.333336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471586248574983:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.353498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471584414568940:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.349673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471587128167030:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(53772697ns) {Configure:104(53764409ns) {ProcessConfigured:135(14389205ns) {StartInitialHandshake:157(14386641ns) {DropHandshakes:470(2419014ns) {DropIncomingHandshake:442(2630ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(2303ns) PrepareNewSessionHandshake:477(3515ns) {SwitchToState:179(1696ns)}}}}}
2025-11-27T18:19:46.343312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471586248574982:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(51632964ns) {Configure:104(51628144ns) {ProcessConfigured:135(9862168ns) {StartInitialHandshake:157(9858399ns) {DropHandshakes:470(2426ns) {DropIncomingHandshake:442(560ns) DropOutgoingHandshake:457(390ns)} GenerateSessionVirtualId:988(924ns) PrepareNewSessionHandshake:477(3999ns) {SwitchToState:179(2313ns)}}}}}
2025-11-27T18:19:46.344121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471581953607622:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(60808746ns) {Configure:104(60803934ns) {ProcessConfigured:135(56220ns) {StartInitialHandshake:157(53943ns) {DropHandshakes:470(1311ns) {DropIncomingHandshake:442(435ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(810ns) PrepareNewSessionHandshake:477(1026ns) {SwitchToState:179(384ns)}}}}}
2025-11-27T18:19:46.354322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471587128167030:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.343424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471580609101644:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.344255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471586248574981:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(58209642ns) {Configure:104(58204588ns) {ProcessConfigured:135(10521076ns) {StartInitialHandshake:157(10518012ns) {DropHandshakes:470(2073ns) {DropIncomingHandshake:442(446ns) DropOutgoingHandshake:457(389ns)} GenerateSessionVirtualId:988(626ns) PrepareNewSessionHandshake:477(3204ns) {SwitchToState:179(853ns)}}}}}
2025-11-27T18:19:46.353560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471584414568940:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.353669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471584904069019:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.356785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471584414568951:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14324
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.361021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471584414568940:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:46.360884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471584904069011:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.364997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471586248574991:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32057
2025-11-27T18:19:46.365162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471584414568949:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.360931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471584904069011:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.365162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471584414568936:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(79473601ns) {Configure:104(76041380ns) {ProcessConfigured:135(3423583ns) {StartInitialHandshake:157(3420100ns) {DropHandshakes:470(3252ns) {DropIncomingHandshake:442(464ns) DropOutgoingHandshake:457(715ns)} GenerateSessionVirtualId:988(519ns) PrepareNewSessionHandshake:477(2793ns) {SwitchToState:179(789ns)}}}}}
2025-11-27T18:19:46.365920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471584414568935:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.364358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471584904069021:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11759
2025-11-27T18:19:46.365403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471584904069019:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.366545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471584904069023:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11759
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.371284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:46.372775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471586248574991:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.372774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471584904069011:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:46.371461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471582568152918:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(84387680ns) {Configure:104(84379765ns) {ProcessConfigured:135(5040999ns) {StartInitialHandshake:157(5037740ns) {DropHandshakes:470(6016ns) {DropIncomingHandshake:442(309ns) DropOutgoingHandshake:457(415ns)} GenerateSessionVirtualId:988(666ns) PrepareNewSessionHandshake:477(2644ns) {SwitchToState:179(1315ns)}}}}}
2025-11-27T18:19:46.372717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471587128167052:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55559
2025-11-27T18:19:46.372871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471584414568935:2109] [node 5] ICP22 created new session: [1:7577471584414568955:2109]
2025-11-27T18:19:46.377525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471586863120285:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53341
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.378790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471587128167052:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.377525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471583597045424:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35443
2025-11-27T18:19:46.374636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471584414568955:2109] [node 5] ICS09 handshake done sender: [1:7577471584414568940:2112] self: [1:7577471584414568939:0] peer: [5:7577471584904069020:0] socket: 212 qp: -1
2025-11-27T18:19:46.379107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471584904069007:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.374577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471586248574998:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32057
2025-11-27T18:19:46.381668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471583597045425:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35443
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.377550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471584414568951:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.379185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:46.381309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471586863120286:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53341
2025-11-27T18:19:46.376091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471581953607622:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.382124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471587128166972:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.381873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471586863120285:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.378723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471586248574998:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.377651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471584414568955:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:46.384267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471583597045416:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.384471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471587128167053:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55559
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.381321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471584904069007:2110] [node 6] ICP22 created new session: [5:7577471584904069025:2110]
2025-11-27T18:19:46.381586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471581953607623:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.377714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:46.384311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471583597045416:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.385152Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471587128167030:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61229) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471587128167037:2111] held: no
2025-11-27T18:19:46.382718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471584904069021:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.386076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471586863120286:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.388566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471586863120289:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53341
2025-11-27T18:19:46.378444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471584414568943:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.389713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471587128167054:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55559
2025-11-27T18:19:46.383209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471584904069025:2110] [node 6] ICS09 handshake done sender: [5:7577471584904069011:2112] self: [5:7577471584904069010:0] peer: [6:7577471587128167051:0] socket: 206 qp: -1
2025-11-27T18:19:46.384496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471583597045418:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.385146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471584904069018:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:46.384523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471583597045418:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.378468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471584414568943:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.389111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471582568152917:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.384801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471584414568943:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:46.385094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471583597045424:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.385966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471584904069025:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:46.405147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471587128167054:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.396105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471586863120289:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.391277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471584414568934:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.411519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471587128166973:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.385335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471583597045425:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.386009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:46.418058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471586248575002:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32057
2025-11-27T18:19:46.391336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:46.405364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471586863120277:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(34761614ns) {Configure:104(34753639ns) {ProcessConfigured:135(5945144ns) {StartInitialHandshake:157(5942746ns) {DropHandshakes:470(477724ns) {DropIncomingHandshake:442(1931ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(1122ns) PrepareNewSessionHandshake:477(3384ns) {SwitchToState:179(1565ns)}}}}}
2025-11-27T18:19:46.389662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471580609101644:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62519) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471584904069018:2115] held: no
2025-11-27T18:19:46.392159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471584414568934:2107] [node 4] ICP22 created new session: [1:7577471584414568962:2107]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:19:46.392609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471584414568959:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14324
2025-11-27T18:19:46.407637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471584904069006:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.421279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471586863120283:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.389354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471579302078049:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.420881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471586248574990:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.396065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471584414568938:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.423028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471584904069026:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11759
2025-11-27T18:19:46.421322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471586863120283:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.392694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471583597045416:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.431297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471584904069026:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.395789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471583597045418:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:46.431047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471586863120299:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53341
2025-11-27T18:19:46.420921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471586248574990:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.405847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471583597045410:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.427755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471587128167056:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55559
2025-11-27T18:19:46.396100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471584414568938:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.431360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471586863120298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53341
2025-11-27T18:19:46.420929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471586248575003:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32057
2025-11-27T18:19:46.432191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471580609101646:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.435414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471584904069031:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11759
2025-11-27T18:19:46.431748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471587128167059:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55559
2025-11-27T18:19:46.396215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471584414568962:2107] [node 4] ICS09 handshake done sender: [1:7577471584414568943:2114] self: [1:7577471584414568942:0] peer: [4:7577471586248575000:0] socket: 214 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.449462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471584904069031:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.431421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471586863120283:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:46.405950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471583597045409:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.433977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471587128167052:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:46.423437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471581953607623:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29578) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471586248574998:2117] held: no
2025-11-27T18:19:46.397808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471584414568962:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:46.455314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471580609101645:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.407127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:46.445546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471586863120298:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.434201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471587128166972:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.397845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471584414568948:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.449596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471586863120276:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.460316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471586248574991:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:46.411470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:46.434274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:46.397858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:46.470566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471584904069007:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(90583202ns) {HandleHandshakeStatus:329(90579268ns) {DropIncomingHandshake:442(14056ns) DropOutgoingHandshake:457(555ns) SwitchToState:179(362ns) ProcessPendingSessionEvents:495(79938075ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 79789637ns max# 7823928ns}}}
2025-11-27T18:19:46.434340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471587128166972:2049] [node 2] ICP22 created new session: [6:7577471587128167060:2049]
2025-11-27T18:19:46.434397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471587128167060:2049] [node 2] ICS09 handshake done sender: [6:7577471587128167052:2117] self: [6:7577471587128167055:0] peer: [2:7577471583597045415:0] socket: 229 qp: -1
2025-11-27T18:19:46.434434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471587128167060:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:46.434474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:46.460661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471587128167059:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.460971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471587128167030:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(24135618ns) {HandleHandshakeStatus:395(24129315ns) {LogHandshakeFail:472(13321ns) {UpdateErrorStateLog:545(5292ns)} DropOutgoingHandshake:457(1100ns) ProcessPendingSessionEvents:495(22116726ns) {unordered DropSessionEvent:512 num# 100 sum# 6509860ns max# 2744791ns}}}
2025-11-27T18:19:46.412222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471583597045410:2109] [node 6] ICP22 created new session: [2:7577471583597045431:2109]
2025-11-27T18:19:46.397878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471584414568948:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.449661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:46.472654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471581953607622:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.463284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471587128166973:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24016) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471587128167054:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.412588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471583597045409:2107] [node 4] ICP22 created new session: [2:7577471583597045430:2107]
2025-11-27T18:19:46.481660Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471580609101644:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(35532268ns) {HandleHandshakeStatus:395(32544325ns) {LogHandshakeFail:472(19919ns) {UpdateErrorStateLog:545(5938ns)} DropOutgoingHandshake:457(1393ns) ProcessPendingSessionEvents:495(31754954ns) {unordered DropSessionEvent:512 num# 100 sum# 10063499ns max# 2786353ns}}}
2025-11-27T18:19:46.449715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471586863120276:2109] [node 6] ICP22 created new session: [3:7577471586863120302:2109]
2025-11-27T18:19:46.414112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471584414568938:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:46.472718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:46.472376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471587128166971:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.414984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471583597045432:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35443
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.481741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471580609101644:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.481694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471586248574990:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:46.421491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471584414568959:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.449764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471586863120302:2109] [node 6] ICS09 handshake done sender: [3:7577471586863120283:2111] self: [3:7577471586863120279:0] peer: [6:7577471587128167057:0] socket: 210 qp: -1
2025-11-27T18:19:46.482071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:46.477763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471587128167037:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:46.482298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471581953607622:2048] [node 1] ICP22 created new session: [4:7577471586248575007:2048]
2025-11-27T18:19:46.417823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471583597045430:2107] [node 4] ICS09 handshake done sender: [2:7577471583597045418:2113] self: [2:7577471583597045414:0] peer: [4:7577471586248575001:0] socket: 225 qp: -1
2025-11-27T18:19:46.449806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471586863120302:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:46.431508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471584414568948:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:46.483094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471580609101644:2048] [node 1] ICP22 created new session: [5:7577471584904069036:2048]
2025-11-27T18:19:46.486854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471584904069037:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11759
2025-11-27T18:19:46.491746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471587128167030:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.420705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471583597045430:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:46.433301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471580119601575:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.482958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471586248575007:2048] [node 1] ICS09 handshake done sender: [4:7577471586248574991:2114] self: [4:7577471586248575000:0] peer: [1:7577471584414568942:0] socket: 221 qp: -1
2025-11-27T18:19:46.449845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:46.491790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.484146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471586248575007:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:46.421291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:46.450906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471586863120285:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:46.436642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:46.491841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471587128167030:2105] [node 5] ICP22 created new session: [6:7577471587128167065:2105]
2025-11-27T18:19:46.487272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471584904069026:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:46.422381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471583597045431:2109] [node 6] ICS09 handshake done sender: [2:7577471583597045416:2111] self: [2:7577471583597045415:0] peer: [6:7577471587128167055:0] socket: 223 qp: -1
2025-11-27T18:19:46.436693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471580119601575:2049] [node 3] ICP22 created new session: [1:7577471584414568967:2049]
2025-11-27T18:19:46.484202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:46.423139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471583597045431:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:19:46.487656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471580609101646:2050] [node 3] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.491883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471587128167065:2105] [node 5] ICS09 handshake done sender: [6:7577471587128167037:2111] self: [6:7577471587128167051:0] peer: [5:7577471584904069010:0] socket: 209 qp: -1
2025-11-27T18:19:46.491918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471587128167065:2105] [node 5] ICS10 traffic start
2025-11-27T18:19:46.491948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:46.496322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471587128167062:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55559
2025-11-27T18:19:46.446317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471584414568967:2049] [node 3] ICS09 handshake done sender: [1:7577471584414568938:2111] self: [1:7577471584414568937:0] peer: [3:7577471586863120293:0] socket: 211 qp: -1
2025-11-27T18:19:46.455012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471586863120284:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.491268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:46.492179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471586248575004:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32057
2025-11-27T18:19:46.423880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:46.451286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471580119601574:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.455045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471586863120284:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.492049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471584904069036:2048] [node 1] ICS09 handshake done sender: [5:7577471584904069018:2115] self: [5:7577471584904069020:0] peer: [1:7577471584414568939:0] socket: 213 qp: -1
2025-11-27T18:19:46.493987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471586248574982:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.510972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471587128167064:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55559
2025-11-27T18:19:46.426001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471583597045432:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.492094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471584904069036:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:46.451383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471584414568967:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:46.495059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:46.457341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471583597045437:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35443
2025-11-27T18:19:46.495109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471586248574982:2108] [node 5] ICP22 created new session: [4:7577471586248575009:2108]
2025-11-27T18:19:46.452899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:46.492117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471580609101646:2050] [node 3] ICP22 created new session: [5:7577471584904069038:2050]
2025-11-27T18:19:46.457689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471582568152917:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.512311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471587128166973:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(32131408ns) {HandleHandshakeStatus:395(32129325ns) {LogHandshakeFail:472(12685ns) {UpdateErrorStateLog:545(6087ns)} DropOutgoingHandshake:457(741ns) ProcessPendingSessionEvents:495(30610041ns) {unordered DropSessionEvent:512 num# 100 sum# 30557268ns max# 8119908ns}}}
2025-11-27T18:19:46.497494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471586248575009:2108] [node 5] ICS09 handshake done sender: [4:7577471586248574990:2113] self: [4:7577471586248574988:0] peer: [5:7577471584904069029:0] socket: 231 qp: -1
No RDMA context found for GID: 2025-11-27T18:19:46.514766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471587128167054:2119] [node 3] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:19:46.492692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:46.460680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471583597045423:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.460718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471583597045423:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.493171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471584904069038:2050] [node 3] ICS09 handshake done sender: [5:7577471584904069026:2119] self: [5:7577471584904069032:0] peer: [3:7577471586863120280:0] socket: 249 qp: -1
2025-11-27T18:19:46.498035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471586248575009:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:46.457734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: 2025-11-27T18:19:46.472043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471583597045425:2116] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:19:46.495025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471584904069038:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:46.498222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:46.453252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:46.515406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471587128166973:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.497041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:46.492139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471579302078049:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.515838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:46.457782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471582568152917:2048] [node 1] ICP22 created new session: [3:7577471586863120305:2048]
2025-11-27T18:19:46.498788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471580609101644:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16487540ns) {HandleHandshakeStatus:329(16485720ns) {DropIncomingHandshake:442(1470ns) DropOutgoingHandshake:457(1115ns) SwitchToState:179(1094ns) ProcessPendingSessionEvents:495(666ns)}}
2025-11-27T18:19:46.454863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471580119601574:2048] [node 2] ICP22 created new session: [1:7577471584414568968:2048]
2025-11-27T18:19:46.498641Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471581953607623:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(34379688ns) {HandleHandshakeStatus:395(34376543ns) {LogHandshakeFail:472(279207ns) {UpdateErrorStateLog:545(6525ns)} DropOutgoingHandshake:457(755ns) ProcessPendingSessionEvents:495(34037950ns) {unordered DropSessionEvent:512 num# 100 sum# 10748429ns max# 4728673ns}}}
2025-11-27T18:19:46.492200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:46.457826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471586863120305:2048] [node 1] ICS09 handshake done sender: [3:7577471586863120285:2113] self: [3:7577471586863120293:0] peer: [1:7577471584414568937:0] socket: 220 qp: -1
2025-11-27T18:19:46.515866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471587128167064:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.513195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471586248575004:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.516835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471587128166973:2050] [node 3] ICP22 created new session: [6:7577471587128167067:2050]
2025-11-27T18:19:46.457861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471586863120305:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:46.515836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471586248574998:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:46.516963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471587128167067:2050] [node 3] ICS09 handshake done sender: [6:7577471587128167054:2119] self: [6:7577471587128167057:0] peer: [3:7577471586863120279:0] socket: 246 qp: -1
2025-11-27T18:19:46.457904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:46.517094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471586248575011:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32057
2025-11-27T18:19:46.492766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471579302078049:2048] [node 1] ICP22 created new session: [2:7577471583597045441:2048]
2025-11-27T18:19:46.518151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471587128167067:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:46.459005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471586863120300:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53341
2025-11-27T18:19:46.518367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471581953607623:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.494968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471583597045441:2048] [node 1] ICS09 handshake done sender: [2:7577471583597045425:2116] self: [2:7577471583597045426:0] peer: [1:7577471584414568947:0] socket: 244 qp: -1
2025-11-27T18:19:46.518588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:46.481659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471586863120300:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.518996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471586248575011:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.482141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471586863120284:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:46.519133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471581953607623:2049] [node 2] ICP22 created new session: [4:7577471586248575013:2049]
2025-11-27T18:19:46.486238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471582568152918:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.519399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471581953607624:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.488804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471586863120278:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.488861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:46.518195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:46.488910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471586863120278:2108] [node 5] ICP22 created new session: [3:7577471586863120311:2108]
2025-11-27T18:19:46.496516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471583597045441:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:46.505449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471584904069034:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11759
2025-11-27T18:19:46.460286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471584414568968:2048] [node 2] ICS09 handshake done sender: [1:7577471584414568948:2115] self: [1:7577471584414568947:0] peer: [2:7577471583597045426:0] socket: 243 qp: -1
2025-11-27T18:19:46.519315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471587128167035:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:46.488955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471586863120311:2108] [node 5] ICS09 handshake done sender: [3:7577471586863120284:2112] self: [3:7577471586863120280:0] peer: [5:7577471584904069032:0] socket: 248 qp: -1
2025-11-27T18:19:46.497007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.488990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471586863120311:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:46.497778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471583597045423:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:46.460736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471584414568968:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:46.510891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471584904069035:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11759
2025-11-27T18:19:46.489028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:46.511329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471584904069006:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14717) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471584904069021:2117] held: no
2025-11-27T18:19:46.508111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471583597045413:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.460778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:46.519752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471586863120296:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.515033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471584904069021:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:46.519817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471586248574992:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.521031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471587128167068:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55559
2025-11-27T18:19:46.508148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471583597045413:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.519121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471584904069006:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.483238Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471584414568935:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(65357856ns) {HandleHandshakeStatus:329(65353532ns) {DropIncomingHandshake:442(15336ns) DropOutgoingHandshake:457(1032ns) SwitchToState:179(442ns) ProcessPendingSessionEvents:495(53367397ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 50813513ns max# 5557921ns}}}
2025-11-27T18:19:46.519852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471586248574992:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.519775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471586863120296:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.523962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471587128167059:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:46.521073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471586248575013:2049] [node 2] ICS09 handshake done sender: [4:7577471586248574998:2117] self: [4:7577471586248575001:0] peer: [2:7577471583597045414:0] socket: 230 qp: -1
2025-11-27T18:19:46.512284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471583597045409:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(89947269ns) {HandleHandshakeStatus:329(89943201ns) {DropIncomingHandshake:442(20041ns) DropOutgoingHandshake:457(280ns) SwitchToState:179(480ns) ProcessPendingSessionEvents:495(72238759ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 72084239ns max# 8661703ns}}}
2025-11-27T18:19:46.519163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:46.524364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471586863120296:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:46.498691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471584414568954:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.521638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471586248575013:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:46.519560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471584904069006:2109] [node 4] ICP22 created new session: [5:7577471584904069041:2109]
2025-11-27T18:19:46.498743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471584414568954:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:46.525096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471586863120277:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.521676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:46.506981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471584414568934:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(91770506ns) {HandleHandshakeStatus:329(91767702ns) {DropIncomingHandshake:442(1595ns) DropOutgoingHandshake:457(899ns) SwitchToState:179(314ns) ProcessPendingSessionEvents:495(76291590ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 68039119ns max# 9086982ns}}}
2025-11-27T18:19:46.513201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471583597045408:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.522201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471586248574992:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:46.525690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:46.513290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:46.519544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471584414568954:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:19:46.514858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471583597045408:2108] [node 5] ICP22 created new session: [2:7577471583597045444:2108]
::ffff:127.1.0.0
2025-11-27T18:19:46.525095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471586248574981:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.519830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471580119601574:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(56869380ns) {HandleHandshakeStatus:329(56866276ns) {DropIncomingHandshake:442(14001ns) DropOutgoingHandshake:457(485ns) SwitchToState:179(3804ns) ProcessPendingSessionEvents:495(35895771ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 35794220ns max# 8988704ns}}}
2025-11-27T18:19:46.526309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471586863120277:2107] [node 4] ICP22 created new session: [3:7577471586863120315:2107]
2025-11-27T18:19:46.519790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471584904069041:2109] [node 4] ICS09 handshake done sender: [5:7577471584904069021:2117] self: [5:7577471584904069029:0] peer: [4:7577471586248574988:0] socket: 235 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.515821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471583597045440:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35443
2025-11-27T18:19:46.525309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:46.520294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471584414568936:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.525746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471587128166971:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.526387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471586863120315:2107] [node 4] ICS09 handshake done sender: [3:7577471586863120296:2117] self: [3:7577471586863120295:0] peer: [4:7577471586248575014:0] socket: 277 qp: -1
2025-11-27T18:19:46.516183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471583597045444:2108] [node 5] ICS09 handshake done sender: [2:7577471583597045423:2114] self: [2:7577471583597045422:0] peer: [5:7577471584904069033:0] socket: 262 qp: -1
2025-11-27T18:19:46.519833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471584904069041:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:46.525797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:46.526738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471586863120315:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:46.525882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471586248574981:2109] [node 6] ICP22 created new session: [4:7577471586248575017:2109]
2025-11-27T18:19:46.521080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:46.519866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:46.516501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471583597045444:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:46.521657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471584414568936:2110] [node 6] ICP22 created new session: [1:7577471584414568972:2110]
2025-11-27T18:19:46.526259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471586248574982:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(26295036ns) {HandleHandshakeStatus:329(26292463ns) {DropIncomingHandshake:442(15084ns) DropOutgoingHandshake:457(172ns) SwitchToState:179(447ns) ProcessPendingSessionEvents:495(19724205ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15890963ns max# 1948743ns}}}
2025-11-27T18:19:46.526230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471587128166971:2048] [node 1] ICP22 created new session: [6:7577471587128167072:2048]
2025-11-27T18:19:46.527164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471586863120314:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53341
2025-11-27T18:19:46.519874Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471584904069031:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:46.516879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:46.527025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471586248575017:2109] [node 6] ICS09 handshake done sender: [4:7577471586248574992:2115] self: [4:7577471586248574989:0] peer: [6:7577471587128167070:0] socket: 227 qp: -1
2025-11-27T18:19:46.523097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471584414568972:2110] [node 6] ICS09 handshake done sender: [1:7577471584414568954:2118] self: [1:7577471584414568952:0] peer: [6:7577471587128167063:0] socket: 257 qp: -1
2025-11-27T18:19:46.527246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:46.530591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471586863120300:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:46.526274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471587128167072:2048] [node 1] ICS09 handshake done sender: [6:7577471587128167059:2121] self: [6:7577471587128167063:0] peer: [1:7577471584414568952:0] socket: 234 qp: -1
2025-11-27T18:19:46.521102Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471583597045410:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(107492325ns) {HandleHandshakeStatus:329(107489564ns) {DropIncomingHandshake:442(671ns) DropOutgoingHandshake:457(683ns) SwitchToState:179(720ns) ProcessPendingSessionEvents:495(81767627ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 77136735ns max# 7914594ns}}}
2025-11-27T18:19:46.524817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471580609101645:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.523421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471584414568972:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:46.527072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471586248575017:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:46.523960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:46.525064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471580119601575:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(83494924ns) {HandleHandshakeStatus:329(83492474ns) {DropIncomingHandshake:442(1319ns) DropOutgoingHandshake:457(744ns) SwitchToState:179(2441ns) ProcessPendingSessionEvents:495(53560589ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 50460150ns max# 9865352ns}}}
2025-11-27T18:19:46.525944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471580119601575:2049] [node 3] ICP18 handshake (actor [1:7577471584414568959:2119]) is too old
2025-11-27T18:19:46.524853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:46.525861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471580609101645:2049] [node 2] ICP22 created new session: [5:7577471584904069043:2049]
2025-11-27T18:19:46.526298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471584904069043:2049] [node 2] ICS09 handshake done sender: [5:7577471584904069031:2120] self: [5:7577471584904069033:0] peer: [2:7577471583597045422:0] socket: 266 qp: -1
2025-11-27T18:19:46.526333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471584904069043:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:46.526406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:46.526307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471587128167072:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:46.526345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:46.530613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471587128167071:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55559
2025-11-27T18:19:46.560180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471587128167035:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14717) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471587128167064:2123] held: no
2025-11-27T18:19:46.541747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471580609101646:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(32780732ns) {HandleHandshakeStatus:329(32778725ns) {DropIncomingHandshake:442(677ns) DropOutgoingHandshake:457(3963ns) SwitchToState:179(548ns) ProcessPendingSessionEvents:495(20969042ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20144719ns max# 4583520ns}}}
2025-11-27T18:19:46.561127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471587128167064:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:46.550649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471580609101645:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(25065079ns) {HandleHandshakeStatus:329(25062674ns) {DropIncomingHandshake:442(840ns) DropOutgoingHandshake:457(4062ns) SwitchToState:179(305ns) ProcessPendingSessionEvents:495(21724360ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21654649ns max# 5672467ns}}}
2025-11-27T18:19:46.523859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471583597045440:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:46.550873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471580609101645:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:46.524023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471583597045413:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:46.527710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471579302078050:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:46.527748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:46.528181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471579302078050:2049] [node 3] ICP22 created new session: [2:7577471583597045446:2049]
2025-11-27T18:19:46.545565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471583597045446:2049] [node 3] ICS09 handshake done sender: [2:7577471583597045413:2110] self: [2:7577471583597045411:0] peer: [3:7577471586863120310:0] socket: 263 qp: -1
2025-11-27T18:19:46.547689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471583597045446:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:46.547740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:46.570839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471583597045408:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(50434087ns) {HandleHandshakeStatus:329(50431032ns) {DropIncomingHandshake:442(16686ns) DropOutgoingHandshake:457(554ns) SwitchToState:179(637ns) ProcessPendingSessionEvents:495(43843594ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 43727543ns max# 6724951ns}}}
2025-11-27T18:19:46.571093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471579302078049:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(42068592ns) {HandleHandshakeStatus:329(42065985ns) {DropIncomingHandshake:442(2304ns) DropOutgoingHandshake:457(8152ns) SwitchToState:179(187ns) ProcessPendingSessionEvents:495(26808523ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26707827ns max# 6031364ns}}}
2025-11-27T18:19:46.547714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471582568152918:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.547792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:46.553402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471582568152918:2049] [node 2] ICP22 created new session: [3:7577471586863120317:2049]
2025-11-27T18:19:46.565584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471586863120317:2049] [node 2] ICS09 handshake done sender: [3:7577471586863120300:2120] self: [3:7577471586863120310:0] peer: [2:7577471583597045411:0] socket: 267 qp: -1
2025-11-27T18:19:46.568534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471586863120317:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:46.569402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:46.571533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471586863120277:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(45542443ns) {HandleHandshakeStatus:329(45540175ns) {DropIncomingHandshake:442(15112ns) DropOutgoingHandshake:457(200ns) SwitchToState:179(171ns) ProcessPendingSessionEvents:495(33556719ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 33472430ns max# 16309430ns}}}
2025-11-27T18:19:46.527105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:46.571635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471587128167035:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.529918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471581953607622:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(46973146ns) {HandleHandshakeStatus:329(46970728ns) {DropIncomingHandshake:442(450ns) DropOutgoingHandshake:457(5542ns) SwitchToState:179(520ns) ProcessPendingSessionEvents:495(22639508ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22540252ns max# 3156735ns}}}
2025-11-27T18:19:46.565766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471581953607624:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24016) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471586248575011:2121] held: no
2025-11-27T18:19:46.570374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471586248575018:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32057
2025-11-27T18:19:46.572046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471587128166971:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(44940921ns) {HandleHandshakeStatus:329(44938339ns) {DropIncomingHandshake:442(413ns) DropOutgoingHandshake:457(8347ns) SwitchToState:179(345ns) ProcessPendingSessionEvents:495(42955237ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 42814963ns max# 8132282ns}}}
2025-11-27T18:19:46.572288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:46.572418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471587128166971:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:46.572942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471587128167035:2108] [node 4] ICP22 created new session: [6:7577471587128167074:2108]
2025-11-27T18:19:46.574572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471587128167074:2108] [node 4] ICS09 handshake done sender: [6:7577471587128167064:2123] self: [6:7577471587128167070:0] peer: [4:7577471586248574989:0] socket: 276 qp: -1
2025-11-27T18:19:46.574633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471587128167074:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:46.574685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:46.575261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471584414568936:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(54164867ns) {HandleHandshakeStatus:329(54162747ns) {DropIncomingHandshake:442(11165ns) DropOutgoingHandshake:457(228ns) SwitchToState:179(272ns) ProcessPendingSessionEvents:495(49796035ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 48702344ns max# 9061449ns}}}
2025-11-27T18:19:46.575542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471582568152918:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(29257376ns) {HandleHandshakeStatus:329(29253497ns) {DropIncomingHandshake:442(2689ns) DropOutgoingHandshake:457(19262ns) SwitchToState:179(889ns) ProcessPendingSessionEvents:495(2442207ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 2364169ns max# 1733131ns}}}
2025-11-27T18:19:46.576594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471586248575011:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:46.576826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471579302078050:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(44015499ns) {HandleHandshakeStatus:329(44013242ns) {DropIncomingHandshake:442(869ns) DropOutgoingHandshake:457(283ns) SwitchToState:179(196ns) ProcessPendingSessionEvents:495(1973868ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 1896982ns max# 1319462ns}}}
2025-11-27T18:19:46.577252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471579302078050:2049] [node 3] ICP18 handshake (actor [2:7577471583597045440:2119]) is too old
2025-11-27T18:19:46.580772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471581953607624:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:46.580842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:46.580889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471581953607624:2050] [node 3] ICP22 created new session: [4:7577471586248575020:2050]
2025-11-27T18:19:46.580938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471586248575020:2050] [node 3] ICS09 handshake done sender: [4:7577471586248575011:2121] self: [4:7577471586248575014:0] peer: [3:7577471586863120295:0] socket: 228 qp: -1
2025-11-27T18:19:46.580981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471586248575020:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:46.581035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:46.624819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:46.625089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471583597045441:2048] [node 1] ICS07 socket disconnect 244 reason# EndOfStream
2025-11-27T18:19:46.625146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471583597045441:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.625296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:46.625371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471583597045441:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:46.625953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:46.626166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471586863120305:2048] [node 1] ICS01 socket: 220 reason# EndOfStream
2025-11-27T18:19:46.626207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471582568152917:2048] [node 1] ICP30 unregister session Session# [3:7577471586863120305:2048] VirtualId# [3:7577471586863120293:0]
2025-11-27T18:19:46.626240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471586863120305:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.626321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:46.626518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:46.627649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471579302078049:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62519) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:46.627710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471583597045441:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:46.628737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471579302078049:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62519) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:46.628795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471583597045441:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:46.629922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471579302078049:2048] [node 1] ICP30 unregister session Session# [2:7577471583597045441:2048] VirtualId# [2:7577471583597045426:0]
2025-11-27T18:19:46.630054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:46.630097Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471579302078049:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:46.630113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:46.630200Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:46.630432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471586248575007:2048] [node 1] ICS07 socket disconnect 221 reason# EndOfStream
2025-11-27T18:19:46.630487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471586248575007:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.630574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:46.630625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471586248575007:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:46.631348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471581953607622:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62519) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:46.631377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:46.631402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471586248575007:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:46.631483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471581953607622:2048] [node 1] ICP30 unregister session Session# [4:7577471586248575007:2048] VirtualId# [4:7577471586248575000:0]
2025-11-27T18:19:46.631628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:46.631669Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471581953607622:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:46.631682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:46.631690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471584904069036:2048] [node 1] ICS01 socket: 213 reason# EndOfStream
2025-11-27T18:19:46.631737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471580609101644:2048] [node 1] ICP30 unregister session Session# [5:7577471584904069036:2048] VirtualId# [5:7577471584904069020:0]
2025-11-27T18:19:46.631769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471584904069036:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.631838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:46.631885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:46.632636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:46.632853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471587128167072:2048] [node 1] ICS01 socket: 234 reason# EndOfStream
2025-11-27T18:19:46.632902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471587128166971:2048] [node 1] ICP30 unregister session Session# [6:7577471587128167072:2048] VirtualId# [6:7577471587128167063:0]
2025-11-27T18:19:46.632929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471587128167072:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.632999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:46.633055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:46.641223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471581953607622:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:46.695766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:46.696014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471586863120317:2049] [node 2] ICS01 socket: 267 reason# EndOfStream
2025-11-27T18:19:46.696079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471582568152918:2049] [node 2] ICP30 unregister session Session# [3:7577471586863120317:2049] VirtualId# [3:7577471586863120310:0]
2025-11-27T18:19:46.696118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471586863120317:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.696200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:46.696249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:46.698717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:46.698956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471586248575013:2049] [node 2] ICS01 socket: 230 reason# EndOfStream
2025-11-27T18:19:46.699005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471581953607623:2049] [node 2] ICP30 unregister session Session# [4:7577471586248575013:2049] VirtualId# [4:7577471586248575001:0]
2025-11-27T18:19:46.699037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471586248575013:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.699116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:46.699174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:46.699836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:46.700037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471584904069043:2049] [node 2] ICS01 socket: 266 reason# EndOfStream
2025-11-27T18:19:46.700083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471580609101645:2049] [node 2] ICP30 unregister session Session# [5:7577471584904069043:2049] VirtualId# [5:7577471584904069033:0]
2025-11-27T18:19:46.700114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471584904069043:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.700189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:46.700244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:46.700949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:46.701147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471587128167060:2049] [node 2] ICS01 socket: 229 reason# EndOfStream
2025-11-27T18:19:46.701195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471587128166972:2049] [node 2] ICP30 unregister session Session# [6:7577471587128167060:2049] VirtualId# [6:7577471587128167055:0]
2025-11-27T18:19:46.701222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471587128167060:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.701349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:46.701396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:46.794468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:46.794719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471586248575020:2050] [node 3] ICS01 socket: 228 reason# EndOfStream
2025-11-27T18:19:46.794771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471581953607624:2050] [node 3] ICP30 unregister session Session# [4:7577471586248575020:2050] VirtualId# [4:7577471586248575014:0]
2025-11-27T18:19:46.794813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471586248575020:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.794887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:46.794936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:46.797384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:46.797580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471587128167067:2050] [node 3] ICS01 socket: 246 reason# EndOfStream
2025-11-27T18:19:46.795595Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:46.795819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471584904069038:2050] [node 3] ICS07 socket disconnect 249 reason# EndOfStream
2025-11-27T18:19:46.795849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471584904069038:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.795919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:46.795954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471584904069038:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:46.801602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471587128166973:2050] [node 3] ICP30 unregister session Session# [6:7577471587128167067:2050] VirtualId# [6:7577471587128167057:0]
2025-11-27T18:19:46.801668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471587128167067:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.801982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:46.802091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:46.802325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471580609101646:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24016) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:46.802466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471584904069038:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:46.802544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471580609101646:2050] [node 3] ICP30 unregister session Session# [5:7577471584904069038:2050] VirtualId# [5:7577471584904069032:0]
2025-11-27T18:19:46.803394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:46.803457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471580609101646:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24016) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:46.803492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24016) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:46.813344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471580609101646:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:46.870704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:46.870874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471584904069041:2109] [node 4] ICS01 socket: 235 reason# EndOfStream
2025-11-27T18:19:46.870935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471584904069006:2109] [node 4] ICP30 unregister session Session# [5:7577471584904069041:2109] VirtualId# [5:7577471584904069029:0]
2025-11-27T18:19:46.870959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471584904069041:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.871019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:46.871061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:46.871921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:46.872131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471587128167074:2108] [node 4] ICS01 socket: 276 reason# EndOfStream
2025-11-27T18:19:46.872184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471587128167035:2108] [node 4] ICP30 unregister session Session# [6:7577471587128167074:2108] VirtualId# [6:7577471587128167070:0]
2025-11-27T18:19:46.872215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471587128167074:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.872289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:46.872336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:46.948831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:46.949108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471587128167065:2105] [node 5] ICS01 socket: 209 reason# EndOfStream
2025-11-27T18:19:46.949166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471587128167030:2105] [node 5] ICP30 unregister session Session# [6:7577471587128167065:2105] VirtualId# [6:7577471587128167051:0]
2025-11-27T18:19:46.949208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471587128167065:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:46.949303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:46.949350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:47.056513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471589965521582:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:47.056552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471589965521581:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:47.077962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471587830890361:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:47.078014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471587830890362:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:47.102535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471589225570790:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:47.102724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471589225570791:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:47.124211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471590696532784:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:47.124240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471590696532785:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:47.124258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471590696532786:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:47.147073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471588658822895:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:47.147138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471588658822896:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:47.147196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471588658822897:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:47.170291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471590526009139:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:47.170336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471590526009141:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:47.170714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471590526009140:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:47.258106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471590526009201:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:47.302258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471590696532847:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:47.300498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471589225570853:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:47.300613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471589965521648:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:47.310614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471587830890428:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:47.334616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471588658822959:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:47.325307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471589225570854:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:47.323188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471589965521651:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:47.350645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471587830890429:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:47.350573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471590526009206:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:47.357533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471588658822960:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.357474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471590696532849:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:47.340777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471589225570855:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:47.350913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471589965521653:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:47.350710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471587830890430:2111] [node 6] ICP01 ready to work
2025-11-27T18:19:47.357506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471589965521582:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32242696ns) {Configure:104(32238533ns) {ProcessConfigured:135(6974902ns) {StartInitialHandshake:157(6971862ns) {DropHandshakes:470(3339ns) {DropIncomingHandshake:442(703ns) DropOutgoingHandshake:457(477ns)} GenerateSessionVirtualId:988(952ns) PrepareNewSessionHandshake:477(4049ns) {SwitchToState:179(1772ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.375010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471588658822956:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52047
2025-11-27T18:19:47.375242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471589225570790:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(102298045ns) {Configure:104(102294545ns) {ProcessConfigured:135(1355295ns) {StartInitialHandshake:157(1352163ns) {DropHandshakes:470(3023ns) {DropIncomingHandshake:442(949ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(1667ns) PrepareNewSessionHandshake:477(17717ns) {SwitchToState:179(1719ns)}}}}}
2025-11-27T18:19:47.376029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471587830890427:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
2025-11-27T18:19:47.388045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471587830890427:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.402614Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471590696532786:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(117380849ns) {Configure:104(117375617ns) {ProcessConfigured:135(2411115ns) {StartInitialHandshake:157(2408428ns) {DropHandshakes:470(2211ns) {DropIncomingHandshake:442(560ns) DropOutgoingHandshake:457(310ns)} GenerateSessionVirtualId:988(1316ns) PrepareNewSessionHandshake:477(2909ns) {SwitchToState:179(1589ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.402932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471589225570853:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27648374ns) {Configure:104(27643634ns) {ProcessConfigured:135(2903436ns) {StartInitialHandshake:157(2901579ns) {DropHandshakes:470(1513ns) {DropIncomingHandshake:442(432ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(749ns) PrepareNewSessionHandshake:477(1877ns) {SwitchToState:179(513ns)}}}}}
2025-11-27T18:19:47.412829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471590526009139:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(36347638ns) {Configure:104(36336704ns) {ProcessConfigured:135(10215812ns) {StartInitialHandshake:157(10213423ns) {DropHandshakes:470(1455ns) {DropIncomingHandshake:442(441ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(466ns) PrepareNewSessionHandshake:477(3155ns) {SwitchToState:179(1621ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.405568Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471590696532849:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29811815ns) {Configure:104(29808319ns) {ProcessConfigured:135(36323ns) {StartInitialHandshake:157(34946ns) {DropHandshakes:470(1064ns) {DropIncomingHandshake:442(303ns) DropOutgoingHandshake:457(103ns)} GenerateSessionVirtualId:988(616ns) PrepareNewSessionHandshake:477(737ns) {SwitchToState:179(330ns)}}}}}
2025-11-27T18:19:47.409651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471589965521655:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46910
2025-11-27T18:19:47.405573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471590696532784:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30578115ns) {Configure:104(30568053ns) {ProcessConfigured:135(3052334ns) {StartInitialHandshake:157(3050235ns) {DropHandshakes:470(1476ns) {DropIncomingHandshake:442(587ns) DropOutgoingHandshake:457(244ns)} GenerateSessionVirtualId:988(447ns) PrepareNewSessionHandshake:477(2521ns) {SwitchToState:179(1185ns)}}}}}
2025-11-27T18:19:47.415999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471589225570858:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55888
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.415326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471590526009207:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48432
2025-11-27T18:19:47.416839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471589965521651:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(34049244ns) {Configure:104(34043283ns) {ProcessConfigured:135(7325321ns) {StartInitialHandshake:157(4676969ns) {DropHandshakes:470(2263ns) {DropIncomingHandshake:442(641ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(1603ns) PrepareNewSessionHandshake:477(2035ns) {SwitchToState:179(1101ns)}}}}}
2025-11-27T18:19:47.417449Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471587830890361:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30702693ns) {Configure:104(30699374ns) {ProcessConfigured:135(5803441ns) {StartInitialHandshake:157(5802063ns) {DropHandshakes:470(1256ns) {DropIncomingHandshake:442(280ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(439ns) PrepareNewSessionHandshake:477(2057ns) {SwitchToState:179(495ns)}}}}}
2025-11-27T18:19:47.416232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471588658822896:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27372812ns) {Configure:104(27366758ns) {ProcessConfigured:135(3161263ns) {StartInitialHandshake:157(3158477ns) {DropHandshakes:470(347256ns) {DropIncomingHandshake:442(389ns) DropOutgoingHandshake:457(593ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(2640ns) {SwitchToState:179(1424ns)}}}}}
2025-11-27T18:19:47.416447Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471590696532847:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(67538904ns) {Configure:104(67530002ns) {ProcessConfigured:135(3364023ns) {StartInitialHandshake:157(3362267ns) {DropHandshakes:470(1555ns) {DropIncomingHandshake:442(427ns) DropOutgoingHandshake:457(351ns)} GenerateSessionVirtualId:988(558ns) PrepareNewSessionHandshake:477(1943ns) {SwitchToState:179(769ns)}}}}}
2025-11-27T18:19:47.419752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471590526009140:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39182970ns) {Configure:104(39178359ns) {ProcessConfigured:135(8608168ns) {StartInitialHandshake:157(8606203ns) {DropHandshakes:470(1340ns) {DropIncomingHandshake:442(376ns) DropOutgoingHandshake:457(175ns)} GenerateSessionVirtualId:988(323ns) PrepareNewSessionHandshake:477(1422ns) {SwitchToState:179(226ns)}}}}}
2025-11-27T18:19:47.419792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471587830890429:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(40866969ns) {Configure:104(40862982ns) {ProcessConfigured:135(6589474ns) {StartInitialHandshake:157(6587114ns) {DropHandshakes:470(1425ns) {DropIncomingHandshake:442(393ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(872ns) PrepareNewSessionHandshake:477(1720ns) {SwitchToState:179(646ns)}}}}}
2025-11-27T18:19:47.417419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471589965521648:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(38880303ns) {Configure:104(38874230ns) {ProcessConfigured:135(9428524ns) {StartInitialHandshake:157(9426114ns) {DropHandshakes:470(1225ns) {DropIncomingHandshake:442(360ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(448ns) PrepareNewSessionHandshake:477(2653ns) {SwitchToState:179(1412ns)}}}}}
2025-11-27T18:19:47.417201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471588658822897:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24232348ns) {Configure:104(24227846ns) {ProcessConfigured:135(3165177ns) {StartInitialHandshake:157(3162920ns) {DropHandshakes:470(1746ns) {DropIncomingHandshake:442(544ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(496ns) PrepareNewSessionHandshake:477(2546ns) {SwitchToState:179(1325ns)}}}}}
2025-11-27T18:19:47.422194Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471589225570791:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40560750ns) {Configure:104(40552851ns) {ProcessConfigured:135(13037676ns) {StartInitialHandshake:157(13035349ns) {DropHandshakes:470(1225ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(641ns) PrepareNewSessionHandshake:477(1075118ns) {SwitchToState:179(3667ns)}}}}}
2025-11-27T18:19:47.416689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471590696532785:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(78029445ns) {Configure:104(78023455ns) {ProcessConfigured:135(2871593ns) {StartInitialHandshake:157(2869663ns) {DropHandshakes:470(2265ns) {DropIncomingHandshake:442(649ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(553ns) PrepareNewSessionHandshake:477(2730ns) {SwitchToState:179(1045ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.421911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471590526009141:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(39418347ns) {Configure:104(39412615ns) {ProcessConfigured:135(12652015ns) {StartInitialHandshake:157(12650118ns) {DropHandshakes:470(1223ns) {DropIncomingHandshake:442(336ns) DropOutgoingHandshake:457(180ns)} GenerateSessionVirtualId:988(538ns) PrepareNewSessionHandshake:477(2604ns) {SwitchToState:179(1415ns)}}}}}
2025-11-27T18:19:47.419830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471588658822956:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.421965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471587830890362:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(41458395ns) {Configure:104(41452968ns) {ProcessConfigured:135(6788676ns) {StartInitialHandshake:157(6786328ns) {DropHandshakes:470(1342ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:477(992487ns) {SwitchToState:179(2397ns)}}}}}
2025-11-27T18:19:47.422145Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471589965521653:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(41103664ns) {Configure:104(41097340ns) {ProcessConfigured:135(6607814ns) {StartInitialHandshake:157(6604746ns) {DropHandshakes:470(1449ns) {DropIncomingHandshake:442(641ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(664ns) PrepareNewSessionHandshake:477(937314ns) {SwitchToState:179(2379ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.421954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471590696532860:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18220
2025-11-27T18:19:47.423342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471589225570855:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(41032189ns) {Configure:104(41027112ns) {ProcessConfigured:135(1952694ns) {StartInitialHandshake:157(1950431ns) {DropHandshakes:470(1464ns) {DropIncomingHandshake:442(360ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(779ns) PrepareNewSessionHandshake:477(2791ns) {SwitchToState:179(1190ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.434441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471590526009207:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.433847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471590696532860:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.422274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471588658822895:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40926725ns) {Configure:104(40921702ns) {ProcessConfigured:135(2826639ns) {StartInitialHandshake:157(2824389ns) {DropHandshakes:470(1242ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:477(774256ns) {SwitchToState:179(2478ns)}}}}}
2025-11-27T18:19:47.423423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471587830890428:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(40247572ns) {Configure:104(40240628ns) {ProcessConfigured:135(9406398ns) {StartInitialHandshake:157(9404343ns) {DropHandshakes:470(905ns) {DropIncomingHandshake:442(235ns) DropOutgoingHandshake:457(156ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:477(3125ns) {SwitchToState:179(1535ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.423559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471589965521655:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.423660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471589225570858:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.442795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471590526009141:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.445579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471589965521662:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46910
2025-11-27T18:19:47.447045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471589965521666:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46910
2025-11-27T18:19:47.447606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471589965521643:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.447638Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471589965521643:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.455721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471589965521652:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.455771Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471589965521652:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.428730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471587830890361:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.438108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471589225570790:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.422906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471588658822960:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(39799590ns) {Configure:104(39794826ns) {ProcessConfigured:135(1759009ns) {StartInitialHandshake:157(1757204ns) {DropHandshakes:470(1743ns) {DropIncomingHandshake:442(449ns) DropOutgoingHandshake:457(372ns)} GenerateSessionVirtualId:988(823ns) PrepareNewSessionHandshake:477(2120ns) {SwitchToState:179(1035ns)}}}}}
2025-11-27T18:19:47.449697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471590696532786:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.477943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471590526009206:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(39625221ns) {Configure:104(39608783ns) {ProcessConfigured:135(62666ns) {StartInitialHandshake:157(59781ns) {DropHandshakes:470(1841ns) {DropIncomingHandshake:442(416ns) DropOutgoingHandshake:457(310ns)} GenerateSessionVirtualId:988(1056ns) PrepareNewSessionHandshake:477(1059ns) {SwitchToState:179(511ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.489755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471590696532865:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18220
2025-11-27T18:19:47.453650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471588658822973:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52047
2025-11-27T18:19:47.467525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471589225570860:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.475325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471587830890430:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(50140146ns) {Configure:104(50132570ns) {ProcessConfigured:135(57162ns) {StartInitialHandshake:157(54976ns) {DropHandshakes:470(2879ns) {DropIncomingHandshake:442(492ns) DropOutgoingHandshake:457(634ns)} GenerateSessionVirtualId:988(908ns) PrepareNewSessionHandshake:477(1949ns) {SwitchToState:179(506ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.467575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471589225570860:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.502321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471587830890444:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
2025-11-27T18:19:47.506476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471589965521643:2108] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:47.461736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471588658822959:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(45525894ns) {Configure:104(43641462ns) {ProcessConfigured:135(1152061ns) {StartInitialHandshake:157(1149505ns) {DropHandshakes:470(2614ns) {DropIncomingHandshake:442(714ns) DropOutgoingHandshake:457(381ns)} GenerateSessionVirtualId:988(1228ns) PrepareNewSessionHandshake:477(1946ns) {SwitchToState:179(735ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.509217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471589965521666:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.507552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471590696532855:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.510910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471589965521664:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46910
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.507599Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471590696532855:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.511254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471590696532866:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18220
2025-11-27T18:19:47.511673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471590696532865:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.470723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471589225570865:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.477935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471588658822973:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.511890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471590526009219:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48432
2025-11-27T18:19:47.470766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471589225570865:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.511906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471587830890446:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.494386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471588658822972:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52047
2025-11-27T18:19:47.512455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471589965521664:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.512297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471587830890444:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.512504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471589965521652:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:47.513015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471590696532867:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18220
2025-11-27T18:19:47.506557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471589225570860:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:47.497576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471588658822960:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.509123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471589225570865:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:47.512995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471589965521662:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.511037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471589225570872:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55888
2025-11-27T18:19:47.509470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471588658822972:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.514207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471590696532866:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.513041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471589965521581:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.515023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471590526009207:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:47.511670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471589225570871:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55888
2025-11-27T18:19:47.515481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471587830890438:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.511030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471588658822895:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.521043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471590526009141:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20698) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471590526009207:2111] held: no
2025-11-27T18:19:47.513839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:47.514880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471590696532785:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.515525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471587830890438:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.511850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471588658822978:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52047
2025-11-27T18:19:47.517519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471590696532860:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:47.511769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471589225570875:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55888
2025-11-27T18:19:47.514296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471589965521660:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.514556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471589225570871:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.513088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471588658822978:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.515531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471587830890427:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:47.517614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471590696532855:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:47.542862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471590526009141:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(21108868ns) {HandleHandshakeStatus:395(21095630ns) {LogHandshakeFail:472(16028ns) {UpdateErrorStateLog:545(4376ns)} DropOutgoingHandshake:457(429ns) ProcessPendingSessionEvents:495(21004728ns) {unordered DropSessionEvent:512 num# 100 sum# 4556607ns max# 1665917ns}}}
2025-11-27T18:19:47.542933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471590526009141:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.514321Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471589965521660:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.542982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:47.518743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471587830890436:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.513826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471588658822896:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.514566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471589225570872:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.522316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471590696532786:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.543037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471590526009141:2050] [node 3] ICP22 created new session: [6:7577471590526009222:2050]
2025-11-27T18:19:47.514529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471589965521581:2048] [node 2] ICP22 created new session: [1:7577471589965521671:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.543083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471590526009222:2050] [node 3] ICS09 handshake done sender: [6:7577471590526009207:2111] self: [6:7577471590526009217:0] peer: [3:7577471589225570859:0] socket: 221 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.533467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471588658822981:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52047
2025-11-27T18:19:47.529481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:47.518768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471587830890436:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.515528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471589225570853:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.529604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471590696532847:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.514643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471589965521668:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46910
2025-11-27T18:19:47.522407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471587830890361:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.516100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.554124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471588658822983:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52047
2025-11-27T18:19:47.543115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471590526009222:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:47.515661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471589965521582:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.529676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:47.517394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471589225570853:2107] [node 4] ICP22 created new session: [3:7577471589225570878:2107]
2025-11-27T18:19:47.586411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471588658822960:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11335) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471588658822973:2117] held: no
2025-11-27T18:19:47.543168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.522458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:47.531201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471590696532786:2050] [node 3] ICP22 created new session: [4:7577471590696532871:2050]
2025-11-27T18:19:47.518611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471589225570854:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.516076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471589965521671:2048] [node 2] ICS09 handshake done sender: [1:7577471589965521643:2108] self: [1:7577471589965521641:0] peer: [2:7577471587830890439:0] socket: 213 qp: -1
2025-11-27T18:19:47.524058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471587830890361:2048] [node 1] ICP22 created new session: [2:7577471587830890448:2048]
2025-11-27T18:19:47.587231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471588658822896:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20290) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471588658822978:2118] held: no
2025-11-27T18:19:47.517436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471589965521671:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:47.531584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471587830890438:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.553650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471590526009221:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48432
2025-11-27T18:19:47.558213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471590526009224:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48432
2025-11-27T18:19:47.520244Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471589225570858:2111] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:47.563663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471590526009141:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(18602235ns) {HandleHandshakeStatus:329(18600622ns) {DropIncomingHandshake:442(682ns) DropOutgoingHandshake:457(251ns) SwitchToState:179(403ns) ProcessPendingSessionEvents:495(1221ns)}}
2025-11-27T18:19:47.593121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471588658822973:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.533516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471590696532871:2050] [node 3] ICS09 handshake done sender: [4:7577471590696532860:2116] self: [4:7577471590696532864:0] peer: [3:7577471589225570863:0] socket: 224 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.518623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.531262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471587830890448:2048] [node 1] ICS09 handshake done sender: [2:7577471587830890427:2113] self: [2:7577471587830890439:0] peer: [1:7577471589965521641:0] socket: 215 qp: -1
2025-11-27T18:19:47.569823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471590526009225:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48432
2025-11-27T18:19:47.533540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471590696532847:2108] [node 5] ICP22 created new session: [4:7577471590696532872:2108]
2025-11-27T18:19:47.520923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471589225570790:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.594463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471588658822972:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:47.520965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:47.521006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471589225570790:2048] [node 1] ICP22 created new session: [3:7577471589225570879:2048]
2025-11-27T18:19:47.521049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471589225570879:2048] [node 1] ICS09 handshake done sender: [3:7577471589225570858:2111] self: [3:7577471589225570870:0] peer: [1:7577471589965521650:0] socket: 217 qp: -1
2025-11-27T18:19:47.521084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471589225570879:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:47.521118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:47.522362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471589225570878:2107] [node 4] ICS09 handshake done sender: [3:7577471589225570865:2113] self: [3:7577471589225570863:0] peer: [4:7577471590696532864:0] socket: 223 qp: -1
2025-11-27T18:19:47.522405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471589225570878:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:47.530862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:47.532242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471589225570854:2109] [node 6] ICP22 created new session: [3:7577471589225570882:2109]
2025-11-27T18:19:47.532250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:47.536164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471589225570882:2109] [node 6] ICS09 handshake done sender: [3:7577471589225570860:2112] self: [3:7577471589225570859:0] peer: [6:7577471590526009217:0] socket: 220 qp: -1
2025-11-27T18:19:47.537325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471589225570882:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:47.545988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:47.599097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471589225570854:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(80864509ns) {HandleHandshakeStatus:329(80863018ns) {DropIncomingHandshake:442(28867ns) DropOutgoingHandshake:457(549ns) SwitchToState:179(322ns) ProcessPendingSessionEvents:495(9243533ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9176344ns max# 8751085ns}}}
2025-11-27T18:19:47.609848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471589225570887:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55888
2025-11-27T18:19:47.624702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471589225570887:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.625054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471589225570869:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.625076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471589225570869:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.635749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471589225570869:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:47.649746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471589225570855:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.650349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:47.652350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471589225570855:2108] [node 5] ICP22 created new session: [3:7577471589225570890:2108]
2025-11-27T18:19:47.652414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471589225570890:2108] [node 5] ICS09 handshake done sender: [3:7577471589225570869:2115] self: [3:7577471589225570868:0] peer: [5:7577471588658822990:0] socket: 269 qp: -1
2025-11-27T18:19:47.652465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471589225570890:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:47.652505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:47.595051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471588658822982:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52047
2025-11-27T18:19:47.602480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471588658822895:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471588658822972:2116] held: no
2025-11-27T18:19:47.603081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471588658822960:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.603134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:47.603181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471588658822960:2109] [node 4] ICP22 created new session: [5:7577471588658822985:2109]
2025-11-27T18:19:47.603227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471588658822985:2109] [node 4] ICS09 handshake done sender: [5:7577471588658822973:2117] self: [5:7577471588658822977:0] peer: [4:7577471590696532853:0] socket: 235 qp: -1
2025-11-27T18:19:47.603267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471588658822985:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:47.603299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:47.603744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471588658822984:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52047
2025-11-27T18:19:47.606528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471588658822895:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.606568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:47.614486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471588658822895:2048] [node 1] ICP22 created new session: [5:7577471588658822989:2048]
2025-11-27T18:19:47.624344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471588658822984:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.624697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471588658822897:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.630785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471588658822978:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:47.631768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471588658822989:2048] [node 1] ICS09 handshake done sender: [5:7577471588658822972:2116] self: [5:7577471588658822979:0] peer: [1:7577471589965521657:0] socket: 234 qp: -1
2025-11-27T18:19:47.631799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471588658822989:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:47.518717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:47.532554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471587830890448:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:47.631843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:47.569986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471590526009224:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.524247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471589965521668:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.522454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471589965521582:2049] [node 3] ICP22 created new session: [1:7577471589965521672:2049]
2025-11-27T18:19:47.532377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471589965521672:2049] [node 3] ICS09 handshake done sender: [1:7577471589965521652:2113] self: [1:7577471589965521650:0] peer: [3:7577471589225570870:0] socket: 216 qp: -1
2025-11-27T18:19:47.532717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471589965521672:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:47.533737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:47.537610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471589965521660:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:47.560602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471589965521582:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(44051369ns) {HandleHandshakeStatus:329(44049136ns) {DropIncomingHandshake:442(10630ns) DropOutgoingHandshake:457(296ns) SwitchToState:179(107ns) ProcessPendingSessionEvents:495(9005646ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8942067ns max# 8522076ns}}}
2025-11-27T18:19:47.573861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471589965521648:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.573941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:47.574000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471589965521648:2109] [node 5] ICP22 created new session: [1:7577471589965521680:2109]
2025-11-27T18:19:47.574051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471589965521680:2109] [node 5] ICS09 handshake done sender: [1:7577471589965521660:2116] self: [1:7577471589965521657:0] peer: [5:7577471588658822979:0] socket: 230 qp: -1
2025-11-27T18:19:47.574094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471589965521680:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:47.574136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:47.613906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471589965521661:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.613949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471589965521661:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.629954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471589965521661:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:47.635989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471589965521653:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.644688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:47.649641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471589965521653:2110] [node 6] ICP22 created new session: [1:7577471589965521683:2110]
2025-11-27T18:19:47.653392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471589965521683:2110] [node 6] ICS09 handshake done sender: [1:7577471589965521661:2117] self: [1:7577471589965521659:0] peer: [6:7577471590526009228:0] socket: 246 qp: -1
2025-11-27T18:19:47.659173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471589965521683:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:47.663576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.534626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471590696532871:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:47.534474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:47.534798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471587830890436:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:47.644470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471588658822896:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.573500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471590526009206:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.581478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471590526009225:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.543538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471587830890429:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.670748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471589965521658:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.534674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:47.543620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:47.594842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471590526009221:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.543677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471587830890429:2109] [node 5] ICP22 created new session: [2:7577471587830890453:2109]
2025-11-27T18:19:47.673524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471589225570886:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55888
2025-11-27T18:19:47.644512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:47.670803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471589965521658:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.536459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471590696532872:2108] [node 5] ICS09 handshake done sender: [4:7577471590696532855:2113] self: [4:7577471590696532853:0] peer: [5:7577471588658822977:0] socket: 231 qp: -1
2025-11-27T18:19:47.543736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471587830890453:2109] [node 5] ICS09 handshake done sender: [2:7577471587830890438:2117] self: [2:7577471587830890434:0] peer: [5:7577471588658822980:0] socket: 250 qp: -1
2025-11-27T18:19:47.677587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471589225570886:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.597558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471590526009140:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.648374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471588658822896:2049] [node 2] ICP22 created new session: [5:7577471588658822992:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.678535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471589225570853:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(69732793ns) {HandleHandshakeStatus:329(69729599ns) {DropIncomingHandshake:442(11659ns) DropOutgoingHandshake:457(346ns) SwitchToState:179(150ns) ProcessPendingSessionEvents:495(39236590ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39134384ns max# 15297747ns}}}
2025-11-27T18:19:47.537501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471590696532872:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:47.651810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471588658822992:2049] [node 2] ICS09 handshake done sender: [5:7577471588658822978:2118] self: [5:7577471588658822980:0] peer: [2:7577471587830890434:0] socket: 253 qp: -1
2025-11-27T18:19:47.610132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471590526009139:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.543772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471587830890453:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:47.689249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471589965521658:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:47.546188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:47.683624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471589225570791:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.615036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471590526009230:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48432
2025-11-27T18:19:47.652548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471588658822971:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.543804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:47.652564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471588658822971:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.585575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471590696532859:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.691062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471589965521651:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.615276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471590526009229:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48432
No RDMA context found for GID: 2025-11-27T18:19:47.553096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471587830890428:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.653384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471588658822992:2049] [node 2] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:19:47.585606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471590696532859:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.630893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471590526009224:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:47.691660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:47.634382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471590526009230:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.641253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471590526009206:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.641321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:47.641379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471590526009206:2110] [node 4] ICP22 created new session: [6:7577471590526009233:2110]
2025-11-27T18:19:47.641419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471590526009233:2110] [node 4] ICS09 handshake done sender: [6:7577471590526009224:2118] self: [6:7577471590526009226:0] peer: [4:7577471590696532858:0] socket: 218 qp: -1
2025-11-27T18:19:47.553144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:47.693277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471589965521651:2107] [node 4] ICP22 created new session: [1:7577471589965521686:2107]
2025-11-27T18:19:47.606462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471590696532859:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:47.641454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471590526009233:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:47.641490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:47.659371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:47.643106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471590526009232:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48432
2025-11-27T18:19:47.553189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471587830890428:2107] [node 4] ICP22 created new session: [2:7577471587830890456:2107]
2025-11-27T18:19:47.701313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471589965521686:2107] [node 4] ICS09 handshake done sender: [1:7577471589965521658:2115] self: [1:7577471589965521656:0] peer: [4:7577471590696532882:0] socket: 264 qp: -1
2025-11-27T18:19:47.660239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471588658822971:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:47.644870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471590526009225:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:47.634773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471590696532849:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.707396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471589965521686:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:47.557521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471587830890456:2107] [node 4] ICS09 handshake done sender: [2:7577471587830890436:2115] self: [2:7577471587830890433:0] peer: [4:7577471590696532870:0] socket: 254 qp: -1
2025-11-27T18:19:47.663206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471588658822993:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52047
2025-11-27T18:19:47.634827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.563625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471587830890456:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:47.650021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471590526009140:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.663662Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471588658822895:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(43829203ns) {HandleHandshakeStatus:329(43826931ns) {DropIncomingHandshake:442(797ns) DropOutgoingHandshake:457(143ns) SwitchToState:179(372ns) ProcessPendingSessionEvents:495(1334ns)}}
2025-11-27T18:19:47.708375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:47.634881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471590696532849:2109] [node 6] ICP22 created new session: [4:7577471590696532880:2109]
2025-11-27T18:19:47.650063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:47.563685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:47.722680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471589225570791:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20290) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471589225570886:2119] held: no
2025-11-27T18:19:47.671509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471588658822959:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.565332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471587830890455:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
2025-11-27T18:19:47.725942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471589225570790:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(172280345ns) {HandleHandshakeStatus:329(172272723ns) {DropIncomingHandshake:442(517ns) DropOutgoingHandshake:457(4515ns) SwitchToState:179(611ns) ProcessPendingSessionEvents:495(161435432ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 161232156ns max# 15552205ns}}}
2025-11-27T18:19:47.634923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471590696532880:2109] [node 6] ICS09 handshake done sender: [4:7577471590696532859:2115] self: [4:7577471590696532858:0] peer: [6:7577471590526009226:0] socket: 244 qp: -1
2025-11-27T18:19:47.729237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471589965521581:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(188027691ns) {HandleHandshakeStatus:329(188024741ns) {DropIncomingHandshake:442(2483ns) DropOutgoingHandshake:457(2549ns) SwitchToState:179(351ns) ProcessPendingSessionEvents:495(167650653ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 167453004ns max# 15321476ns}}}
2025-11-27T18:19:47.672795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471588658822984:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:47.673566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:47.674404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471588658822896:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(26377184ns) {HandleHandshakeStatus:329(26222997ns) {DropIncomingHandshake:442(405ns) DropOutgoingHandshake:457(128ns) SwitchToState:179(207ns) ProcessPendingSessionEvents:495(828ns)}}
2025-11-27T18:19:47.650112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471590526009140:2049] [node 2] ICP22 created new session: [6:7577471590526009236:2049]
2025-11-27T18:19:47.728893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471589225570893:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55888
2025-11-27T18:19:47.732240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471589965521581:2048] [node 2] ICP18 handshake (actor [1:7577471589965521668:2121]) is too old
2025-11-27T18:19:47.565926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471587830890458:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
2025-11-27T18:19:47.634959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471590696532880:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:47.652234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471590526009201:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.569878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471587830890458:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.674489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471588658822897:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20698) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471588658822984:2122] held: no
2025-11-27T18:19:47.737382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471589225570886:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:47.634996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:47.653523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471590526009236:2049] [node 2] ICS09 handshake done sender: [6:7577471590526009225:2119] self: [6:7577471590526009227:0] peer: [2:7577471587830890441:0] socket: 274 qp: -1
2025-11-27T18:19:47.676684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471588658822959:2110] [node 6] ICP22 created new session: [5:7577471588658822996:2110]
2025-11-27T18:19:47.658682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471590526009236:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:47.658732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:47.678507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471590526009139:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471590526009221:2117] held: no
2025-11-27T18:19:47.690887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471590526009231:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48432
2025-11-27T18:19:47.694221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471590526009201:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471590526009230:2121] held: no
2025-11-27T18:19:47.709176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471590526009238:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48432
2025-11-27T18:19:47.715746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471590526009230:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:47.727715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471590526009139:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24397230ns) {HandleHandshakeStatus:395(23473617ns) {LogHandshakeFail:472(17789ns) {UpdateErrorStateLog:545(3798ns)} DropOutgoingHandshake:457(1270ns) ProcessPendingSessionEvents:495(16543718ns) {unordered DropSessionEvent:512 num# 100 sum# 16495567ns max# 3587439ns}}}
2025-11-27T18:19:47.727776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471590526009201:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.727963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:47.728575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471590526009201:2106] [node 5] ICP22 created new session: [6:7577471590526009239:2106]
2025-11-27T18:19:47.728957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471590526009239:2106] [node 5] ICS09 handshake done sender: [6:7577471590526009230:2121] self: [6:7577471590526009235:0] peer: [5:7577471588658822969:0] socket: 279 qp: -1
2025-11-27T18:19:47.729448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471590526009239:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:47.729491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:47.731597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471590526009221:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:47.731782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471590526009139:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.737533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:47.745428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471590526009139:2048] [node 1] ICP22 created new session: [6:7577471590526009241:2048]
2025-11-27T18:19:47.746574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471590526009241:2048] [node 1] ICS09 handshake done sender: [6:7577471590526009221:2117] self: [6:7577471590526009228:0] peer: [1:7577471589965521659:0] socket: 266 qp: -1
2025-11-27T18:19:47.747057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471590526009241:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:47.750612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:47.651199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471590696532877:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18220
2025-11-27T18:19:47.589472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471587830890451:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16975
2025-11-27T18:19:47.653294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471590696532878:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18220
2025-11-27T18:19:47.662504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471590696532878:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.662630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471590696532784:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:47.664786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471590696532866:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:47.670210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471590696532786:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(150017035ns) {HandleHandshakeStatus:329(150014770ns) {DropIncomingHandshake:442(2179ns) DropOutgoingHandshake:457(15264ns) SwitchToState:179(407ns) ProcessPendingSessionEvents:495(98789571ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 98680851ns max# 13569740ns}}}
2025-11-27T18:19:47.671598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471590696532785:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.673689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:47.676886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471590696532785:2049] [node 2] ICP22 created new session: [4:7577471590696532883:2049]
2025-11-27T18:19:47.682258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471590696532883:2049] [node 2] ICS09 handshake done sender: [4:7577471590696532866:2118] self: [4:7577471590696532870:0] peer: [2:7577471587830890433:0] socket: 255 qp: -1
2025-11-27T18:19:47.682301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471590696532883:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:47.687704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:47.711156Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471590696532784:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16055) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471590696532878:2121] held: no
2025-11-27T18:19:47.712046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471590696532849:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(43671702ns) {HandleHandshakeStatus:329(43659602ns) {DropIncomingHandshake:442(352ns) DropOutgoingHandshake:457(504ns) SwitchToState:179(516ns) ProcessPendingSessionEvents:495(43307640ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 43210353ns max# 9780684ns}}}
2025-11-27T18:19:47.723894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471590696532885:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18220
2025-11-27T18:19:47.740919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471590696532847:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(182969870ns) {HandleHandshakeStatus:329(182966300ns) {DropIncomingHandshake:442(26828ns) DropOutgoingHandshake:457(425ns) SwitchToState:179(423ns) ProcessPendingSessionEvents:495(128013702ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 127874822ns max# 13389784ns}}}
2025-11-27T18:19:47.756252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471589965521653:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(115857737ns) {HandleHandshakeStatus:329(115854533ns) {DropIncomingHandshake:442(32608ns) DropOutgoingHandshake:457(700ns) SwitchToState:179(620ns) ProcessPendingSessionEvents:495(73918343ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 73781762ns max# 11747337ns}}}
2025-11-27T18:19:47.604983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471587830890455:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.609364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471587830890451:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.615766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471587830890442:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.615798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471587830890442:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.643513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471587830890442:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:47.653312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471587830890430:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.663317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:47.665124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471587830890430:2111] [node 6] ICP22 created new session: [2:7577471587830890464:2111]
2025-11-27T18:19:47.671176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471587830890464:2111] [node 6] ICS09 handshake done sender: [2:7577471587830890442:2118] self: [2:7577471587830890441:0] peer: [6:7577471590526009227:0] socket: 272 qp: -1
2025-11-27T18:19:47.671223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471587830890464:2111] [node 6] ICS10 traffic start
2025-11-27T18:19:47.671292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:47.689004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471587830890437:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:47.689040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471587830890437:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:47.706445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471587830890428:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(152537662ns) {HandleHandshakeStatus:329(152532102ns) {DropIncomingHandshake:442(443ns) DropOutgoingHandshake:457(516ns) SwitchToState:179(702ns) ProcessPendingSessionEvents:495(99263176ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 97154411ns max# 13692085ns}}}
2025-11-27T18:19:47.706668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471587830890428:2107] [node 4] ICP18 handshake (actor [2:7577471587830890458:2123]) is too old
2025-11-27T18:19:47.707529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471587830890437:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:47.711230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471587830890362:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:47.712650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:47.715457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471587830890362:2049] [node 3] ICP22 created new session: [2:7577471587830890467:2049]
2025-11-27T18:19:47.716107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471587830890467:2049] [node 3] ICS09 handshake done sender: [2:7577471587830890437:2116] self: [2:7577471587830890435:0] peer: [3:7577471589225570892:0] socket: 273 qp: -1
2025-11-27T18:19:47.719396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471587830890467:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:47.719462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:47.732061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471587830890361:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(189626444ns) {HandleHandshakeStatus:329(189623519ns) {DropIncomingHandshake:442(1072ns) DropOutgoingHandshake:457(6215ns) SwitchToState:179(544ns) ProcessPendingSessionEvents:495(162982833ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 158343258ns max# 19213133ns}}}
2025-11-27T18:19:47.678413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471588658822996:2110] [node 6] ICS09 handshake done sender: [5:7577471588658822971:2115] self: [5:7577471588658822969:0] peer: [6:7577471590526009235:0] socket: 277 qp: -1
2025-11-27T18:19:47.682042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471588658822996:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:47.683253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:47.690227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471588658822897:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.690300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:47.690377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471588658822897:2050] [node 3] ICP22 created new session: [5:7577471588658822998:2050]
2025-11-27T18:19:47.690791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471588658822998:2050] [node 3] ICS09 handshake done sender: [5:7577471588658822984:2122] self: [5:7577471588658822990:0] peer: [3:7577471589225570868:0] socket: 275 qp: -1
2025-11-27T18:19:47.690847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471588658822998:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:47.690891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:47.758133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471590696532785:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(64330711ns) {HandleHandshakeStatus:329(64328607ns) {DropIncomingHandshake:442(4044ns) DropOutgoingHandshake:457(12030ns) SwitchToState:179(559ns) ProcessPendingSessionEvents:495(45163304ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 40455307ns max# 5788865ns}}}
2025-11-27T18:19:47.760298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471590696532784:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18866260ns) {HandleHandshakeStatus:395(18863717ns) {LogHandshakeFail:472(1249164ns) {UpdateErrorStateLog:545(5174ns)} DropOutgoingHandshake:457(467ns) ProcessPendingSessionEvents:495(17552691ns) {unordered DropSessionEvent:512 num# 100 sum# 6617590ns max# 2479338ns}}}
2025-11-27T18:19:47.760413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471590696532878:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:47.757982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471590526009139:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24749061ns) {HandleHandshakeStatus:329(24746749ns) {DropIncomingHandshake:442(937ns) DropOutgoingHandshake:457(448ns) SwitchToState:179(547ns) ProcessPendingSessionEvents:495(1381ns)}}
2025-11-27T18:19:47.759825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471587830890362:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(47710812ns) {HandleHandshakeStatus:329(47707949ns) {DropIncomingHandshake:442(14698ns) DropOutgoingHandshake:457(2901ns) SwitchToState:179(571ns) ProcessPendingSessionEvents:495(33823884ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32332637ns max# 5863905ns}}}
2025-11-27T18:19:47.760883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471587830890430:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(70473149ns) {HandleHandshakeStatus:329(70452516ns) {DropIncomingHandshake:442(32014ns) DropOutgoingHandshake:457(520ns) SwitchToState:179(434ns) ProcessPendingSessionEvents:495(46745362ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 45420886ns max# 6062984ns}}}
2025-11-27T18:19:47.761312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471589965521651:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(69507116ns) {HandleHandshakeStatus:329(69504000ns) {DropIncomingHandshake:442(21986ns) DropOutgoingHandshake:457(829ns) SwitchToState:179(358ns) ProcessPendingSessionEvents:495(47601876ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 47483718ns max# 5821001ns}}}
2025-11-27T18:19:47.749675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471589225570791:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.750553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:47.753028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471589225570791:2049] [node 2] ICP22 created new session: [3:7577471589225570894:2049]
2025-11-27T18:19:47.753967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471589225570855:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(78890893ns) {HandleHandshakeStatus:329(78888659ns) {DropIncomingHandshake:442(15083ns) DropOutgoingHandshake:457(356ns) SwitchToState:179(548ns) ProcessPendingSessionEvents:495(75961144ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 75845131ns max# 12048996ns}}}
2025-11-27T18:19:47.754927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471589225570894:2049] [node 2] ICS09 handshake done sender: [3:7577471589225570886:2119] self: [3:7577471589225570892:0] peer: [2:7577471587830890435:0] socket: 276 qp: -1
2025-11-27T18:19:47.756452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471589225570894:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:47.756502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:47.761880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471588658822959:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(74431688ns) {HandleHandshakeStatus:329(74420769ns) {DropIncomingHandshake:442(16062ns) DropOutgoingHandshake:457(437ns) SwitchToState:179(433ns) ProcessPendingSessionEvents:495(56147297ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 56034705ns max# 7684111ns}}}
2025-11-27T18:19:47.763007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471590526009140:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(78020650ns) {HandleHandshakeStatus:329(78018019ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(3362ns) SwitchToState:179(203ns) ProcessPendingSessionEvents:495(56505967ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 56403214ns max# 7707352ns}}}
2025-11-27T18:19:47.763483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471590696532784:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:47.763613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:47.764729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471590696532784:2048] [node 1] ICP22 created new session: [4:7577471590696532886:2048]
2025-11-27T18:19:47.765954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471590696532886:2048] [node 1] ICS09 handshake done sender: [4:7577471590696532878:2121] self: [4:7577471590696532882:0] peer: [1:7577471589965521656:0] socket: 270 qp: -1
2025-11-27T18:19:47.766009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471590696532886:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:47.766073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:47.834523Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:47.833437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:47.833733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471587830890448:2048] [node 1] ICS07 socket disconnect 215 reason# EndOfStream
2025-11-27T18:19:47.833773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471587830890448:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.833869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:47.833915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471587830890448:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:47.834734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471589225570879:2048] [node 1] ICS07 socket disconnect 217 reason# EndOfStream
2025-11-27T18:19:47.834767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471589225570879:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.834837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:47.834884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471589225570879:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:47.838494Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471589225570790:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16055) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:47.838529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471589225570879:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:47.843198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471587830890361:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16055) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:47.842158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471589225570790:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16055) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:47.843286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:47.843526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471590696532886:2048] [node 1] ICS01 socket: 270 reason# EndOfStream
2025-11-27T18:19:47.843566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471590696532784:2048] [node 1] ICP30 unregister session Session# [4:7577471590696532886:2048] VirtualId# [4:7577471590696532882:0]
2025-11-27T18:19:47.843615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471590696532886:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.843685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:47.843832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:47.843922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471587830890448:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:47.845788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:47.843394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471589225570879:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:47.845989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471588658822989:2048] [node 1] ICS01 socket: 234 reason# EndOfStream
2025-11-27T18:19:47.846029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471588658822895:2048] [node 1] ICP30 unregister session Session# [5:7577471588658822989:2048] VirtualId# [5:7577471588658822979:0]
2025-11-27T18:19:47.846071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471588658822989:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.843975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471587830890361:2048] [node 1] ICP30 unregister session Session# [2:7577471587830890448:2048] VirtualId# [2:7577471587830890439:0]
2025-11-27T18:19:47.846138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:47.844062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:47.846180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:47.844098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471587830890361:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16055) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:47.844116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16055) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:47.843450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471589225570790:2048] [node 1] ICP30 unregister session Session# [3:7577471589225570879:2048] VirtualId# [3:7577471589225570870:0]
2025-11-27T18:19:47.843565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:47.843695Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471589225570790:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16055) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:47.843711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16055) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:47.847399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:47.847615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471590526009241:2048] [node 1] ICS01 socket: 266 reason# EndOfStream
2025-11-27T18:19:47.847660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471590526009139:2048] [node 1] ICP30 unregister session Session# [6:7577471590526009241:2048] VirtualId# [6:7577471590526009228:0]
2025-11-27T18:19:47.847690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471590526009241:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.847752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:47.847802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:47.854839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471589225570790:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:47.905488Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:47.905717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471589225570894:2049] [node 2] ICS01 socket: 276 reason# EndOfStream
2025-11-27T18:19:47.905756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471589225570791:2049] [node 2] ICP30 unregister session Session# [3:7577471589225570894:2049] VirtualId# [3:7577471589225570892:0]
2025-11-27T18:19:47.905800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471589225570894:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.905876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:47.905920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:47.910666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:47.910891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471590696532883:2049] [node 2] ICS07 socket disconnect 255 reason# EndOfStream
2025-11-27T18:19:47.910918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471590696532883:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.910995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:47.911037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471590696532883:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:47.911770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471590696532785:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20290) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:47.911788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:47.911828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471590696532883:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:47.911905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471590696532785:2049] [node 2] ICP30 unregister session Session# [4:7577471590696532883:2049] VirtualId# [4:7577471590696532870:0]
2025-11-27T18:19:47.912055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471588658822992:2049] [node 2] ICS01 socket: 253 reason# EndOfStream
2025-11-27T18:19:47.912005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:47.912100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471588658822896:2049] [node 2] ICP30 unregister session Session# [5:7577471588658822992:2049] VirtualId# [5:7577471588658822980:0]
2025-11-27T18:19:47.912313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471590696532785:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20290) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:47.912130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471588658822992:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.912193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:47.912392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20290) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:47.912316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:47.913151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:47.913379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471590526009236:2049] [node 2] ICS07 socket disconnect 274 reason# EndOfStream
2025-11-27T18:19:47.913417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471590526009236:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.913486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:47.913526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471590526009236:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:47.914228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471590526009140:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20290) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:47.914338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471590526009236:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:47.914374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471590526009140:2049] [node 2] ICP30 unregister session Session# [6:7577471590526009236:2049] VirtualId# [6:7577471590526009227:0]
2025-11-27T18:19:47.914482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:47.914512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471590526009140:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20290) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:47.914537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20290) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:47.922258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471590696532785:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:47.924279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471590526009140:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:47.960343Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:47.960566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471590696532871:2050] [node 3] ICS01 socket: 224 reason# EndOfStream
2025-11-27T18:19:47.960612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471590696532786:2050] [node 3] ICP30 unregister session Session# [4:7577471590696532871:2050] VirtualId# [4:7577471590696532864:0]
2025-11-27T18:19:47.960651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471590696532871:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.960748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:47.960821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:47.961306Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:47.961506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471588658822998:2050] [node 3] ICS01 socket: 275 reason# EndOfStream
2025-11-27T18:19:47.961551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471588658822897:2050] [node 3] ICP30 unregister session Session# [5:7577471588658822998:2050] VirtualId# [5:7577471588658822990:0]
2025-11-27T18:19:47.961586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471588658822998:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.961668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:47.961718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:47.962191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:47.962348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471590526009222:2050] [node 3] ICS01 socket: 221 reason# EndOfStream
2025-11-27T18:19:47.962391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471590526009141:2050] [node 3] ICP30 unregister session Session# [6:7577471590526009222:2050] VirtualId# [6:7577471590526009217:0]
2025-11-27T18:19:47.962424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471590526009222:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:47.962487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:47.962549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:48.034930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:48.035193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471588658822985:2109] [node 4] ICS01 socket: 235 reason# EndOfStream
2025-11-27T18:19:48.035243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471588658822960:2109] [node 4] ICP30 unregister session Session# [5:7577471588658822985:2109] VirtualId# [5:7577471588658822977:0]
2025-11-27T18:19:48.035275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471588658822985:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.035365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:48.035423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:48.036128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:48.036293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471590526009233:2110] [node 4] ICS01 socket: 218 reason# EndOfStream
2025-11-27T18:19:48.036334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471590526009206:2110] [node 4] ICP30 unregister session Session# [6:7577471590526009233:2110] VirtualId# [6:7577471590526009226:0]
2025-11-27T18:19:48.036364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471590526009233:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.036454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:48.036507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:48.120733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:48.120965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471590526009239:2106] [node 5] ICS01 socket: 279 reason# EndOfStream
2025-11-27T18:19:48.121014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471590526009201:2106] [node 5] ICP30 unregister session Session# [6:7577471590526009239:2106] VirtualId# [6:7577471590526009235:0]
2025-11-27T18:19:48.121049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471590526009239:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.121123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:48.121182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:48.249431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471596073740155:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:48.249485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471596073740156:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:48.262543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471594917104456:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:48.262577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471594917104457:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:48.283782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471594097450952:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:48.283830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471594097450953:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:48.306421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471593598017897:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:48.306475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471593598017898:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:48.306521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471593598017899:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:48.327936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471592738254653:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:48.327979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471592738254654:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:48.328011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471592738254655:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:48.349044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471595087349307:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:48.349111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471595087349308:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:48.349155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471595087349309:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:48.444257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471595087349365:2104] [node 5] ICP01 ready to work
2025-11-27T18:19:48.480570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471594917104521:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:48.480904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471594097451017:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:48.481042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471596073740218:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:48.481430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471593598017960:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:48.481809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471594097451018:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:48.482969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471594917104522:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:48.482779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471592738254717:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:48.482851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471593598017961:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:48.483450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471596073740219:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:48.487877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471595087349371:2105] [node 4] ICP01 ready to work
2025-11-27T18:19:48.488505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471594097451019:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:48.488647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471594917104523:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:48.488751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471596073740220:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:48.489112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471592738254718:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.519709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471594917104534:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50498
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.525960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471594097451030:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50763
2025-11-27T18:19:48.524539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471596073740231:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10583
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.527216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471594097451034:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50763
2025-11-27T18:19:48.527443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471592738254729:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36880
2025-11-27T18:19:48.527460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471596073740235:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10583
2025-11-27T18:19:48.527632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471596073740231:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.527937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471596073740236:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10583
2025-11-27T18:19:48.528261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471592738254729:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.528897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471593598017972:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.529434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471593598017973:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.530225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471593598017977:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.535993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471594097451035:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50763
2025-11-27T18:19:48.536653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471594097451036:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50763
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.542523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471594917104534:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.542621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471594917104538:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50498
2025-11-27T18:19:48.542749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471594917104539:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50498
2025-11-27T18:19:48.542858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471594917104540:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50498
2025-11-27T18:19:48.542910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471594917104541:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50498
2025-11-27T18:19:48.544340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471593598017972:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.546569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471594097451030:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.547245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471594917104539:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.547482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471594097450953:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.547772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471594917104527:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.547807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471594917104527:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.549495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471594917104538:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.549638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471594097451034:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.549763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471594097450952:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.549655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471594917104540:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.550173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471594917104456:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.551356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471594917104527:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:48.552058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471593598017977:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.552594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471593598017898:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.551870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471594097451035:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.553183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471593598017899:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.549418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471596073740236:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.549530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471596073740238:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10583
2025-11-27T18:19:48.549676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471596073740235:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.549740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471596073740239:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10583
2025-11-27T18:19:48.550125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471596073740224:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.550148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471596073740224:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:48.553549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471594917104541:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.553659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471594917104529:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.550979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471596073740224:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:48.553675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471594917104529:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:48.555012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471594097451025:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.553373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471596073740238:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.555057Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471594097451025:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:48.553512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471596073740222:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.553533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471596073740222:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:48.554293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471596073740156:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.554350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:48.555850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471594917104457:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.554400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471596073740156:2049] [node 3] ICP22 created new session: [1:7577471596073740243:2049]
2025-11-27T18:19:48.555912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:48.554447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471596073740243:2049] [node 3] ICS09 handshake done sender: [1:7577471596073740224:2111] self: [1:7577471596073740223:0] peer: [3:7577471594097451038:0] socket: 229 qp: -1
2025-11-27T18:19:48.554478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471596073740243:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:48.554517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:48.555968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471594917104457:2049] [node 3] ICP22 created new session: [2:7577471594917104547:2049]
2025-11-27T18:19:48.556022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471594917104547:2049] [node 3] ICS09 handshake done sender: [2:7577471594917104527:2113] self: [2:7577471594917104526:0] peer: [3:7577471594097451037:0] socket: 223 qp: -1
2025-11-27T18:19:48.556058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471594917104547:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:48.556116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.557057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471594097451039:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50763
2025-11-27T18:19:48.557382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471593598017973:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.557381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471596073740239:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.557514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471593598017897:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.558919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471594097451036:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.560912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471596073740226:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.560443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471594097451040:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50763
2025-11-27T18:19:48.560949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471596073740226:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.563399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471593598017981:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-27T18:19:48.563422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471594097450952:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471594097451034:2118] held: no
2025-11-27T18:19:48.563500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471592738254734:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36880
2025-11-27T18:19:48.563100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471596073740222:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:48.564085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471594097451030:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.564878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471593598017982:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-27T18:19:48.564940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471596073740226:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:48.564969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471593598017983:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-27T18:19:48.565112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471594917104529:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.566411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471592738254735:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36880
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.566654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471596073740155:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.566643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471594917104550:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50498
2025-11-27T18:19:48.566716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:48.566819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471596073740155:2048] [node 2] ICP22 created new session: [1:7577471596073740250:2048]
2025-11-27T18:19:48.566936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471596073740250:2048] [node 2] ICS09 handshake done sender: [1:7577471596073740222:2110] self: [1:7577471596073740221:0] peer: [2:7577471594917104545:0] socket: 228 qp: -1
2025-11-27T18:19:48.566965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471596073740250:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:48.567010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:48.567358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471594917104521:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.567476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471594097451034:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:48.570229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471596073740218:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.570293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:48.567425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:48.570666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471596073740218:2107] [node 4] ICP22 created new session: [1:7577471596073740252:2107]
2025-11-27T18:19:48.567561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471594917104521:2107] [node 4] ICP22 created new session: [2:7577471594917104553:2107]
2025-11-27T18:19:48.571248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471596073740252:2107] [node 4] ICS09 handshake done sender: [1:7577471596073740226:2112] self: [1:7577471596073740225:0] peer: [4:7577471593598017980:0] socket: 231 qp: -1
2025-11-27T18:19:48.571411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471594097450953:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.571597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471596073740252:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:48.567606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471594917104553:2107] [node 4] ICS09 handshake done sender: [2:7577471594917104529:2114] self: [2:7577471594917104528:0] peer: [4:7577471593598017978:0] socket: 224 qp: -1
2025-11-27T18:19:48.571480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:48.571536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471594097450953:2049] [node 2] ICP22 created new session: [3:7577471594097451044:2049]
2025-11-27T18:19:48.571638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.578357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471595087349385:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56820
2025-11-27T18:19:48.573611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471594097451044:2049] [node 2] ICS09 handshake done sender: [3:7577471594097451030:2117] self: [3:7577471594097451037:0] peer: [2:7577471594917104526:0] socket: 227 qp: -1
2025-11-27T18:19:48.577996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471592738254734:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.567637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471594917104553:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:48.573952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471594097451044:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:48.567670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:48.578157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471592738254653:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.578517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.570826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471594917104538:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:48.571677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471594917104456:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.571715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:48.584222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471596073740228:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.571765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471594917104456:2048] [node 1] ICP22 created new session: [2:7577471594917104556:2048]
2025-11-27T18:19:48.584272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471596073740228:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:48.573996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471594917104556:2048] [node 1] ICS09 handshake done sender: [2:7577471594917104538:2118] self: [2:7577471594917104545:0] peer: [1:7577471596073740221:0] socket: 236 qp: -1
2025-11-27T18:19:48.584920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471595087349386:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56820
2025-11-27T18:19:48.574025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471594917104556:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:48.574064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:48.585229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471593598017973:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:48.585894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471593598017972:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:48.585992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471592738254735:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.586165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471592738254654:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.586119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471594097451025:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:48.586677Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471593598017898:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471593598017972:2115] held: no
2025-11-27T18:19:48.587357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471594097450952:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.587375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471594917104531:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.587410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:48.587408Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471594917104531:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:48.587490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471594097450952:2048] [node 1] ICP22 created new session: [3:7577471594097451046:2048]
2025-11-27T18:19:48.587536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471594097451046:2048] [node 1] ICS09 handshake done sender: [3:7577471594097451034:2118] self: [3:7577471594097451038:0] peer: [1:7577471596073740223:0] socket: 234 qp: -1
2025-11-27T18:19:48.587578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471594097451046:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:48.587616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:48.587820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471593598017898:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.587865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:48.587924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471593598017898:2049] [node 2] ICP22 created new session: [4:7577471593598017984:2049]
2025-11-27T18:19:48.587970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471593598017984:2049] [node 2] ICS09 handshake done sender: [4:7577471593598017972:2115] self: [4:7577471593598017978:0] peer: [2:7577471594917104528:0] socket: 230 qp: -1
2025-11-27T18:19:48.588006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471593598017984:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:48.588041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.588127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471592738254738:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36880
2025-11-27T18:19:48.588272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471596073740228:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:48.588381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471594917104531:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:48.588369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471595087349385:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.588950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471594097450953:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:48.588525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471595087349386:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.589046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471594917104522:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.592080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471594097451017:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.588817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471595087349307:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.589241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:48.588873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471595087349308:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.589360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471594917104522:2109] [node 5] ICP22 created new session: [2:7577471594917104559:2109]
2025-11-27T18:19:48.592139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:48.589593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471594917104559:2109] [node 5] ICS09 handshake done sender: [2:7577471594917104531:2115] self: [2:7577471594917104530:0] peer: [5:7577471592738254737:0] socket: 269 qp: -1
2025-11-27T18:19:48.592191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471594097451017:2107] [node 4] ICP22 created new session: [3:7577471594097451049:2107]
2025-11-27T18:19:48.592338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471594917104559:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:48.592303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471594097451049:2107] [node 4] ICS09 handshake done sender: [3:7577471594097451025:2114] self: [3:7577471594097451024:0] peer: [4:7577471593598017979:0] socket: 239 qp: -1
2025-11-27T18:19:48.592468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:48.592482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471594097451049:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:48.592541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:48.593318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471592738254734:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:48.593085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471596073740219:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.593156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.593290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471596073740219:2108] [node 5] ICP22 created new session: [1:7577471596073740256:2108]
2025-11-27T18:19:48.593333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471596073740256:2108] [node 5] ICS09 handshake done sender: [1:7577471596073740228:2113] self: [1:7577471596073740227:0] peer: [5:7577471592738254736:0] socket: 237 qp: -1
2025-11-27T18:19:48.593911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471592738254739:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36880
2025-11-27T18:19:48.593374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471596073740256:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:48.593411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:48.595321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471592738254653:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.595510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:48.595694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471593598017897:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.595769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:48.595818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471593598017897:2048] [node 1] ICP22 created new session: [4:7577471593598017986:2048]
2025-11-27T18:19:48.595858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471593598017986:2048] [node 1] ICS09 handshake done sender: [4:7577471593598017973:2116] self: [4:7577471593598017980:0] peer: [1:7577471596073740225:0] socket: 233 qp: -1
2025-11-27T18:19:48.595911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471593598017986:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:48.595722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471592738254653:2048] [node 1] ICP22 created new session: [5:7577471592738254740:2048]
2025-11-27T18:19:48.595944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:48.595976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471596073740230:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.595804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471592738254740:2048] [node 1] ICS09 handshake done sender: [5:7577471592738254734:2117] self: [5:7577471592738254736:0] peer: [1:7577471596073740227:0] socket: 262 qp: -1
2025-11-27T18:19:48.595990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471594917104533:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.596022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471594917104533:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:48.595848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471592738254740:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:48.596009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471596073740230:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:48.595885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:48.598992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471593598017897:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.599948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471595087349389:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56820
2025-11-27T18:19:48.600022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471595087349390:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56820
2025-11-27T18:19:48.602213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471596073740230:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:48.602364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471594917104533:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:48.604041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471594917104523:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.604111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:48.604287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471594917104523:2110] [node 6] ICP22 created new session: [2:7577471594917104562:2110]
2025-11-27T18:19:48.604426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471594917104562:2110] [node 6] ICS09 handshake done sender: [2:7577471594917104533:2116] self: [2:7577471594917104532:0] peer: [6:7577471595087349388:0] socket: 254 qp: -1
2025-11-27T18:19:48.604482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471594917104562:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:48.604524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:48.610114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471592738254735:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:48.611572Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471593598017899:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19398) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471593598017977:2117] held: no
2025-11-27T18:19:48.611740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471592738254654:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471592738254735:2118] held: no
2025-11-27T18:19:48.612166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471596073740220:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.612245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:48.612309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471596073740220:2109] [node 6] ICP22 created new session: [1:7577471596073740259:2109]
2025-11-27T18:19:48.612353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471596073740259:2109] [node 6] ICS09 handshake done sender: [1:7577471596073740230:2114] self: [1:7577471596073740229:0] peer: [6:7577471595087349387:0] socket: 225 qp: -1
2025-11-27T18:19:48.612354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471592738254654:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.612392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471596073740259:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:48.612401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:48.612462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471592738254654:2049] [node 2] ICP22 created new session: [5:7577471592738254743:2049]
2025-11-27T18:19:48.612505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471592738254743:2049] [node 2] ICS09 handshake done sender: [5:7577471592738254735:2118] self: [5:7577471592738254737:0] peer: [2:7577471594917104530:0] socket: 270 qp: -1
2025-11-27T18:19:48.612549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471592738254743:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:48.612467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:48.612584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:48.614677Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471593598017977:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:48.615189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471595087349385:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:48.615653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471593598017899:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.615695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:48.615743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471593598017899:2050] [node 3] ICP22 created new session: [4:7577471593598017990:2050]
2025-11-27T18:19:48.615845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471593598017990:2050] [node 3] ICS09 handshake done sender: [4:7577471593598017977:2117] self: [4:7577471593598017979:0] peer: [3:7577471594097451024:0] socket: 242 qp: -1
2025-11-27T18:19:48.615889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471593598017990:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:48.615922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:48.615902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471595087349308:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471595087349386:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.616643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471595087349390:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.616829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471595087349391:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56820
2025-11-27T18:19:48.617451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471595087349307:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.617497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:48.617531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471595087349307:2048] [node 1] ICP22 created new session: [6:7577471595087349392:2048]
2025-11-27T18:19:48.617574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471595087349392:2048] [node 1] ICS09 handshake done sender: [6:7577471595087349385:2115] self: [6:7577471595087349387:0] peer: [1:7577471596073740229:0] socket: 238 qp: -1
2025-11-27T18:19:48.617607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471595087349392:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:48.617635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.618981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471592738254742:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36880
2025-11-27T18:19:48.619486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471592738254745:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36880
2025-11-27T18:19:48.625851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471595087349307:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:48.626901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471595087349386:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:48.623401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471592738254742:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.628989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471595087349309:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.629282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471595087349308:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.629342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:48.629460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471595087349308:2049] [node 2] ICP22 created new session: [6:7577471595087349398:2049]
2025-11-27T18:19:48.626312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471592738254655:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.629577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471595087349398:2049] [node 2] ICS09 handshake done sender: [6:7577471595087349386:2116] self: [6:7577471595087349388:0] peer: [2:7577471594917104532:0] socket: 257 qp: -1
2025-11-27T18:19:48.629616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471595087349398:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:48.629974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:48.631245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471592738254745:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.630142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471594097451027:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.630190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471594097451027:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.631594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471592738254717:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.632117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471593598017992:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.631986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471594097451052:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50763
2025-11-27T18:19:48.632194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471594097451027:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:48.632312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471592738254750:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36880
2025-11-27T18:19:48.632347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471593598017969:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.632928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471595087349394:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56820
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.633063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471595087349395:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56820
2025-11-27T18:19:48.632378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471593598017969:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:48.637852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471594097451018:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.637926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:48.638168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471594097451018:2108] [node 5] ICP22 created new session: [3:7577471594097451054:2108]
2025-11-27T18:19:48.638267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471594097451054:2108] [node 5] ICS09 handshake done sender: [3:7577471594097451027:2115] self: [3:7577471594097451026:0] peer: [5:7577471592738254748:0] socket: 246 qp: -1
2025-11-27T18:19:48.638339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471594097451054:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:48.638394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:48.639637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471592738254742:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:48.639670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471593598017992:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.639979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471593598017971:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.639131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471595087349394:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.639222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471595087349395:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.639345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471595087349371:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.639382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471595087349365:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:48.640002Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471593598017971:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:48.641349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471592738254728:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.641381Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471592738254728:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.641982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471593598017993:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-27T18:19:48.642536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471593598017969:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.643296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471595087349403:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56820
2025-11-27T18:19:48.643362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471594097451029:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.643397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471594097451029:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:48.643182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471592738254655:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.643231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:48.643280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471592738254655:2050] [node 3] ICP22 created new session: [5:7577471592738254751:2050]
2025-11-27T18:19:48.645396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471593598017993:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:48.643323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471592738254751:2050] [node 3] ICS09 handshake done sender: [5:7577471592738254742:2121] self: [5:7577471592738254748:0] peer: [3:7577471594097451026:0] socket: 240 qp: -1
2025-11-27T18:19:48.645481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471593598017960:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.643359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471592738254751:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:48.645539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:48.643391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:48.645607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471593598017960:2108] [node 5] ICP22 created new session: [4:7577471593598017997:2108]
2025-11-27T18:19:48.645653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471593598017997:2108] [node 5] ICS09 handshake done sender: [4:7577471593598017969:2113] self: [4:7577471593598017968:0] peer: [5:7577471592738254749:0] socket: 218 qp: -1
2025-11-27T18:19:48.645693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471593598017997:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:48.645730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:48.645684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471594097451052:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.647797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471595087349404:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56820
2025-11-27T18:19:48.647855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471594097451029:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:48.647950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471593598017971:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:48.648406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471594097451019:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.648507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:48.648552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471594097451019:2110] [node 6] ICP22 created new session: [3:7577471594097451058:2110]
2025-11-27T18:19:48.648621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471594097451058:2110] [node 6] ICS09 handshake done sender: [3:7577471594097451029:2116] self: [3:7577471594097451028:0] peer: [6:7577471595087349397:0] socket: 251 qp: -1
2025-11-27T18:19:48.648666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471594097451058:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:48.648751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:48.648983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471595087349390:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:48.649046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471593598017961:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.649090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:48.649768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471595087349394:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:48.649148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471593598017961:2109] [node 6] ICP22 created new session: [4:7577471593598017999:2109]
2025-11-27T18:19:48.649247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471593598017999:2109] [node 6] ICS09 handshake done sender: [4:7577471593598017971:2114] self: [4:7577471593598017970:0] peer: [6:7577471595087349401:0] socket: 220 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.649707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471593598017999:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:48.649760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:48.650721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471592738254753:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36880
2025-11-27T18:19:48.650908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471595087349309:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.650961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:48.651005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471595087349309:2050] [node 3] ICP22 created new session: [6:7577471595087349406:2050]
2025-11-27T18:19:48.651151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471595087349406:2050] [node 3] ICS09 handshake done sender: [6:7577471595087349390:2118] self: [6:7577471595087349397:0] peer: [3:7577471594097451028:0] socket: 273 qp: -1
2025-11-27T18:19:48.651185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471595087349406:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:48.651213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:48.651446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471592738254728:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:48.651795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471592738254717:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24966) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471592738254745:2122] held: no
2025-11-27T18:19:48.651993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471593598017961:2109] [node 6] ICP18 handshake (actor [4:7577471593598017993:2122]) is too old
2025-11-27T18:19:48.652632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471592738254718:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:48.652692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:48.652735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471592738254718:2110] [node 6] ICP22 created new session: [5:7577471592738254755:2110]
2025-11-27T18:19:48.652782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471592738254755:2110] [node 6] ICS09 handshake done sender: [5:7577471592738254728:2115] self: [5:7577471592738254727:0] peer: [6:7577471595087349402:0] socket: 248 qp: -1
2025-11-27T18:19:48.652823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471592738254755:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:48.652856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:48.653147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471595087349371:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.653187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:48.653222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471595087349371:2105] [node 4] ICP22 created new session: [6:7577471595087349408:2105]
2025-11-27T18:19:48.653295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471595087349408:2105] [node 4] ICS09 handshake done sender: [6:7577471595087349394:2120] self: [6:7577471595087349401:0] peer: [4:7577471593598017970:0] socket: 276 qp: -1
2025-11-27T18:19:48.653327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471595087349408:2105] [node 4] ICS10 traffic start
2025-11-27T18:19:48.653354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:48.656686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471592738254745:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:48.657381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471592738254717:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.657421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:48.657461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471592738254717:2108] [node 4] ICP22 created new session: [5:7577471592738254757:2108]
2025-11-27T18:19:48.657513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471592738254757:2108] [node 4] ICS09 handshake done sender: [5:7577471592738254745:2122] self: [5:7577471592738254749:0] peer: [4:7577471593598017968:0] socket: 221 qp: -1
2025-11-27T18:19:48.657546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471592738254757:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:48.657641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:48.659095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471595087349405:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56820
2025-11-27T18:19:48.659730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471595087349371:2105] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:48.661533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471595087349365:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4240) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471595087349395:2121] held: no
2025-11-27T18:19:48.662120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471595087349395:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:48.662789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471595087349365:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:48.662840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:48.662884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471595087349365:2104] [node 5] ICP22 created new session: [6:7577471595087349410:2104]
2025-11-27T18:19:48.662926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471595087349410:2104] [node 5] ICS09 handshake done sender: [6:7577471595087349395:2121] self: [6:7577471595087349402:0] peer: [5:7577471592738254727:0] socket: 277 qp: -1
2025-11-27T18:19:48.662965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471595087349410:2104] [node 5] ICS10 traffic start
2025-11-27T18:19:48.663014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:48.686366Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:48.686596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471594917104556:2048] [node 1] ICS07 socket disconnect 236 reason# EndOfStream
2025-11-27T18:19:48.686628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471594917104556:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.686715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:48.686762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471594917104556:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:48.688716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:48.689186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471594097451046:2048] [node 1] ICS01 socket: 234 reason# EndOfStream
2025-11-27T18:19:48.689329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471594097450952:2048] [node 1] ICP30 unregister session Session# [3:7577471594097451046:2048] VirtualId# [3:7577471594097451038:0]
2025-11-27T18:19:48.689361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471594097451046:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.689437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:48.689483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:48.694044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471594917104456:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:48.694456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471594917104556:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:48.695777Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471594917104456:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:48.695830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471594917104556:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:48.695867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471594917104456:2048] [node 1] ICP30 unregister session Session# [2:7577471594917104556:2048] VirtualId# [2:7577471594917104545:0]
2025-11-27T18:19:48.695980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:48.695800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:48.696020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471593598017986:2048] [node 1] ICS07 socket disconnect 233 reason# EndOfStream
2025-11-27T18:19:48.696017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471594917104456:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.696049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.696252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471593598017986:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.696355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:48.696392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471593598017986:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:48.696892Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:48.697110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471592738254740:2048] [node 1] ICS07 socket disconnect 262 reason# EndOfStream
2025-11-27T18:19:48.697142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471592738254740:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.697220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:48.697316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471592738254740:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:48.697504Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471593598017897:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:48.697557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471593598017986:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:48.697662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471593598017897:2048] [node 1] ICP30 unregister session Session# [4:7577471593598017986:2048] VirtualId# [4:7577471593598017980:0]
2025-11-27T18:19:48.697973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:48.698028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471593598017897:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.698052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.698463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:48.698674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471595087349392:2048] [node 1] ICS07 socket disconnect 238 reason# EndOfStream
2025-11-27T18:19:48.698703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471595087349392:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.698768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:48.699083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471592738254653:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:48.698807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471595087349392:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:48.699120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471592738254740:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:48.699167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471592738254653:2048] [node 1] ICP30 unregister session Session# [5:7577471592738254740:2048] VirtualId# [5:7577471592738254736:0]
2025-11-27T18:19:48.699250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:48.699287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471592738254653:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.699298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.699552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471595087349307:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:48.699600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471595087349392:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:48.699635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471595087349307:2048] [node 1] ICP30 unregister session Session# [6:7577471595087349392:2048] VirtualId# [6:7577471595087349387:0]
2025-11-27T18:19:48.699810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:48.699846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471595087349307:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.699863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22313) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.705588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471594917104456:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:48.707619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471593598017897:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:48.709184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471592738254653:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:48.709712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471595087349307:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:48.769697Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:48.773235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:48.769927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471594097451044:2049] [node 2] ICS07 socket disconnect 227 reason# EndOfStream
2025-11-27T18:19:48.769969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471594097451044:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.770044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:48.770087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471594097451044:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:48.771603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471594097450953:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17093) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:48.771657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471594097451044:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:48.771689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471594097450953:2049] [node 2] ICP30 unregister session Session# [3:7577471594097451044:2049] VirtualId# [3:7577471594097451037:0]
2025-11-27T18:19:48.773456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471593598017984:2049] [node 2] ICS01 socket: 230 reason# EndOfStream
2025-11-27T18:19:48.771782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:48.773500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471593598017898:2049] [node 2] ICP30 unregister session Session# [4:7577471593598017984:2049] VirtualId# [4:7577471593598017978:0]
2025-11-27T18:19:48.771817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471594097450953:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.773539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471593598017984:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.771850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.773615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:48.773654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:48.775301Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:48.775479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471595087349398:2049] [node 2] ICS01 socket: 257 reason# EndOfStream
2025-11-27T18:19:48.775510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471595087349308:2049] [node 2] ICP30 unregister session Session# [6:7577471595087349398:2049] VirtualId# [6:7577471595087349388:0]
2025-11-27T18:19:48.775543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471595087349398:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.775612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:48.775652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:48.777460Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:48.777759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471592738254743:2049] [node 2] ICS01 socket: 270 reason# EndOfStream
2025-11-27T18:19:48.777851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471592738254654:2049] [node 2] ICP30 unregister session Session# [5:7577471592738254743:2049] VirtualId# [5:7577471592738254737:0]
2025-11-27T18:19:48.777899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471592738254743:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.777987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:48.778090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:48.813704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:48.813942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471592738254751:2050] [node 3] ICS07 socket disconnect 240 reason# EndOfStream
2025-11-27T18:19:48.813975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471592738254751:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.813956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:48.814060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:48.814112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471592738254751:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:48.814181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471593598017990:2050] [node 3] ICS01 socket: 242 reason# EndOfStream
2025-11-27T18:19:48.814247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471593598017899:2050] [node 3] ICP30 unregister session Session# [4:7577471593598017990:2050] VirtualId# [4:7577471593598017979:0]
2025-11-27T18:19:48.814296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471593598017990:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.814445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:48.814548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:48.817226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:48.821511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471595087349406:2050] [node 3] ICS07 socket disconnect 273 reason# EndOfStream
2025-11-27T18:19:48.821549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471595087349406:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.821666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:48.821723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471595087349406:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:48.825460Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471592738254655:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19398) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:48.825528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471592738254751:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:48.825564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471592738254655:2050] [node 3] ICP30 unregister session Session# [5:7577471592738254751:2050] VirtualId# [5:7577471592738254748:0]
2025-11-27T18:19:48.825609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:48.825653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471592738254655:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19398) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.825666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19398) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.826959Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471595087349309:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19398) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:48.827006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471595087349406:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:48.827041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471595087349309:2050] [node 3] ICP30 unregister session Session# [6:7577471595087349406:2050] VirtualId# [6:7577471595087349397:0]
2025-11-27T18:19:48.827120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:48.827144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471595087349309:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19398) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.827156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19398) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.835648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471592738254655:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:48.836701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471595087349309:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:48.868716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:48.868946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471592738254757:2108] [node 4] ICS01 socket: 221 reason# EndOfStream
2025-11-27T18:19:48.868998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471592738254717:2108] [node 4] ICP30 unregister session Session# [5:7577471592738254757:2108] VirtualId# [5:7577471592738254749:0]
2025-11-27T18:19:48.869032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471592738254757:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.869103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:48.869142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:48.873554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:48.873770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471595087349408:2105] [node 4] ICS07 socket disconnect 276 reason# EndOfStream
2025-11-27T18:19:48.873798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471595087349408:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.873873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:48.873916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471595087349408:2105] [node 4] ICS15 start handshake
2025-11-27T18:19:48.874960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471595087349371:2105] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24966) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:48.875013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471595087349408:2105] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:48.875049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471595087349371:2105] [node 4] ICP30 unregister session Session# [6:7577471595087349408:2105] VirtualId# [6:7577471595087349401:0]
2025-11-27T18:19:48.875162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:48.875202Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471595087349371:2105] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24966) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.875222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24966) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:48.885840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471595087349371:2105] [node 4] ICP33 wake up from error state
2025-11-27T18:19:48.904453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:48.904704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471595087349410:2104] [node 5] ICS01 socket: 277 reason# EndOfStream
2025-11-27T18:19:48.904742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471595087349365:2104] [node 5] ICP30 unregister session Session# [6:7577471595087349410:2104] VirtualId# [6:7577471595087349402:0]
2025-11-27T18:19:48.904776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471595087349410:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:48.904872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:48.904989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:49.014544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471597609473650:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:49.017448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471597609473649:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:49.038480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471598653419385:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:49.038534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471598653419386:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:49.074841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471599358700399:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:49.074885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471599358700400:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:49.100896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471597308018617:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:49.100966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471597308018618:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:49.101020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471597308018619:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:49.122758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471596895666307:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:49.122822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471596895666309:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:49.122828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471596895666308:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:49.142070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471599201200977:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:49.142144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471599201200978:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:49.142196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471599201200979:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:49.284050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471598653419449:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:49.260868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471599201201031:2101] [node 5] ICP01 ready to work
2025-11-27T18:19:49.284626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471598653419450:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:49.342139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471599358700473:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:49.352471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471596895666371:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:49.342138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471597609473715:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:49.285138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471598653419451:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:49.366141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471599358700475:2111] [node 6] ICP01 ready to work
2025-11-27T18:19:49.369356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471597609473716:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:49.382285Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471599201200977:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(93006810ns) {Configure:104(92999320ns) {ProcessConfigured:135(80398ns) {StartInitialHandshake:157(77530ns) {DropHandshakes:470(1848ns) {DropIncomingHandshake:442(603ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(660ns) PrepareNewSessionHandshake:477(1343ns) {SwitchToState:179(682ns)}}}}}
2025-11-27T18:19:49.368327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471596895666372:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:49.368374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471599358700474:2110] [node 5] ICP01 ready to work
2025-11-27T18:19:49.382029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471597308018618:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(88919351ns) {Configure:104(88914268ns) {ProcessConfigured:135(75877ns) {StartInitialHandshake:157(73360ns) {DropHandshakes:470(3881ns) {DropIncomingHandshake:442(764ns) DropOutgoingHandshake:457(821ns)} GenerateSessionVirtualId:988(867ns) PrepareNewSessionHandshake:477(2087ns) {SwitchToState:179(799ns)}}}}}
2025-11-27T18:19:49.394858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471599201201031:2101] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(133678524ns) {Configure:104(133675829ns) {ProcessConfigured:135(6738521ns) {StartInitialHandshake:157(6735850ns) {DropHandshakes:470(2067ns) {DropIncomingHandshake:442(504ns) DropOutgoingHandshake:457(514ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:477(2757ns) {SwitchToState:179(1094ns)}}}}}
2025-11-27T18:19:49.387669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471597609473717:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:49.382510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471597308018700:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:49.397383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471599201201043:2103] [node 4] ICP01 ready to work
2025-11-27T18:19:49.383057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471597308018701:2118] [node 6] ICP01 ready to work
2025-11-27T18:19:49.415686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471598653419449:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(89203579ns) {Configure:104(89197121ns) {ProcessConfigured:135(91007ns) {StartInitialHandshake:157(88669ns) {DropHandshakes:470(2944ns) {DropIncomingHandshake:442(714ns) DropOutgoingHandshake:457(460ns)} GenerateSessionVirtualId:988(1027ns) PrepareNewSessionHandshake:477(2400ns) {SwitchToState:179(1157ns)}}}}}
2025-11-27T18:19:49.416756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471596895666307:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17297478ns) {Configure:104(17294098ns) {ProcessConfigured:135(11940460ns) {StartInitialHandshake:157(11938041ns) {DropHandshakes:470(1608ns) {DropIncomingHandshake:442(331ns) DropOutgoingHandshake:457(405ns)} GenerateSessionVirtualId:988(645ns) PrepareNewSessionHandshake:477(6493ns) {SwitchToState:179(3826ns)}}}}}
2025-11-27T18:19:49.423344Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471597609473649:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(164916943ns) {Configure:104(164910498ns) {ProcessConfigured:135(1016357ns) {StartInitialHandshake:157(1013895ns) {DropHandshakes:470(1473ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(288ns)} GenerateSessionVirtualId:988(478ns) PrepareNewSessionHandshake:477(2313ns) {SwitchToState:179(1296ns)}}}}}
2025-11-27T18:19:49.427943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471597308018701:2118] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(26817920ns) {Configure:104(26813866ns) {ProcessConfigured:135(774853ns) {StartInitialHandshake:157(772265ns) {DropHandshakes:470(1199ns) {DropIncomingHandshake:442(217ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(599ns) PrepareNewSessionHandshake:477(4920ns) {SwitchToState:179(3424ns)}}}}}
2025-11-27T18:19:49.428551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471598653419450:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(61997573ns) {Configure:104(61988093ns) {ProcessConfigured:135(276107ns) {StartInitialHandshake:157(273280ns) {DropHandshakes:470(2047ns) {DropIncomingHandshake:442(521ns) DropOutgoingHandshake:457(374ns)} GenerateSessionVirtualId:988(390ns) PrepareNewSessionHandshake:477(6657ns) {SwitchToState:179(2110ns)}}}}}
2025-11-27T18:19:49.434329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471597308018619:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(109962255ns) {Configure:104(109955950ns) {ProcessConfigured:135(463164ns) {StartInitialHandshake:157(461195ns) {DropHandshakes:470(1697ns) {DropIncomingHandshake:442(768ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(445ns) PrepareNewSessionHandshake:477(1572ns) {SwitchToState:179(520ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.436124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471597308018710:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45358
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.458090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471598653419460:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48725
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:19:49.481125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471596895666386:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
::ffff:127.1.0.0
2025-11-27T18:19:49.465718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471599358700400:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(42224026ns) {Configure:104(42213291ns) {ProcessConfigured:135(12074217ns) {StartInitialHandshake:157(12071199ns) {DropHandshakes:470(1497ns) {DropIncomingHandshake:442(368ns) DropOutgoingHandshake:457(289ns)} GenerateSessionVirtualId:988(798ns) PrepareNewSessionHandshake:477(3808ns) {SwitchToState:179(1888ns)}}}}}
2025-11-27T18:19:49.484826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471596895666387:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:19:49.465741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471598653419460:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.476936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471599358700473:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(48469468ns) {Configure:104(48462982ns) {ProcessConfigured:135(50142ns) {StartInitialHandshake:157(47860ns) {DropHandshakes:470(1881ns) {DropIncomingHandshake:442(421ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(637ns) PrepareNewSessionHandshake:477(3208ns) {SwitchToState:179(324ns)}}}}}
2025-11-27T18:19:49.469006Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471598653419385:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40806730ns) {Configure:104(40801085ns) {ProcessConfigured:135(15328133ns) {StartInitialHandshake:157(15324322ns) {DropHandshakes:470(1516ns) {DropIncomingHandshake:442(307ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(611ns) PrepareNewSessionHandshake:477(4399ns) {SwitchToState:179(1829ns)}}}}}
2025-11-27T18:19:49.480579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471599358700476:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43827
2025-11-27T18:19:49.487414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471596895666386:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.495088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471596895666371:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.496745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471597609473717:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(50907394ns) {Configure:104(50902167ns) {ProcessConfigured:135(1452374ns) {StartInitialHandshake:157(1450148ns) {DropHandshakes:470(1541ns) {DropIncomingHandshake:442(378ns) DropOutgoingHandshake:457(308ns)} GenerateSessionVirtualId:988(552ns) PrepareNewSessionHandshake:477(1775ns) {SwitchToState:179(824ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.499644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471596895666387:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.500077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471597308018705:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.501349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471596895666388:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:19:49.500137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471597308018705:2130] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.502031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471597308018710:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.505158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471596895666308:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.502193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471597308018714:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45358
2025-11-27T18:19:49.504529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471597308018714:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.504986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471597308018705:2130] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:49.506985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471597308018715:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45358
2025-11-27T18:19:49.506912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471598653419466:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48725
2025-11-27T18:19:49.509802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471597308018618:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.509900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471597308018700:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.509878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471598653419455:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.509947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471596895666391:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:19:49.509971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.517638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471599201201048:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59689
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.517826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471599201201049:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59689
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.509924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471598653419455:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.510026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471597308018700:2108] [node 5] ICP22 created new session: [4:7577471597308018719:2108]
2025-11-27T18:19:49.527682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471599201201049:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.510078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471596895666388:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.510025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471598653419466:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.527940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471599358700482:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.519788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471597308018719:2108] [node 5] ICS09 handshake done sender: [4:7577471597308018705:2130] self: [4:7577471597308018704:0] peer: [5:7577471596895666389:0] socket: 231 qp: -1
2025-11-27T18:19:49.516777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471596895666309:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.539483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471597609473716:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(34182888ns) {Configure:104(33124302ns) {ProcessConfigured:135(89688ns) {StartInitialHandshake:157(87447ns) {DropHandshakes:470(2316ns) {DropIncomingHandshake:442(535ns) DropOutgoingHandshake:457(339ns)} GenerateSessionVirtualId:988(812ns) PrepareNewSessionHandshake:477(7566ns) {SwitchToState:179(529ns)}}}}}
2025-11-27T18:19:49.522035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471597308018719:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:49.522096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:49.532153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471597308018700:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(21407121ns) {HandleHandshakeStatus:329(21396089ns) {DropIncomingHandshake:442(13621ns) DropOutgoingHandshake:457(710ns) SwitchToState:179(90ns) ProcessPendingSessionEvents:495(579100ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 520506ns max# 88945ns}}}
2025-11-27T18:19:49.558026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471597308018715:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.530382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471599201201048:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.570557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471597308018619:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.534499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471598653419455:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:49.549828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471599201200978:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.541842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471597609473722:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41284
2025-11-27T18:19:49.521874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471596895666393:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.549911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471599201201031:2101] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.561692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471598653419453:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.527992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471599358700482:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.542011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471597609473723:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41284
2025-11-27T18:19:49.561754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471598653419453:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.550330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471599201201056:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59689
2025-11-27T18:19:49.523445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471596895666386:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:49.545401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471599358700482:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:49.575152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471598653419457:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.575177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471598653419457:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.575908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471598653419450:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.575953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:49.576002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471598653419450:2109] [node 5] ICP22 created new session: [2:7577471598653419472:2109]
2025-11-27T18:19:49.576049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471598653419472:2109] [node 5] ICS09 handshake done sender: [2:7577471598653419455:2112] self: [2:7577471598653419454:0] peer: [5:7577471596895666390:0] socket: 235 qp: -1
2025-11-27T18:19:49.576081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471598653419472:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:49.576104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:49.591087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471598653419471:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48725
2025-11-27T18:19:49.591284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471598653419453:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:49.601900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471598653419457:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:49.602050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471598653419470:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48725
2025-11-27T18:19:49.602963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471599358700480:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.544358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471597609473726:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41284
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.622250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471597308018724:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45358
2025-11-27T18:19:49.597441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471599201201043:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(61784930ns) {Configure:104(61776395ns) {ProcessConfigured:135(23476556ns) {StartInitialHandshake:157(23473008ns) {DropHandshakes:470(2134ns) {DropIncomingHandshake:442(460ns) DropOutgoingHandshake:457(349ns)} GenerateSessionVirtualId:988(597ns) PrepareNewSessionHandshake:477(4173ns) {SwitchToState:179(2045ns)}}}}}
2025-11-27T18:19:49.529847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471596895666371:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.625469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471598653419471:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.529900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:49.529962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471596895666371:2108] [node 4] ICP22 created new session: [5:7577471596895666394:2108]
2025-11-27T18:19:49.530009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471596895666394:2108] [node 4] ICS09 handshake done sender: [5:7577471596895666386:2116] self: [5:7577471596895666389:0] peer: [4:7577471597308018704:0] socket: 239 qp: -1
2025-11-27T18:19:49.530045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471596895666394:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:49.530090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:49.634069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471596895666393:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.634191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471596895666376:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.634218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471596895666376:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.635542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471596895666396:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.598757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471597609473726:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.625566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471598653419449:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.637400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:49.637827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471598653419449:2107] [node 4] ICP22 created new session: [2:7577471598653419477:2107]
2025-11-27T18:19:49.602991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471599358700480:2123] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.597977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471599201201060:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59689
2025-11-27T18:19:49.604495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471599358700476:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.636101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471596895666397:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:19:49.598140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471599201201061:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59689
2025-11-27T18:19:49.598845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471597609473723:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.645185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471597308018715:2135] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:49.601475Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471597609473715:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(94962288ns) {Configure:104(94952582ns) {ProcessConfigured:135(2400261ns) {StartInitialHandshake:157(53068ns) {DropHandshakes:470(2127ns) {DropIncomingHandshake:442(566ns) DropOutgoingHandshake:457(561ns)} GenerateSessionVirtualId:988(1110ns) PrepareNewSessionHandshake:477(1034ns) {SwitchToState:179(668ns)}}}}}
2025-11-27T18:19:49.642479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471598653419477:2107] [node 4] ICS09 handshake done sender: [2:7577471598653419453:2111] self: [2:7577471598653419452:0] peer: [4:7577471597308018718:0] socket: 234 qp: -1
2025-11-27T18:19:49.650909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471597308018723:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45358
2025-11-27T18:19:49.599199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471599201201056:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.606811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471597609473710:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.650968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471598653419477:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:49.604549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471599358700474:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.606831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471597609473710:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.655636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:49.663693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471596895666376:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:49.606724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471599201200977:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.604595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:49.620165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471597609473710:2107] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:49.664281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471597308018619:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.650935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471599201201049:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:49.659034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471598653419451:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.604641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471599358700474:2110] [node 5] ICP22 created new session: [3:7577471599358700491:2110]
2025-11-27T18:19:49.663048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:49.664351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:49.663600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471597609473721:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.662299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471599201201060:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.604690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471599358700491:2110] [node 5] ICS09 handshake done sender: [3:7577471599358700482:2124] self: [3:7577471599358700481:0] peer: [5:7577471596895666392:0] socket: 246 qp: -1
2025-11-27T18:19:49.665085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471597308018619:2050] [node 3] ICP22 created new session: [4:7577471597308018725:2050]
2025-11-27T18:19:49.662444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471599201200978:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.663634Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471597609473721:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.604721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471599358700491:2110] [node 5] ICS10 traffic start
2025-11-27T18:19:49.667591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471596895666387:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:49.663359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471598653419451:2110] [node 6] ICP22 created new session: [2:7577471598653419479:2110]
2025-11-27T18:19:49.665169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471597609473722:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.665646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471597308018725:2050] [node 3] ICS09 handshake done sender: [4:7577471597308018715:2135] self: [4:7577471597308018722:0] peer: [3:7577471599358700479:0] socket: 245 qp: -1
2025-11-27T18:19:49.662488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:49.604754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:49.666788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471597308018725:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:49.664046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471598653419479:2110] [node 6] ICS09 handshake done sender: [2:7577471598653419457:2113] self: [2:7577471598653419456:0] peer: [6:7577471599201201057:0] socket: 238 qp: -1
2025-11-27T18:19:49.665254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471597609473650:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.662534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471599201200978:2049] [node 2] ICP22 created new session: [6:7577471599201201064:2049]
2025-11-27T18:19:49.666845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:49.665698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:49.606198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471599358700399:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.662581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471599201201064:2049] [node 2] ICS09 handshake done sender: [6:7577471599201201049:2114] self: [6:7577471599201201057:0] peer: [2:7577471598653419456:0] socket: 242 qp: -1
2025-11-27T18:19:49.667236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471597308018714:2134] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.618859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471599358700480:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:49.666827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471597609473650:2049] [node 3] ICP22 created new session: [1:7577471597609473734:2049]
2025-11-27T18:19:49.649866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471599358700473:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.666908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471597609473734:2049] [node 3] ICS09 handshake done sender: [1:7577471597609473710:2107] self: [1:7577471597609473709:0] peer: [3:7577471599358700494:0] socket: 222 qp: -1
2025-11-27T18:19:49.649925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:49.667298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471597609473734:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:49.649987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471599358700473:2107] [node 4] ICP22 created new session: [3:7577471599358700497:2107]
2025-11-27T18:19:49.650045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471599358700497:2107] [node 4] ICS09 handshake done sender: [3:7577471599358700480:2123] self: [3:7577471599358700479:0] peer: [4:7577471597308018722:0] socket: 243 qp: -1
2025-11-27T18:19:49.667561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:49.650086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471599358700497:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:49.675095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471597609473721:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:49.650124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:49.669026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471597308018618:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.677509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471596895666388:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:49.662618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471599201201064:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:49.664152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471598653419479:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:49.668868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471599358700495:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43827
2025-11-27T18:19:49.669090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:49.662651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:49.678260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471599358700496:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43827
2025-11-27T18:19:49.664194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:49.678428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471597609473717:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.686223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471599201201063:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59689
2025-11-27T18:19:49.669130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471597308018618:2049] [node 2] ICP22 created new session: [4:7577471597308018727:2049]
2025-11-27T18:19:49.687992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471599358700476:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:49.669175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471597308018727:2049] [node 2] ICS09 handshake done sender: [4:7577471597308018714:2134] self: [4:7577471597308018718:0] peer: [2:7577471598653419452:0] socket: 236 qp: -1
2025-11-27T18:19:49.688623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471599201201043:2103] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.678512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:49.669201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471597308018727:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:49.679054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471597609473717:2110] [node 6] ICP22 created new session: [1:7577471597609473739:2110]
2025-11-27T18:19:49.669228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:49.691369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471599358700399:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.685180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471597308018619:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(20077384ns) {HandleHandshakeStatus:329(20074641ns) {DropIncomingHandshake:442(1586ns) DropOutgoingHandshake:457(6251ns) SwitchToState:179(786ns) ProcessPendingSessionEvents:495(16240893ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16166653ns max# 4661826ns}}}
2025-11-27T18:19:49.691664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471596895666372:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.691959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:49.691740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:49.679903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471597609473739:2110] [node 6] ICS09 handshake done sender: [1:7577471597609473721:2112] self: [1:7577471597609473720:0] peer: [6:7577471599201201062:0] socket: 264 qp: -1
2025-11-27T18:19:49.691784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471596895666372:2110] [node 6] ICP22 created new session: [5:7577471596895666401:2110]
2025-11-27T18:19:49.692287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471596895666401:2110] [node 6] ICS09 handshake done sender: [5:7577471596895666376:2112] self: [5:7577471596895666375:0] peer: [6:7577471599201201058:0] socket: 224 qp: -1
2025-11-27T18:19:49.692009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471599358700399:2048] [node 1] ICP22 created new session: [3:7577471599358700501:2048]
2025-11-27T18:19:49.686873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471597609473739:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:49.694025Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471599201200978:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(25858717ns) {HandleHandshakeStatus:329(25852330ns) {DropIncomingHandshake:442(563ns) DropOutgoingHandshake:457(5262ns) SwitchToState:179(359ns) ProcessPendingSessionEvents:495(25498106ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25385373ns max# 5238162ns}}}
2025-11-27T18:19:49.695810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471597308018709:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.686935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:49.695896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471598653419449:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(52002152ns) {HandleHandshakeStatus:329(51998530ns) {DropIncomingHandshake:442(4077084ns) DropOutgoingHandshake:457(1348ns) SwitchToState:179(564ns) ProcessPendingSessionEvents:495(16055909ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12872791ns max# 3015857ns}}}
2025-11-27T18:19:49.692472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471599358700473:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17697687ns) {HandleHandshakeStatus:329(17693257ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(704ns) SwitchToState:179(537ns) ProcessPendingSessionEvents:495(17256203ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17168646ns max# 4822673ns}}}
2025-11-27T18:19:49.694394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471599201201048:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:49.692554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471599358700501:2048] [node 1] ICS09 handshake done sender: [3:7577471599358700476:2121] self: [3:7577471599358700494:0] peer: [1:7577471597609473709:0] socket: 223 qp: -1
2025-11-27T18:19:49.695858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471597308018709:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.698565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471597308018709:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:49.692744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471596895666401:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:49.692585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471599358700501:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:49.693241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:49.695716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471599201200977:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471599201201056:2116] held: no
2025-11-27T18:19:49.693085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:49.700631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471597308018701:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.702637Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471596895666371:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(157429948ns) {HandleHandshakeStatus:329(157426464ns) {DropIncomingHandshake:442(1024ns) DropOutgoingHandshake:457(5090ns) SwitchToState:179(636ns) ProcessPendingSessionEvents:495(109791308ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 109624469ns max# 14240410ns}}}
2025-11-27T18:19:49.701564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:49.702755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471596895666308:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.702603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471597308018701:2118] [node 6] ICP22 created new session: [4:7577471597308018730:2118]
2025-11-27T18:19:49.703962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471598653419470:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.704144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471599201201066:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59689
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.704967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471597609473741:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41284
2025-11-27T18:19:49.703423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:49.706952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471599201201031:2101] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.703743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471597308018730:2118] [node 6] ICS09 handshake done sender: [4:7577471597308018709:2132] self: [4:7577471597308018706:0] peer: [6:7577471599201201067:0] socket: 266 qp: -1
2025-11-27T18:19:49.709665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:49.703880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471597308018730:2118] [node 6] ICS10 traffic start
2025-11-27T18:19:49.704657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:49.704580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471596895666308:2049] [node 2] ICP22 created new session: [5:7577471596895666403:2049]
2025-11-27T18:19:49.711397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471599201201031:2101] [node 5] ICP22 created new session: [6:7577471599201201070:2101]
2025-11-27T18:19:49.705320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471596895666403:2049] [node 2] ICS09 handshake done sender: [5:7577471596895666387:2117] self: [5:7577471596895666390:0] peer: [2:7577471598653419454:0] socket: 240 qp: -1
2025-11-27T18:19:49.711446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471596895666403:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:49.712632Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471596895666372:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(21940370ns) {HandleHandshakeStatus:329(21151671ns) {DropIncomingHandshake:442(16051ns) DropOutgoingHandshake:457(489ns) SwitchToState:179(826ns) ProcessPendingSessionEvents:495(15867000ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15798460ns max# 6128738ns}}}
2025-11-27T18:19:49.712791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471596895666309:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.712833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:49.712875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471596895666309:2050] [node 3] ICP22 created new session: [5:7577471596895666406:2050]
2025-11-27T18:19:49.712921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471596895666406:2050] [node 3] ICS09 handshake done sender: [5:7577471596895666388:2118] self: [5:7577471596895666392:0] peer: [3:7577471599358700481:0] socket: 248 qp: -1
2025-11-27T18:19:49.712947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471596895666406:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:49.712977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:49.715223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:49.717318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471599201201070:2101] [node 5] ICS09 handshake done sender: [6:7577471599201201048:2113] self: [6:7577471599201201058:0] peer: [5:7577471596895666375:0] socket: 237 qp: -1
2025-11-27T18:19:49.717375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471599201201070:2101] [node 5] ICS10 traffic start
2025-11-27T18:19:49.728797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471598653419451:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(44060846ns) {HandleHandshakeStatus:329(44057696ns) {DropIncomingHandshake:442(1220ns) DropOutgoingHandshake:457(1895ns) SwitchToState:179(484ns) ProcessPendingSessionEvents:495(36398860ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 34996620ns max# 6338496ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.717419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:49.731940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471598653419385:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.735053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471598653419481:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48725
2025-11-27T18:19:49.738995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471598653419484:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48725
2025-11-27T18:19:49.737657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471597609473719:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.744787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471597308018731:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45358
2025-11-27T18:19:49.744686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471599201201031:2101] [node 5] ICP35 event processing took too much time PendingConnection:279(21752205ns) {HandleHandshakeStatus:329(21748759ns) {DropIncomingHandshake:442(502ns) DropOutgoingHandshake:457(5699ns) SwitchToState:179(3550ns) ProcessPendingSessionEvents:495(3906034ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3840895ns max# 2157242ns}}}
2025-11-27T18:19:49.737696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471597609473719:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.747513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471597308018701:2118] [node 6] ICP35 event processing took too much time PendingConnection:279(31253103ns) {HandleHandshakeStatus:329(31249757ns) {DropIncomingHandshake:442(484ns) DropOutgoingHandshake:457(1955ns) SwitchToState:179(618ns) ProcessPendingSessionEvents:495(12864070ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12773816ns max# 5389576ns}}}
2025-11-27T18:19:49.738695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471597609473743:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41284
2025-11-27T18:19:49.744958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471597609473719:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.745879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471597609473717:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(52270811ns) {HandleHandshakeStatus:329(52266113ns) {DropIncomingHandshake:442(16348ns) DropOutgoingHandshake:457(377ns) SwitchToState:179(671ns) ProcessPendingSessionEvents:495(41157227ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39580646ns max# 3971527ns}}}
2025-11-27T18:19:49.748466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471597609473743:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.748513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471597609473649:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.749175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471599201201068:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59689
2025-11-27T18:19:49.748715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:49.750975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471598653419471:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:49.749183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471599201201056:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:49.748942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471597609473649:2048] [node 2] ICP22 created new session: [1:7577471597609473745:2048]
2025-11-27T18:19:49.748997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471597609473745:2048] [node 2] ICS09 handshake done sender: [1:7577471597609473719:2111] self: [1:7577471597609473718:0] peer: [2:7577471598653419483:0] socket: 263 qp: -1
2025-11-27T18:19:49.749094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471597609473745:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:49.753038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471599201200977:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.751526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471599358700399:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(55204095ns) {HandleHandshakeStatus:329(55201510ns) {DropIncomingHandshake:442(795ns) DropOutgoingHandshake:457(7615ns) SwitchToState:179(112ns) ProcessPendingSessionEvents:495(49234238ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 47972754ns max# 6222294ns}}}
2025-11-27T18:19:49.749141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:49.753121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:49.753169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471599201200977:2048] [node 1] ICP22 created new session: [6:7577471599201201074:2048]
2025-11-27T18:19:49.753209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471599201201074:2048] [node 1] ICS09 handshake done sender: [6:7577471599201201056:2116] self: [6:7577471599201201062:0] peer: [1:7577471597609473720:0] socket: 265 qp: -1
2025-11-27T18:19:49.753245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471599201201074:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:49.753292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:49.754221Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471596895666308:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(44959139ns) {HandleHandshakeStatus:329(44956972ns) {DropIncomingHandshake:442(566ns) DropOutgoingHandshake:457(4081ns) SwitchToState:179(481ns) ProcessPendingSessionEvents:495(30314420ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 30231046ns max# 6264222ns}}}
2025-11-27T18:19:49.753961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471598653419451:2110] [node 6] ICP18 handshake (actor [2:7577471598653419470:2118]) is too old
2025-11-27T18:19:49.754151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471598653419385:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.754210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:49.754543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471598653419385:2048] [node 1] ICP22 created new session: [2:7577471598653419485:2048]
2025-11-27T18:19:49.755305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471598653419485:2048] [node 1] ICS09 handshake done sender: [2:7577471598653419471:2119] self: [2:7577471598653419483:0] peer: [1:7577471597609473718:0] socket: 267 qp: -1
2025-11-27T18:19:49.758025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471598653419485:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:49.758394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:49.760885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471597308018731:2138] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.762281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471597308018617:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.779549Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471598653419385:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24695943ns) {HandleHandshakeStatus:329(24692690ns) {DropIncomingHandshake:442(785ns) DropOutgoingHandshake:457(5717ns) SwitchToState:179(574ns) ProcessPendingSessionEvents:495(19823181ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19732285ns max# 7614968ns}}}
2025-11-27T18:19:49.769972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471597609473731:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.776652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471597308018733:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45358
2025-11-27T18:19:49.770031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471597609473731:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.783273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471597609473649:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(33931138ns) {HandleHandshakeStatus:329(33928902ns) {DropIncomingHandshake:442(479ns) DropOutgoingHandshake:457(492ns) SwitchToState:179(548ns) ProcessPendingSessionEvents:495(31817743ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27596834ns max# 11150690ns}}}
2025-11-27T18:19:49.783476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471597609473649:2048] [node 2] ICP18 handshake (actor [1:7577471597609473743:2119]) is too old
2025-11-27T18:19:49.793426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471599358700495:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.786511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471599201201072:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59689
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.792454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471597609473731:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:49.794762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471597308018735:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45358
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.814649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471596895666405:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:19:49.812999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471597308018731:2138] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:49.815700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471597609473715:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.816806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471597308018617:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.815944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:49.816011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471597609473715:2108] [node 4] ICP22 created new session: [1:7577471597609473748:2108]
2025-11-27T18:19:49.816443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471597609473748:2108] [node 4] ICS09 handshake done sender: [1:7577471597609473731:2117] self: [1:7577471597609473730:0] peer: [4:7577471597308018734:0] socket: 233 qp: -1
2025-11-27T18:19:49.819147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471597609473748:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:49.819195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:49.820704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471599201201060:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.816864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:49.822109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471599201201043:2103] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.822196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:49.822254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471599201201043:2103] [node 4] ICP22 created new session: [6:7577471599201201076:2103]
2025-11-27T18:19:49.822301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471599201201076:2103] [node 4] ICS09 handshake done sender: [6:7577471599201201060:2118] self: [6:7577471599201201067:0] peer: [4:7577471597308018706:0] socket: 271 qp: -1
2025-11-27T18:19:49.822338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471599201201076:2103] [node 4] ICS10 traffic start
2025-11-27T18:19:49.822387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:49.816920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471597308018617:2048] [node 1] ICP22 created new session: [4:7577471597308018736:2048]
2025-11-27T18:19:49.816966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471597308018736:2048] [node 1] ICS09 handshake done sender: [4:7577471597308018731:2138] self: [4:7577471597308018734:0] peer: [1:7577471597609473730:0] socket: 286 qp: -1
2025-11-27T18:19:49.828264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471599358700503:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43827
2025-11-27T18:19:49.817013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471597308018736:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:49.817046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:49.829267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471596895666405:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.829413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471596895666307:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.829922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471599358700400:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.831178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471598653419459:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.831219Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471598653419459:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.837192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471598653419459:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:49.831528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471597609473725:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.831589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471597609473725:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.837562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471599358700505:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43827
2025-11-27T18:19:49.837611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471599201201068:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.838003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471597609473725:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:49.840182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471596895666410:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14889
2025-11-27T18:19:49.837755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471599358700503:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.838302Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471599358700489:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:49.838341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471599358700489:2126] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:49.839602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471599358700495:2127] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:49.840278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471597609473716:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.840850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471598653419386:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.837773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471599201200979:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:49.840360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471599358700489:2126] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:49.844858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:49.842305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471599201201079:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59689
2025-11-27T18:19:49.845779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471596895666405:2123] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:49.840339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:49.841537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471599358700400:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.842660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471599201201068:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:49.841589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:49.841637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471599358700400:2049] [node 2] ICP22 created new session: [3:7577471599358700508:2049]
2025-11-27T18:19:49.841699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471599358700508:2049] [node 2] ICS09 handshake done sender: [3:7577471599358700495:2127] self: [3:7577471599358700504:0] peer: [2:7577471598653419458:0] socket: 279 qp: -1
2025-11-27T18:19:49.845434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471598653419386:2049] [node 3] ICP22 created new session: [2:7577471598653419488:2049]
2025-11-27T18:19:49.854142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471597308018701:2118] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:49.840389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471597609473716:2109] [node 5] ICP22 created new session: [1:7577471597609473751:2109]
2025-11-27T18:19:49.841737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471599358700508:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:49.842963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471599201200979:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.846645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471598653419488:2049] [node 3] ICS09 handshake done sender: [2:7577471598653419459:2114] self: [2:7577471598653419458:0] peer: [3:7577471599358700504:0] socket: 274 qp: -1
2025-11-27T18:19:49.844512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471597609473751:2109] [node 5] ICS09 handshake done sender: [1:7577471597609473725:2115] self: [1:7577471597609473724:0] peer: [5:7577471596895666409:0] socket: 283 qp: -1
2025-11-27T18:19:49.846685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471598653419488:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:49.841774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:49.857561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471596895666307:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:49.851151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471599358700475:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:49.851221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:49.851280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471599358700475:2111] [node 6] ICP22 created new session: [3:7577471599358700510:2111]
2025-11-27T18:19:49.851339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471599358700510:2111] [node 6] ICS09 handshake done sender: [3:7577471599358700489:2126] self: [3:7577471599358700488:0] peer: [6:7577471599201201078:0] socket: 284 qp: -1
2025-11-27T18:19:49.851375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471599358700510:2111] [node 6] ICS10 traffic start
2025-11-27T18:19:49.851411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:49.844880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471597609473751:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:49.845362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:49.859559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471597609473716:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(18381626ns) {HandleHandshakeStatus:329(18378806ns) {DropIncomingHandshake:442(13235ns) DropOutgoingHandshake:457(465ns) SwitchToState:179(210ns) ProcessPendingSessionEvents:495(5874311ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5803847ns max# 946162ns}}}
2025-11-27T18:19:49.850195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:49.859914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471598653419386:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(18078287ns) {HandleHandshakeStatus:329(18076251ns) {DropIncomingHandshake:442(18355ns) DropOutgoingHandshake:457(659ns) SwitchToState:179(3074ns) ProcessPendingSessionEvents:495(3077621ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 2934329ns max# 1352247ns}}}
2025-11-27T18:19:49.857611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:49.857897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471596895666307:2048] [node 1] ICP22 created new session: [5:7577471596895666411:2048]
2025-11-27T18:19:49.843001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:49.843050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471599201200979:2050] [node 3] ICP22 created new session: [6:7577471599201201080:2050]
2025-11-27T18:19:49.843086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471599201201080:2050] [node 3] ICS09 handshake done sender: [6:7577471599201201068:2122] self: [6:7577471599201201078:0] peer: [3:7577471599358700488:0] socket: 228 qp: -1
2025-11-27T18:19:49.843122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471599201201080:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:49.843150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:49.861418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471599201200979:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17568895ns) {HandleHandshakeStatus:329(17567122ns) {DropIncomingHandshake:442(458ns) DropOutgoingHandshake:457(4096ns) SwitchToState:179(189ns) ProcessPendingSessionEvents:495(17280375ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17204069ns max# 4791346ns}}}
2025-11-27T18:19:49.861488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471596895666411:2048] [node 1] ICS09 handshake done sender: [5:7577471596895666405:2123] self: [5:7577471596895666409:0] peer: [1:7577471597609473724:0] socket: 285 qp: -1
2025-11-27T18:19:49.861557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471596895666411:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:49.861606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:49.862623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471596895666307:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:49.888289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:49.888595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471598653419485:2048] [node 1] ICS07 socket disconnect 267 reason# EndOfStream
2025-11-27T18:19:49.888640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471598653419485:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:49.888747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:49.888801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471598653419485:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:49.889618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:49.889838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471599358700501:2048] [node 1] ICS07 socket disconnect 223 reason# EndOfStream
2025-11-27T18:19:49.889865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471599358700501:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:49.889939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:49.889986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471599358700501:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:49.891732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471598653419385:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:49.891782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471598653419485:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:49.894483Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:49.894853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471599358700399:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:49.894874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471597308018736:2048] [node 1] ICS01 socket: 286 reason# EndOfStream
2025-11-27T18:19:49.894913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471599358700501:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:49.894918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471597308018617:2048] [node 1] ICP30 unregister session Session# [4:7577471597308018736:2048] VirtualId# [4:7577471597308018734:0]
2025-11-27T18:19:49.894959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471597308018736:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:49.895056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:49.895928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471598653419385:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:49.895107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:49.896446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471598653419485:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:49.896682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471598653419385:2048] [node 1] ICP30 unregister session Session# [2:7577471598653419485:2048] VirtualId# [2:7577471598653419483:0]
2025-11-27T18:19:49.896844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:49.896873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471598653419385:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:49.896921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471599358700399:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:49.896883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:49.896977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471599358700501:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:49.897018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471599358700399:2048] [node 1] ICP30 unregister session Session# [3:7577471599358700501:2048] VirtualId# [3:7577471599358700494:0]
2025-11-27T18:19:49.897107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:49.897140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471599358700399:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:49.897177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:49.897430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:49.897646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471596895666411:2048] [node 1] ICS07 socket disconnect 285 reason# EndOfStream
2025-11-27T18:19:49.897679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471596895666411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:49.897742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:49.897780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471596895666411:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:49.898450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471596895666307:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:49.898510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471596895666411:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:49.898543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471596895666307:2048] [node 1] ICP30 unregister session Session# [5:7577471596895666411:2048] VirtualId# [5:7577471596895666409:0]
2025-11-27T18:19:49.898594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:49.898631Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471596895666307:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:49.898645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17569) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:49.898725Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:49.898957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471599201201074:2048] [node 1] ICS01 socket: 265 reason# EndOfStream
2025-11-27T18:19:49.898996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471599201200977:2048] [node 1] ICP30 unregister session Session# [6:7577471599201201074:2048] VirtualId# [6:7577471599201201062:0]
2025-11-27T18:19:49.899025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471599201201074:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:49.899089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:49.899127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:49.904672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471599358700399:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:49.904689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471598653419385:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:49.908756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471596895666307:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:49.977445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:49.977494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:49.977704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471599358700508:2049] [node 2] ICS01 socket: 279 reason# EndOfStream
2025-11-27T18:19:49.977702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471597308018727:2049] [node 2] ICS01 socket: 236 reason# EndOfStream
2025-11-27T18:19:49.977741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471599358700400:2049] [node 2] ICP30 unregister session Session# [3:7577471599358700508:2049] VirtualId# [3:7577471599358700504:0]
2025-11-27T18:19:49.977742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471597308018618:2049] [node 2] ICP30 unregister session Session# [4:7577471597308018727:2049] VirtualId# [4:7577471597308018718:0]
2025-11-27T18:19:49.977775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471597308018727:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:49.977774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471599358700508:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:49.977853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:49.977852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:49.977899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:49.977899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:49.978679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:49.978864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471596895666403:2049] [node 2] ICS07 socket disconnect 240 reason# EndOfStream
2025-11-27T18:19:49.978890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471596895666403:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:49.978939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:49.978979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471596895666403:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:49.979721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471596895666308:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21950) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:49.979850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471596895666403:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:49.979889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471596895666308:2049] [node 2] ICP30 unregister session Session# [5:7577471596895666403:2049] VirtualId# [5:7577471596895666390:0]
2025-11-27T18:19:49.979981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:49.980027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471596895666308:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21950) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:49.980042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21950) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:49.980486Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:49.980708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471599201201064:2049] [node 2] ICS01 socket: 242 reason# EndOfStream
2025-11-27T18:19:49.980766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471599201200978:2049] [node 2] ICP30 unregister session Session# [6:7577471599201201064:2049] VirtualId# [6:7577471599201201057:0]
2025-11-27T18:19:49.980794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471599201201064:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:49.980873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:49.980929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:49.989427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471596895666308:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:50.034662Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:50.034953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471597308018725:2050] [node 3] ICS01 socket: 245 reason# EndOfStream
2025-11-27T18:19:50.035016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471597308018619:2050] [node 3] ICP30 unregister session Session# [4:7577471597308018725:2050] VirtualId# [4:7577471597308018722:0]
2025-11-27T18:19:50.035062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471597308018725:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.035168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:50.035237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:50.035868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:50.036072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471596895666406:2050] [node 3] ICS07 socket disconnect 248 reason# EndOfStream
2025-11-27T18:19:50.036110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471596895666406:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.036182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:50.036234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471596895666406:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:50.037070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:50.037245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471599201201080:2050] [node 3] ICS07 socket disconnect 228 reason# EndOfStream
2025-11-27T18:19:50.037294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471599201201080:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.037354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:50.037397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471599201201080:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:50.037537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471596895666309:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13227) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:50.037804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471596895666406:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:50.037871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471596895666309:2050] [node 3] ICP30 unregister session Session# [5:7577471596895666406:2050] VirtualId# [5:7577471596895666392:0]
2025-11-27T18:19:50.037920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:50.037979Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471596895666309:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13227) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.038006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13227) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.038366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471599201200979:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13227) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:50.038473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471599201201080:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:50.038517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471599201200979:2050] [node 3] ICP30 unregister session Session# [6:7577471599201201080:2050] VirtualId# [6:7577471599201201078:0]
2025-11-27T18:19:50.038648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:50.038695Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471599201200979:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13227) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.038710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13227) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.048104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471596895666309:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:50.048109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471599201200979:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:50.104654Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:50.105072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:50.105337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471596895666394:2108] [node 4] ICS07 socket disconnect 239 reason# EndOfStream
2025-11-27T18:19:50.105379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471596895666394:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.105477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:50.105531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471596895666394:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:50.105004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471599201201076:2103] [node 4] ICS07 socket disconnect 271 reason# EndOfStream
2025-11-27T18:19:50.105054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471599201201076:2103] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.105145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:50.105190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471599201201076:2103] [node 4] ICS15 start handshake
2025-11-27T18:19:50.106007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471599201201043:2103] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11953) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:50.106116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471599201201076:2103] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:50.106238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471599201201043:2103] [node 4] ICP30 unregister session Session# [6:7577471599201201076:2103] VirtualId# [6:7577471599201201067:0]
2025-11-27T18:19:50.106304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:50.106338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471599201201043:2103] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11953) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.106355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11953) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.106508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471596895666371:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11953) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:50.106611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471596895666394:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:50.106658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471596895666371:2108] [node 4] ICP30 unregister session Session# [5:7577471596895666394:2108] VirtualId# [5:7577471596895666389:0]
2025-11-27T18:19:50.106805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:50.110543Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471596895666371:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11953) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.110589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11953) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.116683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471599201201043:2103] [node 4] ICP33 wake up from error state
2025-11-27T18:19:50.124274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471596895666371:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:19:50.176435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:50.176688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471599201201070:2101] [node 5] ICS01 socket: 237 reason# EndOfStream
2025-11-27T18:19:50.176735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471599201201031:2101] [node 5] ICP30 unregister session Session# [6:7577471599201201070:2101] VirtualId# [6:7577471599201201058:0]
2025-11-27T18:19:50.176767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471599201201070:2101] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.176843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:50.176896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:50.277827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471602971900333:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:50.277880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471602971900334:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:50.298426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471602352619880:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:50.298554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471602352619881:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:50.323393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471603031375654:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:50.323437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471603031375655:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:50.349917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471600720798683:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:50.349955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471600720798685:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:50.350191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471600720798684:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:50.372267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471604671576628:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:50.372374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471604671576627:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:50.372623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471604671576629:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:50.394203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471601533514510:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:50.394262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471601533514512:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:50.395133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471601533514511:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:50.455097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471601533514572:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:50.480815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471602352619944:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:50.497647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471602971900396:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:50.538960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471600720798764:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:50.546471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471604671576692:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:50.538559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471602352619945:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:50.538588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471603031375720:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:50.521322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471601533514572:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25817506ns) {Configure:104(25812235ns) {ProcessConfigured:135(22133166ns) {StartInitialHandshake:157(22129861ns) {DropHandshakes:470(2987ns) {DropIncomingHandshake:442(764ns) DropOutgoingHandshake:457(527ns)} GenerateSessionVirtualId:988(759ns) PrepareNewSessionHandshake:477(2933ns) {SwitchToState:179(1708ns)}}}}}
2025-11-27T18:19:50.557459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471600720798765:2117] [node 6] ICP01 ready to work
2025-11-27T18:19:50.557908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471603031375721:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:50.553794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471604671576693:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:50.538644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471602352619946:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:50.551192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471602971900398:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:50.553518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471602971900397:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:50.558206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471603031375722:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:50.558908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471601533514576:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:50.566302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471602971900333:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(106641081ns) {Configure:104(106636037ns) {ProcessConfigured:135(5143967ns) {StartInitialHandshake:157(5132212ns) {DropHandshakes:470(85003ns) {DropIncomingHandshake:442(469ns) DropOutgoingHandshake:457(682ns)} GenerateSessionVirtualId:988(1443ns) PrepareNewSessionHandshake:477(4745ns) {SwitchToState:179(2584ns)}}}}}
2025-11-27T18:19:50.577310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471602352619880:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17934116ns) {Configure:104(17931715ns) {ProcessConfigured:135(4399849ns) {StartInitialHandshake:157(4397757ns) {DropHandshakes:470(1115ns) {DropIncomingHandshake:442(299ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(599ns) PrepareNewSessionHandshake:477(2321ns) {SwitchToState:179(353ns)}}}}}
2025-11-27T18:19:50.580616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471602352619946:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19132883ns) {Configure:104(19125569ns) {ProcessConfigured:135(1242666ns) {StartInitialHandshake:157(1240480ns) {DropHandshakes:470(1568ns) {DropIncomingHandshake:442(467ns) DropOutgoingHandshake:457(200ns)} GenerateSessionVirtualId:988(489ns) PrepareNewSessionHandshake:477(1903ns) {SwitchToState:179(825ns)}}}}}
2025-11-27T18:19:50.581409Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471603031375654:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20393720ns) {Configure:104(20390394ns) {ProcessConfigured:135(885511ns) {StartInitialHandshake:157(884054ns) {DropHandshakes:470(806ns) {DropIncomingHandshake:442(268ns) DropOutgoingHandshake:457(127ns)} GenerateSessionVirtualId:988(267ns) PrepareNewSessionHandshake:477(1564ns) {SwitchToState:179(638ns)}}}}}
2025-11-27T18:19:50.587030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471602352619945:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20331013ns) {Configure:104(20327456ns) {ProcessConfigured:135(1559684ns) {StartInitialHandshake:157(1557795ns) {DropHandshakes:470(1194ns) {DropIncomingHandshake:442(310ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(350494ns) {SwitchToState:179(1615ns)}}}}}
2025-11-27T18:19:50.586691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471600720798684:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19838285ns) {Configure:104(19834400ns) {ProcessConfigured:135(993602ns) {StartInitialHandshake:157(991905ns) {DropHandshakes:470(861ns) {DropIncomingHandshake:442(177ns) DropOutgoingHandshake:457(149ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:477(1450ns) {SwitchToState:179(653ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.592086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471603031375729:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38395
2025-11-27T18:19:50.592512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471603031375655:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20942729ns) {Configure:104(20938440ns) {ProcessConfigured:135(571204ns) {StartInitialHandshake:157(569401ns) {DropHandshakes:470(1160ns) {DropIncomingHandshake:442(263ns) DropOutgoingHandshake:457(160ns)} GenerateSessionVirtualId:988(523ns) PrepareNewSessionHandshake:477(1572ns) {SwitchToState:179(516ns)}}}}}
2025-11-27T18:19:50.594525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471603031375729:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.607625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471601533514576:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(26610782ns) {Configure:104(26602120ns) {ProcessConfigured:135(70236ns) {StartInitialHandshake:157(67344ns) {DropHandshakes:470(1575ns) {DropIncomingHandshake:442(455ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(674ns) PrepareNewSessionHandshake:477(2005ns) {SwitchToState:179(904ns)}}}}}
2025-11-27T18:19:50.607836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471603031375721:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25310852ns) {Configure:104(25308231ns) {ProcessConfigured:135(49831ns) {StartInitialHandshake:157(47161ns) {DropHandshakes:470(1501ns) {DropIncomingHandshake:442(333ns) DropOutgoingHandshake:457(182ns)} GenerateSessionVirtualId:988(454ns) PrepareNewSessionHandshake:477(1688ns) {SwitchToState:179(435ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.618602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471602971900407:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55798
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.618929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471602971900412:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55798
2025-11-27T18:19:50.615808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471602352619959:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9084
2025-11-27T18:19:50.611770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471604671576693:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30940648ns) {Configure:104(30933188ns) {ProcessConfigured:135(3083799ns) {StartInitialHandshake:157(3081121ns) {DropHandshakes:470(2092ns) {DropIncomingHandshake:442(456ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(976ns) PrepareNewSessionHandshake:477(2289ns) {SwitchToState:179(1345ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.630846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471604671576702:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-27T18:19:50.631190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471604671576709:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-27T18:19:50.631223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471604671576710:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.619000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471602971900413:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55798
2025-11-27T18:19:50.619060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471602971900414:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55798
2025-11-27T18:19:50.617646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471603031375738:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38395
2025-11-27T18:19:50.620586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471602352619960:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9084
2025-11-27T18:19:50.633825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471604671576702:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.617750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471603031375739:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38395
2025-11-27T18:19:50.624994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471602352619881:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(30927568ns) {Configure:104(30923203ns) {ProcessConfigured:135(59068ns) {StartInitialHandshake:157(56795ns) {DropHandshakes:470(2644ns) {DropIncomingHandshake:442(611ns) DropOutgoingHandshake:457(689ns)} GenerateSessionVirtualId:988(885ns) PrepareNewSessionHandshake:477(1858ns) {SwitchToState:179(583ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.640053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471602971900407:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.638684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471602352619963:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9084
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.646280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471601533514588:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64025
2025-11-27T18:19:50.649902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471603031375738:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.649343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471602352619953:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9084
2025-11-27T18:19:50.649991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471604671576706:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.649961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471601533514588:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.653401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471602971900397:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(52724995ns) {Configure:104(52716801ns) {ProcessConfigured:135(8565703ns) {StartInitialHandshake:157(8562222ns) {DropHandshakes:470(2487ns) {DropIncomingHandshake:442(661ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:477(519383ns) {SwitchToState:179(3263ns)}}}}}
2025-11-27T18:19:50.649929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471603031375740:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38395
2025-11-27T18:19:50.661437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471602352619959:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.657401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471604671576709:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.663383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471602971900412:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.668045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471602352619953:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.661629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471601533514511:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.668214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471602971900408:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55798
2025-11-27T18:19:50.661695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471604671576628:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.668205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471602352619952:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.665399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471604671576706:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.668249Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471602352619952:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.668319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471604671576629:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.668980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471602971900414:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.669171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471602971900408:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.669115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471600720798764:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(44092376ns) {Configure:104(44081973ns) {ProcessConfigured:135(372721ns) {StartInitialHandshake:157(369816ns) {DropHandshakes:470(1678ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(353ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:477(2217ns) {SwitchToState:179(858ns)}}}}}
2025-11-27T18:19:50.669173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471602352619963:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.670252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471604671576710:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.671270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471603031375728:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.671613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471604671576627:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.670186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471602352619950:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.671733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471602971900400:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.671772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471602971900400:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.672993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471602971900406:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.671308Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471603031375728:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.670216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471602352619950:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.673024Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471602971900406:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.674008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471602971900404:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.674036Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471602971900404:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.670288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471602352619880:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.671568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471603031375740:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.673062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471602352619950:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:50.674859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471602971900400:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:50.675581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471604671576716:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-27T18:19:50.673206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471602352619952:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:50.675808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471602971900404:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:50.675583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471604671576715:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-27T18:19:50.675670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471602352619946:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.675716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.671848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471603031375654:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.676214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471602352619945:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.676229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471600720798765:2117] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36355645ns) {Configure:104(36346210ns) {ProcessConfigured:135(1755770ns) {StartInitialHandshake:157(1753332ns) {DropHandshakes:470(108693ns) {DropIncomingHandshake:442(523ns) DropOutgoingHandshake:457(385ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:477(1910ns) {SwitchToState:179(680ns)}}}}}
2025-11-27T18:19:50.674047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471603031375728:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:50.676262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:50.676362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471602352619946:2108] [node 5] ICP22 created new session: [2:7577471602352619969:2108]
2025-11-27T18:19:50.676827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471604671576717:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-27T18:19:50.676896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471602971900413:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.675170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471603031375739:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.676819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471601533514590:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64025
2025-11-27T18:19:50.677287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471602971900406:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:50.675223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471603031375720:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.676970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471604671576709:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:50.677471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471602971900334:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.675308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:50.676539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471602352619945:2109] [node 6] ICP22 created new session: [2:7577471602352619971:2109]
2025-11-27T18:19:50.677231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471604671576706:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:50.677552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:50.675818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471603031375720:2108] [node 5] ICP22 created new session: [3:7577471603031375744:2108]
2025-11-27T18:19:50.676607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471602352619969:2108] [node 5] ICS09 handshake done sender: [2:7577471602352619950:2112] self: [2:7577471602352619948:0] peer: [5:7577471604671576712:0] socket: 254 qp: -1
2025-11-27T18:19:50.677710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471602971900333:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.676233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471603031375744:2108] [node 5] ICS09 handshake done sender: [3:7577471603031375728:2115] self: [3:7577471603031375725:0] peer: [5:7577471604671576713:0] socket: 238 qp: -1
2025-11-27T18:19:50.677751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471602971900334:2049] [node 3] ICP22 created new session: [1:7577471602971900424:2049]
2025-11-27T18:19:50.680883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471601533514588:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:50.679335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471604671576628:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:50.676275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471603031375744:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:50.676981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471602352619969:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:50.679383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:50.676534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:50.680194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471604671576628:2049] [node 2] ICP22 created new session: [5:7577471604671576718:2049]
2025-11-27T18:19:50.677015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:50.677808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471602971900424:2049] [node 3] ICS09 handshake done sender: [1:7577471602971900404:2112] self: [1:7577471602971900403:0] peer: [3:7577471603031375742:0] socket: 259 qp: -1
2025-11-27T18:19:50.682184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471603031375745:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38395
2025-11-27T18:19:50.682299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471601533514511:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:50.677023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471602352619971:2109] [node 6] ICS09 handshake done sender: [2:7577471602352619952:2113] self: [2:7577471602352619951:0] peer: [6:7577471601533514589:0] socket: 253 qp: -1
2025-11-27T18:19:50.677956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:50.680327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471604671576629:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64405) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471604671576706:2117] held: no
2025-11-27T18:19:50.682348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:50.677972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471602971900424:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:50.677295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471602352619971:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:50.682137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471604671576629:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.678854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:50.682702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471601533514511:2049] [node 2] ICP22 created new session: [6:7577471601533514592:2049]
2025-11-27T18:19:50.677328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:50.682178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:50.678991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471602971900333:2048] [node 2] ICP22 created new session: [1:7577471602971900425:2048]
2025-11-27T18:19:50.683637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471601533514592:2049] [node 2] ICS09 handshake done sender: [6:7577471601533514588:2115] self: [6:7577471601533514589:0] peer: [2:7577471602352619951:0] socket: 262 qp: -1
2025-11-27T18:19:50.684022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471602352619960:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.679431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471602971900425:2048] [node 2] ICS09 handshake done sender: [1:7577471602971900400:2110] self: [1:7577471602971900399:0] peer: [2:7577471602352619966:0] socket: 258 qp: -1
2025-11-27T18:19:50.682233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471604671576718:2049] [node 2] ICS09 handshake done sender: [5:7577471604671576709:2119] self: [5:7577471604671576712:0] peer: [2:7577471602352619948:0] socket: 264 qp: -1
2025-11-27T18:19:50.683679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471601533514592:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:50.680224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471602971900425:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:50.682268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471604671576718:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:50.683720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:50.680262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:50.686053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471603031375654:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63193) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471603031375740:2121] held: no
2025-11-27T18:19:50.682307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471604671576629:2050] [node 3] ICP22 created new session: [5:7577471604671576719:2050]
2025-11-27T18:19:50.682681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: 2025-11-27T18:19:50.686099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471602971900398:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.687024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471603031375740:2121] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:19:50.683338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471604671576719:2050] [node 3] ICS09 handshake done sender: [5:7577471604671576706:2117] self: [5:7577471604671576713:0] peer: [3:7577471603031375725:0] socket: 239 qp: -1
2025-11-27T18:19:50.686154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:50.685715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471601533514591:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64025
2025-11-27T18:19:50.683531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471604671576719:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:50.686842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471602971900398:2108] [node 5] ICP22 created new session: [1:7577471602971900428:2108]
2025-11-27T18:19:50.683790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:50.687335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471602971900428:2108] [node 5] ICS09 handshake done sender: [1:7577471602971900406:2113] self: [1:7577471602971900405:0] peer: [5:7577471604671576714:0] socket: 260 qp: -1
2025-11-27T18:19:50.687288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471602352619970:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9084
2025-11-27T18:19:50.687102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471604671576710:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:50.687481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471602971900428:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:50.687697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471601533514591:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.687729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471603031375654:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:50.687784Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471602352619880:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63193) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471602352619963:2119] held: no
2025-11-27T18:19:50.687530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.687931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:50.688323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471601533514512:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.688381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471603031375654:2048] [node 1] ICP22 created new session: [3:7577471603031375749:2048]
2025-11-27T18:19:50.689234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471604671576627:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:19:50.689308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
::ffff:127.1.0.0
2025-11-27T18:19:50.689087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471603031375749:2048] [node 1] ICS09 handshake done sender: [3:7577471603031375740:2121] self: [3:7577471603031375742:0] peer: [1:7577471602971900403:0] socket: 263 qp: -1
2025-11-27T18:19:50.689351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471604671576627:2048] [node 1] ICP22 created new session: [5:7577471604671576724:2048]
2025-11-27T18:19:50.689324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471603031375749:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.689077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471600720798779:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-27T18:19:50.689391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:50.690204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471604671576628:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.690293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471604671576724:2048] [node 1] ICS09 handshake done sender: [5:7577471604671576710:2120] self: [5:7577471604671576714:0] peer: [1:7577471602971900405:0] socket: 267 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.690772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471603031375731:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.689723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471600720798780:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-27T18:19:50.690332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471604671576724:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:50.690799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471603031375731:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.690286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471600720798779:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.692382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471600720798781:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-27T18:19:50.693882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471601533514596:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64025
2025-11-27T18:19:50.690744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:50.695761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471602352619963:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:50.693820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471600720798783:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-27T18:19:50.697903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471602352619880:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.698073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:50.700996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471603031375731:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:50.698122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471602352619880:2048] [node 1] ICP22 created new session: [2:7577471602352619977:2048]
2025-11-27T18:19:50.706148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471600720798780:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.705612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471601533514597:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64025
2025-11-27T18:19:50.700003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471602352619977:2048] [node 1] ICS09 handshake done sender: [2:7577471602352619963:2119] self: [2:7577471602352619966:0] peer: [1:7577471602971900399:0] socket: 265 qp: -1
2025-11-27T18:19:50.705629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471603031375751:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38395
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.700452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471602352619977:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:50.708107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471600720798684:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.709068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471603031375722:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.700496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:50.708440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471600720798783:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.709139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:50.709004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471601533514597:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.709941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471602971900402:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.708969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471600720798683:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.709482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471603031375722:2110] [node 6] ICP22 created new session: [3:7577471603031375754:2110]
2025-11-27T18:19:50.709762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471601533514510:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.709988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471602971900402:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.710424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471603031375754:2110] [node 6] ICS09 handshake done sender: [3:7577471603031375731:2117] self: [3:7577471603031375730:0] peer: [6:7577471601533514595:0] socket: 243 qp: -1
2025-11-27T18:19:50.709065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471602352619958:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.709585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471600720798784:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-27T18:19:50.710537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471603031375754:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:50.710508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471601533514598:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64025
2025-11-27T18:19:50.711085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471602971900416:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.709109Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471602352619958:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.711950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471601533514512:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64405) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471601533514591:2117] held: no
2025-11-27T18:19:50.711120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471602971900416:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.711329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:50.714318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471600720798784:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.711634Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471602352619958:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:50.716444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471600720798685:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.711678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471602971900402:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:50.715195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471602352619944:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.715181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471603031375751:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.716360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471602971900396:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.715259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:50.717052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:50.718207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471600720798684:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31779) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471600720798780:2134] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.715313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471602352619944:2107] [node 4] ICP22 created new session: [2:7577471602352619980:2107]
2025-11-27T18:19:50.719297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471603031375655:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.717046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471602352619980:2107] [node 4] ICS09 handshake done sender: [2:7577471602352619958:2115] self: [2:7577471602352619957:0] peer: [4:7577471600720798785:0] socket: 244 qp: -1
2025-11-27T18:19:50.718749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471602971900396:2107] [node 4] ICP22 created new session: [1:7577471602971900434:2107]
2025-11-27T18:19:50.718418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471602352619980:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:50.718471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:50.719758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471603031375737:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.719008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471602971900416:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:50.719630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471601533514600:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64025
2025-11-27T18:19:50.719088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471602971900434:2107] [node 4] ICS09 handshake done sender: [1:7577471602971900402:2111] self: [1:7577471602971900401:0] peer: [4:7577471600720798786:0] socket: 285 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:19:50.719802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471603031375737:2118] [node 4] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-27T18:19:50.721479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471601533514591:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:50.719338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471602971900434:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:50.724722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471603031375737:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:50.723173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471601533514597:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:50.724148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471600720798787:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-27T18:19:50.723920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471602352619962:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.719522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:50.724540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471600720798788:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-27T18:19:50.724158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471601533514512:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:50.723966Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471602352619962:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.720719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471602971900398:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(24341459ns) {HandleHandshakeStatus:329(24338558ns) {DropIncomingHandshake:442(12727ns) DropOutgoingHandshake:457(296ns) SwitchToState:179(508ns) ProcessPendingSessionEvents:495(22223370ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22091097ns max# 2450529ns}}}
2025-11-27T18:19:50.724868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471600720798683:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63193) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471600720798783:2136] held: no
2025-11-27T18:19:50.724359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471601533514510:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:50.721462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471602971900397:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.725272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471604671576627:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(29028682ns) {HandleHandshakeStatus:329(28898705ns) {DropIncomingHandshake:442(942ns) DropOutgoingHandshake:457(17531ns) SwitchToState:179(262ns) ProcessPendingSessionEvents:495(25448804ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25305059ns max# 4637565ns}}}
2025-11-27T18:19:50.725181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471603031375721:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.724421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:50.726192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471602352619962:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:50.721796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:50.722213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471602971900397:2109] [node 6] ICP22 created new session: [1:7577471602971900437:2109]
2025-11-27T18:19:50.725505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:50.724498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471601533514510:2048] [node 1] ICP22 created new session: [6:7577471601533514601:2048]
2025-11-27T18:19:50.727118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471603031375721:2107] [node 4] ICP22 created new session: [3:7577471603031375758:2107]
2025-11-27T18:19:50.727964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471603031375758:2107] [node 4] ICS09 handshake done sender: [3:7577471603031375737:2118] self: [3:7577471603031375736:0] peer: [4:7577471600720798789:0] socket: 242 qp: -1
2025-11-27T18:19:50.728146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471603031375758:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:50.728191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:50.724819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:50.723903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471602971900437:2109] [node 6] ICS09 handshake done sender: [1:7577471602971900416:2119] self: [1:7577471602971900415:0] peer: [6:7577471601533514599:0] socket: 286 qp: -1
2025-11-27T18:19:50.728639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471602352619881:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.725120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471601533514601:2048] [node 1] ICS09 handshake done sender: [6:7577471601533514597:2119] self: [6:7577471601533514599:0] peer: [1:7577471602971900415:0] socket: 288 qp: -1
2025-11-27T18:19:50.728701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:50.728542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471600720798781:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.724177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471602971900437:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:50.728917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471602352619881:2049] [node 3] ICP22 created new session: [2:7577471602352619983:2049]
2025-11-27T18:19:50.725136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471601533514512:2050] [node 3] ICP22 created new session: [6:7577471601533514602:2050]
2025-11-27T18:19:50.725113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:50.729066Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471600720798780:2134] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:50.725174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471601533514601:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:50.729312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471602352619983:2049] [node 3] ICS09 handshake done sender: [2:7577471602352619962:2118] self: [2:7577471602352619961:0] peer: [3:7577471603031375756:0] socket: 250 qp: -1
2025-11-27T18:19:50.730796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471604671576708:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.725180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471601533514602:2050] [node 3] ICS09 handshake done sender: [6:7577471601533514591:2117] self: [6:7577471601533514595:0] peer: [3:7577471603031375730:0] socket: 245 qp: -1
2025-11-27T18:19:50.730831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471604671576708:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.729545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471602352619983:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:50.725209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471601533514602:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:50.729746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:50.725242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:50.725489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:50.728795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471601533514596:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.729721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471601533514572:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.731313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471600720798684:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:50.731371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:50.732419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471600720798684:2049] [node 2] ICP22 created new session: [4:7577471600720798793:2049]
2025-11-27T18:19:50.732488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471600720798783:2136] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:50.734147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471600720798793:2049] [node 2] ICS09 handshake done sender: [4:7577471600720798780:2134] self: [4:7577471600720798785:0] peer: [2:7577471602352619957:0] socket: 257 qp: -1
2025-11-27T18:19:50.734188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471600720798793:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:50.734229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:50.737108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471603031375722:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(19450866ns) {HandleHandshakeStatus:329(19448462ns) {DropIncomingHandshake:442(13000ns) DropOutgoingHandshake:457(393ns) SwitchToState:179(546ns) ProcessPendingSessionEvents:495(13768796ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13657168ns max# 2365489ns}}}
No RDMA context found for GID: 2025-11-27T18:19:50.736824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471600720798685:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64405) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471600720798784:2137] held: no
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:19:50.737365Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471604671576708:2118] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:19:50.737209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471603031375655:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31779) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471603031375751:2123] held: no
2025-11-27T18:19:50.741386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471604671576727:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-27T18:19:50.741477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471600720798683:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:50.741738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:50.741787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471600720798683:2048] [node 1] ICP22 created new session: [4:7577471600720798796:2048]
2025-11-27T18:19:50.741948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471600720798796:2048] [node 1] ICS09 handshake done sender: [4:7577471600720798783:2136] self: [4:7577471600720798786:0] peer: [1:7577471602971900401:0] socket: 287 qp: -1
2025-11-27T18:19:50.741983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471600720798790:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26687
2025-11-27T18:19:50.741991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471600720798796:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:50.742034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:50.742164Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471602352619944:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16366652ns) {HandleHandshakeStatus:329(16363888ns) {DropIncomingHandshake:442(17760ns) DropOutgoingHandshake:457(329ns) SwitchToState:179(495ns) ProcessPendingSessionEvents:495(11809084ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11698999ns max# 1394516ns}}}
2025-11-27T18:19:50.742662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471604671576693:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.742719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:50.743253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471604671576693:2110] [node 6] ICP22 created new session: [5:7577471604671576728:2110]
2025-11-27T18:19:50.743758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471603031375760:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38395
2025-11-27T18:19:50.744051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471604671576728:2110] [node 6] ICS09 handshake done sender: [5:7577471604671576708:2118] self: [5:7577471604671576707:0] peer: [6:7577471601533514605:0] socket: 282 qp: -1
2025-11-27T18:19:50.744452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471604671576728:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:50.744502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:50.746368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471603031375751:2123] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:50.746431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471601533514510:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17795916ns) {HandleHandshakeStatus:329(17793276ns) {DropIncomingHandshake:442(527ns) DropOutgoingHandshake:457(4660ns) SwitchToState:179(356ns) ProcessPendingSessionEvents:495(14190671ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12449831ns max# 4146295ns}}}
2025-11-27T18:19:50.747062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471600720798784:2137] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:50.747251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471603031375655:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.756426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471601533514606:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64025
2025-11-27T18:19:50.747355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:50.750447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471600720798685:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:50.750495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:50.755533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471600720798685:2050] [node 3] ICP22 created new session: [4:7577471600720798798:2050]
2025-11-27T18:19:50.747868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471603031375655:2049] [node 2] ICP22 created new session: [3:7577471603031375761:2049]
2025-11-27T18:19:50.757105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471600720798798:2050] [node 3] ICS09 handshake done sender: [4:7577471600720798784:2137] self: [4:7577471600720798789:0] peer: [3:7577471603031375736:0] socket: 289 qp: -1
2025-11-27T18:19:50.748475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471603031375761:2049] [node 2] ICS09 handshake done sender: [3:7577471603031375751:2123] self: [3:7577471603031375756:0] peer: [2:7577471602352619961:0] socket: 248 qp: -1
2025-11-27T18:19:50.757275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471600720798798:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:50.757317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.753357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471603031375761:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:50.753418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:50.758232Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471601533514572:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1416) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471601533514596:2118] held: no
2025-11-27T18:19:50.758484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471601533514607:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64025
2025-11-27T18:19:50.758888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471601533514596:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:50.759827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471601533514572:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:50.759881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:50.760373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471601533514572:2107] [node 5] ICP22 created new session: [6:7577471601533514608:2107]
2025-11-27T18:19:50.760445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471601533514608:2107] [node 5] ICS09 handshake done sender: [6:7577471601533514596:2118] self: [6:7577471601533514605:0] peer: [5:7577471604671576707:0] socket: 284 qp: -1
2025-11-27T18:19:50.760567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471601533514608:2107] [node 5] ICS10 traffic start
2025-11-27T18:19:50.760621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:50.760896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471604671576727:2124] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.761069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471604671576692:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.761370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471600720798776:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.761404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471600720798776:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.761706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471601533514607:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.761840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471601533514576:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:50.762053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471600720798778:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:50.762097Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471600720798778:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:50.762255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471600720798776:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:50.762701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471600720798764:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.762791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:50.762878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471600720798764:2108] [node 5] ICP22 created new session: [4:7577471600720798802:2108]
2025-11-27T18:19:50.762936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471600720798802:2108] [node 5] ICS09 handshake done sender: [4:7577471600720798776:2131] self: [4:7577471600720798775:0] peer: [5:7577471604671576730:0] socket: 233 qp: -1
2025-11-27T18:19:50.762965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471600720798802:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:50.763006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:50.762936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471604671576731:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-27T18:19:50.763070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471600720798778:2132] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:50.763183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471604671576727:2124] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:50.763319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471601533514611:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64025
2025-11-27T18:19:50.763762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471604671576692:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471604671576727:2124] held: no
2025-11-27T18:19:50.763624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471600720798765:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:50.763670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:50.763911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471601533514607:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:50.763712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471600720798765:2117] [node 6] ICP22 created new session: [4:7577471600720798804:2117]
2025-11-27T18:19:50.763943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471600720798804:2117] [node 6] ICS09 handshake done sender: [4:7577471600720798778:2132] self: [4:7577471600720798777:0] peer: [6:7577471601533514610:0] socket: 256 qp: -1
2025-11-27T18:19:50.765100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471601533514576:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471601533514607:2123] held: no
2025-11-27T18:19:50.764030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471600720798804:2117] [node 6] ICS10 traffic start
2025-11-27T18:19:50.764085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:50.766934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471604671576692:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:50.767028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:50.767100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471604671576692:2108] [node 4] ICP22 created new session: [5:7577471604671576732:2108]
2025-11-27T18:19:50.767170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471604671576732:2108] [node 4] ICS09 handshake done sender: [5:7577471604671576727:2124] self: [5:7577471604671576730:0] peer: [4:7577471600720798775:0] socket: 241 qp: -1
2025-11-27T18:19:50.767206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471604671576732:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:50.767258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471601533514576:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:50.767297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:50.767339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471601533514576:2110] [node 4] ICP22 created new session: [6:7577471601533514612:2110]
2025-11-27T18:19:50.767379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471601533514612:2110] [node 4] ICS09 handshake done sender: [6:7577471601533514607:2123] self: [6:7577471601533514610:0] peer: [4:7577471600720798777:0] socket: 296 qp: -1
2025-11-27T18:19:50.767420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471601533514612:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:50.767452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:50.767244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:50.824586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:50.824845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471602352619977:2048] [node 1] ICS01 socket: 265 reason# EndOfStream
2025-11-27T18:19:50.824896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471602352619880:2048] [node 1] ICP30 unregister session Session# [2:7577471602352619977:2048] VirtualId# [2:7577471602352619966:0]
2025-11-27T18:19:50.824927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471602352619977:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.824996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:50.825045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:50.826095Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:50.826332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471603031375749:2048] [node 1] ICS01 socket: 263 reason# EndOfStream
2025-11-27T18:19:50.826377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471603031375654:2048] [node 1] ICP30 unregister session Session# [3:7577471603031375749:2048] VirtualId# [3:7577471603031375742:0]
2025-11-27T18:19:50.826420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471603031375749:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.826501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:50.826554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:50.828996Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:50.829213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471600720798796:2048] [node 1] ICS01 socket: 287 reason# EndOfStream
2025-11-27T18:19:50.829284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471600720798683:2048] [node 1] ICP30 unregister session Session# [4:7577471600720798796:2048] VirtualId# [4:7577471600720798786:0]
2025-11-27T18:19:50.829321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471600720798796:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.829407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:50.829473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:50.830997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:50.831208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471601533514601:2048] [node 1] ICS07 socket disconnect 288 reason# EndOfStream
2025-11-27T18:19:50.831259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471601533514601:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.831334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:50.831373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471601533514601:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:50.830053Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:50.830330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471604671576724:2048] [node 1] ICS01 socket: 267 reason# EndOfStream
2025-11-27T18:19:50.830402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471604671576627:2048] [node 1] ICP30 unregister session Session# [5:7577471604671576724:2048] VirtualId# [5:7577471604671576714:0]
2025-11-27T18:19:50.830470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471604671576724:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.830552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:50.830608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:50.832767Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471601533514510:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63193) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:50.832830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471601533514601:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:50.832886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471601533514510:2048] [node 1] ICP30 unregister session Session# [6:7577471601533514601:2048] VirtualId# [6:7577471601533514599:0]
2025-11-27T18:19:50.833023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:50.833073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471601533514510:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63193) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.833090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63193) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.842749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471601533514510:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:50.904143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:50.904363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471603031375761:2049] [node 2] ICS01 socket: 248 reason# EndOfStream
2025-11-27T18:19:50.904421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471603031375655:2049] [node 2] ICP30 unregister session Session# [3:7577471603031375761:2049] VirtualId# [3:7577471603031375756:0]
2025-11-27T18:19:50.904476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471603031375761:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.904553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:50.904591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:50.907499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:50.907784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471600720798793:2049] [node 2] ICS01 socket: 257 reason# EndOfStream
2025-11-27T18:19:50.907846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471600720798684:2049] [node 2] ICP30 unregister session Session# [4:7577471600720798793:2049] VirtualId# [4:7577471600720798785:0]
2025-11-27T18:19:50.907885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471600720798793:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.907808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:50.907989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:50.908049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471604671576718:2049] [node 2] ICS07 socket disconnect 264 reason# EndOfStream
2025-11-27T18:19:50.908053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:50.908085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471604671576718:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.908145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:50.908189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471604671576718:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:50.908826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:50.909097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471601533514592:2049] [node 2] ICS07 socket disconnect 262 reason# EndOfStream
2025-11-27T18:19:50.909115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471601533514592:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.909182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:50.909122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471604671576628:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31779) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:50.909235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471601533514592:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:50.909210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471604671576718:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:50.909247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471604671576628:2049] [node 2] ICP30 unregister session Session# [5:7577471604671576718:2049] VirtualId# [5:7577471604671576712:0]
2025-11-27T18:19:50.909375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:50.909412Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471604671576628:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.909428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.910049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471601533514511:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31779) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:50.910174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471601533514592:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:50.910211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471601533514511:2049] [node 2] ICP30 unregister session Session# [6:7577471601533514592:2049] VirtualId# [6:7577471601533514589:0]
2025-11-27T18:19:50.910294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:50.910355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471601533514511:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.910370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:50.919547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471604671576628:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:50.920477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471601533514511:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:50.960010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:50.960218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471600720798798:2050] [node 3] ICS01 socket: 289 reason# EndOfStream
2025-11-27T18:19:50.960263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471600720798685:2050] [node 3] ICP30 unregister session Session# [4:7577471600720798798:2050] VirtualId# [4:7577471600720798789:0]
2025-11-27T18:19:50.960295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471600720798798:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.960389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:50.960454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:50.960994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:50.961227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471604671576719:2050] [node 3] ICS01 socket: 239 reason# EndOfStream
2025-11-27T18:19:50.961297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471604671576629:2050] [node 3] ICP30 unregister session Session# [5:7577471604671576719:2050] VirtualId# [5:7577471604671576713:0]
2025-11-27T18:19:50.961332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471604671576719:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.961404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:50.961458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:50.961988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:50.962126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471601533514602:2050] [node 3] ICS01 socket: 245 reason# EndOfStream
2025-11-27T18:19:50.962161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471601533514512:2050] [node 3] ICP30 unregister session Session# [6:7577471601533514602:2050] VirtualId# [6:7577471601533514595:0]
2025-11-27T18:19:50.962178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471601533514602:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:50.962239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:50.962275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:51.027834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:51.028056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471604671576732:2108] [node 4] ICS01 socket: 241 reason# EndOfStream
2025-11-27T18:19:51.028099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471604671576692:2108] [node 4] ICP30 unregister session Session# [5:7577471604671576732:2108] VirtualId# [5:7577471604671576730:0]
2025-11-27T18:19:51.028130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471604671576732:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.028197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:51.028235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:51.028811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:51.029030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471601533514612:2110] [node 4] ICS01 socket: 296 reason# EndOfStream
2025-11-27T18:19:51.029071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471601533514576:2110] [node 4] ICP30 unregister session Session# [6:7577471601533514612:2110] VirtualId# [6:7577471601533514610:0]
2025-11-27T18:19:51.029117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471601533514612:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.029195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:51.029245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:51.073866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:51.074114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471601533514608:2107] [node 5] ICS01 socket: 284 reason# EndOfStream
2025-11-27T18:19:51.074161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471601533514572:2107] [node 5] ICP30 unregister session Session# [6:7577471601533514608:2107] VirtualId# [6:7577471601533514605:0]
2025-11-27T18:19:51.074209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471601533514608:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.074304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:51.074353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:51.182141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471608313148128:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:51.182356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471608313148129:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:51.204243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471607361972149:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:51.204286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471607361972150:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:51.241180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471605486321083:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:51.241477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471605486321082:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:51.269593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471608531892154:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:51.269677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471608531892155:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:51.269734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471608531892156:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:51.285208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471606560245037:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:51.285289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471606560245038:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:51.285355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471606560245039:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:51.310249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471606470501203:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:51.310317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471606470501204:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:51.310477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471606470501205:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:51.432853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471608313148193:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:51.447787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471608313148192:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:51.476262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471606560245100:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:51.475194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471606470501264:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:51.468955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471605486321154:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:51.475208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471608531892217:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:51.477982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471607361972212:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:51.475513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471605486321155:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:51.478402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471608531892218:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:51.479239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471607361972213:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:51.461614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471608313148191:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:51.479986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471607361972214:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:51.480073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471606470501267:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:51.479279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471605486321156:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:51.480413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471606560245101:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:51.485902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471608313148129:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(98815727ns) {Configure:104(98805220ns) {ProcessConfigured:135(523215ns) {StartInitialHandshake:157(521014ns) {DropHandshakes:470(2314ns) {DropIncomingHandshake:442(576ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(1123ns) PrepareNewSessionHandshake:477(2055ns) {SwitchToState:179(570ns)}}}}}
2025-11-27T18:19:51.498441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471605486321083:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19222502ns) {Configure:104(19212775ns) {ProcessConfigured:135(478240ns) {StartInitialHandshake:157(476489ns) {DropHandshakes:470(2063ns) {DropIncomingHandshake:442(621ns) DropOutgoingHandshake:457(283ns)} GenerateSessionVirtualId:988(742ns) PrepareNewSessionHandshake:477(1203ns) {SwitchToState:179(537ns)}}}}}
2025-11-27T18:19:51.499092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471608531892156:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16812981ns) {Configure:104(16804064ns) {ProcessConfigured:135(228756ns) {StartInitialHandshake:157(226901ns) {DropHandshakes:470(1042ns) {DropIncomingHandshake:442(316ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(488ns) PrepareNewSessionHandshake:477(870ns) {SwitchToState:179(148ns)}}}}}
2025-11-27T18:19:51.500066Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471607361972149:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17317736ns) {Configure:104(17313704ns) {ProcessConfigured:135(795569ns) {StartInitialHandshake:157(793426ns) {DropHandshakes:470(1477ns) {DropIncomingHandshake:442(613ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(555ns) PrepareNewSessionHandshake:477(1433ns) {SwitchToState:179(510ns)}}}}}
2025-11-27T18:19:51.500426Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471606470501203:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17659811ns) {Configure:104(17655360ns) {ProcessConfigured:135(451612ns) {StartInitialHandshake:157(449901ns) {DropHandshakes:470(757ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(242ns) PrepareNewSessionHandshake:477(1342ns) {SwitchToState:179(565ns)}}}}}
2025-11-27T18:19:51.500406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471608531892154:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29090115ns) {Configure:104(29080043ns) {ProcessConfigured:135(1072497ns) {StartInitialHandshake:157(1070966ns) {DropHandshakes:470(1347ns) {DropIncomingHandshake:442(366ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:477(1508ns) {SwitchToState:179(736ns)}}}}}
2025-11-27T18:19:51.500392Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471605486321082:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17974247ns) {Configure:104(17970042ns) {ProcessConfigured:135(958308ns) {StartInitialHandshake:157(957274ns) {DropHandshakes:470(926ns) {DropIncomingHandshake:442(291ns) DropOutgoingHandshake:457(168ns)} GenerateSessionVirtualId:988(502ns) PrepareNewSessionHandshake:477(1305ns) {SwitchToState:179(418ns)}}}}}
2025-11-27T18:19:51.512769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471608531892155:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18823814ns) {Configure:104(18820389ns) {ProcessConfigured:135(986585ns) {StartInitialHandshake:157(984679ns) {DropHandshakes:470(1103ns) {DropIncomingHandshake:442(298ns) DropOutgoingHandshake:457(368ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(1357ns) {SwitchToState:179(319ns)}}}}}
2025-11-27T18:19:51.500613Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471607361972150:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18208522ns) {Configure:104(18205220ns) {ProcessConfigured:135(997394ns) {StartInitialHandshake:157(996032ns) {DropHandshakes:470(650ns) {DropIncomingHandshake:442(191ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(129ns) PrepareNewSessionHandshake:477(1327ns) {SwitchToState:179(558ns)}}}}}
2025-11-27T18:19:51.505494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471606470501204:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16537064ns) {Configure:104(16531504ns) {ProcessConfigured:135(633337ns) {StartInitialHandshake:157(631864ns) {DropHandshakes:470(1324ns) {DropIncomingHandshake:442(350ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(415ns) PrepareNewSessionHandshake:477(1006ns) {SwitchToState:179(112ns)}}}}}
2025-11-27T18:19:51.514398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471608313148191:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17785785ns) {Configure:104(17779507ns) {ProcessConfigured:135(144072ns) {StartInitialHandshake:157(141537ns) {DropHandshakes:470(1373ns) {DropIncomingHandshake:442(316ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(432ns) PrepareNewSessionHandshake:477(1307ns) {SwitchToState:179(570ns)}}}}}
2025-11-27T18:19:51.513697Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471606470501264:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18038375ns) {Configure:104(18033727ns) {ProcessConfigured:135(2087203ns) {StartInitialHandshake:157(2085641ns) {DropHandshakes:470(1250ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(177ns) PrepareNewSessionHandshake:477(1668ns) {SwitchToState:179(882ns)}}}}}
2025-11-27T18:19:51.514192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471606560245037:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30630844ns) {Configure:104(30625304ns) {ProcessConfigured:135(11150190ns) {StartInitialHandshake:157(11148046ns) {DropHandshakes:470(497ns) {DropIncomingHandshake:442(142ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(2032ns) {SwitchToState:179(899ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.514766Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471606470501267:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25367940ns) {Configure:104(25359866ns) {ProcessConfigured:135(11264836ns) {StartInitialHandshake:157(11262398ns) {DropHandshakes:470(1144ns) {DropIncomingHandshake:442(340ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(441ns) PrepareNewSessionHandshake:477(2508ns) {SwitchToState:179(1078ns)}}}}}
2025-11-27T18:19:51.514529Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471606560245100:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27584883ns) {Configure:104(27580064ns) {ProcessConfigured:135(7754107ns) {StartInitialHandshake:157(7752461ns) {DropHandshakes:470(1269ns) {DropIncomingHandshake:442(270ns) DropOutgoingHandshake:457(161ns)} GenerateSessionVirtualId:988(543ns) PrepareNewSessionHandshake:477(3554ns) {SwitchToState:179(1589ns)}}}}}
2025-11-27T18:19:51.521808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471605486321154:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25625148ns) {Configure:104(25620547ns) {ProcessConfigured:135(968481ns) {StartInitialHandshake:157(965902ns) {DropHandshakes:470(1112ns) {DropIncomingHandshake:442(298ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(539ns) PrepareNewSessionHandshake:477(2122ns) {SwitchToState:179(1293ns)}}}}}
2025-11-27T18:19:51.514998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471606560245039:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31192806ns) {Configure:104(31188296ns) {ProcessConfigured:135(3854922ns) {StartInitialHandshake:157(3853105ns) {DropHandshakes:470(1386ns) {DropIncomingHandshake:442(348ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(520ns) PrepareNewSessionHandshake:477(1462ns) {SwitchToState:179(580ns)}}}}}
2025-11-27T18:19:51.515618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471606560245038:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(33417919ns) {Configure:104(33414655ns) {ProcessConfigured:135(1542992ns) {StartInitialHandshake:157(1541109ns) {DropHandshakes:470(1446ns) {DropIncomingHandshake:442(430ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(408ns) PrepareNewSessionHandshake:477(502988ns) {SwitchToState:179(1345ns)}}}}}
2025-11-27T18:19:51.522178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471607361972225:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42510
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.528134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471607361972225:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.528779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471605486321165:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14396
2025-11-27T18:19:51.528947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471606470501276:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31780
2025-11-27T18:19:51.528779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471607361972149:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.529620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471608313148198:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.529114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471605486321166:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14396
2025-11-27T18:19:51.528976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471606470501205:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26755563ns) {Configure:104(26748480ns) {ProcessConfigured:135(1430365ns) {StartInitialHandshake:157(1427925ns) {DropHandshakes:470(2997ns) {DropIncomingHandshake:442(678ns) DropOutgoingHandshake:457(580ns)} GenerateSessionVirtualId:988(599ns) PrepareNewSessionHandshake:477(4018ns) {SwitchToState:179(1943ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.530230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471606560245110:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56665
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.529668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471608313148198:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.531545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471608531892230:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57465
2025-11-27T18:19:51.531584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471606470501281:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31780
2025-11-27T18:19:51.531040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471605486321165:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.535701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471607361972227:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42510
2025-11-27T18:19:51.533908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471605486321166:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.535926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471608313148207:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15347
2025-11-27T18:19:51.536171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471608531892230:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.538072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471608313148198:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:19:51.536278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471606470501276:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:19:51.539226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471608531892155:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:51.537847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471606560245110:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.539547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471608313148209:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15347
2025-11-27T18:19:51.540727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471606560245037:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:51.540954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471608531892231:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57465
2025-11-27T18:19:51.540238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471605486321082:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-27T18:19:51.539556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471608313148208:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15347
::ffff:127.1.0.0
2025-11-27T18:19:51.541134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471607361972217:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.541141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471606470501284:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31780
2025-11-27T18:19:51.541039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471608313148210:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15347
2025-11-27T18:19:51.541044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471605486321083:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.541176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471607361972217:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.541224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471606470501204:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:51.542664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471605486321171:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14396
2025-11-27T18:19:51.542671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471607361972229:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42510
2025-11-27T18:19:51.542861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471608313148207:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.543002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471605486321170:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14396
2025-11-27T18:19:51.543381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471607361972232:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42510
2025-11-27T18:19:51.543725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471607361972217:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:51.542611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471606470501283:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31780
2025-11-27T18:19:51.542869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471608313148199:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.543398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471608531892231:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.543711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471605486321171:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.543758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471607361972222:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.542750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471606470501281:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.545064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471606560245107:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.544511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471608531892224:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.544538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471605486321170:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.543243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471606470501283:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.545097Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471606560245107:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.542905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471608313148199:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.543780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471607361972222:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.544539Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471608531892224:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.543848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471607361972221:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.542921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471608313148128:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.549626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471608531892224:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.543760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471606470501267:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:51.543872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471607361972221:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-27T18:19:51.550792Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471606560245107:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:51.543764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471606470501264:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-27T18:19:51.543050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.549828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471607361972222:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:51.553497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471606560245100:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.550816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471606470501284:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.551210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471607361972221:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:51.557482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471608531892235:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57465
2025-11-27T18:19:51.558125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:51.543208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471608313148209:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.551409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471606470501205:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:51.557460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471605486321162:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.559573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471606560245100:2109] [node 6] ICP22 created new session: [5:7577471606560245119:2109]
2025-11-27T18:19:51.559512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471608531892218:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.557506Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471605486321162:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.543443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471608313148128:2048] [node 2] ICP22 created new session: [1:7577471608313148212:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.559593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.543743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471608313148208:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.544142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471608313148212:2048] [node 2] ICS09 handshake done sender: [1:7577471608313148198:2111] self: [1:7577471608313148197:0] peer: [2:7577471607361972228:0] socket: 234 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.544142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471608313148195:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.544173Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471608313148195:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.560381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471606470501289:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31780
2025-11-27T18:19:51.560326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471605486321179:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14396
2025-11-27T18:19:51.544547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471608313148212:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:51.560435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471606470501291:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31780
2025-11-27T18:19:51.560391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471605486321178:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14396
2025-11-27T18:19:51.545064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:51.550188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471608313148199:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:51.551379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471607361972213:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.560652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471606560245117:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56665
2025-11-27T18:19:51.552499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:51.560668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471608531892218:2109] [node 6] ICP22 created new session: [4:7577471608531892238:2109]
2025-11-27T18:19:51.550478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471608313148195:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:51.560850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471605486321162:2121] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:51.560722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471606560245119:2109] [node 6] ICS09 handshake done sender: [5:7577471606560245107:2111] self: [5:7577471606560245103:0] peer: [6:7577471606470501286:0] socket: 266 qp: -1
2025-11-27T18:19:51.552760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471607361972213:2107] [node 4] ICP22 created new session: [2:7577471607361972236:2107]
2025-11-27T18:19:51.560949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471606560245119:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:51.558319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471608313148192:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.553470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471607361972227:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.560980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:51.557507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471607361972236:2107] [node 4] ICS09 handshake done sender: [2:7577471607361972217:2110] self: [2:7577471607361972215:0] peer: [4:7577471608531892234:0] socket: 241 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.558777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471608313148129:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.557510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471607361972150:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.559093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:51.561424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471606470501288:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31780
2025-11-27T18:19:51.563773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471605486321166:2124] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:51.561467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471608531892238:2109] [node 6] ICS09 handshake done sender: [4:7577471608531892224:2112] self: [4:7577471608531892222:0] peer: [6:7577471606470501287:0] socket: 251 qp: -1
2025-11-27T18:19:51.563690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471606470501292:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31780
2025-11-27T18:19:51.562172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471608531892238:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:51.563824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471605486321155:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.559066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:51.559553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:51.567006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471606560245037:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471606560245110:2114] held: no
2025-11-27T18:19:51.559312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471607361972236:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:51.562810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471608531892230:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:51.564934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:51.559505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471607361972229:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.563711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:51.559629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471607361972150:2049] [node 3] ICP22 created new session: [2:7577471607361972237:2049]
2025-11-27T18:19:51.566009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471605486321155:2109] [node 6] ICP22 created new session: [3:7577471605486321181:2109]
2025-11-27T18:19:51.564928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471608531892155:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.560325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471607361972212:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.564972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:51.567037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471605486321083:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.560329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471607361972237:2049] [node 3] ICS09 handshake done sender: [2:7577471607361972222:2113] self: [2:7577471607361972220:0] peer: [3:7577471605486321173:0] socket: 238 qp: -1
2025-11-27T18:19:51.568135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471606470501289:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.560258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471608313148129:2049] [node 3] ICP22 created new session: [1:7577471608313148220:2049]
2025-11-27T18:19:51.568526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471606560245110:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.568371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471606470501267:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31200) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471606470501281:2115] held: no
2025-11-27T18:19:51.560361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:51.560311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471608313148220:2049] [node 3] ICS09 handshake done sender: [1:7577471608313148195:2110] self: [1:7577471608313148194:0] peer: [3:7577471605486321172:0] socket: 235 qp: -1
2025-11-27T18:19:51.571318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471606560245122:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56665
2025-11-27T18:19:51.568111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:51.566947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471608531892155:2049] [node 2] ICP22 created new session: [4:7577471608531892241:2049]
2025-11-27T18:19:51.560366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471607361972237:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:51.560357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471608313148220:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:51.568335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471605486321181:2109] [node 6] ICS09 handshake done sender: [3:7577471605486321162:2121] self: [3:7577471605486321161:0] peer: [6:7577471606470501290:0] socket: 269 qp: -1
2025-11-27T18:19:51.569372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471606470501276:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:51.568107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471608531892241:2049] [node 2] ICS09 handshake done sender: [4:7577471608531892230:2114] self: [4:7577471608531892234:0] peer: [2:7577471607361972215:0] socket: 247 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.569536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471606470501281:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:51.560409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:51.568337Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471605486321165:2123] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:51.560355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471608313148192:2108] [node 5] ICP22 created new session: [1:7577471608313148221:2108]
2025-11-27T18:19:51.568378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471608531892241:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:51.568435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471605486321083:2049] [node 2] ICP22 created new session: [3:7577471605486321182:2049]
2025-11-27T18:19:51.560404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:51.560414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471607361972212:2109] [node 6] ICP22 created new session: [2:7577471607361972240:2109]
2025-11-27T18:19:51.568583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.568722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471605486321182:2049] [node 2] ICS09 handshake done sender: [3:7577471605486321166:2124] self: [3:7577471605486321173:0] peer: [2:7577471607361972220:0] socket: 244 qp: -1
2025-11-27T18:19:51.570985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471606470501283:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:51.560418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471608313148221:2108] [node 5] ICS09 handshake done sender: [1:7577471608313148199:2112] self: [1:7577471608313148196:0] peer: [5:7577471606560245114:0] socket: 236 qp: -1
2025-11-27T18:19:51.573671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471606560245037:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.568866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471605486321181:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:51.571088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471606470501204:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.574235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: 2025-11-27T18:19:51.568917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
::ffff:127.1.0.0
2025-11-27T18:19:51.560458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471608313148221:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:51.574421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471606560245037:2048] [node 1] ICP22 created new session: [5:7577471606560245125:2048]
2025-11-27T18:19:51.560489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:51.562528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471608313148210:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.571143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:51.560456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:51.568954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471605486321182:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.560466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471607361972240:2109] [node 6] ICS09 handshake done sender: [2:7577471607361972221:2112] self: [2:7577471607361972218:0] peer: [6:7577471606470501285:0] socket: 237 qp: -1
2025-11-27T18:19:51.572726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471606470501204:2049] [node 2] ICP22 created new session: [6:7577471606470501294:2049]
2025-11-27T18:19:51.576358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471608531892244:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57465
2025-11-27T18:19:51.576409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471606560245125:2048] [node 1] ICS09 handshake done sender: [5:7577471606560245110:2114] self: [5:7577471606560245114:0] peer: [1:7577471608313148196:0] socket: 246 qp: -1
2025-11-27T18:19:51.573054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471608313148128:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(19917650ns) {HandleHandshakeStatus:329(19915214ns) {DropIncomingHandshake:442(220ns) DropOutgoingHandshake:457(414ns) SwitchToState:179(302ns) ProcessPendingSessionEvents:495(13298029ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13206137ns max# 4788881ns}}}
2025-11-27T18:19:51.568983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:51.560494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471607361972240:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:51.578686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471606560245125:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:51.573733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471608313148224:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15347
2025-11-27T18:19:51.560533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:51.572071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471605486321082:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.573393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471606470501294:2049] [node 2] ICS09 handshake done sender: [6:7577471606470501276:2114] self: [6:7577471606470501285:0] peer: [2:7577471607361972218:0] socket: 245 qp: -1
2025-11-27T18:19:51.581511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:51.582439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471608531892244:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.573561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471606470501294:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:51.572441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:51.563339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471607361972225:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:51.574293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:51.582412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471606560245123:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56665
2025-11-27T18:19:51.573027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471605486321082:2048] [node 1] ICP22 created new session: [3:7577471605486321185:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.572419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471607361972149:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.574965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471606470501203:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:51.586148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471606560245123:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.573614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471605486321185:2048] [node 1] ICS09 handshake done sender: [3:7577471605486321165:2123] self: [3:7577471605486321172:0] peer: [1:7577471608313148194:0] socket: 242 qp: -1
2025-11-27T18:19:51.586574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471606560245101:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:51.573653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471605486321185:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:51.574980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471606470501267:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.573683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:51.575027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:51.575069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471606470501267:2109] [node 4] ICP22 created new session: [6:7577471606470501297:2109]
2025-11-27T18:19:51.572459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:51.575113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471606470501297:2109] [node 4] ICS09 handshake done sender: [6:7577471606470501281:2115] self: [6:7577471606470501287:0] peer: [4:7577471608531892222:0] socket: 253 qp: -1
2025-11-27T18:19:51.586800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471608531892154:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:51.573374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471607361972149:2048] [node 1] ICP22 created new session: [2:7577471607361972246:2048]
2025-11-27T18:19:51.576766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471606470501293:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31780
2025-11-27T18:19:51.587271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471608531892245:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57465
2025-11-27T18:19:51.587653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471608313148206:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.577079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471606470501297:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:51.573737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471607361972246:2048] [node 1] ICS09 handshake done sender: [2:7577471607361972225:2114] self: [2:7577471607361972228:0] peer: [1:7577471608313148197:0] socket: 239 qp: -1
2025-11-27T18:19:51.587698Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471608313148206:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.587317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471608531892233:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.577125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:51.575154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471607361972246:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:51.587342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471608531892233:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.588407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471608313148204:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.581617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471606470501264:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.576387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:51.588432Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471608313148204:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.582463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:51.589744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471608313148206:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:51.590884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471606560245100:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(29552465ns) {HandleHandshakeStatus:329(29549567ns) {DropIncomingHandshake:442(240ns) DropOutgoingHandshake:457(563ns) SwitchToState:179(360ns) ProcessPendingSessionEvents:495(19116323ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17468635ns max# 4129371ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.591677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471607361972212:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(23375238ns) {HandleHandshakeStatus:329(23372696ns) {DropIncomingHandshake:442(246ns) DropOutgoingHandshake:457(222ns) SwitchToState:179(407ns) ProcessPendingSessionEvents:495(20217637ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18341710ns max# 1368748ns}}}
2025-11-27T18:19:51.585895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471606470501264:2106] [node 5] ICP22 created new session: [6:7577471606470501299:2106]
2025-11-27T18:19:51.590430Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471608313148129:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(24367236ns) {HandleHandshakeStatus:329(24364068ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(273ns) SwitchToState:179(429ns) ProcessPendingSessionEvents:495(22567810ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20940582ns max# 2236242ns}}}
2025-11-27T18:19:51.592077Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471607361972213:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(35004289ns) {HandleHandshakeStatus:329(35001793ns) {DropIncomingHandshake:442(432ns) DropOutgoingHandshake:457(521ns) SwitchToState:179(140ns) ProcessPendingSessionEvents:495(25393178ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23242099ns max# 2701617ns}}}
2025-11-27T18:19:51.586265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471606470501299:2106] [node 5] ICS09 handshake done sender: [6:7577471606470501283:2117] self: [6:7577471606470501286:0] peer: [5:7577471606560245103:0] socket: 271 qp: -1
2025-11-27T18:19:51.590948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471608313148204:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:51.586303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471606470501299:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:51.592162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471608313148191:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:19:51.592448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471606560245128:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56665
2025-11-27T18:19:51.586342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
::ffff:127.1.0.0
2025-11-27T18:19:51.592213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:51.588329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471606470501284:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:51.592487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471608313148191:2107] [node 4] ICP22 created new session: [1:7577471608313148228:2107]
2025-11-27T18:19:51.590670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471606470501205:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.590713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:51.592554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471608313148228:2107] [node 4] ICS09 handshake done sender: [1:7577471608313148206:2114] self: [1:7577471608313148205:0] peer: [4:7577471608531892246:0] socket: 301 qp: -1
2025-11-27T18:19:51.591925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471608531892245:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.592590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471608313148228:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:51.593290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471607361972150:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(29610038ns) {HandleHandshakeStatus:329(29607512ns) {DropIncomingHandshake:442(225ns) DropOutgoingHandshake:457(1471ns) SwitchToState:179(425ns) ProcessPendingSessionEvents:495(26563542ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26429440ns max# 2140451ns}}}
2025-11-27T18:19:51.591889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471606470501205:2050] [node 3] ICP22 created new session: [6:7577471606470501302:2050]
2025-11-27T18:19:51.592667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471608531892218:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(28467620ns) {HandleHandshakeStatus:329(27585983ns) {DropIncomingHandshake:442(16094ns) DropOutgoingHandshake:457(278ns) SwitchToState:179(418ns) ProcessPendingSessionEvents:495(20304704ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20199435ns max# 2140796ns}}}
2025-11-27T18:19:51.593419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:51.592933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471606470501302:2050] [node 3] ICS09 handshake done sender: [6:7577471606470501284:2118] self: [6:7577471606470501290:0] peer: [3:7577471605486321161:0] socket: 274 qp: -1
2025-11-27T18:19:51.593805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471607361972212:2109] [node 6] ICP18 handshake (actor [2:7577471607361972227:2115]) is too old
2025-11-27T18:19:51.592948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471608531892247:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57465
2025-11-27T18:19:51.593515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471608313148193:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.593450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471608531892233:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:51.592968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471606470501302:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:51.593576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:51.594157Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471608531892155:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(22634935ns) {HandleHandshakeStatus:329(22632353ns) {DropIncomingHandshake:442(988ns) DropOutgoingHandshake:457(5753ns) SwitchToState:179(345ns) ProcessPendingSessionEvents:495(16192769ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15527599ns max# 2948896ns}}}
2025-11-27T18:19:51.592999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:51.593762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471608313148193:2109] [node 6] ICP22 created new session: [1:7577471608313148230:2109]
No RDMA context found for GID: 2025-11-27T18:19:51.593867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471608313148129:2049] [node 3] ICP18 handshake (actor [1:7577471608313148210:2118]) is too old
::ffff:127.1.0.0
2025-11-27T18:19:51.594244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471608313148230:2109] [node 6] ICS09 handshake done sender: [1:7577471608313148204:2113] self: [1:7577471608313148200:0] peer: [6:7577471606470501296:0] socket: 282 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.596087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471607361972150:2049] [node 3] ICP18 handshake (actor [2:7577471607361972229:2116]) is too old
2025-11-27T18:19:51.596465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471608531892154:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471608531892244:2118] held: no
2025-11-27T18:19:51.596437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471606560245129:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56665
2025-11-27T18:19:51.596601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471608531892217:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.596651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.594744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471608313148230:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:51.596651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471605486321083:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23373691ns) {HandleHandshakeStatus:329(23370458ns) {DropIncomingHandshake:442(730ns) DropOutgoingHandshake:457(4732ns) SwitchToState:179(2244ns) ProcessPendingSessionEvents:495(20553630ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18892802ns max# 2056510ns}}}
2025-11-27T18:19:51.595305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:51.596694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471608531892217:2108] [node 5] ICP22 created new session: [4:7577471608531892251:2108]
2025-11-27T18:19:51.597134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471606560245128:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.596500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471606470501303:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31780
2025-11-27T18:19:51.597161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471607361972244:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42510
2025-11-27T18:19:51.597498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471608531892251:2108] [node 5] ICS09 handshake done sender: [4:7577471608531892233:2116] self: [4:7577471608531892232:0] peer: [5:7577471606560245127:0] socket: 295 qp: -1
2025-11-27T18:19:51.596732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471606470501204:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20778460ns) {HandleHandshakeStatus:329(20775998ns) {DropIncomingHandshake:442(1264ns) DropOutgoingHandshake:457(5487ns) SwitchToState:179(452ns) ProcessPendingSessionEvents:495(13211366ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13114735ns max# 3979285ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.598212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471606560245039:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:51.598466Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471605486321155:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(29216442ns) {HandleHandshakeStatus:329(29214766ns) {DropIncomingHandshake:442(604ns) DropOutgoingHandshake:457(313ns) SwitchToState:179(445ns) ProcessPendingSessionEvents:495(22751563ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22628026ns max# 2487088ns}}}
2025-11-27T18:19:51.599136Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471607361972149:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20039337ns) {HandleHandshakeStatus:329(20035533ns) {DropIncomingHandshake:442(554ns) DropOutgoingHandshake:457(3893ns) SwitchToState:179(510ns) ProcessPendingSessionEvents:495(10967558ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10883673ns max# 2108832ns}}}
2025-11-27T18:19:51.598058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471608531892247:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.598944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471605486321175:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.598758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471606560245123:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:51.598372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471608531892251:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:51.599644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471606560245101:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.598971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471605486321175:2127] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.598409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:51.599806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.599648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471608531892156:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:51.600279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471606560245101:2108] [node 4] ICP22 created new session: [5:7577471606560245131:2108]
2025-11-27T18:19:51.599722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471608531892249:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57465
2025-11-27T18:19:51.600839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471606560245131:2108] [node 4] ICS09 handshake done sender: [5:7577471606560245123:2118] self: [5:7577471606560245127:0] peer: [4:7577471608531892232:0] socket: 258 qp: -1
2025-11-27T18:19:51.601009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471608531892244:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:51.601515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471605486321164:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.600999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471606560245131:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:51.602032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471607361972245:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42510
2025-11-27T18:19:51.601546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471605486321164:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.601490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:51.602089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471608531892154:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.602046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471606560245122:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.602306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:51.602528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471608531892154:2048] [node 1] ICP22 created new session: [4:7577471608531892254:2048]
2025-11-27T18:19:51.602743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471606560245038:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:51.606280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471605486321082:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25976684ns) {HandleHandshakeStatus:329(25974142ns) {DropIncomingHandshake:442(467ns) DropOutgoingHandshake:457(4056ns) SwitchToState:179(319ns) ProcessPendingSessionEvents:495(20465924ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20322662ns max# 3191535ns}}}
2025-11-27T18:19:51.606597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471608531892254:2048] [node 1] ICS09 handshake done sender: [4:7577471608531892244:2118] self: [4:7577471608531892246:0] peer: [1:7577471608313148205:0] socket: 302 qp: -1
2025-11-27T18:19:51.606746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471606470501203:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471606470501289:2120] held: no
2025-11-27T18:19:51.606644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471608531892254:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:51.606496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471605486321175:2127] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:51.607567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471605486321164:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:51.607004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:51.615436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471605486321156:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.615514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:51.617416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471605486321156:2108] [node 5] ICP22 created new session: [3:7577471605486321191:2108]
2025-11-27T18:19:51.618910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471605486321154:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.619307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:51.620093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471606560245134:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56665
2025-11-27T18:19:51.620264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471607361972245:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.619988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471605486321191:2108] [node 5] ICS09 handshake done sender: [3:7577471605486321175:2127] self: [3:7577471605486321174:0] peer: [5:7577471606560245130:0] socket: 270 qp: -1
2025-11-27T18:19:51.620865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471606560245128:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:51.620834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471607361972231:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:51.620035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471605486321191:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.621491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471606560245039:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.620865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471607361972231:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:51.622081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:51.620108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471605486321154:2107] [node 4] ICP22 created new session: [3:7577471605486321192:2107]
2025-11-27T18:19:51.622306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471606560245039:2050] [node 3] ICP22 created new session: [5:7577471606560245135:2050]
2025-11-27T18:19:51.622068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471607361972213:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:51.620243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:51.622956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471606470501289:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:51.622819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471608531892256:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57465
2025-11-27T18:19:51.620894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471605486321192:2107] [node 4] ICS09 handshake done sender: [3:7577471605486321164:2122] self: [3:7577471605486321163:0] peer: [4:7577471608531892253:0] socket: 249 qp: -1
2025-11-27T18:19:51.623322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471606560245135:2050] [node 3] ICS09 handshake done sender: [5:7577471606560245128:2119] self: [5:7577471606560245130:0] peer: [3:7577471605486321174:0] socket: 306 qp: -1
2025-11-27T18:19:51.623755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471606560245135:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:51.621046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471605486321192:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:51.624955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471608531892217:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(16937167ns) {HandleHandshakeStatus:329(16932210ns) {DropIncomingHandshake:442(10474ns) DropOutgoingHandshake:457(281ns) SwitchToState:179(267ns) ProcessPendingSessionEvents:495(14606644ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14520423ns max# 3868685ns}}}
2025-11-27T18:19:51.621079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:51.623982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:51.625352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471607361972231:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:51.625720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471606470501203:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.626592Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471608531892156:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15416) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471608531892247:2120] held: no
2025-11-27T18:19:51.626096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:51.626690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471608531892247:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:51.626659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471607361972214:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:51.626286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471606470501203:2048] [node 1] ICP22 created new session: [6:7577471606470501306:2048]
2025-11-27T18:19:51.627079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:51.627910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471606560245101:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(23927856ns) {HandleHandshakeStatus:329(23925800ns) {DropIncomingHandshake:442(1312ns) DropOutgoingHandshake:457(3844ns) SwitchToState:179(361ns) ProcessPendingSessionEvents:495(19154880ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19043455ns max# 3420638ns}}}
2025-11-27T18:19:51.627136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471607361972214:2108] [node 5] ICP22 created new session: [2:7577471607361972250:2108]
2025-11-27T18:19:51.626515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471606470501306:2048] [node 1] ICS09 handshake done sender: [6:7577471606470501289:2120] self: [6:7577471606470501296:0] peer: [1:7577471608313148200:0] socket: 290 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:51.627635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471607361972250:2108] [node 5] ICS09 handshake done sender: [2:7577471607361972231:2117] self: [2:7577471607361972230:0] peer: [5:7577471606560245133:0] socket: 298 qp: -1
2025-11-27T18:19:51.627993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471606560245101:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:51.626545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471606470501306:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:51.627916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471607361972250:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:51.626574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:51.627952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:51.631160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471606560245137:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56665
2025-11-27T18:19:51.631800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471606560245038:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3750) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471606560245122:2117] held: no
2025-11-27T18:19:51.631649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471608531892156:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.632095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471606560245039:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:51.631845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:51.632098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471608531892156:2050] [node 3] ICP22 created new session: [4:7577471608531892257:2050]
2025-11-27T18:19:51.632470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471608531892257:2050] [node 3] ICS09 handshake done sender: [4:7577471608531892247:2120] self: [4:7577471608531892253:0] peer: [3:7577471605486321163:0] socket: 267 qp: -1
2025-11-27T18:19:51.632530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471608531892257:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:51.632572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:51.633026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471606560245122:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:51.633199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471606560245038:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:51.633303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:51.633382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471606560245038:2049] [node 2] ICP22 created new session: [5:7577471606560245138:2049]
2025-11-27T18:19:51.633453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471606560245138:2049] [node 2] ICS09 handshake done sender: [5:7577471606560245122:2117] self: [5:7577471606560245133:0] peer: [2:7577471607361972230:0] socket: 299 qp: -1
2025-11-27T18:19:51.633558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471606560245138:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:51.633602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:51.673665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:51.673859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471607361972246:2048] [node 1] ICS07 socket disconnect 239 reason# EndOfStream
2025-11-27T18:19:51.673887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471607361972246:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.673950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:51.673988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471607361972246:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:51.674792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:51.674993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471605486321185:2048] [node 1] ICS07 socket disconnect 242 reason# EndOfStream
2025-11-27T18:19:51.675031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471605486321185:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.675111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:51.675186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471605486321185:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:51.676657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471607361972149:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62267) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:51.676702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471607361972246:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:51.677981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471605486321082:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62267) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:51.678017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471605486321185:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:51.679691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:51.679821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471607361972149:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62267) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:51.679901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471608531892254:2048] [node 1] ICS01 socket: 302 reason# EndOfStream
2025-11-27T18:19:51.679906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471607361972246:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:51.679962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471608531892154:2048] [node 1] ICP30 unregister session Session# [4:7577471608531892254:2048] VirtualId# [4:7577471608531892246:0]
2025-11-27T18:19:51.679990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471608531892254:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.680051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:51.679945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471607361972149:2048] [node 1] ICP30 unregister session Session# [2:7577471607361972246:2048] VirtualId# [2:7577471607361972228:0]
2025-11-27T18:19:51.680101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:51.680079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:51.680163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471607361972149:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62267) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.680178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62267) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.680271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471605486321082:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62267) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:51.680326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471605486321185:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:51.680382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471605486321082:2048] [node 1] ICP30 unregister session Session# [3:7577471605486321185:2048] VirtualId# [3:7577471605486321172:0]
2025-11-27T18:19:51.680528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:51.680568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471605486321082:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62267) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.680583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62267) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.681350Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:51.681548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471606560245125:2048] [node 1] ICS01 socket: 246 reason# EndOfStream
2025-11-27T18:19:51.681597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471606560245037:2048] [node 1] ICP30 unregister session Session# [5:7577471606560245125:2048] VirtualId# [5:7577471606560245114:0]
2025-11-27T18:19:51.681636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471606560245125:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.681703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:51.681745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:51.685489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:51.685851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471606470501306:2048] [node 1] ICS01 socket: 290 reason# EndOfStream
2025-11-27T18:19:51.685918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471606470501203:2048] [node 1] ICP30 unregister session Session# [6:7577471606470501306:2048] VirtualId# [6:7577471606470501296:0]
2025-11-27T18:19:51.685969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471606470501306:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.686082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:51.686139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:51.690705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471605486321082:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:51.741634Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:51.741883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471605486321182:2049] [node 2] ICS07 socket disconnect 244 reason# EndOfStream
2025-11-27T18:19:51.741923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471605486321182:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.742022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:51.742081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471605486321182:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:51.744822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471605486321083:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3750) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:51.744866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471605486321182:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:51.744984Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:51.745198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471608531892241:2049] [node 2] ICS01 socket: 247 reason# EndOfStream
2025-11-27T18:19:51.745442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471608531892155:2049] [node 2] ICP30 unregister session Session# [4:7577471608531892241:2049] VirtualId# [4:7577471608531892234:0]
2025-11-27T18:19:51.745501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471608531892241:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.745597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:51.745654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:51.746042Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471605486321083:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3750) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:51.746259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471605486321182:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:51.746307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471605486321083:2049] [node 2] ICP30 unregister session Session# [3:7577471605486321182:2049] VirtualId# [3:7577471605486321173:0]
2025-11-27T18:19:51.746422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:51.746470Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471605486321083:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.746485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.746861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:51.747015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471606560245138:2049] [node 2] ICS01 socket: 299 reason# EndOfStream
2025-11-27T18:19:51.747076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471606560245038:2049] [node 2] ICP30 unregister session Session# [5:7577471606560245138:2049] VirtualId# [5:7577471606560245133:0]
2025-11-27T18:19:51.747096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471606560245138:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.747161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:51.747204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:51.747818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:51.748031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471606470501294:2049] [node 2] ICS07 socket disconnect 245 reason# EndOfStream
2025-11-27T18:19:51.748065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471606470501294:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.748145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:51.748212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471606470501294:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:51.748836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471606470501204:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3750) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:51.748935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471606470501294:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:51.748969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471606470501204:2049] [node 2] ICP30 unregister session Session# [6:7577471606470501294:2049] VirtualId# [6:7577471606470501285:0]
2025-11-27T18:19:51.749056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:51.749095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471606470501204:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.749107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.756085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471605486321083:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:51.759160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471606470501204:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:51.824066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:51.825269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:51.824343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471608531892257:2050] [node 3] ICS01 socket: 267 reason# EndOfStream
2025-11-27T18:19:51.824386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471608531892156:2050] [node 3] ICP30 unregister session Session# [4:7577471608531892257:2050] VirtualId# [4:7577471608531892253:0]
2025-11-27T18:19:51.824439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471608531892257:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.824526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:51.824568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:51.825488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471606560245135:2050] [node 3] ICS07 socket disconnect 306 reason# EndOfStream
2025-11-27T18:19:51.825525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471606560245135:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.825587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:51.825633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471606560245135:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:51.826419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:51.826455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471606560245039:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15416) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:51.826532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471606560245135:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:51.826568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471606560245039:2050] [node 3] ICP30 unregister session Session# [5:7577471606560245135:2050] VirtualId# [5:7577471606560245130:0]
2025-11-27T18:19:51.826652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471606470501302:2050] [node 3] ICS07 socket disconnect 274 reason# EndOfStream
2025-11-27T18:19:51.826681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471606470501302:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.826689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:51.826729Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471606560245039:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15416) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.826733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:51.826745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15416) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.826759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471606470501302:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:51.827445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471606470501205:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15416) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:51.827522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471606470501302:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:51.827562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471606470501205:2050] [node 3] ICP30 unregister session Session# [6:7577471606470501302:2050] VirtualId# [6:7577471606470501290:0]
2025-11-27T18:19:51.827650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:51.827696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471606470501205:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15416) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.827709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15416) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.837188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471606560245039:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:51.838215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471606470501205:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:51.892332Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:51.892523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471606560245131:2108] [node 4] ICS07 socket disconnect 258 reason# EndOfStream
2025-11-27T18:19:51.892548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471606560245131:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.892608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:51.892641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471606560245131:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:51.893322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:51.893385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471606560245101:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31200) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:51.893440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471606560245131:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:51.893531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471606470501297:2109] [node 4] ICS01 socket: 253 reason# EndOfStream
2025-11-27T18:19:51.893513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471606560245101:2108] [node 4] ICP30 unregister session Session# [5:7577471606560245131:2108] VirtualId# [5:7577471606560245127:0]
2025-11-27T18:19:51.893587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471606470501267:2109] [node 4] ICP30 unregister session Session# [6:7577471606470501297:2109] VirtualId# [6:7577471606470501287:0]
2025-11-27T18:19:51.893613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:51.893629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471606470501297:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.893657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471606560245101:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31200) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.893706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:51.893693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31200) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.893766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:51.903531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471606560245101:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:19:51.963191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:51.963444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471606470501299:2106] [node 5] ICS07 socket disconnect 271 reason# EndOfStream
2025-11-27T18:19:51.963481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471606470501299:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:51.963577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:51.963627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471606470501299:2106] [node 5] ICS15 start handshake
2025-11-27T18:19:51.964385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471606470501264:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23005) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:51.964517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471606470501299:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:51.964579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471606470501264:2106] [node 5] ICP30 unregister session Session# [6:7577471606470501299:2106] VirtualId# [6:7577471606470501286:0]
2025-11-27T18:19:51.964696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:19:51.964731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471606470501264:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23005) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:51.964749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23005) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:52.073945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471611821485293:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:52.073945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471611821485292:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:52.093883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471611798584286:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:52.094479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471611798584285:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:52.118505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471609478510925:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:52.118605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471609478510926:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:52.138891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471610523793909:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:52.138958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471610523793910:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:52.139062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471610523793911:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:52.163307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471610192364818:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:52.163509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471610192364820:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:52.163533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471610192364819:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:52.183291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471613054219647:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:52.183437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471613054219648:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:52.183581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471613054219649:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:52.271823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471613054219708:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:52.308421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471611821485355:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:52.319978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471609478510988:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:52.327260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471610192364881:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:52.327236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471611821485358:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:52.327221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471609478510989:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:52.329158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471610523793972:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:52.329204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471610192364882:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:52.329492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471609478510990:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:52.329706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471610523793973:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:52.329499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471613054219711:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:52.328737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471611821485359:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:52.327217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471611798584348:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:52.329198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471611798584350:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:52.330080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471611821485292:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(57000439ns) {Configure:104(56994169ns) {ProcessConfigured:135(20229807ns) {StartInitialHandshake:157(20227385ns) {DropHandshakes:470(3684ns) {DropIncomingHandshake:442(689ns) DropOutgoingHandshake:457(578ns)} GenerateSessionVirtualId:988(954ns) PrepareNewSessionHandshake:477(3164ns) {SwitchToState:179(1471ns)}}}}}
2025-11-27T18:19:52.329877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471611798584349:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:52.331603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471613054219708:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(57958376ns) {Configure:104(57952606ns) {ProcessConfigured:135(2804938ns) {StartInitialHandshake:157(2802373ns) {DropHandshakes:470(1424664ns) {DropIncomingHandshake:442(1544ns) DropOutgoingHandshake:457(323ns)} GenerateSessionVirtualId:988(1630ns) PrepareNewSessionHandshake:477(2531ns) {SwitchToState:179(781ns)}}}}}
2025-11-27T18:19:52.334983Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471611821485293:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52889927ns) {Configure:104(52880894ns) {ProcessConfigured:135(1537511ns) {StartInitialHandshake:157(1438724ns) {DropHandshakes:470(2150ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(350ns)} GenerateSessionVirtualId:988(2276ns) PrepareNewSessionHandshake:477(1988ns) {SwitchToState:179(1023ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.346083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471611821485370:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7293
2025-11-27T18:19:52.362639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471610523793909:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32514332ns) {Configure:104(32511134ns) {ProcessConfigured:135(54062ns) {StartInitialHandshake:157(51825ns) {DropHandshakes:470(1055ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(167ns)} GenerateSessionVirtualId:988(761ns) PrepareNewSessionHandshake:477(653ns) {SwitchToState:179(262ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.363807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471609478510925:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33148280ns) {Configure:104(33126740ns) {ProcessConfigured:135(1609188ns) {StartInitialHandshake:157(1606623ns) {DropHandshakes:470(2760ns) {DropIncomingHandshake:442(884ns) DropOutgoingHandshake:457(374ns)} GenerateSessionVirtualId:988(1149ns) PrepareNewSessionHandshake:477(3370ns) {SwitchToState:179(1399ns)}}}}}
2025-11-27T18:19:52.364864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471611798584286:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32924832ns) {Configure:104(32919856ns) {ProcessConfigured:135(4316684ns) {StartInitialHandshake:157(4314505ns) {DropHandshakes:470(1228ns) {DropIncomingHandshake:442(442ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(427ns) PrepareNewSessionHandshake:477(1979ns) {SwitchToState:179(342ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.365074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471610192364881:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30975307ns) {Configure:104(30971088ns) {ProcessConfigured:135(4242410ns) {StartInitialHandshake:157(4240380ns) {DropHandshakes:470(1369ns) {DropIncomingHandshake:442(391ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(340ns) PrepareNewSessionHandshake:477(2316ns) {SwitchToState:179(758ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.377470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471611821485370:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.369563Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471610523793910:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39375820ns) {Configure:104(39368651ns) {ProcessConfigured:135(5626822ns) {StartInitialHandshake:157(5624929ns) {DropHandshakes:470(869ns) {DropIncomingHandshake:442(249ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(3436ns) {SwitchToState:179(1406ns)}}}}}
2025-11-27T18:19:52.364664Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471609478510926:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(74080848ns) {Configure:104(74060166ns) {ProcessConfigured:135(3809326ns) {StartInitialHandshake:157(3806504ns) {DropHandshakes:470(418297ns) {DropIncomingHandshake:442(1295ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(1287ns) PrepareNewSessionHandshake:477(2342ns) {SwitchToState:179(1056ns)}}}}}
2025-11-27T18:19:52.371351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471610192364818:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31750056ns) {Configure:104(31745995ns) {ProcessConfigured:135(8472988ns) {StartInitialHandshake:157(8470610ns) {DropHandshakes:470(1367ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(496ns) PrepareNewSessionHandshake:477(2727ns) {SwitchToState:179(863ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.377606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471611798584348:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32138805ns) {Configure:104(32133656ns) {ProcessConfigured:135(1967024ns) {StartInitialHandshake:157(1964656ns) {DropHandshakes:470(1894ns) {DropIncomingHandshake:442(406ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:477(1873ns) {SwitchToState:179(703ns)}}}}}
2025-11-27T18:19:52.371230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471609478510990:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(26555217ns) {Configure:104(26550819ns) {ProcessConfigured:135(5399472ns) {StartInitialHandshake:157(5396407ns) {DropHandshakes:470(1356ns) {DropIncomingHandshake:442(345ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(466ns) PrepareNewSessionHandshake:477(6925ns) {SwitchToState:179(4194ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.370448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471610523793984:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36094
2025-11-27T18:19:52.384351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471611798584350:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31522864ns) {Configure:104(31518408ns) {ProcessConfigured:135(4461591ns) {StartInitialHandshake:157(4459353ns) {DropHandshakes:470(590ns) {DropIncomingHandshake:442(250ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:477(1593ns) {SwitchToState:179(117ns)}}}}}
2025-11-27T18:19:52.384288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471610192364819:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(33471102ns) {Configure:104(33466956ns) {ProcessConfigured:135(6921197ns) {StartInitialHandshake:157(6917446ns) {DropHandshakes:470(1294ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(276ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:477(2733ns) {SwitchToState:179(678ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.384388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471611798584285:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33897889ns) {Configure:104(33892711ns) {ProcessConfigured:135(5227913ns) {StartInitialHandshake:157(5226092ns) {DropHandshakes:470(683ns) {DropIncomingHandshake:442(241ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(514ns) PrepareNewSessionHandshake:477(1504ns) {SwitchToState:179(547ns)}}}}}
2025-11-27T18:19:52.384423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471611798584349:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(32964010ns) {Configure:104(32959005ns) {ProcessConfigured:135(6109258ns) {StartInitialHandshake:157(6106926ns) {DropHandshakes:470(1253ns) {DropIncomingHandshake:442(426ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(370ns) PrepareNewSessionHandshake:477(2157ns) {SwitchToState:179(188ns)}}}}}
2025-11-27T18:19:52.384334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471610523793911:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33642702ns) {Configure:104(33635347ns) {ProcessConfigured:135(63887ns) {StartInitialHandshake:157(60987ns) {DropHandshakes:470(1375ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(501ns) PrepareNewSessionHandshake:477(809ns) {SwitchToState:179(320ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.371244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471609478510989:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23698798ns) {Configure:104(23691849ns) {ProcessConfigured:135(3640271ns) {StartInitialHandshake:157(3637004ns) {DropHandshakes:470(3237ns) {DropIncomingHandshake:442(828ns) DropOutgoingHandshake:457(313ns)} GenerateSessionVirtualId:988(1023ns) PrepareNewSessionHandshake:477(3110ns) {SwitchToState:179(777ns)}}}}}
2025-11-27T18:19:52.384362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471610192364820:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33060022ns) {Configure:104(33055629ns) {ProcessConfigured:135(6305039ns) {StartInitialHandshake:157(6302338ns) {DropHandshakes:470(1581ns) {DropIncomingHandshake:442(404ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(795ns) PrepareNewSessionHandshake:477(2477ns) {SwitchToState:179(692ns)}}}}}
2025-11-27T18:19:52.410986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471611821485373:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7293
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.389426Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471610523793973:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(33300195ns) {Configure:104(33294806ns) {ProcessConfigured:135(5665511ns) {StartInitialHandshake:157(5662435ns) {DropHandshakes:470(1020ns) {DropIncomingHandshake:442(351ns) DropOutgoingHandshake:457(218ns)} GenerateSessionVirtualId:988(303ns) PrepareNewSessionHandshake:477(10263ns) {SwitchToState:179(8593ns)}}}}}
2025-11-27T18:19:52.404692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471611798584361:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:86
2025-11-27T18:19:52.374308Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471609478510988:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31052007ns) {Configure:104(31047527ns) {ProcessConfigured:135(8446949ns) {StartInitialHandshake:157(8444170ns) {DropHandshakes:470(2035816ns) {DropIncomingHandshake:442(518ns) DropOutgoingHandshake:457(1885ns)} GenerateSessionVirtualId:988(1146ns) PrepareNewSessionHandshake:477(2767ns) {SwitchToState:179(1323ns)}}}}}
2025-11-27T18:19:52.386476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471610192364893:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56877
2025-11-27T18:19:52.412761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471611821485366:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.383349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471609478511001:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58633
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:19:52.404820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471611798584362:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:86
2025-11-27T18:19:52.389517Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471610523793972:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32450844ns) {Configure:104(32446569ns) {ProcessConfigured:135(2554900ns) {StartInitialHandshake:157(2553144ns) {DropHandshakes:470(1621ns) {DropIncomingHandshake:442(375ns) DropOutgoingHandshake:457(446ns)} GenerateSessionVirtualId:988(585ns) PrepareNewSessionHandshake:477(1391ns) {SwitchToState:179(225ns)}}}}}
2025-11-27T18:19:52.384668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471609478511001:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.412785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471611821485366:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.428421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471613054219725:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8313
2025-11-27T18:19:52.386625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471610192364894:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56877
2025-11-27T18:19:52.403795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471610523793984:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.389347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471610192364882:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31367271ns) {Configure:104(31361045ns) {ProcessConfigured:135(4172175ns) {StartInitialHandshake:157(4170309ns) {DropHandshakes:470(498ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(2788ns) {SwitchToState:179(555ns)}}}}}
2025-11-27T18:19:52.414218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471611798584361:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.403842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471609478510925:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.413693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471611821485366:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:52.416185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471611798584363:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:86
2025-11-27T18:19:52.404071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471610523793909:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.418906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471611821485363:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.425804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471609478511003:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58633
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.442616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471613054219725:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.398307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471610192364893:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.416388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471611798584362:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.406332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471610523793982:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.405934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471610192364894:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.444571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471613054219711:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.437349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471609478511006:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58633
2025-11-27T18:19:52.418940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471611821485363:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.419090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471611798584285:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.427136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471611821485355:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.440159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471609478511006:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.406351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471610523793982:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.446371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471613054219726:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8313
2025-11-27T18:19:52.406082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471610192364882:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.441193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471611798584363:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.416669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471610523793990:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36094
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.441174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471609478511003:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 
::ffff:127.1.0.0
2025-11-27T18:19:52.449133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471613054219727:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8313
2025-11-27T18:19:52.427200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:52.440754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471610192364900:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56877
2025-11-27T18:19:52.418891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471610523793982:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:52.453291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471613054219726:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.452283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471610192364894:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:52.453970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471609478510925:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471609478511001:2115] held: no
2025-11-27T18:19:52.453917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471613054219727:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.427249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471611821485355:2107] [node 4] ICP22 created new session: [1:7577471611821485377:2107]
2025-11-27T18:19:52.454742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471609478511008:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58633
2025-11-27T18:19:52.453966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471610192364882:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.454163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471613054219728:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8313
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.433383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471611821485377:2107] [node 4] ICS09 handshake done sender: [1:7577471611821485366:2113] self: [1:7577471611821485364:0] peer: [4:7577471610523793989:0] socket: 247 qp: -1
2025-11-27T18:19:52.454951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471611798584369:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:86
2025-11-27T18:19:52.454217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471613054219708:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.433419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471611821485377:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:52.455547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471613054219648:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.433463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:52.456948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471613054219728:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.437346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471611821485360:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.437384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471611821485360:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.457469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471611798584371:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:86
2025-11-27T18:19:52.457029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.437477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471610523793992:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36094
2025-11-27T18:19:52.457089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471610192364882:2108] [node 4] ICP22 created new session: [5:7577471610192364902:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.439762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471611821485363:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:52.459161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471609478511000:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.458836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471613054219647:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.440930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471610523793972:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.461304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471611798584359:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.441189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:52.440193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471611821485373:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.459211Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471609478511000:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.457433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471610192364890:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.457472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471610192364890:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.464092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471613054219732:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8313
2025-11-27T18:19:52.461346Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471611798584359:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.441534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471611821485360:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:52.467494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471609478511001:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:52.441450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471610523793984:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:52.462176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471611798584361:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:52.459920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471610192364902:2108] [node 4] ICS09 handshake done sender: [5:7577471610192364894:2116] self: [5:7577471610192364898:0] peer: [4:7577471610523793975:0] socket: 258 qp: -1
2025-11-27T18:19:52.444329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471611821485293:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.444402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:52.444669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471611821485292:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.444701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:52.444981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471611821485292:2048] [node 2] ICP22 created new session: [1:7577471611821485384:2048]
2025-11-27T18:19:52.445011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471611821485293:2049] [node 3] ICP22 created new session: [1:7577471611821485383:2049]
2025-11-27T18:19:52.446429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471611821485384:2048] [node 2] ICS09 handshake done sender: [1:7577471611821485360:2111] self: [1:7577471611821485357:0] peer: [2:7577471611798584365:0] socket: 240 qp: -1
2025-11-27T18:19:52.449155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471611821485383:2049] [node 3] ICS09 handshake done sender: [1:7577471611821485363:2112] self: [1:7577471611821485362:0] peer: [3:7577471609478511007:0] socket: 246 qp: -1
2025-11-27T18:19:52.449369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471611821485383:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:52.452187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471611821485384:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:52.452483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:52.452505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:52.453502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471611821485379:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7293
2025-11-27T18:19:52.454797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471611821485379:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.463510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471611821485369:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.463543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471611821485369:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.470022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471611821485369:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:52.478525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471611821485359:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.478592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:52.478638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471611821485359:2109] [node 6] ICP22 created new session: [1:7577471611821485390:2109]
2025-11-27T18:19:52.478681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471611821485390:2109] [node 6] ICS09 handshake done sender: [1:7577471611821485369:2115] self: [1:7577471611821485367:0] peer: [6:7577471613054219733:0] socket: 289 qp: -1
2025-11-27T18:19:52.478710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471611821485390:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:52.484274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:52.484355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471611821485368:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.484389Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471611821485368:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.443623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471610523793988:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36094
2025-11-27T18:19:52.478845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471613054219708:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11742) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471613054219726:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.460899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471610192364890:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:52.468962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471609478511000:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.465042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471611798584285:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.492270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471611821485355:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(34936616ns) {HandleHandshakeStatus:329(34932936ns) {DropIncomingHandshake:442(2158ns) DropOutgoingHandshake:457(1200ns) SwitchToState:179(909ns) ProcessPendingSessionEvents:495(23398786ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22042095ns max# 3578008ns}}}
2025-11-27T18:19:52.443696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471610523793972:2108] [node 5] ICP22 created new session: [4:7577471610523793994:2108]
2025-11-27T18:19:52.462125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471610192364901:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56877
2025-11-27T18:19:52.473503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471609478510925:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.487188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471613054219725:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:52.465075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471611798584358:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.444127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471610523793909:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.473548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:52.502008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471613054219711:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.465106Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471611798584358:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.503701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471611821485389:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7293
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.462205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471610192364902:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:52.444144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471610523793990:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.474597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471609478510925:2048] [node 1] ICP22 created new session: [3:7577471609478511013:2048]
2025-11-27T18:19:52.506168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471611821485368:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:52.463600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:52.502173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:52.465751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:52.444186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:52.478846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471609478511013:2048] [node 1] ICS09 handshake done sender: [3:7577471609478511001:2115] self: [3:7577471609478511007:0] peer: [1:7577471611821485362:0] socket: 249 qp: -1
2025-11-27T18:19:52.466638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471610192364881:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.502698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471613054219734:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8313
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.508443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471611821485389:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.465775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471611798584359:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:52.481792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471609478510988:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.444683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471610523793994:2108] [node 5] ICS09 handshake done sender: [4:7577471610523793982:2113] self: [4:7577471610523793975:0] peer: [5:7577471610192364898:0] socket: 253 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.467249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471610192364901:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.503595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471613054219711:2109] [node 4] ICP22 created new session: [6:7577471613054219737:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.514376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471611821485358:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.467260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471611798584285:2048] [node 1] ICP22 created new session: [2:7577471611798584374:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.514440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:52.444727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471610523793994:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:52.481844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:52.505577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471613054219737:2109] [node 4] ICS09 handshake done sender: [6:7577471613054219725:2115] self: [6:7577471613054219729:0] peer: [4:7577471610523793974:0] socket: 254 qp: -1
2025-11-27T18:19:52.467486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:52.467314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471611798584374:2048] [node 1] ICS09 handshake done sender: [2:7577471611798584361:2115] self: [2:7577471611798584365:0] peer: [1:7577471611821485357:0] socket: 257 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.521461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471611821485358:2108] [node 5] ICP22 created new session: [1:7577471611821485394:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.481892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471609478510988:2107] [node 4] ICP22 created new session: [3:7577471609478511014:2107]
2025-11-27T18:19:52.444768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:52.506204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471613054219737:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:52.526239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471611821485394:2108] [node 5] ICS09 handshake done sender: [1:7577471611821485368:2114] self: [1:7577471611821485365:0] peer: [5:7577471610192364908:0] socket: 260 qp: -1
2025-11-27T18:19:52.467345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471611798584374:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:52.467535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471610192364881:2109] [node 6] ICP22 created new session: [5:7577471610192364906:2109]
2025-11-27T18:19:52.510102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:52.446109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471610523793993:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36094
2025-11-27T18:19:52.481933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471609478511014:2107] [node 4] ICS09 handshake done sender: [3:7577471609478511000:2114] self: [3:7577471609478510998:0] peer: [4:7577471610523793999:0] socket: 268 qp: -1
2025-11-27T18:19:52.526432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471611821485394:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:52.469794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:52.446410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471610523793909:2048] [node 1] ICP22 created new session: [4:7577471610523793995:2048]
2025-11-27T18:19:52.527235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:52.517715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471613054219647:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471613054219728:2118] held: no
2025-11-27T18:19:52.481964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471609478511014:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:52.473623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471611798584371:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.449145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471610523793980:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.526889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471613054219735:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8313
2025-11-27T18:19:52.469537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471610192364818:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.473771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471611798584348:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.481992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:52.449177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471610523793980:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-27T18:19:52.474567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471610192364906:2109] [node 6] ICS09 handshake done sender: [5:7577471610192364890:2112] self: [5:7577471610192364885:0] peer: [6:7577471613054219730:0] socket: 278 qp: -1
::ffff:127.1.0.0
2025-11-27T18:19:52.528279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471613054219648:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22016) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471613054219727:2117] held: no
2025-11-27T18:19:52.449516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471610523793995:2048] [node 1] ICS09 handshake done sender: [4:7577471610523793984:2115] self: [4:7577471610523793989:0] peer: [1:7577471611821485364:0] socket: 250 qp: -1
2025-11-27T18:19:52.529123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471613054219738:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8313
2025-11-27T18:19:52.452215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471610523793995:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:52.529557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471613054219736:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8313
2025-11-27T18:19:52.452257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:52.532015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471613054219735:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.476753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471610192364906:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:52.452515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471610523793911:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.453644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471610523793988:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.476803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:52.453937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471610523793993:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.509082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471610192364912:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56877
2025-11-27T18:19:52.454780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471610523793910:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.521321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471610192364901:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:52.456949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471610523793980:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:52.526473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471610192364818:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.462166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471610523793973:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.526962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:52.527509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471610192364818:2048] [node 1] ICP22 created new session: [5:7577471610192364913:2048]
2025-11-27T18:19:52.463488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:52.527529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471610192364910:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56877
2025-11-27T18:19:52.463544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471610523793973:2109] [node 6] ICP22 created new session: [4:7577471610523794002:2109]
2025-11-27T18:19:52.528207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471610192364913:2048] [node 1] ICS09 handshake done sender: [5:7577471610192364901:2118] self: [5:7577471610192364908:0] peer: [1:7577471611821485365:0] socket: 294 qp: -1
2025-11-27T18:19:52.464985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471610523794002:2109] [node 6] ICS09 handshake done sender: [4:7577471610523793980:2112] self: [4:7577471610523793974:0] peer: [6:7577471613054219729:0] socket: 252 qp: -1
2025-11-27T18:19:52.528710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471610192364913:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:52.466651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471610523794002:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:52.528751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:52.467271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:52.529197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471610192364911:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56877
2025-11-27T18:19:52.506213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471610523793911:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2533) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471610523793990:2117] held: no
2025-11-27T18:19:52.529948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471610192364910:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.527767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471610523794003:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36094
2025-11-27T18:19:52.531243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471610192364820:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.531926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471610192364911:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.532386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471610192364819:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.489602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471609478511013:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:52.489653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:52.497605Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471609478510925:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16296269ns) {HandleHandshakeStatus:329(16293708ns) {DropIncomingHandshake:442(624ns) DropOutgoingHandshake:457(256ns) SwitchToState:179(354ns) ProcessPendingSessionEvents:495(1084ns)}}
2025-11-27T18:19:52.473803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:52.532405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471613054219728:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:52.532380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471611821485292:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(65741817ns) {HandleHandshakeStatus:329(65739265ns) {DropIncomingHandshake:442(464ns) DropOutgoingHandshake:457(395ns) SwitchToState:179(136ns) ProcessPendingSessionEvents:495(55735992ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 48556502ns max# 5352169ns}}}
2025-11-27T18:19:52.473852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471611798584348:2107] [node 4] ICP22 created new session: [2:7577471611798584377:2107]
2025-11-27T18:19:52.526590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471609478511018:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58633
2025-11-27T18:19:52.532416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471613054219708:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(48158958ns) {HandleHandshakeStatus:395(48156056ns) {LogHandshakeFail:472(16313ns) {UpdateErrorStateLog:545(5941ns)} DropOutgoingHandshake:457(1500ns) ProcessPendingSessionEvents:495(46758413ns) {unordered DropSessionEvent:512 num# 100 sum# 46699541ns max# 11148888ns}}}
2025-11-27T18:19:52.533980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471610523794005:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36094
2025-11-27T18:19:52.528053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471609478511018:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.473895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471611798584377:2107] [node 4] ICS09 handshake done sender: [2:7577471611798584359:2113] self: [2:7577471611798584355:0] peer: [4:7577471610523794001:0] socket: 275 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.538482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471610523793972:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(86241852ns) {HandleHandshakeStatus:329(86239873ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(561ns) SwitchToState:179(305ns) ProcessPendingSessionEvents:495(77744938ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 77597185ns max# 8586565ns}}}
2025-11-27T18:19:52.532467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471613054219727:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:52.539192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471611821485293:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(83605208ns) {HandleHandshakeStatus:329(83602970ns) {DropIncomingHandshake:442(13704ns) DropOutgoingHandshake:457(278ns) SwitchToState:179(120ns) ProcessPendingSessionEvents:495(71874718ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 68751683ns max# 10600703ns}}}
2025-11-27T18:19:52.473930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471611798584377:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:52.473965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:52.532494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471613054219649:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.490076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471611798584358:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:52.532519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471613054219726:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:52.528842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471609478510926:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:52.534723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471613054219648:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.492960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471611798584373:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:86
2025-11-27T18:19:52.546008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471610523793993:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:52.539950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471611821485292:2048] [node 2] ICP18 handshake (actor [1:7577471611821485379:2118]) is too old
2025-11-27T18:19:52.532943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471609478510996:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.502450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471611798584349:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.534864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471613054219647:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.546026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471610523793911:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(22792364ns) {HandleHandshakeStatus:395(22790012ns) {LogHandshakeFail:472(531577ns) {UpdateErrorStateLog:545(6202ns)} DropOutgoingHandshake:457(406ns) ProcessPendingSessionEvents:495(22197349ns) {unordered DropSessionEvent:512 num# 100 sum# 5138406ns max# 2074239ns}}}
2025-11-27T18:19:52.532969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471609478510996:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.540338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471611821485359:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(53768843ns) {HandleHandshakeStatus:329(53766298ns) {DropIncomingHandshake:442(25996ns) DropOutgoingHandshake:457(401ns) SwitchToState:179(485ns) ProcessPendingSessionEvents:495(33169487ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29925916ns max# 6904044ns}}}
2025-11-27T18:19:52.503591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:52.547098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471610523793990:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:52.537586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471609478510997:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.534916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:52.551926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471610192364918:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56877
2025-11-27T18:19:52.505466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471611798584349:2109] [node 6] ICP22 created new session: [2:7577471611798584381:2109]
2025-11-27T18:19:52.537613Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471609478510997:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.535134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:52.551987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471610192364882:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(79029696ns) {HandleHandshakeStatus:329(78764769ns) {DropIncomingHandshake:442(925ns) DropOutgoingHandshake:457(12709ns) SwitchToState:179(667ns) ProcessPendingSessionEvents:495(63605302ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 62339966ns max# 7177610ns}}}
2025-11-27T18:19:52.550876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471610523793909:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(85038136ns) {HandleHandshakeStatus:329(85036026ns) {DropIncomingHandshake:442(415ns) DropOutgoingHandshake:457(5393ns) SwitchToState:179(274ns) ProcessPendingSessionEvents:495(75140366ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 69512202ns max# 7143250ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.535535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471613054219647:2048] [node 1] ICP22 created new session: [6:7577471613054219743:2048]
2025-11-27T18:19:52.507662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471611798584373:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.541328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471609478510996:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:52.563752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471610192364910:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:52.551168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471610523793973:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(71399079ns) {HandleHandshakeStatus:329(70423196ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(485ns) SwitchToState:179(678ns) ProcessPendingSessionEvents:495(55857270ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 54727839ns max# 7755859ns}}}
2025-11-27T18:19:52.551937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471610523793910:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.508274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471611798584381:2109] [node 6] ICS09 handshake done sender: [2:7577471611798584358:2112] self: [2:7577471611798584351:0] peer: [6:7577471613054219731:0] socket: 281 qp: -1
2025-11-27T18:19:52.537587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471613054219648:2049] [node 2] ICP22 created new session: [6:7577471613054219742:2049]
2025-11-27T18:19:52.567628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471610192364820:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.542387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471609478510926:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22016) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471609478511018:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:52.508324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471611798584381:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:52.552803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:52.538032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471613054219708:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.567697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:52.569608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471610192364820:2050] [node 3] ICP22 created new session: [5:7577471610192364919:2050]
2025-11-27T18:19:52.551820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471609478510997:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:52.573975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471610192364919:2050] [node 3] ICS09 handshake done sender: [5:7577471610192364910:2119] self: [5:7577471610192364915:0] peer: [3:7577471609478510992:0] socket: 309 qp: -1
2025-11-27T18:19:52.538070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:52.557613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471609478510989:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.538895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471613054219708:2106] [node 5] ICP22 created new session: [6:7577471613054219744:2106]
2025-11-27T18:19:52.557667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:52.574813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471610192364919:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:52.538928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471613054219743:2048] [node 1] ICS09 handshake done sender: [6:7577471613054219728:2118] self: [6:7577471613054219733:0] peer: [1:7577471611821485367:0] socket: 290 qp: -1
2025-11-27T18:19:52.561553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471609478510989:2108] [node 5] ICP22 created new session: [3:7577471609478511024:2108]
2025-11-27T18:19:52.576137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:52.538998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471613054219742:2049] [node 2] ICS09 handshake done sender: [6:7577471613054219727:2117] self: [6:7577471613054219731:0] peer: [2:7577471611798584351:0] socket: 284 qp: -1
2025-11-27T18:19:52.563072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471609478511024:2108] [node 5] ICS09 handshake done sender: [3:7577471609478510996:2111] self: [3:7577471609478510992:0] peer: [5:7577471610192364915:0] socket: 256 qp: -1
2025-11-27T18:19:52.510413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: 2025-11-27T18:19:52.552851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471610523793910:2049] [node 2] ICP22 created new session: [4:7577471610523794006:2049]
::ffff:127.1.0.0
2025-11-27T18:19:52.539953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471613054219743:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:52.564133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471609478511024:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:52.578035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471610192364881:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(86130203ns) {HandleHandshakeStatus:329(86127340ns) {DropIncomingHandshake:442(16723ns) DropOutgoingHandshake:457(200ns) SwitchToState:179(697ns) ProcessPendingSessionEvents:495(54837313ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 54679881ns max# 5699404ns}}}
2025-11-27T18:19:52.539960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471613054219742:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:52.564672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:52.540403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:52.564886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471609478510990:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.564941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:52.541460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471613054219747:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8313
2025-11-27T18:19:52.530063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471611798584356:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.530092Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471611798584356:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.565194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471609478511020:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58633
2025-11-27T18:19:52.542109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:52.552915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471610523794006:2049] [node 2] ICS09 handshake done sender: [4:7577471610523793993:2119] self: [4:7577471610523794001:0] peer: [2:7577471611798584355:0] socket: 277 qp: -1
2025-11-27T18:19:52.552951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471610523794006:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:52.564983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471609478510990:2109] [node 6] ICP22 created new session: [3:7577471609478511026:2109]
2025-11-27T18:19:52.532533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471611798584356:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:52.543194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471613054219744:2106] [node 5] ICS09 handshake done sender: [6:7577471613054219726:2116] self: [6:7577471613054219730:0] peer: [5:7577471610192364885:0] socket: 280 qp: -1
2025-11-27T18:19:52.579375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471610192364917:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56877
2025-11-27T18:19:52.567609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471609478511026:2109] [node 6] ICS09 handshake done sender: [3:7577471609478510997:2112] self: [3:7577471609478510994:0] peer: [6:7577471613054219741:0] socket: 303 qp: -1
2025-11-27T18:19:52.552991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:52.585343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471611821485358:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(59652737ns) {HandleHandshakeStatus:329(59647480ns) {DropIncomingHandshake:442(15712ns) DropOutgoingHandshake:457(419ns) SwitchToState:179(492ns) ProcessPendingSessionEvents:495(46149402ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 45100330ns max# 8291153ns}}}
2025-11-27T18:19:52.532660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471611798584357:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:52.532680Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471611798584357:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:52.573148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471609478511026:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:52.555169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471610523793911:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.535519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471611798584286:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.537550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:52.545929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471613054219744:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:52.538586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471611798584286:2049] [node 3] ICP22 created new session: [2:7577471611798584386:2049]
2025-11-27T18:19:52.540058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471611798584348:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(53434352ns) {HandleHandshakeStatus:329(53431685ns) {DropIncomingHandshake:442(349ns) DropOutgoingHandshake:457(270ns) SwitchToState:179(400ns) ProcessPendingSessionEvents:495(53113515ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 53003194ns max# 7580792ns}}}
2025-11-27T18:19:52.546392Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471613054219711:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(30488764ns) {HandleHandshakeStatus:329(30485172ns) {DropIncomingHandshake:442(1535ns) DropOutgoingHandshake:457(5818ns) SwitchToState:179(744ns) ProcessPendingSessionEvents:495(7074672ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6996783ns max# 1014107ns}}}
2025-11-27T18:19:52.555204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:52.555246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471610523793911:2050] [node 3] ICP22 created new session: [4:7577471610523794008:2050]
2025-11-27T18:19:52.555287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471610523794008:2050] [node 3] ICS09 handshake done sender: [4:7577471610523793990:2117] self: [4:7577471610523793999:0] peer: [3:7577471609478510998:0] socket: 269 qp: -1
2025-11-27T18:19:52.555319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471610523794008:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:52.555360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:52.575948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471610523793972:2108] [node 5] ICP18 handshake (actor [4:7577471610523793988:2116]) is too old
2025-11-27T18:19:52.589393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471610192364819:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22016) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471610192364911:2120] held: no
2025-11-27T18:19:52.546813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:52.561503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471613054219735:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:52.564412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471613054219649:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.567123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:52.567190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471613054219649:2050] [node 3] ICP22 created new session: [6:7577471613054219749:2050]
2025-11-27T18:19:52.572560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471613054219749:2050] [node 3] ICS09 handshake done sender: [6:7577471613054219735:2121] self: [6:7577471613054219741:0] peer: [3:7577471609478510994:0] socket: 305 qp: -1
2025-11-27T18:19:52.573392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471613054219749:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:52.573460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:52.573199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:52.575861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471609478511021:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58633
2025-11-27T18:19:52.544084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471611798584386:2049] [node 3] ICS09 handshake done sender: [2:7577471611798584356:2110] self: [2:7577471611798584352:0] peer: [3:7577471609478511019:0] socket: 308 qp: -1
2025-11-27T18:19:52.590169Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471609478510989:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(31395551ns) {HandleHandshakeStatus:329(31392911ns) {DropIncomingHandshake:442(14097ns) DropOutgoingHandshake:457(352ns) SwitchToState:179(659ns) ProcessPendingSessionEvents:495(21969520ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21865983ns max# 5659483ns}}}
2025-11-27T18:19:52.590385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471609478510926:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23582566ns) {HandleHandshakeStatus:395(23580161ns) {LogHandshakeFail:472(23000ns) {UpdateErrorStateLog:545(5907ns)} DropOutgoingHandshake:457(1444ns) ProcessPendingSessionEvents:495(21677385ns) {unordered DropSessionEvent:512 num# 100 sum# 21631070ns max# 5366342ns}}}
2025-11-27T18:19:52.591128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471609478511018:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:52.598073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471609478510926:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.590905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471613054219649:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(24983783ns) {HandleHandshakeStatus:329(24981781ns) {DropIncomingHandshake:442(795ns) DropOutgoingHandshake:457(7204ns) SwitchToState:179(756ns) ProcessPendingSessionEvents:495(15066548ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14974163ns max# 3763732ns}}}
2025-11-27T18:19:52.598146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:52.598217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471609478510926:2049] [node 2] ICP22 created new session: [3:7577471609478511028:2049]
2025-11-27T18:19:52.598268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471609478511028:2049] [node 2] ICS09 handshake done sender: [3:7577471609478511018:2119] self: [3:7577471609478511019:0] peer: [2:7577471611798584352:0] socket: 310 qp: -1
2025-11-27T18:19:52.544285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471611798584357:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:52.601588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471610192364820:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(22248154ns) {HandleHandshakeStatus:329(22245345ns) {DropIncomingHandshake:442(463ns) DropOutgoingHandshake:457(4125ns) SwitchToState:179(427ns) ProcessPendingSessionEvents:495(11933061ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11851123ns max# 5412933ns}}}
2025-11-27T18:19:52.598308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471609478511028:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:52.609925Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471610192364820:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:52.598356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:52.544479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471611798584386:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:52.544525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:52.546708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471611798584285:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(67421089ns) {HandleHandshakeStatus:329(67418510ns) {DropIncomingHandshake:442(436ns) DropOutgoingHandshake:457(11388ns) SwitchToState:179(463ns) ProcessPendingSessionEvents:495(37637342ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 37534043ns max# 6126211ns}}}
2025-11-27T18:19:52.563849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471611798584350:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:52.563915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:52.564216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471611798584350:2108] [node 5] ICP22 created new session: [2:7577471611798584388:2108]
2025-11-27T18:19:52.565820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471611798584388:2108] [node 5] ICS09 handshake done sender: [2:7577471611798584357:2111] self: [2:7577471611798584354:0] peer: [5:7577471610192364916:0] socket: 311 qp: -1
2025-11-27T18:19:52.567029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471611798584388:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:52.567071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:52.573516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471611798584348:2107] [node 4] ICP18 handshake (actor [2:7577471611798584371:2119]) is too old
2025-11-27T18:19:52.578398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471611798584349:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(60581712ns) {HandleHandshakeStatus:329(60578680ns) {DropIncomingHandshake:442(13568ns) DropOutgoingHandshake:457(289ns) SwitchToState:179(814ns) ProcessPendingSessionEvents:495(36975250ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36038017ns max# 5450669ns}}}
2025-11-27T18:19:52.589978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471611798584286:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(42834476ns) {HandleHandshakeStatus:329(42831642ns) {DropIncomingHandshake:442(11241ns) DropOutgoingHandshake:457(314ns) SwitchToState:179(3346ns) ProcessPendingSessionEvents:495(31312609ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 31219480ns max# 5350197ns}}}
2025-11-27T18:19:52.590205Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471611798584350:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(25894647ns) {HandleHandshakeStatus:329(25370287ns) {DropIncomingHandshake:442(16402ns) DropOutgoingHandshake:457(522ns) SwitchToState:179(601ns) ProcessPendingSessionEvents:495(15042099ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14703420ns max# 4120187ns}}}
2025-11-27T18:19:52.611550Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471609478510990:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(46197999ns) {HandleHandshakeStatus:329(46185523ns) {DropIncomingHandshake:442(642ns) DropOutgoingHandshake:457(395ns) SwitchToState:179(369ns) ProcessPendingSessionEvents:495(35767341ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 34684444ns max# 4764855ns}}}
2025-11-27T18:19:52.612330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471609478510990:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:52.612803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471610192364819:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(22705671ns) {HandleHandshakeStatus:395(22702123ns) {LogHandshakeFail:472(16124ns) {UpdateErrorStateLog:545(6023ns)} DropOutgoingHandshake:457(600ns) ProcessPendingSessionEvents:495(22606462ns) {unordered DropSessionEvent:512 num# 100 sum# 5205827ns max# 2621339ns}}}
2025-11-27T18:19:52.612939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471610192364911:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:52.613087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471610192364819:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:52.618208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471610192364818:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(53324363ns) {HandleHandshakeStatus:329(53321119ns) {DropIncomingHandshake:442(1119ns) DropOutgoingHandshake:457(4963ns) SwitchToState:179(1651ns) ProcessPendingSessionEvents:495(49836093ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 49665726ns max# 5341400ns}}}
2025-11-27T18:19:52.622336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:52.624449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471610192364819:2049] [node 2] ICP22 created new session: [5:7577471610192364921:2049]
2025-11-27T18:19:52.624524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471610192364921:2049] [node 2] ICS09 handshake done sender: [5:7577471610192364911:2120] self: [5:7577471610192364916:0] peer: [2:7577471611798584354:0] socket: 283 qp: -1
2025-11-27T18:19:52.624573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471610192364921:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:52.624632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:52.691415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:52.691665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471611798584374:2048] [node 1] ICS07 socket disconnect 257 reason# EndOfStream
2025-11-27T18:19:52.691704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471611798584374:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.691764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:52.691809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:52.691878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471611798584374:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:52.691998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471609478511013:2048] [node 1] ICS01 socket: 249 reason# EndOfStream
2025-11-27T18:19:52.692048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471609478510925:2048] [node 1] ICP30 unregister session Session# [3:7577471609478511013:2048] VirtualId# [3:7577471609478511007:0]
2025-11-27T18:19:52.692083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471609478511013:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.692143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:52.692203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:52.694947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471611798584285:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:52.694999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471611798584374:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:52.695427Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:52.695658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471610523793995:2048] [node 1] ICS07 socket disconnect 250 reason# EndOfStream
2025-11-27T18:19:52.695702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471610523793995:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.695770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:52.695821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471610523793995:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:52.696029Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471611798584285:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:52.696109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471611798584374:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:52.696171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471611798584285:2048] [node 1] ICP30 unregister session Session# [2:7577471611798584374:2048] VirtualId# [2:7577471611798584365:0]
2025-11-27T18:19:52.696275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:52.696314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471611798584285:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:52.696330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:52.696492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:52.696566Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471610523793909:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:52.696637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471610523793995:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:52.696775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471610192364913:2048] [node 1] ICS07 socket disconnect 294 reason# EndOfStream
2025-11-27T18:19:52.696679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471610523793909:2048] [node 1] ICP30 unregister session Session# [4:7577471610523793995:2048] VirtualId# [4:7577471610523793989:0]
2025-11-27T18:19:52.696824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471610192364913:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.696899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:52.696916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:52.696942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471610192364913:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:52.696960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471610523793909:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:52.696978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:52.697566Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471610192364818:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:52.697679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471610192364913:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:52.697719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471610192364818:2048] [node 1] ICP30 unregister session Session# [5:7577471610192364913:2048] VirtualId# [5:7577471610192364908:0]
2025-11-27T18:19:52.697743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:52.697805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:52.697841Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471610192364818:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:52.697855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32028) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:52.697942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471613054219743:2048] [node 1] ICS01 socket: 290 reason# EndOfStream
2025-11-27T18:19:52.697982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471613054219647:2048] [node 1] ICP30 unregister session Session# [6:7577471613054219743:2048] VirtualId# [6:7577471613054219733:0]
2025-11-27T18:19:52.698010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471613054219743:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.698102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:52.698164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:52.705345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471610523793909:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:52.707384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471610192364818:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:52.778863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:52.779131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471609478511028:2049] [node 2] ICS01 socket: 310 reason# EndOfStream
2025-11-27T18:19:52.779182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471609478510926:2049] [node 2] ICP30 unregister session Session# [3:7577471609478511028:2049] VirtualId# [3:7577471609478511019:0]
2025-11-27T18:19:52.779220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471609478511028:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.779295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:52.779370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:52.781360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:52.781596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471610523794006:2049] [node 2] ICS01 socket: 277 reason# EndOfStream
2025-11-27T18:19:52.781649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471610523793910:2049] [node 2] ICP30 unregister session Session# [4:7577471610523794006:2049] VirtualId# [4:7577471610523794001:0]
2025-11-27T18:19:52.781697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471610523794006:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.781761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:52.781804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:52.782528Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:52.782743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471610192364921:2049] [node 2] ICS01 socket: 283 reason# EndOfStream
2025-11-27T18:19:52.782780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471610192364819:2049] [node 2] ICP30 unregister session Session# [5:7577471610192364921:2049] VirtualId# [5:7577471610192364916:0]
2025-11-27T18:19:52.782808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471610192364921:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.782871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:52.782929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:52.783526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:52.783667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471613054219742:2049] [node 2] ICS01 socket: 284 reason# EndOfStream
2025-11-27T18:19:52.783698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471613054219648:2049] [node 2] ICP30 unregister session Session# [6:7577471613054219742:2049] VirtualId# [6:7577471613054219731:0]
2025-11-27T18:19:52.783715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471613054219742:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.783771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:52.783811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:52.874547Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:52.874825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471610523794008:2050] [node 3] ICS01 socket: 269 reason# EndOfStream
2025-11-27T18:19:52.874896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471610523793911:2050] [node 3] ICP30 unregister session Session# [4:7577471610523794008:2050] VirtualId# [4:7577471610523793999:0]
2025-11-27T18:19:52.874942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471610523794008:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.875031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:52.875103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:52.875750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:52.875980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471610192364919:2050] [node 3] ICS01 socket: 309 reason# EndOfStream
2025-11-27T18:19:52.876020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471610192364820:2050] [node 3] ICP30 unregister session Session# [5:7577471610192364919:2050] VirtualId# [5:7577471610192364915:0]
2025-11-27T18:19:52.876052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471610192364919:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.876117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:52.876165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:52.876916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:52.877117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471613054219749:2050] [node 3] ICS01 socket: 305 reason# EndOfStream
2025-11-27T18:19:52.877154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471613054219649:2050] [node 3] ICP30 unregister session Session# [6:7577471613054219749:2050] VirtualId# [6:7577471613054219741:0]
2025-11-27T18:19:52.877179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471613054219749:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.877243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:52.877352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:52.993293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:52.993538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471610192364902:2108] [node 4] ICS07 socket disconnect 258 reason# EndOfStream
2025-11-27T18:19:52.993578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471610192364902:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.993667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:52.993709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471610192364902:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:52.994990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471610192364882:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65164) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:52.995138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471610192364902:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:52.995186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471610192364882:2108] [node 4] ICP30 unregister session Session# [5:7577471610192364902:2108] VirtualId# [5:7577471610192364898:0]
2025-11-27T18:19:52.995296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:52.995338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471610192364882:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65164) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:52.995356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65164) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:52.994953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:52.995154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471613054219737:2109] [node 4] ICS07 socket disconnect 254 reason# EndOfStream
2025-11-27T18:19:52.995179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471613054219737:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:52.995242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:52.995278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471613054219737:2109] [node 4] ICS15 start handshake
2025-11-27T18:19:52.999457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471613054219711:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65164) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:52.999533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471613054219737:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:52.999581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471613054219711:2109] [node 4] ICP30 unregister session Session# [6:7577471613054219737:2109] VirtualId# [6:7577471613054219729:0]
2025-11-27T18:19:52.999694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:52.999746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471613054219711:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65164) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:52.999761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65164) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:53.010673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471613054219711:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:19:53.123949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:53.124188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471613054219744:2106] [node 5] ICS01 socket: 280 reason# EndOfStream
2025-11-27T18:19:53.124240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471613054219708:2106] [node 5] ICP30 unregister session Session# [6:7577471613054219744:2106] VirtualId# [6:7577471613054219730:0]
2025-11-27T18:19:53.124276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471613054219744:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:53.124385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:53.124450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:53.248346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471613621747892:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:53.248418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471613621747893:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:53.270985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471614509184163:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:53.271018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471614509184164:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:53.291980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471613909287142:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:53.292047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471613909287143:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:53.313948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471614430619646:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:53.314003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471614430619647:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:53.314130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471614430619648:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:53.335283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471616421290197:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:53.335327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471616421290198:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:53.335357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471616421290199:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:53.361318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471617215943254:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:53.361363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471617215943255:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:53.361396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471617215943256:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:53.484978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471617215943305:2096] [node 5] ICP01 ready to work
2025-11-27T18:19:53.485250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471617215943306:2097] [node 4] ICP01 ready to work
2025-11-27T18:19:53.605891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471613621747892:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(104081168ns) {Configure:104(98254221ns) {ProcessConfigured:135(78378ns) {StartInitialHandshake:157(74538ns) {DropHandshakes:470(2766ns) {DropIncomingHandshake:442(592ns) DropOutgoingHandshake:457(494ns)} GenerateSessionVirtualId:988(1080ns) PrepareNewSessionHandshake:477(2087ns) {SwitchToState:179(959ns)}}}}}
2025-11-27T18:19:53.606394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471613621747958:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:53.602282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471616421290278:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:53.606985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471613621747959:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:53.608629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471614509184228:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:53.608497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471613909287223:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:53.607510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471613621747960:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:53.609618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471614430619727:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:53.609741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471614509184229:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:53.622473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471616421290283:2126] [node 6] ICP01 ready to work
2025-11-27T18:19:53.623447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471614430619728:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:53.624995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471614509184230:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:53.626154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471617215943305:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(36141747ns) {Configure:104(36131835ns) {ProcessConfigured:135(82106ns) {StartInitialHandshake:157(80006ns) {DropHandshakes:470(3930ns) {DropIncomingHandshake:442(375ns) DropOutgoingHandshake:457(182ns)} GenerateSessionVirtualId:988(376ns) PrepareNewSessionHandshake:477(1776ns) {SwitchToState:179(548ns)}}}}}
2025-11-27T18:19:53.636490Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471613621747893:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18569748ns) {Configure:104(18564764ns) {ProcessConfigured:135(1344936ns) {StartInitialHandshake:157(1340230ns) {DropHandshakes:470(2866ns) {DropIncomingHandshake:442(837ns) DropOutgoingHandshake:457(335ns)} GenerateSessionVirtualId:988(856ns) PrepareNewSessionHandshake:477(2417ns) {SwitchToState:179(1407ns)}}}}}
2025-11-27T18:19:53.639417Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471613909287142:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(26610619ns) {Configure:104(26605827ns) {ProcessConfigured:135(10038084ns) {StartInitialHandshake:157(10034551ns) {DropHandshakes:470(1312ns) {DropIncomingHandshake:442(360ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(591ns) PrepareNewSessionHandshake:477(3774ns) {SwitchToState:179(1870ns)}}}}}
2025-11-27T18:19:53.639965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471613909287226:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:53.646209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471613909287227:2127] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.677449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471616421290290:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9577
2025-11-27T18:19:53.687394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471614509184229:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16439715ns) {Configure:104(16434132ns) {ProcessConfigured:135(60121ns) {StartInitialHandshake:157(57812ns) {DropHandshakes:470(1444ns) {DropIncomingHandshake:442(344ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(287ns) PrepareNewSessionHandshake:477(2248ns) {SwitchToState:179(585ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.688532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471613621747969:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36918
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.689151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471613621747972:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36918
2025-11-27T18:19:53.688564Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471614430619646:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20032195ns) {Configure:104(20027205ns) {ProcessConfigured:135(2924730ns) {StartInitialHandshake:157(2921204ns) {DropHandshakes:470(1924ns) {DropIncomingHandshake:442(343ns) DropOutgoingHandshake:457(380ns)} GenerateSessionVirtualId:988(607ns) PrepareNewSessionHandshake:477(3071ns) {SwitchToState:179(1862ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.705601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471613621747972:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.703207Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471613909287227:2127] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19243767ns) {Configure:104(19237670ns) {ProcessConfigured:135(1566002ns) {StartInitialHandshake:157(1563686ns) {DropHandshakes:470(3042ns) {DropIncomingHandshake:442(725ns) DropOutgoingHandshake:457(382ns)} GenerateSessionVirtualId:988(900ns) PrepareNewSessionHandshake:477(3744ns) {SwitchToState:179(1901ns)}}}}}
2025-11-27T18:19:53.705731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471613621747974:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36918
2025-11-27T18:19:53.707357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471614509184230:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19515857ns) {Configure:104(19511587ns) {ProcessConfigured:135(55982ns) {StartInitialHandshake:157(50969ns) {DropHandshakes:470(1856ns) {DropIncomingHandshake:442(314ns) DropOutgoingHandshake:457(424ns)} GenerateSessionVirtualId:988(570ns) PrepareNewSessionHandshake:477(821ns) {SwitchToState:179(278ns)}}}}}
2025-11-27T18:19:53.704654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471613909287236:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59140
2025-11-27T18:19:53.704780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471613909287237:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59140
2025-11-27T18:19:53.709184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471613909287236:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.709375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471613909287237:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.711555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471616421290290:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.714474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471613909287142:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.724432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471613621747969:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.724685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471613621747962:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.723230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471614430619737:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38988
2025-11-27T18:19:53.724714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471613621747962:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.723389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471614430619738:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38988
2025-11-27T18:19:53.723489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471614430619739:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38988
2025-11-27T18:19:53.727228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471614430619737:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.727428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471614430619738:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.727524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471614430619739:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.727839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471614430619646:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.727881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471614430619648:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.730854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471614509184241:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13932
2025-11-27T18:19:53.731001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471614509184242:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13932
2025-11-27T18:19:53.731110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471614509184243:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13932
2025-11-27T18:19:53.731654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471614509184244:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13932
2025-11-27T18:19:53.737553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471614509184242:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.726459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471613621747976:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36918
2025-11-27T18:19:53.726710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471613621747962:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:53.730339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471613909287231:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.730377Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471613909287231:2130] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.746367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471613621747974:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.747066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471613909287243:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59140
2025-11-27T18:19:53.747278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471613909287244:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59140
2025-11-27T18:19:53.746521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471613621747964:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.746545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471613621747964:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.749044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471613909287243:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.749390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471613909287236:2133] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.746026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471614430619748:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38988
2025-11-27T18:19:53.750079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471613909287142:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.750135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:53.750549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471613909287142:2048] [node 1] ICP22 created new session: [3:7577471613909287247:2048]
2025-11-27T18:19:53.750599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471613909287247:2048] [node 1] ICS09 handshake done sender: [3:7577471613909287236:2133] self: [3:7577471613909287241:0] peer: [1:7577471613621747961:0] socket: 261 qp: -1
2025-11-27T18:19:53.750710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471613909287247:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:53.750207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471614430619749:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38988
2025-11-27T18:19:53.754686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471613621747893:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.754741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:53.754793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471613621747893:2049] [node 3] ICP22 created new session: [1:7577471613621747980:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.756802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471614430619750:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38988
2025-11-27T18:19:53.754849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471613621747980:2049] [node 3] ICS09 handshake done sender: [1:7577471613621747962:2112] self: [1:7577471613621747961:0] peer: [3:7577471613909287241:0] socket: 253 qp: -1
2025-11-27T18:19:53.754892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471613621747980:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:53.754930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:53.756514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471613621747893:2049] [node 3] ICP18 handshake (actor [1:7577471613621747974:2118]) is too old
2025-11-27T18:19:53.757024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471614509184243:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.757167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471614509184241:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.757517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471614509184244:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.757759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471614509184163:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.753872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:53.758329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471614430619748:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.758440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471614430619647:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.758714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471613621747979:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36918
2025-11-27T18:19:53.759574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471614509184236:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.759606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471614509184236:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.758878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471613909287245:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59140
2025-11-27T18:19:53.761695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471614509184236:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:53.761701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471613621747976:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.761986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471613621747964:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:53.761351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471614430619739:2134] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:53.759080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471613909287231:2130] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:53.763099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471614509184228:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.763164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:53.762235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471613909287223:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.762286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:53.762427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471613909287223:2107] [node 4] ICP22 created new session: [3:7577471613909287250:2107]
2025-11-27T18:19:53.762645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471613909287250:2107] [node 4] ICS09 handshake done sender: [3:7577471613909287231:2130] self: [3:7577471613909287230:0] peer: [4:7577471614430619747:0] socket: 265 qp: -1
2025-11-27T18:19:53.762679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471613909287250:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:53.762741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:53.768928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471613909287245:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.763209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471614509184228:2107] [node 4] ICP22 created new session: [2:7577471614509184253:2107]
2025-11-27T18:19:53.767811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471614509184253:2107] [node 4] ICS09 handshake done sender: [2:7577471614509184236:2114] self: [2:7577471614509184235:0] peer: [4:7577471614430619751:0] socket: 285 qp: -1
2025-11-27T18:19:53.768055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471614509184253:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:53.768120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.771601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471616421290295:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9577
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.771513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471613909287143:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.772931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471616421290296:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9577
2025-11-27T18:19:53.770829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471614430619648:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.770898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:53.770952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471614430619648:2050] [node 3] ICP22 created new session: [4:7577471614430619753:2050]
2025-11-27T18:19:53.771004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471614430619753:2050] [node 3] ICS09 handshake done sender: [4:7577471614430619739:2134] self: [4:7577471614430619747:0] peer: [3:7577471613909287230:0] socket: 273 qp: -1
2025-11-27T18:19:53.771038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471614430619753:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:53.771079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.777001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471616421290297:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9577
2025-11-27T18:19:53.775136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471614509184234:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.781872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471613621747957:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.786752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471617215943332:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55098
2025-11-27T18:19:53.775170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471614509184234:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.781913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471613621747957:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.776964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471614509184234:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:53.784179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471613621747979:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.790004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471614509184164:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.784353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471613621747958:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.790081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:53.812054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471617215943333:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55098
2025-11-27T18:19:53.784407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:53.790147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471614509184164:2049] [node 3] ICP22 created new session: [2:7577471614509184257:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.790222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471614509184257:2049] [node 3] ICS09 handshake done sender: [2:7577471614509184234:2113] self: [2:7577471614509184233:0] peer: [3:7577471613909287254:0] socket: 284 qp: -1
2025-11-27T18:19:53.790256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471614509184257:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:53.790298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:53.812682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471614509184259:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13932
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.814925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471617215943334:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55098
2025-11-27T18:19:53.784454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471613621747958:2107] [node 4] ICP22 created new session: [1:7577471613621747985:2107]
2025-11-27T18:19:53.784498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471613621747985:2107] [node 4] ICS09 handshake done sender: [1:7577471613621747964:2113] self: [1:7577471613621747963:0] peer: [4:7577471614430619746:0] socket: 258 qp: -1
2025-11-27T18:19:53.784545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471613621747985:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:53.784580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:53.797589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471613621747958:2107] [node 4] ICP18 handshake (actor [1:7577471613621747976:2119]) is too old
2025-11-27T18:19:53.815066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471614430619648:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:53.804425Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471613621747957:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:53.815175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471614430619752:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38988
2025-11-27T18:19:53.813422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471613621747892:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.813501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:53.813561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471613621747892:2048] [node 2] ICP22 created new session: [1:7577471613621747989:2048]
2025-11-27T18:19:53.813618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471613621747989:2048] [node 2] ICS09 handshake done sender: [1:7577471613621747957:2111] self: [1:7577471613621747956:0] peer: [2:7577471614509184250:0] socket: 252 qp: -1
2025-11-27T18:19:53.813657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471613621747989:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:53.813694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:53.818127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471614430619738:2133] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:53.819038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471614509184241:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:53.819316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471616421290295:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.824846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471614430619647:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27702) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471614430619748:2136] held: no
2025-11-27T18:19:53.821686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471616421290297:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.826440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471614430619646:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.826513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:53.826571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471614430619646:2048] [node 1] ICP22 created new session: [4:7577471614430619755:2048]
2025-11-27T18:19:53.826618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471614430619755:2048] [node 1] ICS09 handshake done sender: [4:7577471614430619738:2133] self: [4:7577471614430619746:0] peer: [1:7577471613621747963:0] socket: 272 qp: -1
2025-11-27T18:19:53.826655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471614430619755:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:53.826697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:53.827405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471614430619646:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:53.836470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471617215943334:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.836625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471617215943254:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.834701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471614430619748:2136] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:53.837482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471616421290197:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.837584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471616421290198:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.841739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471613621747968:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.844534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471617215943336:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55098
2025-11-27T18:19:53.844834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471614509184163:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.844907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:53.844972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471614509184163:2048] [node 1] ICP22 created new session: [2:7577471614509184261:2048]
2025-11-27T18:19:53.845041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471614509184261:2048] [node 1] ICS09 handshake done sender: [2:7577471614509184241:2117] self: [2:7577471614509184250:0] peer: [1:7577471613621747956:0] socket: 267 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.841784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471613621747968:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.842928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471613621747966:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.845077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471614509184261:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:53.842954Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471613621747966:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.850492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471613621747968:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:53.849298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471616421290296:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.849666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471616421290300:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9577
2025-11-27T18:19:53.849774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471616421290199:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.850394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471617215943332:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.845115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:53.852023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471614509184163:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:53.854046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471617215943334:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:53.853738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471613621747966:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:53.854230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471614430619647:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.854285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:53.860189Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471613909287143:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27702) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471613909287245:2137] held: no
2025-11-27T18:19:53.854342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471614430619647:2049] [node 2] ICP22 created new session: [4:7577471614430619759:2049]
2025-11-27T18:19:53.857138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471614430619759:2049] [node 2] ICS09 handshake done sender: [4:7577471614430619748:2136] self: [4:7577471614430619751:0] peer: [2:7577471614509184235:0] socket: 287 qp: -1
2025-11-27T18:19:53.857207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471614430619759:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:53.862755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:53.861460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471613621747960:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.866418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471616421290295:2134] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:53.861540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:53.864935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471613909287233:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.866618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471614509184238:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.867209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471617215943256:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.864973Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471613909287233:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.866671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471614509184238:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.861597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471613621747960:2110] [node 6] ICP22 created new session: [1:7577471613621747993:2110]
2025-11-27T18:19:53.861642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471613621747993:2110] [node 6] ICS09 handshake done sender: [1:7577471613621747968:2115] self: [1:7577471613621747967:0] peer: [6:7577471617215943335:0] socket: 278 qp: -1
2025-11-27T18:19:53.869311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471614509184238:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:53.867269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471617215943254:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.866167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471613909287255:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59140
2025-11-27T18:19:53.894240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471616421290197:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.861678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471613621747993:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:53.861712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:53.862676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471613621747959:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.863214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:53.864276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471613621747959:2109] [node 5] ICP22 created new session: [1:7577471613621747995:2109]
2025-11-27T18:19:53.876421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471613621747995:2109] [node 5] ICS09 handshake done sender: [1:7577471613621747966:2114] self: [1:7577471613621747965:0] peer: [5:7577471616421290298:0] socket: 266 qp: -1
2025-11-27T18:19:53.876490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471613621747995:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:53.876555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:53.889023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:53.898607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471614509184229:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.894313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:53.867743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471613909287245:2137] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:53.898671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:53.898724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471614509184229:2108] [node 5] ICP22 created new session: [2:7577471614509184264:2108]
2025-11-27T18:19:53.898765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471614509184264:2108] [node 5] ICS09 handshake done sender: [2:7577471614509184238:2115] self: [2:7577471614509184237:0] peer: [5:7577471616421290299:0] socket: 299 qp: -1
2025-11-27T18:19:53.898804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471614509184264:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:53.898837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:53.894365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471616421290197:2048] [node 1] ICP22 created new session: [5:7577471616421290302:2048]
2025-11-27T18:19:53.889087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471617215943254:2048] [node 1] ICP22 created new session: [6:7577471617215943338:2048]
2025-11-27T18:19:53.893353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471613909287233:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:53.900314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471613909287143:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.900368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:53.900443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471613909287143:2049] [node 2] ICP22 created new session: [3:7577471613909287257:2049]
2025-11-27T18:19:53.900532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471613909287257:2049] [node 2] ICS09 handshake done sender: [3:7577471613909287245:2137] self: [3:7577471613909287254:0] peer: [2:7577471614509184233:0] socket: 288 qp: -1
2025-11-27T18:19:53.900579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471613909287257:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:53.900636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:53.901765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471613909287235:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.901795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471613909287235:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.902453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471613909287226:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.902506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:53.902555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471613909287226:2108] [node 5] ICP22 created new session: [3:7577471613909287259:2108]
2025-11-27T18:19:53.902596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471613909287259:2108] [node 5] ICS09 handshake done sender: [3:7577471613909287233:2131] self: [3:7577471613909287232:0] peer: [5:7577471616421290301:0] socket: 263 qp: -1
2025-11-27T18:19:53.902634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471613909287259:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:53.902666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:53.889128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471617215943338:2048] [node 1] ICS09 handshake done sender: [6:7577471617215943334:2117] self: [6:7577471617215943335:0] peer: [1:7577471613621747967:0] socket: 302 qp: -1
2025-11-27T18:19:53.889163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471617215943338:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:53.889203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:53.890717Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471617215943254:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22625028ns) {HandleHandshakeStatus:329(22623583ns) {DropIncomingHandshake:442(207ns) DropOutgoingHandshake:457(4357ns) SwitchToState:179(165ns) ProcessPendingSessionEvents:495(573883ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 508056ns max# 66982ns}}}
2025-11-27T18:19:53.890903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471617215943333:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.894412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471616421290302:2048] [node 1] ICS09 handshake done sender: [5:7577471616421290295:2134] self: [5:7577471616421290298:0] peer: [1:7577471613621747965:0] socket: 296 qp: -1
2025-11-27T18:19:53.894447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471616421290302:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:53.894487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:53.906401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471616421290197:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.906700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471616421290304:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9577
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.910259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471617215943255:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.910380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471617215943340:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55098
2025-11-27T18:19:53.911631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471614509184240:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.911683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471614509184240:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.911376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471616421290305:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9577
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.912574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471616421290306:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9577
2025-11-27T18:19:53.912275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471617215943340:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.916650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471613909287235:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:53.917018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471613909287227:2127] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.917067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:53.917120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471613909287227:2127] [node 6] ICP22 created new session: [3:7577471613909287262:2127]
2025-11-27T18:19:53.917182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471613909287262:2127] [node 6] ICS09 handshake done sender: [3:7577471613909287235:2132] self: [3:7577471613909287234:0] peer: [6:7577471617215943337:0] socket: 280 qp: -1
2025-11-27T18:19:53.918219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471616421290199:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1255) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471616421290296:2135] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.920936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471617215943343:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55098
2025-11-27T18:19:53.917418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471613909287262:2127] [node 6] ICS10 traffic start
2025-11-27T18:19:53.917462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.923076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471614509184266:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13932
2025-11-27T18:19:53.923253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471614509184240:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:53.923421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471614430619736:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.923453Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471614430619736:2131] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.924730Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471614430619736:2131] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:53.922834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471616421290198:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27702) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471616421290297:2136] held: no
2025-11-27T18:19:53.924505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471616421290306:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.922764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471617215943306:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.924593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471616421290296:2135] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:53.922989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471617215943332:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:53.926018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471614430619728:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.926075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:53.925857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471614509184230:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.925919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:53.926727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471614430619728:2109] [node 6] ICP22 created new session: [4:7577471614430619763:2109]
2025-11-27T18:19:53.926690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471616421290297:2136] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:53.925973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471614509184230:2109] [node 6] ICP22 created new session: [2:7577471614509184268:2109]
2025-11-27T18:19:53.926016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471614509184268:2109] [node 6] ICS09 handshake done sender: [2:7577471614509184240:2116] self: [2:7577471614509184239:0] peer: [6:7577471617215943341:0] socket: 274 qp: -1
2025-11-27T18:19:53.926050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471614509184268:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:53.926103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:53.926780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471614430619763:2109] [node 6] ICS09 handshake done sender: [4:7577471614430619736:2131] self: [4:7577471614430619735:0] peer: [6:7577471617215943344:0] socket: 306 qp: -1
2025-11-27T18:19:53.928232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471614509184266:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.928342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471614509184230:2109] [node 6] ICP18 handshake (actor [2:7577471614509184266:2122]) is too old
2025-11-27T18:19:53.926809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471614430619763:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:53.926843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:53.927225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471617215943256:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.927289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:53.927568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471617215943256:2050] [node 3] ICP22 created new session: [6:7577471617215943347:2050]
2025-11-27T18:19:53.927615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471617215943347:2050] [node 3] ICS09 handshake done sender: [6:7577471617215943332:2115] self: [6:7577471617215943337:0] peer: [3:7577471613909287234:0] socket: 297 qp: -1
2025-11-27T18:19:53.928350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471616421290199:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.927649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471617215943347:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:53.927685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:53.928412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:53.928450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471616421290199:2050] [node 3] ICP22 created new session: [5:7577471616421290308:2050]
2025-11-27T18:19:53.928485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471616421290308:2050] [node 3] ICS09 handshake done sender: [5:7577471616421290296:2135] self: [5:7577471616421290301:0] peer: [3:7577471613909287232:0] socket: 298 qp: -1
2025-11-27T18:19:53.928511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471616421290308:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:53.928535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:53.930478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471616421290198:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.930516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:53.930553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471616421290198:2049] [node 2] ICP22 created new session: [5:7577471616421290310:2049]
2025-11-27T18:19:53.930590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471616421290310:2049] [node 2] ICS09 handshake done sender: [5:7577471616421290297:2136] self: [5:7577471616421290299:0] peer: [2:7577471614509184237:0] socket: 300 qp: -1
2025-11-27T18:19:53.930617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471616421290310:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:53.930649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.934265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471614430619762:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38988
2025-11-27T18:19:53.937626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471616421290278:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.940431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471614430619741:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.940464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471614430619741:2135] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.941602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471614430619741:2135] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:53.942859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471614430619727:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.942917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:53.942968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471614430619727:2108] [node 5] ICP22 created new session: [4:7577471614430619766:2108]
2025-11-27T18:19:53.943015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471614430619766:2108] [node 5] ICS09 handshake done sender: [4:7577471614430619741:2135] self: [4:7577471614430619740:0] peer: [5:7577471616421290313:0] socket: 307 qp: -1
2025-11-27T18:19:53.943049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471614430619766:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:53.943083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.947546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471617215943345:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55098
2025-11-27T18:19:53.948108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471617215943346:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55098
2025-11-27T18:19:53.948502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471617215943255:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27702) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471617215943333:2116] held: no
2025-11-27T18:19:53.949217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471617215943306:2097] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19608) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471617215943340:2119] held: no
2025-11-27T18:19:53.950230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471617215943333:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:53.950998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471617215943340:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:53.954060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471617215943255:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.954126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:53.954179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471617215943255:2049] [node 2] ICP22 created new session: [6:7577471617215943350:2049]
2025-11-27T18:19:53.954224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471617215943350:2049] [node 2] ICS09 handshake done sender: [6:7577471617215943333:2116] self: [6:7577471617215943341:0] peer: [2:7577471614509184239:0] socket: 275 qp: -1
2025-11-27T18:19:53.954261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471617215943350:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:53.954302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:53.955291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471617215943306:2097] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.955331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:53.955374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471617215943306:2097] [node 4] ICP22 created new session: [6:7577471617215943352:2097]
2025-11-27T18:19:53.955420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471617215943352:2097] [node 4] ICS09 handshake done sender: [6:7577471617215943340:2119] self: [6:7577471617215943344:0] peer: [4:7577471614430619735:0] socket: 248 qp: -1
2025-11-27T18:19:53.955449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471617215943352:2097] [node 4] ICS10 traffic start
2025-11-27T18:19:53.955485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:53.953520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471614430619762:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.960343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471617215943349:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55098
2025-11-27T18:19:53.953686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471614430619727:2108] [node 5] ICP18 handshake (actor [4:7577471614430619762:2140]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.962663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471616421290314:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9577
2025-11-27T18:19:53.963093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471616421290278:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19608) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471616421290306:2140] held: no
2025-11-27T18:19:53.964011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471617215943349:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.964145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471617215943305:2096] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:53.964286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471616421290306:2140] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:53.964734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471616421290289:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:53.964786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471616421290289:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:53.965684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471616421290278:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.965735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:53.965781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471616421290278:2108] [node 4] ICP22 created new session: [5:7577471616421290315:2108]
2025-11-27T18:19:53.965844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471616421290315:2108] [node 4] ICS09 handshake done sender: [5:7577471616421290306:2140] self: [5:7577471616421290313:0] peer: [4:7577471614430619740:0] socket: 308 qp: -1
2025-11-27T18:19:53.965897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471616421290315:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:53.965932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:53.967606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471617215943355:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55098
2025-11-27T18:19:53.968653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471616421290289:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:53.969053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471617215943349:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:53.969462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471616421290283:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:53.969516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:53.969560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471616421290283:2126] [node 6] ICP22 created new session: [5:7577471616421290318:2126]
2025-11-27T18:19:53.969608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471616421290318:2126] [node 6] ICS09 handshake done sender: [5:7577471616421290289:2132] self: [5:7577471616421290288:0] peer: [6:7577471617215943354:0] socket: 256 qp: -1
2025-11-27T18:19:53.969638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471616421290318:2126] [node 6] ICS10 traffic start
2025-11-27T18:19:53.969669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:53.973490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471617215943305:2096] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:53.973550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:53.973609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471617215943305:2096] [node 5] ICP22 created new session: [6:7577471617215943356:2096]
2025-11-27T18:19:53.973670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471617215943356:2096] [node 5] ICS09 handshake done sender: [6:7577471617215943349:2123] self: [6:7577471617215943354:0] peer: [5:7577471616421290288:0] socket: 247 qp: -1
2025-11-27T18:19:53.973707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471617215943356:2096] [node 5] ICS10 traffic start
2025-11-27T18:19:53.973753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:53.978795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471617215943305:2096] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:54.006822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:54.007102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471614509184261:2048] [node 1] ICS07 socket disconnect 267 reason# EndOfStream
2025-11-27T18:19:54.007159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471614509184261:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.007248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:54.007309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471614509184261:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:54.007446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:54.007641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471613909287247:2048] [node 1] ICS07 socket disconnect 261 reason# EndOfStream
2025-11-27T18:19:54.007682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471613909287247:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.007760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:54.007797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471613909287247:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:54.010282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471614509184163:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:54.010324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471614509184261:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:54.011535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471613909287142:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:54.011575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471613909287247:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:54.012597Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471614509184163:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.012552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471613909287142:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.011759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:54.012681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471613909287247:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.012040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471614430619755:2048] [node 1] ICS07 socket disconnect 272 reason# EndOfStream
2025-11-27T18:19:54.012721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471614509184261:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.012074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471614430619755:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.012177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:54.012219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471614430619755:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:54.012768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471614509184163:2048] [node 1] ICP30 unregister session Session# [2:7577471614509184261:2048] VirtualId# [2:7577471614509184250:0]
2025-11-27T18:19:54.012816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:54.012977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471614509184163:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.013007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.012792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471613909287142:2048] [node 1] ICP30 unregister session Session# [3:7577471613909287247:2048] VirtualId# [3:7577471613909287241:0]
2025-11-27T18:19:54.012919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:54.012965Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471613909287142:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.012980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.013376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471614430619646:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.013450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471614430619755:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.013486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471614430619646:2048] [node 1] ICP30 unregister session Session# [4:7577471614430619755:2048] VirtualId# [4:7577471614430619746:0]
2025-11-27T18:19:54.014003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:54.014073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471614430619646:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.014106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.016855Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:54.017097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471616421290302:2048] [node 1] ICS07 socket disconnect 296 reason# EndOfStream
2025-11-27T18:19:54.017135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471616421290302:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.017222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:54.017478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471616421290302:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:54.018608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:54.019482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471617215943338:2048] [node 1] ICS07 socket disconnect 302 reason# EndOfStream
2025-11-27T18:19:54.019911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471617215943338:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.020288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471616421290197:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.019998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:54.020468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471616421290302:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.020278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471617215943338:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:54.020564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471616421290197:2048] [node 1] ICP30 unregister session Session# [5:7577471616421290302:2048] VirtualId# [5:7577471616421290298:0]
2025-11-27T18:19:54.020685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:54.020726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471616421290197:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.020749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.023203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471614430619646:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:54.023203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471613909287142:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:54.022221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471614509184163:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:54.025679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471617215943254:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.025751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471617215943338:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.025787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471617215943254:2048] [node 1] ICP30 unregister session Session# [6:7577471617215943338:2048] VirtualId# [6:7577471617215943335:0]
2025-11-27T18:19:54.025906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:54.025960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471617215943254:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.025976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13968) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.029371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471616421290197:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:54.036516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471617215943254:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:54.092335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:54.092612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471613909287257:2049] [node 2] ICS01 socket: 288 reason# EndOfStream
2025-11-27T18:19:54.092670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471613909287143:2049] [node 2] ICP30 unregister session Session# [3:7577471613909287257:2049] VirtualId# [3:7577471613909287254:0]
2025-11-27T18:19:54.092710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471613909287257:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.092823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:54.092882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:54.096424Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:54.096569Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:54.096704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471614430619759:2049] [node 2] ICS01 socket: 287 reason# EndOfStream
2025-11-27T18:19:54.096756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471614430619647:2049] [node 2] ICP30 unregister session Session# [4:7577471614430619759:2049] VirtualId# [4:7577471614430619751:0]
2025-11-27T18:19:54.096790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471614430619759:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.096900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:54.096958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:54.097062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:54.097607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471617215943350:2049] [node 2] ICS01 socket: 275 reason# EndOfStream
2025-11-27T18:19:54.097654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471617215943255:2049] [node 2] ICP30 unregister session Session# [6:7577471617215943350:2049] VirtualId# [6:7577471617215943341:0]
2025-11-27T18:19:54.097681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471617215943350:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.097745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:54.097782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:54.096802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471616421290310:2049] [node 2] ICS01 socket: 300 reason# EndOfStream
2025-11-27T18:19:54.096852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471616421290198:2049] [node 2] ICP30 unregister session Session# [5:7577471616421290310:2049] VirtualId# [5:7577471616421290299:0]
2025-11-27T18:19:54.096890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471616421290310:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.096971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:54.097014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:54.133848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:54.134098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471616421290308:2050] [node 3] ICS01 socket: 298 reason# EndOfStream
2025-11-27T18:19:54.134141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471616421290199:2050] [node 3] ICP30 unregister session Session# [5:7577471616421290308:2050] VirtualId# [5:7577471616421290301:0]
2025-11-27T18:19:54.134176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471616421290308:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.134258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:54.134317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:54.137123Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:54.137365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471614430619753:2050] [node 3] ICS07 socket disconnect 273 reason# EndOfStream
2025-11-27T18:19:54.137403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471614430619753:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.137484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:54.137530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471614430619753:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:54.138051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:54.138191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471617215943347:2050] [node 3] ICS07 socket disconnect 297 reason# EndOfStream
2025-11-27T18:19:54.138226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471617215943347:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.138301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:54.138356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471617215943347:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:54.139245Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471617215943256:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1255) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.139253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471614430619648:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1255) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.139298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471617215943347:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.139308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471614430619753:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.139337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471617215943256:2050] [node 3] ICP30 unregister session Session# [6:7577471617215943347:2050] VirtualId# [6:7577471617215943337:0]
2025-11-27T18:19:54.139348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471614430619648:2050] [node 3] ICP30 unregister session Session# [4:7577471614430619753:2050] VirtualId# [4:7577471614430619747:0]
2025-11-27T18:19:54.139435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:54.139475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471614430619648:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1255) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.139494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1255) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.139423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:54.139459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471617215943256:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1255) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.139473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1255) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.149207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471614430619648:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:54.149360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471617215943256:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:54.194741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:54.194996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471616421290315:2108] [node 4] ICS01 socket: 308 reason# EndOfStream
2025-11-27T18:19:54.195040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471616421290278:2108] [node 4] ICP30 unregister session Session# [5:7577471616421290315:2108] VirtualId# [5:7577471616421290313:0]
2025-11-27T18:19:54.195072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471616421290315:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.195165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:54.195227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:54.196062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:54.196274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471617215943352:2097] [node 4] ICS01 socket: 248 reason# EndOfStream
2025-11-27T18:19:54.196307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471617215943306:2097] [node 4] ICP30 unregister session Session# [6:7577471617215943352:2097] VirtualId# [6:7577471617215943344:0]
2025-11-27T18:19:54.196333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471617215943352:2097] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.196416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:54.196470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:54.230471Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:54.230725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471617215943356:2096] [node 5] ICS07 socket disconnect 247 reason# EndOfStream
2025-11-27T18:19:54.230764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471617215943356:2096] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.230841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:54.230889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471617215943356:2096] [node 5] ICS15 start handshake
2025-11-27T18:19:54.231812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471617215943305:2096] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26917) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.231874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471617215943356:2096] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.231911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471617215943305:2096] [node 5] ICP30 unregister session Session# [6:7577471617215943356:2096] VirtualId# [6:7577471617215943354:0]
2025-11-27T18:19:54.232019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:19:54.232060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471617215943305:2096] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26917) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.232095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26917) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.242431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471617215943305:2096] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:54.317248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471618311074555:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:54.317314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471618311074556:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:54.339377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471619292941207:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:54.339417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471619292941208:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:54.362617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471620361881666:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:54.362681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471620361881667:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:54.385126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471621344792441:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:54.385212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471621344792443:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:54.385296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471621344792442:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:54.416824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471619351506843:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:54.416910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471619351506844:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:54.416946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471619351506845:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:54.439859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471619772438667:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:54.439924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471619772438668:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:54.440427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471619772438666:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:54.532853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471619772438726:2105] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.536127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471619351506904:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59387
2025-11-27T18:19:54.600236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471618311074620:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:54.593639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471619772438733:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:54.600213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471619292941270:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:54.602870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471621344792504:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:54.600292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471620361881730:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:54.607953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471619292941271:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:54.602921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471618311074619:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:54.606948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471620361881731:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:54.608798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471621344792505:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:54.608294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471620361881732:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:54.608593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471619351506908:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:54.609133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471618311074621:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:54.609005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471619351506909:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:54.609384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471619292941272:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:54.622711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471619351506904:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.631099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471621344792514:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30228
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.633649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471619351506924:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59387
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.635096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471620361881741:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32052
2025-11-27T18:19:54.636103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471621344792514:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.636436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471620361881741:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.638575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471621344792520:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30228
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.639961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471619351506924:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:19:54.640106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471618311074630:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7275
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.640134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471619292941282:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29556
::ffff:127.1.0.0
2025-11-27T18:19:54.640221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471619351506908:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.640107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471618311074631:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7275
2025-11-27T18:19:54.640546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471619292941281:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29556
2025-11-27T18:19:54.640295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471618311074634:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7275
2025-11-27T18:19:54.640847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471621344792512:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.641148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471619292941282:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.642599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471619292941281:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.642394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471618311074631:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.640895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471621344792512:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.644673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471619292941285:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29556
2025-11-27T18:19:54.643853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471618311074630:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.645303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471621344792522:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30228
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:19:54.644846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471618311074634:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.645571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471621344792512:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:54.645770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471618311074638:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7275
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.645859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471619351506926:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59387
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.647558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471619292941289:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29556
2025-11-27T18:19:54.647387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471621344792504:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.647460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.648017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471618311074639:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7275
2025-11-27T18:19:54.647670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471621344792504:2108] [node 5] ICP22 created new session: [4:7577471621344792524:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.653376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471620361881748:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32052
2025-11-27T18:19:54.655516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471619292941285:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.655842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471619351506927:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59387
2025-11-27T18:19:54.648044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471621344792524:2108] [node 5] ICS09 handshake done sender: [4:7577471621344792512:2112] self: [4:7577471621344792511:0] peer: [5:7577471619351506925:0] socket: 264 qp: -1
2025-11-27T18:19:54.648126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471621344792524:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:54.655884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471618311074638:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.656151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471620361881749:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32052
2025-11-27T18:19:54.648172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:54.659226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471620361881749:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.659434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471618311074639:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.659448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471621344792520:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.659623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471619351506908:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5238) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471619351506924:2117] held: no
No RDMA context found for GID: 2025-11-27T18:19:54.659656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471621344792522:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:19:54.659667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471620361881667:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.659778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471619292941289:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.659887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471620361881750:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32052
2025-11-27T18:19:54.659942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471621344792443:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.660019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471620361881748:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.660191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471619292941207:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.660967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471619292941278:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.660992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471619292941278:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.661860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471619292941278:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:54.660526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471621344792442:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.660377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471620361881666:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.660356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471619351506924:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:54.662816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471619292941276:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.662866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471619292941276:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.667208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471618311074625:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.665599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471619351506908:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.667251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471618311074625:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.666672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471620361881740:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.665650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:54.667829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471618311074623:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.667861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471618311074623:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.668786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471618311074623:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:54.671596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471618311074625:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:54.666718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471620361881740:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.665707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471619351506908:2109] [node 4] ICP22 created new session: [5:7577471619351506928:2109]
2025-11-27T18:19:54.665753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471619351506928:2109] [node 4] ICS09 handshake done sender: [5:7577471619351506924:2117] self: [5:7577471619351506925:0] peer: [4:7577471621344792511:0] socket: 265 qp: -1
2025-11-27T18:19:54.673032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471619292941270:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.665793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471619351506928:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:54.665827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:54.671325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471619351506926:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.673138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:54.673094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471618311074556:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.673152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:54.676706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471619351506843:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.673189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471619292941270:2107] [node 4] ICP22 created new session: [2:7577471619292941297:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.677140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471619292941297:2107] [node 4] ICS09 handshake done sender: [2:7577471619292941278:2112] self: [2:7577471619292941277:0] peer: [4:7577471621344792528:0] socket: 284 qp: -1
2025-11-27T18:19:54.673205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471618311074556:2049] [node 3] ICP22 created new session: [1:7577471618311074647:2049]
2025-11-27T18:19:54.677799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471619292941276:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:54.673506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471618311074647:2049] [node 3] ICS09 handshake done sender: [1:7577471618311074623:2111] self: [1:7577471618311074622:0] peer: [3:7577471620361881752:0] socket: 280 qp: -1
2025-11-27T18:19:54.677855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471619292941297:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:54.674601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471618311074647:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:54.678123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:54.676991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:54.677996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471620361881740:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:54.679383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471618311074555:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.678791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471620361881753:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32052
2025-11-27T18:19:54.679444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:54.679452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471620361881750:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.679524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471620361881730:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.679730Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471621344792504:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(23615857ns) {HandleHandshakeStatus:329(23612250ns) {DropIncomingHandshake:442(12359ns) DropOutgoingHandshake:457(554ns) SwitchToState:179(538ns) ProcessPendingSessionEvents:495(17405374ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16097660ns max# 3803445ns}}}
2025-11-27T18:19:54.679587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471618311074555:2048] [node 2] ICP22 created new session: [1:7577471618311074649:2048]
2025-11-27T18:19:54.679687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.679695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471618311074649:2048] [node 2] ICS09 handshake done sender: [1:7577471618311074625:2112] self: [1:7577471618311074624:0] peer: [2:7577471619292941293:0] socket: 283 qp: -1
2025-11-27T18:19:54.679838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471620361881730:2107] [node 4] ICP22 created new session: [3:7577471620361881756:2107]
2025-11-27T18:19:54.679734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471618311074649:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:54.680064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471620361881756:2107] [node 4] ICS09 handshake done sender: [3:7577471620361881740:2114] self: [3:7577471620361881737:0] peer: [4:7577471621344792527:0] socket: 275 qp: -1
2025-11-27T18:19:54.679773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:54.680059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471621344792529:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30228
2025-11-27T18:19:54.680096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471620361881756:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:54.680195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:54.684611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471619292941208:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.684579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471618311074627:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.684637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471618311074627:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.684821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471620361881754:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32052
2025-11-27T18:19:54.684563Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471621344792442:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471621344792522:2117] held: no
2025-11-27T18:19:54.684685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:54.685049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471619292941208:2049] [node 3] ICP22 created new session: [2:7577471619292941300:2049]
2025-11-27T18:19:54.685291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471619292941300:2049] [node 3] ICS09 handshake done sender: [2:7577471619292941276:2111] self: [2:7577471619292941274:0] peer: [3:7577471620361881751:0] socket: 282 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.685425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471619292941300:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:54.686664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:54.687277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471621344792530:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30228
2025-11-27T18:19:54.687370Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471620361881666:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27420) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471620361881748:2117] held: no
2025-11-27T18:19:54.687581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471621344792522:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.688287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471619292941296:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29556
2025-11-27T18:19:54.688599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471621344792442:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.688652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:54.688669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471621344792520:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:54.688695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471621344792442:2049] [node 2] ICP22 created new session: [4:7577471621344792531:2049]
2025-11-27T18:19:54.688750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471620361881748:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:54.689075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471621344792531:2049] [node 2] ICS09 handshake done sender: [4:7577471621344792522:2117] self: [4:7577471621344792528:0] peer: [2:7577471619292941277:0] socket: 289 qp: -1
2025-11-27T18:19:54.689111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471621344792531:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:54.689839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471618311074627:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:54.689153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.690452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471621344792443:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.690392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471618311074620:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.690517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:54.690734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471619292941298:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29556
2025-11-27T18:19:54.690594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471620361881730:2107] [node 4] ICP18 handshake (actor [3:7577471620361881750:2119]) is too old
2025-11-27T18:19:54.690475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:54.690706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471620361881666:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.690721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471618311074620:2108] [node 5] ICP22 created new session: [1:7577471618311074654:2108]
2025-11-27T18:19:54.690752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471619351506932:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59387
No RDMA context found for GID: 2025-11-27T18:19:54.690767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471618311074654:2108] [node 5] ICS09 handshake done sender: [1:7577471618311074627:2113] self: [1:7577471618311074626:0] peer: [5:7577471619351506931:0] socket: 286 qp: -1
::ffff:127.1.0.02025-11-27T18:19:54.690701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471621344792443:2050] [node 3] ICP22 created new session: [4:7577471621344792535:2050]
2025-11-27T18:19:54.690717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471620361881749:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:54.690744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:54.690807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471620361881666:2048] [node 1] ICP22 created new session: [3:7577471620361881760:2048]
2025-11-27T18:19:54.690802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471618311074654:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:54.690754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471621344792535:2050] [node 3] ICS09 handshake done sender: [4:7577471621344792520:2116] self: [4:7577471621344792527:0] peer: [3:7577471620361881737:0] socket: 276 qp: -1
2025-11-27T18:19:54.690888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471621344792535:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:54.691514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471619351506926:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:54.692421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471619772438744:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51279
2025-11-27T18:19:54.691151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471620361881760:2048] [node 1] ICS09 handshake done sender: [3:7577471620361881748:2117] self: [3:7577471620361881752:0] peer: [1:7577471618311074622:0] socket: 290 qp: -1
2025-11-27T18:19:54.691255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:54.691095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:54.691186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471620361881760:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:54.693993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471619351506843:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.694147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:54.691322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:54.694700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471619292941207:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27420) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471619292941289:2118] held: no
2025-11-27T18:19:54.694619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471619351506843:2048] [node 1] ICP22 created new session: [5:7577471619351506933:2048]
2025-11-27T18:19:54.694929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471619351506933:2048] [node 1] ICS09 handshake done sender: [5:7577471619351506926:2118] self: [5:7577471619351506931:0] peer: [1:7577471618311074626:0] socket: 287 qp: -1
2025-11-27T18:19:54.695298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471619351506933:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.695712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:54.697820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471619292941298:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.698511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471619292941289:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:54.698880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471619351506934:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59387
2025-11-27T18:19:54.699152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471619292941207:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.699191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:54.700344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471619292941207:2048] [node 1] ICP22 created new session: [2:7577471619292941304:2048]
2025-11-27T18:19:54.701900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471620361881667:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.700433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471619292941304:2048] [node 1] ICS09 handshake done sender: [2:7577471619292941289:2118] self: [2:7577471619292941293:0] peer: [1:7577471618311074624:0] socket: 285 qp: -1
2025-11-27T18:19:54.701963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:54.700470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471619292941304:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:54.700520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:54.702007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471620361881667:2049] [node 2] ICP22 created new session: [3:7577471620361881762:2049]
2025-11-27T18:19:54.702048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471620361881762:2049] [node 2] ICS09 handshake done sender: [3:7577471620361881749:2118] self: [3:7577471620361881751:0] peer: [2:7577471619292941274:0] socket: 292 qp: -1
2025-11-27T18:19:54.702085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471620361881762:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:54.702121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.705289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471619351506937:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59387
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.705383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471620361881667:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:54.705283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471619772438746:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51279
2025-11-27T18:19:54.705474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471621344792534:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30228
2025-11-27T18:19:54.705295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471619772438745:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51279
2025-11-27T18:19:54.705384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471619772438747:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51279
2025-11-27T18:19:54.706332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471621344792534:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.709225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471619772438744:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.710731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471619772438666:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.711611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471618311074633:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.714602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471621344792441:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.711651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471618311074633:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.714891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471619772438747:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.714611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471618311074633:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:54.715759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471619772438748:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51279
2025-11-27T18:19:54.716859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471618311074621:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.716930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:54.717184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471619772438726:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.717001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471618311074621:2109] [node 6] ICP22 created new session: [1:7577471618311074657:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.717243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471619772438746:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.717368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471619772438745:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.717600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471618311074657:2109] [node 6] ICS09 handshake done sender: [1:7577471618311074633:2117] self: [1:7577471618311074632:0] peer: [6:7577471619772438749:0] socket: 279 qp: -1
2025-11-27T18:19:54.717205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471619351506934:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.717652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471618311074657:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:54.717485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471619772438748:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.717693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:54.717543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471619351506922:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.717565Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471619351506922:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.717754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471619772438750:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51279
2025-11-27T18:19:54.717798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471619351506844:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.718430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471619772438668:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.718434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471619351506937:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.718586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471619772438733:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.719086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471618311074629:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.719045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471619292941280:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.719072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471619351506845:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.719085Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471619292941280:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.719126Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471618311074629:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.719600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471619351506922:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:54.720231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471620361881739:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.720250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471620361881743:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.720155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471619772438666:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27420) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471619772438744:2115] held: no
2025-11-27T18:19:54.720262Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471620361881739:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.721519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471619351506909:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.720287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471620361881743:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.722331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:54.722399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471619351506909:2110] [node 6] ICP22 created new session: [5:7577471619351506941:2110]
2025-11-27T18:19:54.721684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471618311074629:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:54.722931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471620361881743:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:54.722716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471619292941280:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:54.723006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471619351506941:2110] [node 6] ICS09 handshake done sender: [5:7577471619351506922:2116] self: [5:7577471619351506920:0] peer: [6:7577471619772438751:0] socket: 268 qp: -1
2025-11-27T18:19:54.723614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471619351506941:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:54.723640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471618311074619:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.724222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471620361881732:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.724063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471619292941272:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.723687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:54.723664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:54.724118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:54.725119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471619772438667:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:54.724223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471618311074619:2107] [node 4] ICP22 created new session: [1:7577471618311074660:2107]
2025-11-27T18:19:54.724266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.724247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471619292941272:2108] [node 5] ICP22 created new session: [2:7577471619292941308:2108]
2025-11-27T18:19:54.724898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471620361881739:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:54.725175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471621344792519:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.724626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471618311074660:2107] [node 4] ICS09 handshake done sender: [1:7577471618311074629:2114] self: [1:7577471618311074628:0] peer: [4:7577471621344792537:0] socket: 272 qp: -1
2025-11-27T18:19:54.724535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471619292941308:2108] [node 5] ICS09 handshake done sender: [2:7577471619292941280:2113] self: [2:7577471619292941279:0] peer: [5:7577471619351506938:0] socket: 278 qp: -1
2025-11-27T18:19:54.724659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471618311074660:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:54.725553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471619772438744:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:54.725004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:54.725101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471619292941308:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:54.724956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471620361881732:2109] [node 6] ICP22 created new session: [3:7577471620361881765:2109]
2025-11-27T18:19:54.725211Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471621344792519:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.725144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:54.725359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471621344792538:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30228
2025-11-27T18:19:54.725123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471620361881765:2109] [node 6] ICS09 handshake done sender: [3:7577471620361881743:2116] self: [3:7577471620361881742:0] peer: [6:7577471619772438752:0] socket: 281 qp: -1
2025-11-27T18:19:54.726647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471621344792534:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:54.725176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471620361881765:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:54.725212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:54.726116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471620361881731:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.727175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471621344792441:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.728347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471621344792519:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:54.726158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:54.728353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471619292941284:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:54.731585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471619772438666:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.728370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:54.726554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471620361881731:2108] [node 5] ICP22 created new session: [3:7577471620361881768:2108]
2025-11-27T18:19:54.731664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:54.729644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471621344792441:2048] [node 1] ICP22 created new session: [4:7577471621344792540:2048]
2025-11-27T18:19:54.731725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471619772438666:2048] [node 1] ICP22 created new session: [6:7577471619772438757:2048]
2025-11-27T18:19:54.726633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471620361881768:2108] [node 5] ICS09 handshake done sender: [3:7577471620361881739:2113] self: [3:7577471620361881738:0] peer: [5:7577471619351506940:0] socket: 311 qp: -1
2025-11-27T18:19:54.729772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471621344792540:2048] [node 1] ICS09 handshake done sender: [4:7577471621344792534:2120] self: [4:7577471621344792537:0] peer: [1:7577471618311074628:0] socket: 308 qp: -1
2025-11-27T18:19:54.726661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471620361881768:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:54.728400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471619292941284:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:54.729884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471621344792540:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:54.732461Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471619351506844:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471619351506934:2121] held: no
2025-11-27T18:19:54.729921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:54.732471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471619772438757:2048] [node 1] ICS09 handshake done sender: [6:7577471619772438744:2115] self: [6:7577471619772438749:0] peer: [1:7577471618311074632:0] socket: 288 qp: -1
2025-11-27T18:19:54.726697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:54.733067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471619292941284:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.732490Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471619772438726:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64487) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471619772438747:2118] held: no
2025-11-27T18:19:54.732514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471619772438757:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:54.732548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:54.738773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471619292941271:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.738834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:54.738940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471619292941271:2109] [node 6] ICP22 created new session: [2:7577471619292941311:2109]
2025-11-27T18:19:54.738980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471619292941311:2109] [node 6] ICS09 handshake done sender: [2:7577471619292941284:2116] self: [2:7577471619292941283:0] peer: [6:7577471619772438754:0] socket: 260 qp: -1
2025-11-27T18:19:54.739012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471619292941311:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:54.739201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.739453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471621344792505:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:54.739597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:54.739731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471621344792505:2109] [node 6] ICP22 created new session: [4:7577471621344792542:2109]
2025-11-27T18:19:54.739775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471621344792542:2109] [node 6] ICS09 handshake done sender: [4:7577471621344792519:2115] self: [4:7577471621344792518:0] peer: [6:7577471619772438753:0] socket: 255 qp: -1
2025-11-27T18:19:54.739819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471621344792542:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:54.739852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:54.738695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471619351506943:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59387
2025-11-27T18:19:54.739440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471619351506944:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59387
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.740428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471619351506934:2121] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.741070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471619351506845:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471619351506937:2122] held: no
2025-11-27T18:19:54.741232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471619772438755:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51279
2025-11-27T18:19:54.741287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471619772438756:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51279
2025-11-27T18:19:54.741677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471619351506844:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.742391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:54.743598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471619351506844:2049] [node 2] ICP22 created new session: [5:7577471619351506945:2049]
2025-11-27T18:19:54.743918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471619351506945:2049] [node 2] ICS09 handshake done sender: [5:7577471619351506934:2121] self: [5:7577471619351506938:0] peer: [2:7577471619292941279:0] socket: 309 qp: -1
2025-11-27T18:19:54.743959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471619351506945:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:54.744022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:54.744062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471619351506937:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:54.745578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471619772438667:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471619772438748:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.746062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471620361881770:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32052
2025-11-27T18:19:54.746131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471619772438759:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51279
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.746393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471619351506845:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.746468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:54.746512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471619772438760:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51279
2025-11-27T18:19:54.746521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471619351506845:2050] [node 3] ICP22 created new session: [5:7577471619351506947:2050]
2025-11-27T18:19:54.746614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471619351506947:2050] [node 3] ICS09 handshake done sender: [5:7577471619351506937:2122] self: [5:7577471619351506940:0] peer: [3:7577471620361881738:0] socket: 296 qp: -1
2025-11-27T18:19:54.746668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471619351506947:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:54.746663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471619772438747:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:54.746721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:54.746935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471619772438746:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:54.747417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471619772438726:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.747418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471619772438668:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:54.747458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:54.747461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:54.747503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471619772438726:2105] [node 5] ICP22 created new session: [6:7577471619772438764:2105]
2025-11-27T18:19:54.747506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471619772438668:2050] [node 3] ICP22 created new session: [6:7577471619772438763:2050]
2025-11-27T18:19:54.747541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471619772438764:2105] [node 5] ICS09 handshake done sender: [6:7577471619772438747:2118] self: [6:7577471619772438751:0] peer: [5:7577471619351506920:0] socket: 312 qp: -1
2025-11-27T18:19:54.747614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471621344792544:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30228
2025-11-27T18:19:54.747562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471619772438763:2050] [node 3] ICS09 handshake done sender: [6:7577471619772438746:2117] self: [6:7577471619772438752:0] peer: [3:7577471620361881742:0] socket: 310 qp: -1
2025-11-27T18:19:54.747563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471619772438764:2105] [node 5] ICS10 traffic start
2025-11-27T18:19:54.747587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:54.747592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471619772438763:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:54.747619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:54.748462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471619772438745:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:54.749812Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471619772438748:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:54.750657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471619772438733:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.750713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:54.750779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471619772438733:2110] [node 4] ICP22 created new session: [6:7577471619772438767:2110]
2025-11-27T18:19:54.750837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471619772438767:2110] [node 4] ICS09 handshake done sender: [6:7577471619772438745:2116] self: [6:7577471619772438753:0] peer: [4:7577471621344792518:0] socket: 274 qp: -1
2025-11-27T18:19:54.750883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471619772438767:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:54.750922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:54.751062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471619772438667:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:54.751102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:54.751156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471619772438667:2049] [node 2] ICP22 created new session: [6:7577471619772438769:2049]
2025-11-27T18:19:54.751198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471619772438769:2049] [node 2] ICS09 handshake done sender: [6:7577471619772438748:2119] self: [6:7577471619772438754:0] peer: [2:7577471619292941283:0] socket: 313 qp: -1
2025-11-27T18:19:54.751167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471620361881732:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:54.751239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471619772438769:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:54.751269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:54.774912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:54.775168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471619292941304:2048] [node 1] ICS01 socket: 285 reason# EndOfStream
2025-11-27T18:19:54.775216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471619292941207:2048] [node 1] ICP30 unregister session Session# [2:7577471619292941304:2048] VirtualId# [2:7577471619292941293:0]
2025-11-27T18:19:54.775248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471619292941304:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.775325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:54.775370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:54.777163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:54.777426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471620361881760:2048] [node 1] ICS01 socket: 290 reason# EndOfStream
2025-11-27T18:19:54.777499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471620361881666:2048] [node 1] ICP30 unregister session Session# [3:7577471620361881760:2048] VirtualId# [3:7577471620361881752:0]
2025-11-27T18:19:54.777532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471620361881760:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.777627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:54.777681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:54.778903Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:54.779124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471621344792540:2048] [node 1] ICS07 socket disconnect 308 reason# EndOfStream
2025-11-27T18:19:54.779172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471621344792540:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.779237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:54.779281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471621344792540:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:54.779975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471621344792441:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27420) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.780032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471621344792540:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.780099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471621344792441:2048] [node 1] ICP30 unregister session Session# [4:7577471621344792540:2048] VirtualId# [4:7577471621344792537:0]
2025-11-27T18:19:54.780157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:54.780232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:54.780285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471621344792441:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27420) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.780306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27420) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.780396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471619351506933:2048] [node 1] ICS01 socket: 287 reason# EndOfStream
2025-11-27T18:19:54.780446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471619351506843:2048] [node 1] ICP30 unregister session Session# [5:7577471619351506933:2048] VirtualId# [5:7577471619351506931:0]
2025-11-27T18:19:54.780478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471619351506933:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.780551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:54.780601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:54.781414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:54.781599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471619772438757:2048] [node 1] ICS01 socket: 288 reason# EndOfStream
2025-11-27T18:19:54.781645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471619772438666:2048] [node 1] ICP30 unregister session Session# [6:7577471619772438757:2048] VirtualId# [6:7577471619772438749:0]
2025-11-27T18:19:54.781669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471619772438757:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.781732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:54.781767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:54.790237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471621344792441:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:54.878545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:54.878764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471620361881762:2049] [node 2] ICS07 socket disconnect 292 reason# EndOfStream
2025-11-27T18:19:54.878798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471620361881762:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.878874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:54.878919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471620361881762:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:54.882358Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471620361881667:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29811) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:54.882402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471620361881762:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:54.883257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471620361881667:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29811) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.882826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:54.883047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471621344792531:2049] [node 2] ICS01 socket: 289 reason# EndOfStream
2025-11-27T18:19:54.883087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471621344792442:2049] [node 2] ICP30 unregister session Session# [4:7577471621344792531:2049] VirtualId# [4:7577471621344792528:0]
2025-11-27T18:19:54.883412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471620361881762:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.883119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471621344792531:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.883222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:54.883460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471620361881667:2049] [node 2] ICP30 unregister session Session# [3:7577471620361881762:2049] VirtualId# [3:7577471620361881751:0]
2025-11-27T18:19:54.883285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:54.883578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:54.883625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471620361881667:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29811) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.883644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29811) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.885678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:54.885872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471619772438769:2049] [node 2] ICS01 socket: 313 reason# EndOfStream
2025-11-27T18:19:54.885903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471619772438667:2049] [node 2] ICP30 unregister session Session# [6:7577471619772438769:2049] VirtualId# [6:7577471619772438754:0]
2025-11-27T18:19:54.885928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471619772438769:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.885996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:54.886044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:54.884709Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:54.884909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471619351506945:2049] [node 2] ICS01 socket: 309 reason# EndOfStream
2025-11-27T18:19:54.884947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471619351506844:2049] [node 2] ICP30 unregister session Session# [5:7577471619351506945:2049] VirtualId# [5:7577471619351506938:0]
2025-11-27T18:19:54.884989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471619351506945:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.885074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:54.885113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:54.936781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:54.936984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471621344792535:2050] [node 3] ICS07 socket disconnect 276 reason# EndOfStream
2025-11-27T18:19:54.937008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471621344792535:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.937066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:54.937096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471621344792535:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:54.938116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471621344792443:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13437) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.938236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471621344792535:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.938276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471621344792443:2050] [node 3] ICP30 unregister session Session# [4:7577471621344792535:2050] VirtualId# [4:7577471621344792527:0]
2025-11-27T18:19:54.938377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:54.938420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471621344792443:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13437) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.938436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13437) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.938611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:54.938850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471619351506947:2050] [node 3] ICS01 socket: 296 reason# EndOfStream
2025-11-27T18:19:54.938894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471619351506845:2050] [node 3] ICP30 unregister session Session# [5:7577471619351506947:2050] VirtualId# [5:7577471619351506940:0]
2025-11-27T18:19:54.938923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471619351506947:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.938992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:54.939029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:54.940734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:54.940950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471619772438763:2050] [node 3] ICS07 socket disconnect 310 reason# EndOfStream
2025-11-27T18:19:54.940979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471619772438763:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.941064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:54.941111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471619772438763:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:54.942090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471619772438668:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13437) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.942227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471619772438763:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.942363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471619772438668:2050] [node 3] ICP30 unregister session Session# [6:7577471619772438763:2050] VirtualId# [6:7577471619772438752:0]
2025-11-27T18:19:54.942765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:54.942842Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471619772438668:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13437) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.942876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13437) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.947908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471621344792443:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:54.952979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471619772438668:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:54.988567Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:54.988787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471619351506928:2109] [node 4] ICS01 socket: 265 reason# EndOfStream
2025-11-27T18:19:54.988835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471619351506908:2109] [node 4] ICP30 unregister session Session# [5:7577471619351506928:2109] VirtualId# [5:7577471619351506925:0]
2025-11-27T18:19:54.988872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471619351506928:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.988955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:54.989000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:54.989668Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:54.989868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471619772438767:2110] [node 4] ICS07 socket disconnect 274 reason# EndOfStream
2025-11-27T18:19:54.989894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471619772438767:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:54.989962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:54.989996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471619772438767:2110] [node 4] ICS15 start handshake
2025-11-27T18:19:54.990914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471619772438733:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5238) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:54.991028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471619772438767:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:54.991068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471619772438733:2110] [node 4] ICP30 unregister session Session# [6:7577471619772438767:2110] VirtualId# [6:7577471619772438753:0]
2025-11-27T18:19:54.991212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:54.991249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471619772438733:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5238) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:54.991262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5238) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:55.001456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471619772438733:2110] [node 4] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:55.179077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471624914578349:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:55.179107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471624914578350:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:55.200776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471624262360904:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:55.200998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471624262360905:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:55.223672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471623097449372:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:55.223733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471623097449373:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:55.254473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471624087252521:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:55.254507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471624087252522:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:55.254534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471624087252523:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:55.278488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471622752358846:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:55.278537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471622752358848:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:55.278750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471622752358847:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:55.300044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471623935797289:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:55.300094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471623935797290:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:55.300139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471623935797291:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:55.404175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471623935797351:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:55.414998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471624914578414:2109] [node 5] ICP01 ready to work
2025-11-27T18:19:55.427980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471624914578416:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:55.449400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471624914578417:2111] [node 6] ICP01 ready to work
2025-11-27T18:19:55.492501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471622752358927:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:55.476489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471623097449439:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:55.460335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471624262360968:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:55.482273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471623935797355:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:55.490740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471624087252584:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:55.495289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471624087252585:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:55.490826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471623097449438:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:55.488084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471624262360969:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:55.490806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471624262360970:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:55.495488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471622752358928:2118] [node 6] ICP01 ready to work
2025-11-27T18:19:55.495375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471623097449440:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:55.504599Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471624262360904:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(80862676ns) {Configure:104(80855158ns) {ProcessConfigured:135(461434ns) {StartInitialHandshake:157(458485ns) {DropHandshakes:470(145963ns) {DropIncomingHandshake:442(1366ns) DropOutgoingHandshake:457(734ns)} GenerateSessionVirtualId:988(1826ns) PrepareNewSessionHandshake:477(2620ns) {SwitchToState:179(1292ns)}}}}}
2025-11-27T18:19:55.511172Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471624262360905:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17776450ns) {Configure:104(17766631ns) {ProcessConfigured:135(5915606ns) {StartInitialHandshake:157(5913578ns) {DropHandshakes:470(1318ns) {DropIncomingHandshake:442(429ns) DropOutgoingHandshake:457(272ns)} GenerateSessionVirtualId:988(632ns) PrepareNewSessionHandshake:477(2921ns) {SwitchToState:179(1263ns)}}}}}
2025-11-27T18:19:55.517965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471624914578414:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18570051ns) {Configure:104(18564973ns) {ProcessConfigured:135(6266958ns) {StartInitialHandshake:157(6265306ns) {DropHandshakes:470(1123ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(174ns)} GenerateSessionVirtualId:988(584ns) PrepareNewSessionHandshake:477(2665ns) {SwitchToState:179(1461ns)}}}}}
2025-11-27T18:19:55.520492Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471622752358847:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22151808ns) {Configure:104(22147972ns) {ProcessConfigured:135(7939834ns) {StartInitialHandshake:157(7937666ns) {DropHandshakes:470(1588653ns) {DropIncomingHandshake:442(1661ns) DropOutgoingHandshake:457(281ns)} GenerateSessionVirtualId:988(360ns) PrepareNewSessionHandshake:477(5735ns) {SwitchToState:179(4259ns)}}}}}
2025-11-27T18:19:55.518264Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471624087252521:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21830933ns) {Configure:104(21827016ns) {ProcessConfigured:135(808566ns) {StartInitialHandshake:157(806239ns) {DropHandshakes:470(852ns) {DropIncomingHandshake:442(247ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(1303ns) {SwitchToState:179(169ns)}}}}}
2025-11-27T18:19:55.521363Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471622752358848:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20759207ns) {Configure:104(20754042ns) {ProcessConfigured:135(8480026ns) {StartInitialHandshake:157(8478386ns) {DropHandshakes:470(1329ns) {DropIncomingHandshake:442(666ns) DropOutgoingHandshake:457(224ns)} GenerateSessionVirtualId:988(490ns) PrepareNewSessionHandshake:477(2530ns) {SwitchToState:179(1026ns)}}}}}
2025-11-27T18:19:55.521915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471624262360968:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20853837ns) {Configure:104(20848140ns) {ProcessConfigured:135(6678500ns) {StartInitialHandshake:157(6676810ns) {DropHandshakes:470(3936954ns) {DropIncomingHandshake:442(2313ns) DropOutgoingHandshake:457(139ns)} GenerateSessionVirtualId:988(1603ns) PrepareNewSessionHandshake:477(2313ns) {SwitchToState:179(1052ns)}}}}}
2025-11-27T18:19:55.521867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471623097449373:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21420424ns) {Configure:104(21416498ns) {ProcessConfigured:135(3230654ns) {StartInitialHandshake:157(3229001ns) {DropHandshakes:470(1144ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(248ns)} GenerateSessionVirtualId:988(379ns) PrepareNewSessionHandshake:477(1818ns) {SwitchToState:179(150ns)}}}}}
2025-11-27T18:19:55.521300Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471624087252522:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21425675ns) {Configure:104(21413071ns) {ProcessConfigured:135(691048ns) {StartInitialHandshake:157(688955ns) {DropHandshakes:470(1427ns) {DropIncomingHandshake:442(419ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(429ns) PrepareNewSessionHandshake:477(1894ns) {SwitchToState:179(1107ns)}}}}}
2025-11-27T18:19:55.520390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471623935797291:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19452215ns) {Configure:104(19446826ns) {ProcessConfigured:135(320514ns) {StartInitialHandshake:157(318866ns) {DropHandshakes:470(1394ns) {DropIncomingHandshake:442(296ns) DropOutgoingHandshake:457(477ns)} GenerateSessionVirtualId:988(376ns) PrepareNewSessionHandshake:477(2438ns) {SwitchToState:179(1153ns)}}}}}
2025-11-27T18:19:55.529040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471623097449439:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24737357ns) {Configure:104(24733294ns) {ProcessConfigured:135(473128ns) {StartInitialHandshake:157(470465ns) {DropHandshakes:470(1374ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(509ns) PrepareNewSessionHandshake:477(2184ns) {SwitchToState:179(1007ns)}}}}}
2025-11-27T18:19:55.521803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471623935797290:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20760727ns) {Configure:104(20756631ns) {ProcessConfigured:135(814504ns) {StartInitialHandshake:157(812335ns) {DropHandshakes:470(1524ns) {DropIncomingHandshake:442(688ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(424ns) PrepareNewSessionHandshake:477(1825ns) {SwitchToState:179(836ns)}}}}}
2025-11-27T18:19:55.535985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471623097449438:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17667727ns) {Configure:104(17662175ns) {ProcessConfigured:135(560524ns) {StartInitialHandshake:157(557992ns) {DropHandshakes:470(2168ns) {DropIncomingHandshake:442(638ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(583ns) PrepareNewSessionHandshake:477(2175ns) {SwitchToState:179(1104ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.542130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471624087252585:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18911601ns) {Configure:104(18906187ns) {ProcessConfigured:135(44025ns) {StartInitialHandshake:157(42134ns) {DropHandshakes:470(2323ns) {DropIncomingHandshake:442(541ns) DropOutgoingHandshake:457(273ns)} GenerateSessionVirtualId:988(426ns) PrepareNewSessionHandshake:477(1192ns) {SwitchToState:179(478ns)}}}}}
2025-11-27T18:19:55.539697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471623097449449:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25960
2025-11-27T18:19:55.539987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471623097449450:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25960
No RDMA context found for GID: 2025-11-27T18:19:55.544298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471623097449450:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:19:55.548135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471623097449372:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.548750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471622752358935:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:299
2025-11-27T18:19:55.549096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471622752358928:2118] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(26252210ns) {Configure:104(26246788ns) {ProcessConfigured:135(364695ns) {StartInitialHandshake:157(361687ns) {DropHandshakes:470(4893ns) {DropIncomingHandshake:442(714ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(1766ns) PrepareNewSessionHandshake:477(1752ns) {SwitchToState:179(685ns)}}}}}
2025-11-27T18:19:55.551512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471623935797367:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.552231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471624914578429:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39533
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.553197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471623097449449:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.553353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471622752358935:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.553394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471624914578421:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.555262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471624087252597:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3396
2025-11-27T18:19:55.558419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471623935797367:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.553446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471622752358939:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:299
2025-11-27T18:19:55.553416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471624914578421:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.562189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471623935797291:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.562668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471622752358939:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.558413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471624914578421:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.563349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471624262360977:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.562520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471624087252598:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3396
2025-11-27T18:19:55.562829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471622752358848:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.574519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471623097449446:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.562788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471624087252597:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.577381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471624087252599:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3396
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.576682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471624914578430:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39533
2025-11-27T18:19:55.574574Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471623097449446:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.577864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471624914578350:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.578902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471624262360978:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
2025-11-27T18:19:55.578519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471623097449457:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25960
2025-11-27T18:19:55.577994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:55.579047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471624087252584:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29781990ns) {Configure:104(29776114ns) {ProcessConfigured:135(14401955ns) {StartInitialHandshake:157(14398183ns) {DropHandshakes:470(2282ns) {DropIncomingHandshake:442(381ns) DropOutgoingHandshake:457(700ns)} GenerateSessionVirtualId:988(423ns) PrepareNewSessionHandshake:477(6199ns) {SwitchToState:179(3562ns)}}}}}
2025-11-27T18:19:55.579460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471623097449459:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25960
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.579619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471624087252598:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.578566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471624914578350:2049] [node 3] ICP22 created new session: [1:7577471624914578433:2049]
2025-11-27T18:19:55.579555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471623097449448:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.579588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471623097449448:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.581103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471624262360979:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.579433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471624914578433:2049] [node 3] ICS09 handshake done sender: [1:7577471624914578421:2112] self: [1:7577471624914578419:0] peer: [3:7577471623097449456:0] socket: 264 qp: -1
2025-11-27T18:19:55.582366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471624087252521:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.605672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471623935797369:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-27T18:19:55.582734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471624262360977:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.610793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471623097449457:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.582639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471624087252599:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.579475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471624914578433:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:55.622760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471623935797367:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:55.625364Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471623935797291:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26725) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471623935797367:2115] held: no
2025-11-27T18:19:55.613484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471623097449446:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:55.627398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471623935797291:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.627480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:55.627532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471623935797291:2050] [node 3] ICP22 created new session: [6:7577471623935797370:2050]
2025-11-27T18:19:55.602818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471624262360978:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.579696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:55.624920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471622752358946:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:299
2025-11-27T18:19:55.617326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471623097449450:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:55.627586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471623935797370:2050] [node 3] ICS09 handshake done sender: [6:7577471623935797367:2115] self: [6:7577471623935797368:0] peer: [3:7577471623097449445:0] socket: 278 qp: -1
2025-11-27T18:19:55.605806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471624262360981:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
2025-11-27T18:19:55.581824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471624914578430:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:19:55.617781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471623097449460:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25960
::ffff:127.1.0.0
2025-11-27T18:19:55.614015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471624262360979:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.592927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471624914578429:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.627621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471623935797370:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:55.622622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471624262360969:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(58789538ns) {Configure:104(58782811ns) {ProcessConfigured:135(20715135ns) {StartInitialHandshake:157(20710864ns) {DropHandshakes:470(2431ns) {DropIncomingHandshake:442(744ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(1167ns) PrepareNewSessionHandshake:477(3568ns) {SwitchToState:179(1621ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.617931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471623097449448:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:55.605914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471624914578432:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39533
2025-11-27T18:19:55.642798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471622752358947:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:299
2025-11-27T18:19:55.623471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471624262360904:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.628076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:55.648729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471622752358946:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.618117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471623097449439:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.609692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471624914578422:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.618176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:55.624923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471624262360981:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.609737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471624914578422:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.618229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471623097449439:2110] [node 6] ICP22 created new session: [3:7577471623097449463:2110]
No RDMA context found for GID: 2025-11-27T18:19:55.649776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471622752358948:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:299
::ffff:127.1.0.0
2025-11-27T18:19:55.618273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471623097449463:2110] [node 6] ICS09 handshake done sender: [3:7577471623097449446:2115] self: [3:7577471623097449445:0] peer: [6:7577471623935797368:0] socket: 271 qp: -1
2025-11-27T18:19:55.635903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471624914578422:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:55.651565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471622752358947:2136] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.647338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471624262360992:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.636207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471624914578411:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.659878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471624087252521:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28058) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471624087252598:2115] held: no
2025-11-27T18:19:55.657532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471624262360976:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.651712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471622752358846:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.618307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471623097449463:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:55.653237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471622752358847:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.618349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:55.660454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471624087252605:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3396
2025-11-27T18:19:55.636245Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471624914578411:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.665873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471623935797373:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-27T18:19:55.657559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471624262360976:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.651444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471624914578416:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.660595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471622752358939:2133] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:55.658802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471624262360974:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.619414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471623097449372:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.662690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471622752358848:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.651502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:55.658830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471624262360974:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.619453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:55.651552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471624914578416:2107] [node 4] ICP22 created new session: [1:7577471624914578439:2107]
2025-11-27T18:19:55.663559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471624262360993:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15400
2025-11-27T18:19:55.619491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471623097449372:2048] [node 1] ICP22 created new session: [3:7577471623097449465:2048]
2025-11-27T18:19:55.651593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471624914578439:2107] [node 4] ICS09 handshake done sender: [1:7577471624914578422:2113] self: [1:7577471624914578420:0] peer: [4:7577471624087252603:0] socket: 263 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.662737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.673941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471624087252606:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3396
2025-11-27T18:19:55.674444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471624087252606:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.676092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471624087252523:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.663567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471624262360974:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:55.651627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471624914578439:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:55.651669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:55.655712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471624914578424:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.663720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471622752358848:2050] [node 3] ICP22 created new session: [5:7577471622752358952:2050]
2025-11-27T18:19:55.619528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471623097449465:2048] [node 1] ICS09 handshake done sender: [3:7577471623097449450:2118] self: [3:7577471623097449456:0] peer: [1:7577471624914578419:0] socket: 269 qp: -1
2025-11-27T18:19:55.667745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471624262360992:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.619567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471623097449465:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:55.676592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471624087252598:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:55.673547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471624262360977:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:55.655739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471624914578424:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.619594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:55.667744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471622752358952:2050] [node 3] ICS09 handshake done sender: [5:7577471622752358939:2133] self: [5:7577471622752358945:0] peer: [3:7577471623097449447:0] socket: 282 qp: -1
2025-11-27T18:19:55.677315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471624262360976:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:55.657545Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471624914578411:2108] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:55.667797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471622752358952:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:55.627499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471623097449438:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.677836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471624262360905:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.677445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471624087252521:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.627534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:55.659258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471624914578349:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.668086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:55.677501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:55.677940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471624262360904:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.676964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471622752358951:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:299
2025-11-27T18:19:55.659782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:55.627588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471623097449438:2108] [node 5] ICP22 created new session: [3:7577471623097449468:2108]
2025-11-27T18:19:55.678159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471624087252521:2048] [node 1] ICP22 created new session: [4:7577471624087252609:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.677972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:55.680183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471622752358947:2136] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:55.642681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471623097449468:2108] [node 5] ICS09 handshake done sender: [3:7577471623097449448:2116] self: [3:7577471623097449447:0] peer: [5:7577471622752358945:0] socket: 274 qp: -1
2025-11-27T18:19:55.680565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471624087252609:2048] [node 1] ICS09 handshake done sender: [4:7577471624087252598:2115] self: [4:7577471624087252603:0] peer: [1:7577471624914578420:0] socket: 275 qp: -1
2025-11-27T18:19:55.660536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471624914578349:2048] [node 2] ICP22 created new session: [1:7577471624914578444:2048]
2025-11-27T18:19:55.685525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471623935797375:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-27T18:19:55.677998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471624262360904:2048] [node 1] ICP22 created new session: [2:7577471624262360998:2048]
2025-11-27T18:19:55.684722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471622752358846:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.645153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471623097449468:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:55.661620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471624914578444:2048] [node 2] ICS09 handshake done sender: [1:7577471624914578411:2108] self: [1:7577471624914578410:0] peer: [2:7577471624262360988:0] socket: 259 qp: -1
2025-11-27T18:19:55.687663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471623935797374:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-27T18:19:55.678177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:55.682268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471624087252609:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:55.685014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:55.662537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471624914578424:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:55.649571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471623097449460:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.682326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:55.685059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471622752358846:2048] [node 1] ICP22 created new session: [5:7577471622752358956:2048]
2025-11-27T18:19:55.679135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471624262360905:2049] [node 3] ICP22 created new session: [2:7577471624262360997:2049]
2025-11-27T18:19:55.662560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471624914578444:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.653226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:55.654176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471623097449373:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.703422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471623935797374:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.686817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471622752358956:2048] [node 1] ICS09 handshake done sender: [5:7577471622752358947:2136] self: [5:7577471622752358949:0] peer: [1:7577471624914578423:0] socket: 302 qp: -1
2025-11-27T18:19:55.687680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471622752358956:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:55.687719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:55.702873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471622752358846:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17754375ns) {HandleHandshakeStatus:329(17751633ns) {DropIncomingHandshake:442(1219ns) DropOutgoingHandshake:457(5230ns) SwitchToState:179(774ns) ProcessPendingSessionEvents:495(6303737ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6249336ns max# 5868964ns}}}
2025-11-27T18:19:55.703020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471622752358955:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:299
2025-11-27T18:19:55.662634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:55.679142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471624262360998:2048] [node 1] ICS09 handshake done sender: [2:7577471624262360977:2114] self: [2:7577471624262360988:0] peer: [1:7577471624914578410:0] socket: 267 qp: -1
2025-11-27T18:19:55.713560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471623935797355:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.670209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471624914578432:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.717518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471624087252611:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3396
2025-11-27T18:19:55.680471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471624262360998:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:55.676779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471623097449473:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25960
2025-11-27T18:19:55.673535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471624914578414:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.726261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471624087252596:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.680517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:55.680570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471624262360997:2049] [node 3] ICS09 handshake done sender: [2:7577471624262360974:2112] self: [2:7577471624262360973:0] peer: [3:7577471623097449472:0] socket: 296 qp: -1
2025-11-27T18:19:55.682096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471624262360997:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:55.682140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:55.729976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471624262360968:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.734552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:55.736522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471624262360905:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(58114942ns) {HandleHandshakeStatus:329(58112508ns) {DropIncomingHandshake:442(17244ns) DropOutgoingHandshake:457(323ns) SwitchToState:179(320ns) ProcessPendingSessionEvents:495(51724370ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 51625605ns max# 7551247ns}}}
2025-11-27T18:19:55.737708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471622752358942:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.737771Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471622752358942:2134] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.730141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471623935797375:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.677160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471623097449455:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.676574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:55.738249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471624262360968:2108] [node 5] ICP22 created new session: [2:7577471624262361001:2108]
2025-11-27T18:19:55.726326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471624087252596:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.734417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471623935797351:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.752473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471623935797373:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.677201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471623097449455:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.676957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471624914578414:2109] [node 5] ICP22 created new session: [1:7577471624914578446:2109]
2025-11-27T18:19:55.739618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471624262361001:2108] [node 5] ICS09 handshake done sender: [2:7577471624262360976:2113] self: [2:7577471624262360975:0] peer: [5:7577471622752358950:0] socket: 298 qp: -1
2025-11-27T18:19:55.732650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471624087252606:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:55.682783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471623097449460:2122] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.677019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471624914578446:2109] [node 5] ICS09 handshake done sender: [1:7577471624914578424:2114] self: [1:7577471624914578423:0] peer: [5:7577471622752358949:0] socket: 301 qp: -1
2025-11-27T18:19:55.732925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471624087252523:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.755298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471622752358946:2135] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.742330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471624262361001:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:55.755651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471623935797289:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.683747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471623097449455:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:55.732974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:55.751779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:55.755475Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471622752358848:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(61084283ns) {HandleHandshakeStatus:329(61080553ns) {DropIncomingHandshake:442(797ns) DropOutgoingHandshake:457(4950ns) SwitchToState:179(656ns) ProcessPendingSessionEvents:495(55364495ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 55231581ns max# 11593500ns}}}
2025-11-27T18:19:55.733031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471624087252523:2050] [node 3] ICP22 created new session: [4:7577471624087252612:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.782311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471624262360968:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(51434802ns) {HandleHandshakeStatus:329(51428810ns) {DropIncomingHandshake:442(15026ns) DropOutgoingHandshake:457(391ns) SwitchToState:179(626ns) ProcessPendingSessionEvents:495(10983115ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10916270ns max# 10497916ns}}}
2025-11-27T18:19:55.760941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471622752358847:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10300) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471622752358946:2135] held: no
2025-11-27T18:19:55.677193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471624914578446:2109] [node 5] ICS10 traffic start
2025-11-27T18:19:55.733076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471624087252612:2050] [node 3] ICS09 handshake done sender: [4:7577471624087252606:2119] self: [4:7577471624087252608:0] peer: [3:7577471623097449454:0] socket: 309 qp: -1
2025-11-27T18:19:55.677243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:55.774761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471623935797380:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-27T18:19:55.688325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471623097449373:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.684531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471624914578350:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(98688091ns) {HandleHandshakeStatus:329(98684894ns) {DropIncomingHandshake:442(464ns) DropOutgoingHandshake:457(458ns) SwitchToState:179(833ns) ProcessPendingSessionEvents:495(85206642ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 85025221ns max# 21789449ns}}}
2025-11-27T18:19:55.791469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471623935797374:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:55.749960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471624914578349:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(67056853ns) {HandleHandshakeStatus:329(67051108ns) {DropIncomingHandshake:442(514ns) DropOutgoingHandshake:457(470ns) SwitchToState:179(821ns) ProcessPendingSessionEvents:495(59727656ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 56073441ns max# 9791750ns}}}
2025-11-27T18:19:55.797413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471623935797381:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-27T18:19:55.733111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471624087252612:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:55.811589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471623935797355:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.819009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:55.824084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471623935797355:2110] [node 4] ICP22 created new session: [6:7577471623935797382:2110]
2025-11-27T18:19:55.825525Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471623935797373:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:55.825786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471623935797382:2110] [node 4] ICS09 handshake done sender: [6:7577471623935797374:2118] self: [6:7577471623935797376:0] peer: [4:7577471624087252595:0] socket: 321 qp: -1
2025-11-27T18:19:55.825839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471623935797382:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:55.827184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:55.688504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:55.751124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471624914578349:2048] [node 2] ICP18 handshake (actor [1:7577471624914578432:2118]) is too old
2025-11-27T18:19:55.733148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.765752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471624914578414:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(67297657ns) {HandleHandshakeStatus:329(64093025ns) {DropIncomingHandshake:442(17781ns) DropOutgoingHandshake:457(356ns) SwitchToState:179(1053ns) ProcessPendingSessionEvents:495(57407817ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 57288419ns max# 9138350ns}}}
2025-11-27T18:19:55.688517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471623097449440:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.834327Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471624262360904:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(83352724ns) {HandleHandshakeStatus:329(83348315ns) {DropIncomingHandshake:442(371ns) DropOutgoingHandshake:457(2659ns) SwitchToState:179(332ns) ProcessPendingSessionEvents:495(77345684ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 77199579ns max# 8821230ns}}}
2025-11-27T18:19:55.834513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471624262360987:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.834547Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471624262360987:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.762314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471624087252615:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3396
2025-11-27T18:19:55.765946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471624914578428:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.770649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471624087252596:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:55.765972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471624914578428:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.809759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471624087252615:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.768442Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471624914578428:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:55.809843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471624087252585:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.768780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471624914578417:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.824427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:55.768837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:55.824487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471624087252585:2109] [node 6] ICP22 created new session: [4:7577471624087252617:2109]
2025-11-27T18:19:55.768910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471624914578417:2111] [node 6] ICP22 created new session: [1:7577471624914578449:2111]
2025-11-27T18:19:55.824563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471624087252617:2109] [node 6] ICS09 handshake done sender: [4:7577471624087252596:2113] self: [4:7577471624087252595:0] peer: [6:7577471623935797376:0] socket: 320 qp: -1
2025-11-27T18:19:55.768979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471624914578449:2111] [node 6] ICS09 handshake done sender: [1:7577471624914578428:2115] self: [1:7577471624914578427:0] peer: [6:7577471623935797378:0] socket: 312 qp: -1
2025-11-27T18:19:55.824606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471624087252617:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:55.769022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471624914578449:2111] [node 6] ICS10 traffic start
2025-11-27T18:19:55.824644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:55.769074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:55.831262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471624087252522:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.836291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471624262360987:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:55.829059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471623935797289:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.688568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.844115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471624087252620:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3396
2025-11-27T18:19:55.690292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471623097449373:2049] [node 2] ICP22 created new session: [3:7577471623097449475:2049]
2025-11-27T18:19:55.833420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:55.695213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471623097449475:2049] [node 2] ICS09 handshake done sender: [3:7577471623097449460:2122] self: [3:7577471623097449472:0] peer: [2:7577471624262360973:0] socket: 297 qp: -1
2025-11-27T18:19:55.833662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471623935797379:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-27T18:19:55.695424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471623097449440:2107] [node 4] ICP22 created new session: [3:7577471623097449476:2107]
2025-11-27T18:19:55.844607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471624262360982:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.844651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471624262360982:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.846806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471624262360969:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.847653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.850560Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471624087252615:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:55.836602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471623935797355:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(24182659ns) {HandleHandshakeStatus:329(24177804ns) {DropIncomingHandshake:442(550ns) DropOutgoingHandshake:457(5712ns) SwitchToState:179(3914ns) ProcessPendingSessionEvents:495(6821113ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6765552ns max# 6366560ns}}}
2025-11-27T18:19:55.703297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471623097449475:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:55.852412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471624262360969:2107] [node 4] ICP22 created new session: [2:7577471624262361006:2107]
2025-11-27T18:19:55.852512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471624087252522:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.703731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471623097449476:2107] [node 4] ICS09 handshake done sender: [3:7577471623097449455:2119] self: [3:7577471623097449454:0] peer: [4:7577471624087252608:0] socket: 308 qp: -1
2025-11-27T18:19:55.852465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471624262361006:2107] [node 4] ICS09 handshake done sender: [2:7577471624262360987:2119] self: [2:7577471624262360986:0] peer: [4:7577471624087252619:0] socket: 279 qp: -1
2025-11-27T18:19:55.853217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:55.838286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471623935797289:2048] [node 1] ICP22 created new session: [6:7577471623935797384:2048]
2025-11-27T18:19:55.866569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471622752358847:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(72291914ns) {HandleHandshakeStatus:395(72288975ns) {LogHandshakeFail:472(13481ns) {UpdateErrorStateLog:545(5470ns)} DropOutgoingHandshake:457(2477ns) ProcessPendingSessionEvents:495(66835461ns) {unordered DropSessionEvent:512 num# 100 sum# 66749683ns max# 11556741ns}}}
2025-11-27T18:19:55.703926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.841170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471623935797384:2048] [node 1] ICS09 handshake done sender: [6:7577471623935797373:2117] self: [6:7577471623935797378:0] peer: [1:7577471624914578427:0] socket: 314 qp: -1
2025-11-27T18:19:55.841216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471623935797384:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:55.841275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:55.842044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471623935797379:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.842626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471623935797290:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.853801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471623935797388:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-27T18:19:55.853905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471623935797389:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32595
2025-11-27T18:19:55.868394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471623935797379:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:55.878053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471623935797290:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.853301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471624087252522:2049] [node 2] ICP22 created new session: [4:7577471624087252621:2049]
2025-11-27T18:19:55.852506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471624262361006:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:55.703998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471623097449476:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:55.854676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471624087252621:2049] [node 2] ICS09 handshake done sender: [4:7577471624087252615:2121] self: [4:7577471624087252619:0] peer: [2:7577471624262360986:0] socket: 284 qp: -1
2025-11-27T18:19:55.873505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471622752358847:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.854052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:55.893975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:55.905555Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471624914578417:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(74296513ns) {HandleHandshakeStatus:329(74293107ns) {DropIncomingHandshake:442(650ns) DropOutgoingHandshake:457(598ns) SwitchToState:179(920ns) ProcessPendingSessionEvents:495(73871534ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 73742806ns max# 14964728ns}}}
2025-11-27T18:19:55.861415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471624087252621:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:55.713221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:55.873558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:55.855318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471624262360982:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:55.861496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:55.722126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471623097449440:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(32883331ns) {HandleHandshakeStatus:329(32880719ns) {DropIncomingHandshake:442(13430ns) DropOutgoingHandshake:457(282ns) SwitchToState:179(373ns) ProcessPendingSessionEvents:495(647386ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 574107ns max# 73801ns}}}
2025-11-27T18:19:55.905729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471623935797290:2049] [node 2] ICP22 created new session: [6:7577471623935797390:2049]
2025-11-27T18:19:55.877674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471622752358847:2049] [node 2] ICP22 created new session: [5:7577471622752358960:2049]
2025-11-27T18:19:55.885427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471622752358960:2049] [node 2] ICS09 handshake done sender: [5:7577471622752358946:2135] self: [5:7577471622752358950:0] peer: [2:7577471624262360975:0] socket: 300 qp: -1
2025-11-27T18:19:55.731526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471623097449438:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(57245348ns) {HandleHandshakeStatus:329(57221757ns) {DropIncomingHandshake:442(266ns) DropOutgoingHandshake:457(270ns) SwitchToState:179(362ns) ProcessPendingSessionEvents:495(30832303ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29191309ns max# 3554347ns}}}
2025-11-27T18:19:55.816115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471623097449373:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(127224403ns) {HandleHandshakeStatus:329(126873393ns) {DropIncomingHandshake:442(713ns) DropOutgoingHandshake:457(4400ns) SwitchToState:179(614ns) ProcessPendingSessionEvents:495(94686035ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 93673078ns max# 10822042ns}}}
2025-11-27T18:19:55.817038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471623097449373:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:55.885478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471622752358960:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:55.885529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:55.885849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471622752358942:2134] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:55.877399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471624262360970:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.901617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471622752358959:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:299
2025-11-27T18:19:55.882623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:55.906126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471623935797390:2049] [node 2] ICS09 handshake done sender: [6:7577471623935797379:2120] self: [6:7577471623935797387:0] peer: [2:7577471624262360980:0] socket: 276 qp: -1
2025-11-27T18:19:55.906247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471623935797390:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:55.890469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471624262360970:2110] [node 6] ICP22 created new session: [2:7577471624262361009:2110]
2025-11-27T18:19:55.906789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:55.909063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471622752358928:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.909134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:55.909191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471622752358928:2118] [node 6] ICP22 created new session: [5:7577471622752358962:2118]
2025-11-27T18:19:55.909136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471623935797375:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:55.899810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471624262361009:2110] [node 6] ICS09 handshake done sender: [2:7577471624262360982:2118] self: [2:7577471624262360980:0] peer: [6:7577471623935797387:0] socket: 272 qp: -1
2025-11-27T18:19:55.909240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471622752358962:2118] [node 6] ICS09 handshake done sender: [5:7577471622752358942:2134] self: [5:7577471622752358938:0] peer: [6:7577471623935797377:0] socket: 322 qp: -1
2025-11-27T18:19:55.909303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471622752358962:2118] [node 6] ICS10 traffic start
2025-11-27T18:19:55.909335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:55.899865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471624262361009:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:55.913934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471624087252522:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(37667897ns) {HandleHandshakeStatus:329(37664652ns) {DropIncomingHandshake:442(2658ns) DropOutgoingHandshake:457(15169ns) SwitchToState:179(875ns) ProcessPendingSessionEvents:495(7590891ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7521963ns max# 6926295ns}}}
2025-11-27T18:19:55.925166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471623935797351:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.913620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471622752358959:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.925234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471623935797290:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(31049227ns) {HandleHandshakeStatus:329(31047069ns) {DropIncomingHandshake:442(391ns) DropOutgoingHandshake:457(5468ns) SwitchToState:179(381ns) ProcessPendingSessionEvents:495(1937564ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 1690093ns max# 868890ns}}}
2025-11-27T18:19:55.905998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:55.932648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471624262360970:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(54364323ns) {HandleHandshakeStatus:329(54358803ns) {DropIncomingHandshake:442(22648ns) DropOutgoingHandshake:457(684ns) SwitchToState:179(771ns) ProcessPendingSessionEvents:495(25055588ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24969397ns max# 10066090ns}}}
2025-11-27T18:19:55.925241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:55.938280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471622752358927:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:55.934195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471623935797351:2107] [node 5] ICP22 created new session: [6:7577471623935797392:2107]
2025-11-27T18:19:55.938678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471624087252601:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:55.938591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471623935797392:2107] [node 5] ICS09 handshake done sender: [6:7577471623935797375:2119] self: [6:7577471623935797377:0] peer: [5:7577471622752358938:0] socket: 323 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:55.938646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471623935797392:2107] [node 5] ICS10 traffic start
2025-11-27T18:19:55.938727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471624087252601:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:55.939802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471624087252601:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:55.938705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:55.940023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471622752358965:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:299
2025-11-27T18:19:55.941151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471624262360969:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(61061094ns) {HandleHandshakeStatus:329(61057262ns) {DropIncomingHandshake:442(20107ns) DropOutgoingHandshake:457(652ns) SwitchToState:179(649ns) ProcessPendingSessionEvents:495(52350758ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 48250449ns max# 11339862ns}}}
2025-11-27T18:19:55.942758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471622752358959:2140] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:55.943139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471622752358927:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:55.943204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:55.943262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471622752358927:2108] [node 4] ICP22 created new session: [5:7577471622752358966:2108]
2025-11-27T18:19:55.943308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471622752358966:2108] [node 4] ICS09 handshake done sender: [5:7577471622752358959:2140] self: [5:7577471622752358964:0] peer: [4:7577471624087252600:0] socket: 299 qp: -1
2025-11-27T18:19:55.943353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471622752358966:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:55.943385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:55.944207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471624087252584:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:55.954159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471623935797351:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(18892812ns) {HandleHandshakeStatus:329(18888969ns) {DropIncomingHandshake:442(1128ns) DropOutgoingHandshake:457(6001ns) SwitchToState:179(680ns) ProcessPendingSessionEvents:495(1765401ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 1691098ns max# 477629ns}}}
2025-11-27T18:19:55.949414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:55.944235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471622752358928:2118] [node 6] ICP35 event processing took too much time PendingConnection:279(31227345ns) {HandleHandshakeStatus:329(31223911ns) {DropIncomingHandshake:442(16964ns) DropOutgoingHandshake:457(669ns) SwitchToState:179(696ns) ProcessPendingSessionEvents:495(25547324ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25447073ns max# 10069761ns}}}
2025-11-27T18:19:55.957778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471624087252584:2108] [node 5] ICP22 created new session: [4:7577471624087252624:2108]
2025-11-27T18:19:55.958179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471624087252624:2108] [node 5] ICS09 handshake done sender: [4:7577471624087252601:2117] self: [4:7577471624087252600:0] peer: [5:7577471622752358964:0] socket: 288 qp: -1
2025-11-27T18:19:55.958730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471624087252624:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:55.958786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:55.963132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471622752358927:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(19026271ns) {HandleHandshakeStatus:329(19022670ns) {DropIncomingHandshake:442(1080ns) DropOutgoingHandshake:457(6039ns) SwitchToState:179(661ns) ProcessPendingSessionEvents:495(18604548ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18487647ns max# 6154523ns}}}
2025-11-27T18:19:55.963174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471624087252584:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18043621ns) {HandleHandshakeStatus:329(18041708ns) {DropIncomingHandshake:442(22178ns) DropOutgoingHandshake:457(345ns) SwitchToState:179(238ns) ProcessPendingSessionEvents:495(817609ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 753855ns max# 323856ns}}}
2025-11-27T18:19:56.014104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:56.014305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471624262360998:2048] [node 1] ICS07 socket disconnect 267 reason# EndOfStream
2025-11-27T18:19:56.014346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471624262360998:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.014433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:56.014489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471624262360998:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:56.015311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:56.015498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471623097449465:2048] [node 1] ICS01 socket: 269 reason# EndOfStream
2025-11-27T18:19:56.015544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471623097449372:2048] [node 1] ICP30 unregister session Session# [3:7577471623097449465:2048] VirtualId# [3:7577471623097449456:0]
2025-11-27T18:19:56.015582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471623097449465:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.015660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:56.015719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:56.016803Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471624262360904:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28058) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:56.016845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471624262360998:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:56.018644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471624262360904:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28058) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:56.019829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:56.018722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471624262360998:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:56.020016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471624262360904:2048] [node 1] ICP30 unregister session Session# [2:7577471624262360998:2048] VirtualId# [2:7577471624262360988:0]
2025-11-27T18:19:56.020147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:56.020173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471624087252609:2048] [node 1] ICS01 socket: 275 reason# EndOfStream
2025-11-27T18:19:56.020194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471624262360904:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.020217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28058) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.020222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471624087252521:2048] [node 1] ICP30 unregister session Session# [4:7577471624087252609:2048] VirtualId# [4:7577471624087252603:0]
2025-11-27T18:19:56.020258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471624087252609:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.020329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:56.020392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:56.021195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:56.021398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471622752358956:2048] [node 1] ICS01 socket: 302 reason# EndOfStream
2025-11-27T18:19:56.021435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471622752358846:2048] [node 1] ICP30 unregister session Session# [5:7577471622752358956:2048] VirtualId# [5:7577471622752358949:0]
2025-11-27T18:19:56.021467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471622752358956:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.021523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:56.021574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:56.025482Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:56.025869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471623935797384:2048] [node 1] ICS01 socket: 314 reason# EndOfStream
2025-11-27T18:19:56.025964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471623935797289:2048] [node 1] ICP30 unregister session Session# [6:7577471623935797384:2048] VirtualId# [6:7577471623935797378:0]
2025-11-27T18:19:56.026031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471623935797384:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.026140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:56.026206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:56.121050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:56.121308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471623097449475:2049] [node 2] ICS07 socket disconnect 297 reason# EndOfStream
2025-11-27T18:19:56.121351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471623097449475:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.121434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:56.121480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471623097449475:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:56.123084Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471623097449373:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10300) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:56.123152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471623097449475:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:56.123192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471623097449373:2049] [node 2] ICP30 unregister session Session# [3:7577471623097449475:2049] VirtualId# [3:7577471623097449472:0]
2025-11-27T18:19:56.123278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:56.123314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471623097449373:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10300) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.123331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10300) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.126436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:56.126684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471624087252621:2049] [node 2] ICS01 socket: 284 reason# EndOfStream
2025-11-27T18:19:56.126737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471624087252522:2049] [node 2] ICP30 unregister session Session# [4:7577471624087252621:2049] VirtualId# [4:7577471624087252619:0]
2025-11-27T18:19:56.126771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471624087252621:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.126846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:56.126886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:56.127520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:56.127708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471622752358960:2049] [node 2] ICS01 socket: 300 reason# EndOfStream
2025-11-27T18:19:56.133494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471622752358847:2049] [node 2] ICP30 unregister session Session# [5:7577471622752358960:2049] VirtualId# [5:7577471622752358950:0]
2025-11-27T18:19:56.133529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471622752358960:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.133605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:56.134156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:56.132749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:56.132984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471623935797390:2049] [node 2] ICS07 socket disconnect 276 reason# EndOfStream
2025-11-27T18:19:56.134439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471623097449373:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:56.133017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471623935797390:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.133103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:56.133144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471623935797390:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:56.134707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471623935797290:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10300) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:56.134845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471623935797390:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:56.134947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471623935797290:2049] [node 2] ICP30 unregister session Session# [6:7577471623935797390:2049] VirtualId# [6:7577471623935797387:0]
2025-11-27T18:19:56.135064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:56.135104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471623935797290:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10300) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.135126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10300) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.149992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471623935797290:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:56.194864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:56.195119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471624087252612:2050] [node 3] ICS07 socket disconnect 309 reason# EndOfStream
2025-11-27T18:19:56.195149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471624087252612:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.195227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:56.195269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471624087252612:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:56.196261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:56.196059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471624087252523:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26725) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:56.196106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471624087252612:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:56.196205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471624087252523:2050] [node 3] ICP30 unregister session Session# [4:7577471624087252612:2050] VirtualId# [4:7577471624087252608:0]
2025-11-27T18:19:56.196268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:56.196317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471624087252523:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26725) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.196487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471622752358952:2050] [node 3] ICS07 socket disconnect 282 reason# EndOfStream
2025-11-27T18:19:56.196334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26725) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.196520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471622752358952:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.196588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:56.196637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471622752358952:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:56.197448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:56.197986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471623935797370:2050] [node 3] ICS01 socket: 278 reason# EndOfStream
2025-11-27T18:19:56.197457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471622752358848:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26725) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:56.197578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471622752358952:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:56.198053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471622752358848:2050] [node 3] ICP30 unregister session Session# [5:7577471622752358952:2050] VirtualId# [5:7577471622752358945:0]
2025-11-27T18:19:56.198216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471623935797291:2050] [node 3] ICP30 unregister session Session# [6:7577471623935797370:2050] VirtualId# [6:7577471623935797368:0]
2025-11-27T18:19:56.198258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471623935797370:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.198348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:56.198401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:56.198389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:56.198452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471622752358848:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26725) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.198475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26725) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.206406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471624087252523:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:56.208438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471622752358848:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:56.269522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:56.269780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471622752358966:2108] [node 4] ICS07 socket disconnect 299 reason# EndOfStream
2025-11-27T18:19:56.269819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471622752358966:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.269895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:56.269951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471622752358966:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:56.271253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471622752358927:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17421) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:56.271550Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:56.271407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471622752358966:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:56.271468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471622752358927:2108] [node 4] ICP30 unregister session Session# [5:7577471622752358966:2108] VirtualId# [5:7577471622752358964:0]
2025-11-27T18:19:56.271585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:56.271619Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471622752358927:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17421) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.271646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17421) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.271770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471623935797382:2110] [node 4] ICS07 socket disconnect 321 reason# EndOfStream
2025-11-27T18:19:56.271808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471623935797382:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.271903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:56.271940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471623935797382:2110] [node 4] ICS15 start handshake
2025-11-27T18:19:56.272928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471623935797355:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17421) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:56.273054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471623935797382:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:56.273100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471623935797355:2110] [node 4] ICP30 unregister session Session# [6:7577471623935797382:2110] VirtualId# [6:7577471623935797376:0]
2025-11-27T18:19:56.273187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:56.273222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471623935797355:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17421) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.273238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17421) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:56.283377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471623935797355:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:19:56.349600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:56.349866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471623935797392:2107] [node 5] ICS01 socket: 323 reason# EndOfStream
2025-11-27T18:19:56.349910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471623935797351:2107] [node 5] ICP30 unregister session Session# [6:7577471623935797392:2107] VirtualId# [6:7577471623935797377:0]
2025-11-27T18:19:56.349937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471623935797392:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:56.350014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:56.350076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:56.501562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471626266687611:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:56.501603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471626266687612:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:56.524794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471626297790627:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:56.525015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471626297790628:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:56.547190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471628102002570:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:56.547294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471628102002571:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:56.568854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471628626488926:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:56.568899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471628626488927:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:56.568935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471628626488928:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:56.595599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471629228371341:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:56.595649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471629228371342:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:56.596641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471629228371340:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:56.619254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471630195624029:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:56.619300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471630195624030:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:56.619333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471630195624031:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:56.843676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471630195624091:2106] [node 5] ICP01 ready to work
2025-11-27T18:19:56.911066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471626297790690:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:56.910743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471626266687611:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28457962ns) {Configure:104(28448032ns) {ProcessConfigured:135(12085263ns) {StartInitialHandshake:157(12080712ns) {DropHandshakes:470(2248ns) {DropIncomingHandshake:442(465ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(564ns) PrepareNewSessionHandshake:477(6165ns) {SwitchToState:179(3031ns)}}}}}
2025-11-27T18:19:56.881956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471626266687677:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:56.937976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471628102002634:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:56.940610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471628626488989:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:56.923741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471626266687678:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:56.940416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471626266687679:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:56.938347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471628102002633:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:56.937935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471626297790692:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:56.940480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471628626488990:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:56.939069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471628102002635:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:56.939187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471626297790693:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:56.938010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471630195624093:2109] [node 4] ICP01 ready to work
2025-11-27T18:19:56.940890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471629228371404:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:56.953760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471629228371405:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:56.986856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471628626488928:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28159539ns) {Configure:104(28155213ns) {ProcessConfigured:135(5324976ns) {StartInitialHandshake:157(5321558ns) {DropHandshakes:470(2086ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(308ns)} GenerateSessionVirtualId:988(486ns) PrepareNewSessionHandshake:477(3002ns) {SwitchToState:179(1021ns)}}}}}
2025-11-27T18:19:56.997681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471630195624091:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(48277244ns) {Configure:104(48268462ns) {ProcessConfigured:135(50494ns) {StartInitialHandshake:157(47390ns) {DropHandshakes:470(2432ns) {DropIncomingHandshake:442(591ns) DropOutgoingHandshake:457(454ns)} GenerateSessionVirtualId:988(479ns) PrepareNewSessionHandshake:477(1563ns) {SwitchToState:179(546ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.000047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471630195624100:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-27T18:19:56.989899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471628102002633:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29763606ns) {Configure:104(29754850ns) {ProcessConfigured:135(48848ns) {StartInitialHandshake:157(44510ns) {DropHandshakes:470(1207ns) {DropIncomingHandshake:442(337ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(426ns) PrepareNewSessionHandshake:477(844ns) {SwitchToState:179(358ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.002143Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471630195624029:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33228054ns) {Configure:104(33218937ns) {ProcessConfigured:135(58913ns) {StartInitialHandshake:157(57345ns) {DropHandshakes:470(1267ns) {DropIncomingHandshake:442(313ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:477(1806ns) {SwitchToState:179(707ns)}}}}}
2025-11-27T18:19:57.009300Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471626297790628:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(142005638ns) {Configure:104(142001057ns) {ProcessConfigured:135(2994406ns) {StartInitialHandshake:157(2991667ns) {DropHandshakes:470(2688ns) {DropIncomingHandshake:442(696ns) DropOutgoingHandshake:457(371ns)} GenerateSessionVirtualId:988(768ns) PrepareNewSessionHandshake:477(618469ns) {SwitchToState:179(3634ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.015804Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471626266687679:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39062956ns) {Configure:104(39055874ns) {ProcessConfigured:135(13407020ns) {StartInitialHandshake:157(10156656ns) {DropHandshakes:470(2117ns) {DropIncomingHandshake:442(509ns) DropOutgoingHandshake:457(358ns)} GenerateSessionVirtualId:988(1854ns) PrepareNewSessionHandshake:477(4610ns) {SwitchToState:179(2008ns)}}}}}
2025-11-27T18:19:57.016594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471630195624100:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.038258Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471626297790627:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(91240389ns) {Configure:104(91230002ns) {ProcessConfigured:135(12545627ns) {StartInitialHandshake:157(12542319ns) {DropHandshakes:470(1359899ns) {DropIncomingHandshake:442(2248ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(2267ns) PrepareNewSessionHandshake:477(3946ns) {SwitchToState:179(1459ns)}}}}}
2025-11-27T18:19:57.016321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471632921456298:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8291
2025-11-27T18:19:57.033710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471628102002570:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(64694543ns) {Configure:104(62068809ns) {ProcessConfigured:135(17086712ns) {StartInitialHandshake:157(17084590ns) {DropHandshakes:470(1436ns) {DropIncomingHandshake:442(487ns) DropOutgoingHandshake:457(397ns)} GenerateSessionVirtualId:988(390ns) PrepareNewSessionHandshake:477(3046ns) {SwitchToState:179(1414ns)}}}}}
2025-11-27T18:19:57.065580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471629228371342:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(65894474ns) {Configure:104(65886903ns) {ProcessConfigured:135(20240453ns) {StartInitialHandshake:157(20238344ns) {DropHandshakes:470(595ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:477(4893ns) {SwitchToState:179(2272ns)}}}}}
2025-11-27T18:19:57.025790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471626266687612:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(60381131ns) {Configure:104(60372974ns) {ProcessConfigured:135(13799641ns) {StartInitialHandshake:157(13797273ns) {DropHandshakes:470(3284254ns) {DropIncomingHandshake:442(1656ns) DropOutgoingHandshake:457(136ns)} GenerateSessionVirtualId:988(667ns) PrepareNewSessionHandshake:477(2503ns) {SwitchToState:179(944ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.040604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471634490591403:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-27T18:19:57.049097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471628626488994:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.069008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471630592757992:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8240
2025-11-27T18:19:57.065868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471628102002571:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(92296360ns) {Configure:104(92285712ns) {ProcessConfigured:135(10432494ns) {StartInitialHandshake:157(10430909ns) {DropHandshakes:470(670ns) {DropIncomingHandshake:442(232ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(564ns) PrepareNewSessionHandshake:477(2314ns) {SwitchToState:179(689ns)}}}}}
2025-11-27T18:19:57.043133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471626266687681:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:92
No RDMA context found for GID: 2025-11-27T18:19:57.070380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471628102002635:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(75781078ns) {Configure:104(75768427ns) {ProcessConfigured:135(23947044ns) {StartInitialHandshake:157(23944778ns) {DropHandshakes:470(1067ns) {DropIncomingHandshake:442(292ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(377ns) PrepareNewSessionHandshake:477(12423170ns) {SwitchToState:179(4414ns)}}}}}
::ffff:127.1.0.0
2025-11-27T18:19:57.043200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471630561654984:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:92
2025-11-27T18:19:57.043713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471630561654984:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.048315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471630195624093:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(71433187ns) {Configure:104(71427364ns) {ProcessConfigured:135(17822101ns) {StartInitialHandshake:157(17819508ns) {DropHandshakes:470(2555ns) {DropIncomingHandshake:442(701ns) DropOutgoingHandshake:457(407ns)} GenerateSessionVirtualId:988(725ns) PrepareNewSessionHandshake:477(2527ns) {SwitchToState:179(1276ns)}}}}}
2025-11-27T18:19:57.049122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471628626488994:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.069078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471630592757994:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8240
2025-11-27T18:19:57.066004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471626266687677:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(49639170ns) {Configure:104(49632897ns) {ProcessConfigured:135(14141357ns) {StartInitialHandshake:157(14139217ns) {DropHandshakes:470(1901ns) {DropIncomingHandshake:442(503ns) DropOutgoingHandshake:457(321ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:477(2213ns) {SwitchToState:179(102ns)}}}}}
2025-11-27T18:19:57.083380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471632396969945:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53269
2025-11-27T18:19:57.048719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471630195624093:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.079302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471634490591406:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-27T18:19:57.065889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471628626488994:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:57.085999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471632396969945:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.078298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471626266687681:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.082121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471630592757994:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.084235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471629228371405:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(115282874ns) {Configure:104(115277755ns) {ProcessConfigured:135(2396492ns) {StartInitialHandshake:157(2394793ns) {DropHandshakes:470(2032ns) {DropIncomingHandshake:442(452ns) DropOutgoingHandshake:457(244ns)} GenerateSessionVirtualId:988(483ns) PrepareNewSessionHandshake:477(2350ns) {SwitchToState:179(292ns)}}}}}
2025-11-27T18:19:57.081984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471630195624031:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(74854299ns) {Configure:104(74850935ns) {ProcessConfigured:135(13385236ns) {StartInitialHandshake:157(13382944ns) {DropHandshakes:470(3706ns) {DropIncomingHandshake:442(1127ns) DropOutgoingHandshake:457(588ns)} GenerateSessionVirtualId:988(654ns) PrepareNewSessionHandshake:477(1781ns) {SwitchToState:179(209ns)}}}}}
2025-11-27T18:19:57.077709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471628626488989:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.086875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471628102002570:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.086079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471629228371341:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(103953911ns) {Configure:104(103949607ns) {ProcessConfigured:135(2069796ns) {StartInitialHandshake:157(2068282ns) {DropHandshakes:470(1109ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(340ns) PrepareNewSessionHandshake:477(1698ns) {SwitchToState:179(849ns)}}}}}
2025-11-27T18:19:57.084595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471630561654990:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:92
2025-11-27T18:19:57.082358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471626297790692:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(112381694ns) {Configure:104(112376377ns) {ProcessConfigured:135(15736353ns) {StartInitialHandshake:157(15733560ns) {DropHandshakes:470(1096ns) {DropIncomingHandshake:442(214ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(459ns) PrepareNewSessionHandshake:477(4005ns) {SwitchToState:179(1547ns)}}}}}
2025-11-27T18:19:57.082323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471630195624100:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:57.077774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:57.084656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471630592757992:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.083329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471628626488989:2109] [node 6] ICP22 created new session: [4:7577471632921456302:2109]
2025-11-27T18:19:57.090155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471626266687676:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.082499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471630195624093:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.086933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471626297790627:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.090203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471626266687676:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.083769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471628626488927:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(130044597ns) {Configure:104(130037476ns) {ProcessConfigured:135(8114780ns) {StartInitialHandshake:157(8111813ns) {DropHandshakes:470(3727ns) {DropIncomingHandshake:442(1066ns) DropOutgoingHandshake:457(379ns)} GenerateSessionVirtualId:988(970ns) PrepareNewSessionHandshake:477(2913ns) {SwitchToState:179(873ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.094794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471632396969946:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53269
2025-11-27T18:19:57.083405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:57.084237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471630195624093:2109] [node 4] ICP22 created new session: [6:7577471634490591407:2109]
2025-11-27T18:19:57.084737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471634490591407:2109] [node 4] ICS09 handshake done sender: [6:7577471630195624100:2113] self: [6:7577471634490591405:0] peer: [4:7577471628626488993:0] socket: 271 qp: -1
2025-11-27T18:19:57.085018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471634490591407:2109] [node 4] ICS10 traffic start
2025-11-27T18:19:57.093585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471630561654982:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.093619Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471630561654982:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.116351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471629228371404:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(112535846ns) {Configure:104(105159375ns) {ProcessConfigured:135(12215430ns) {StartInitialHandshake:157(12211518ns) {DropHandshakes:470(2998ns) {DropIncomingHandshake:442(441ns) DropOutgoingHandshake:457(496ns)} GenerateSessionVirtualId:988(606ns) PrepareNewSessionHandshake:477(4622298ns) {SwitchToState:179(4518ns)}}}}}
2025-11-27T18:19:57.086340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:57.086026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471632921456302:2109] [node 6] ICS09 handshake done sender: [4:7577471628626488994:2111] self: [4:7577471628626488993:0] peer: [6:7577471634490591405:0] socket: 265 qp: -1
2025-11-27T18:19:57.106091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471628102002638:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.086065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471632921456302:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:57.093627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471630561654990:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.112689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471634490591403:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.106134Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471628102002638:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.126730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471629228371408:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20854
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.112682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471626266687676:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.121876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471628102002640:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.086903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:57.113696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471630195624031:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.127448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471630592757998:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.113504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471630561654982:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:57.121922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471628102002640:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.131189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471633523338711:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20854
2025-11-27T18:19:57.126199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471632396969946:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.118038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471630561654992:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:92
2025-11-27T18:19:57.092932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471632921456298:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.129510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471632396969950:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53269
2025-11-27T18:19:57.127492Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471630592757998:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.093837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471628626488928:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.131927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471629228371408:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.146206Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471630195624093:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(62871729ns) {HandleHandshakeStatus:329(62869047ns) {DropIncomingHandshake:442(509ns) DropOutgoingHandshake:457(9188ns) SwitchToState:179(322ns) ProcessPendingSessionEvents:495(55864639ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 55733053ns max# 11162493ns}}}
2025-11-27T18:19:57.125405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471626266687611:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.129716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471632396969949:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53269
2025-11-27T18:19:57.130891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471630592758004:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8240
2025-11-27T18:19:57.116696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471632921456304:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8291
2025-11-27T18:19:57.125771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471630561654992:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.132553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471633523338712:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20854
2025-11-27T18:19:57.132680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471629228371404:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.130139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471632396969948:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53269
2025-11-27T18:19:57.142536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471626297790693:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(38384070ns) {Configure:104(38374133ns) {ProcessConfigured:135(867316ns) {StartInitialHandshake:157(327529ns) {DropHandshakes:470(2418ns) {DropIncomingHandshake:442(825ns) DropOutgoingHandshake:457(353ns)} GenerateSessionVirtualId:988(914ns) PrepareNewSessionHandshake:477(1265ns) {SwitchToState:179(682ns)}}}}}
2025-11-27T18:19:57.126092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:57.122264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471632921456304:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.132159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471632396969948:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.126286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471626266687611:2048] [node 2] ICP22 created new session: [1:7577471630561654996:2048]
2025-11-27T18:19:57.129635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471630561654996:2048] [node 2] ICS09 handshake done sender: [1:7577471626266687676:2111] self: [1:7577471626266687674:0] peer: [2:7577471630592758002:0] socket: 268 qp: -1
2025-11-27T18:19:57.132977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471633523338711:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.155502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471626297790627:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23552) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471630592757992:2112] held: no
2025-11-27T18:19:57.125415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471628626488927:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.129673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471626266687612:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.156964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471633523338712:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.156129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471630592757992:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.147723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471628102002638:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.130784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471632921456308:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8291
2025-11-27T18:19:57.162291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471626297790627:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.156071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471628626488992:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.162343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:57.156118Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471628626488992:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.162393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471626297790627:2048] [node 1] ICP22 created new session: [2:7577471630592758009:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.158937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471632921456298:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:57.162438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471630592758009:2048] [node 1] ICS09 handshake done sender: [2:7577471630592757992:2112] self: [2:7577471630592758002:0] peer: [1:7577471626266687674:0] socket: 279 qp: -1
2025-11-27T18:19:57.158800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471629228371340:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.156139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471632396969949:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.129737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:57.162478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471630592758009:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:57.161605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471628626488928:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.171898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471634490591413:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-27T18:19:57.157119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471628102002640:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:57.130107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471630561654996:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:57.161659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:57.162528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:57.158685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471632396969945:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:57.173496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471634490591412:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.163400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471630592757998:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:57.130281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471626266687612:2049] [node 3] ICP22 created new session: [1:7577471630561654998:2049]
2025-11-27T18:19:57.163335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471628626488928:2050] [node 3] ICP22 created new session: [4:7577471632921456310:2050]
2025-11-27T18:19:57.131187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471630561654998:2049] [node 3] ICS09 handshake done sender: [1:7577471630561654982:2114] self: [1:7577471630561654980:0] peer: [3:7577471632396969947:0] socket: 286 qp: -1
2025-11-27T18:19:57.161323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471628102002570:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23552) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471632396969945:2115] held: no
2025-11-27T18:19:57.173700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471634490591411:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-27T18:19:57.173423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471626297790690:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.173516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471632921456310:2050] [node 3] ICS09 handshake done sender: [4:7577471632921456298:2114] self: [4:7577471632921456306:0] peer: [3:7577471628102002637:0] socket: 277 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.173635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:57.164708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471628102002633:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.131330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:57.173915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471630195624031:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5584) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471634490591403:2115] held: no
2025-11-27T18:19:57.175220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471632921456310:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:57.131345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471630561654998:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:57.164814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:57.173696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471626297790690:2107] [node 4] ICP22 created new session: [2:7577471630592758012:2107]
2025-11-27T18:19:57.178625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:57.186594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471634490591413:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.131927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:57.198977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471629228371340:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23552) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471633523338712:2117] held: no
2025-11-27T18:19:57.164859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471628102002633:2107] [node 4] ICP22 created new session: [3:7577471632396969956:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.181574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471630592758012:2107] [node 4] ICS09 handshake done sender: [2:7577471630592757998:2115] self: [2:7577471630592757997:0] peer: [4:7577471632921456307:0] socket: 296 qp: -1
2025-11-27T18:19:57.193314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471632921456309:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8291
2025-11-27T18:19:57.143880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471626266687611:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(17788713ns) {HandleHandshakeStatus:329(17611258ns) {DropIncomingHandshake:442(152ns) DropOutgoingHandshake:457(610ns) SwitchToState:179(198ns) ProcessPendingSessionEvents:495(10221449ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10158373ns max# 9739126ns}}}
2025-11-27T18:19:57.187651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471634490591412:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.164912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471632396969956:2107] [node 4] ICS09 handshake done sender: [3:7577471628102002638:2110] self: [3:7577471628102002637:0] peer: [4:7577471632921456306:0] socket: 276 qp: -1
2025-11-27T18:19:57.196548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471628626488928:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(34522515ns) {HandleHandshakeStatus:329(34519759ns) {DropIncomingHandshake:442(1526ns) DropOutgoingHandshake:457(6311ns) SwitchToState:179(576ns) ProcessPendingSessionEvents:495(9672074ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9609021ns max# 3772925ns}}}
2025-11-27T18:19:57.185878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471630592758012:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:57.197460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471630195624029:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.196657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471632921456311:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8291
2025-11-27T18:19:57.164952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471632396969956:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:57.186522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.197865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471630195624091:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.143981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471626266687611:2048] [node 2] ICP18 handshake (actor [1:7577471630561654992:2119]) is too old
2025-11-27T18:19:57.197474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471628626488992:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:57.204563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471633523338709:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.197725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471626297790695:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.165017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:57.164642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471630561655002:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:92
2025-11-27T18:19:57.203755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471634490591414:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-27T18:19:57.197766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471626297790695:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.168636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471630561654981:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.205336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471630592758015:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8240
2025-11-27T18:19:57.206758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471630592758015:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.168676Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471630561654981:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:19:57.206852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471626297790695:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:57.204601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471633523338709:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.198046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471632921456304:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:57.175243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471630561655002:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.166095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471628102002634:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.204619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471634490591414:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.207115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471633523338722:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20854
2025-11-27T18:19:57.201342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471628626488989:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(98294130ns) {HandleHandshakeStatus:329(98290547ns) {DropIncomingHandshake:442(2214ns) DropOutgoingHandshake:457(1567ns) SwitchToState:179(528ns) ProcessPendingSessionEvents:495(76868544ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 76692297ns max# 9293480ns}}}
2025-11-27T18:19:57.205969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471630195624030:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.207680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471633523338721:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20854
2025-11-27T18:19:57.178336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471630561654981:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:57.166171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:19:57.201398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471628626488927:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.201445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:57.202018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471632921456309:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.207895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471630592758006:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.206147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471634490591417:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-27T18:19:57.166217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471628102002634:2109] [node 6] ICP22 created new session: [3:7577471632396969958:2109]
2025-11-27T18:19:57.189571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471626266687679:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.203053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471628626488927:2049] [node 2] ICP22 created new session: [4:7577471632921456315:2049]
2025-11-27T18:19:57.207933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471630592758006:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.207871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471630195624029:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23552) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471634490591413:2120] held: no
2025-11-27T18:19:57.189633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:57.209109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471633523338724:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20854
2025-11-27T18:19:57.166254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471632396969958:2109] [node 6] ICS09 handshake done sender: [3:7577471628102002640:2111] self: [3:7577471628102002639:0] peer: [6:7577471634490591410:0] socket: 280 qp: -1
2025-11-27T18:19:57.203060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471628626488990:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.207996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471626297790628:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.210274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471633523338723:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20854
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.207983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471634490591413:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:57.189987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471626266687679:2108] [node 5] ICP22 created new session: [1:7577471630561655005:2108]
2025-11-27T18:19:57.166283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471632396969958:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:57.203540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471632921456315:2049] [node 2] ICS09 handshake done sender: [4:7577471632921456304:2116] self: [4:7577471632921456307:0] peer: [2:7577471630592757997:0] socket: 297 qp: -1
2025-11-27T18:19:57.211676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471633523338712:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:57.208145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:57.197001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471630561655005:2108] [node 5] ICS09 handshake done sender: [1:7577471630561654981:2113] self: [1:7577471630561654979:0] peer: [5:7577471633523338720:0] socket: 285 qp: -1
2025-11-27T18:19:57.211736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471630195624031:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(29160124ns) {HandleHandshakeStatus:395(29156961ns) {LogHandshakeFail:472(13401ns) {UpdateErrorStateLog:545(5468ns)} DropOutgoingHandshake:457(302ns) ProcessPendingSessionEvents:495(24862600ns) {unordered DropSessionEvent:512 num# 100 sum# 4865039ns max# 1158805ns}}}
2025-11-27T18:19:57.203756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471632921456315:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:57.166310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:57.212564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471633523338709:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:57.209231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471626297790628:2049] [node 3] ICP22 created new session: [2:7577471630592758018:2049]
2025-11-27T18:19:57.212536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471634490591403:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:57.203540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:57.181673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471628102002571:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.197457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471630561655005:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:57.210386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471630592758018:2049] [node 3] ICS09 handshake done sender: [2:7577471626297790695:2111] self: [2:7577471626297790694:0] peer: [3:7577471632396969960:0] socket: 302 qp: -1
2025-11-27T18:19:57.213146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471629228371408:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:57.213893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471630195624031:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.198158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471632396969955:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53269
2025-11-27T18:19:57.204485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:57.213940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:57.211147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471630592758018:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:57.206698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471628102002570:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24954056ns) {HandleHandshakeStatus:395(24938570ns) {LogHandshakeFail:472(13122ns) {UpdateErrorStateLog:545(4856ns)} DropOutgoingHandshake:457(1080ns) ProcessPendingSessionEvents:495(24524291ns) {unordered DropSessionEvent:512 num# 100 sum# 2942680ns max# 1575901ns}}}
2025-11-27T18:19:57.216736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471629228371340:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.198062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.204505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471628626488990:2108] [node 5] ICP22 created new session: [4:7577471632921456316:2108]
2025-11-27T18:19:57.201248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471630561654986:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.214169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471630195624031:2050] [node 3] ICP22 created new session: [6:7577471634490591420:2050]
2025-11-27T18:19:57.211709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:57.216773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:57.206951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471632396969955:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.205033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471632921456316:2108] [node 5] ICS09 handshake done sender: [4:7577471628626488992:2110] self: [4:7577471628626488991:0] peer: [5:7577471633523338718:0] socket: 264 qp: -1
2025-11-27T18:19:57.214549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471630592758016:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8240
2025-11-27T18:19:57.217284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471634490591420:2050] [node 3] ICS09 handshake done sender: [6:7577471634490591403:2115] self: [6:7577471634490591410:0] peer: [3:7577471628102002639:0] socket: 284 qp: -1
2025-11-27T18:19:57.207325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471628102002570:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.201299Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471630561654986:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.217253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471629228371340:2048] [node 1] ICP22 created new session: [5:7577471633523338727:2048]
2025-11-27T18:19:57.203786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471630561654986:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:57.205327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471632921456316:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:57.217415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471630195624029:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.207688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:57.217465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471629228371405:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.218631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471630592758006:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:57.217531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:57.205938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:57.204630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471626266687678:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.207965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471628102002570:2048] [node 1] ICP22 created new session: [3:7577471632396969964:2048]
2025-11-27T18:19:57.217874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:57.217726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471633523338727:2048] [node 1] ICS09 handshake done sender: [5:7577471633523338712:2117] self: [5:7577471633523338720:0] peer: [1:7577471630561654979:0] socket: 301 qp: -1
2025-11-27T18:19:57.229384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471630592758020:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8240
2025-11-27T18:19:57.208032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471632396969963:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53269
2025-11-27T18:19:57.205135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:57.218043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471634490591420:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:57.208771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471632396969964:2048] [node 1] ICS09 handshake done sender: [3:7577471632396969945:2115] self: [3:7577471632396969947:0] peer: [1:7577471630561654980:0] socket: 291 qp: -1
2025-11-27T18:19:57.205387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471626266687678:2109] [node 6] ICP22 created new session: [1:7577471630561655008:2109]
2025-11-27T18:19:57.217739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471629228371405:2110] [node 6] ICP22 created new session: [5:7577471633523338728:2110]
2025-11-27T18:19:57.241453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471630592758000:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.208822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471632396969964:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.217759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471633523338724:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.218057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471630195624029:2048] [node 1] ICP22 created new session: [6:7577471634490591421:2048]
2025-11-27T18:19:57.241505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471630592758000:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.205668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471630561655008:2109] [node 6] ICS09 handshake done sender: [1:7577471630561654986:2117] self: [1:7577471630561654985:0] peer: [6:7577471634490591415:0] socket: 319 qp: -1
2025-11-27T18:19:57.257428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471628626488927:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17962182ns) {HandleHandshakeStatus:329(17959338ns) {DropIncomingHandshake:442(624ns) DropOutgoingHandshake:457(4457ns) SwitchToState:179(697ns) ProcessPendingSessionEvents:495(13170664ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13075331ns max# 2293382ns}}}
2025-11-27T18:19:57.205969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471630561655008:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:19:57.217959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471629228371404:2108] [node 4] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:19:57.218102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:57.218978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471634490591421:2048] [node 1] ICS09 handshake done sender: [6:7577471634490591413:2120] self: [6:7577471634490591415:0] peer: [1:7577471630561654985:0] socket: 321 qp: -1
2025-11-27T18:19:57.244348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471630592758020:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.208858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:57.219628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471634490591421:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:57.213816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471632396969949:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:57.244957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471626297790693:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.214781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471628102002571:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.245007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:57.216839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:57.217898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471628102002571:2049] [node 2] ICP22 created new session: [3:7577471632396969967:2049]
2025-11-27T18:19:57.218070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471633523338727:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:57.245509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471626297790693:2109] [node 6] ICP22 created new session: [2:7577471630592758024:2109]
2025-11-27T18:19:57.223433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471634490591419:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-27T18:19:57.206780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:57.218935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471632396969967:2049] [node 2] ICS09 handshake done sender: [3:7577471632396969949:2118] self: [3:7577471632396969960:0] peer: [2:7577471626297790694:0] socket: 304 qp: -1
2025-11-27T18:19:57.218289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471633523338728:2110] [node 6] ICS09 handshake done sender: [5:7577471633523338709:2114] self: [5:7577471633523338707:0] peer: [6:7577471634490591416:0] socket: 316 qp: -1
2025-11-27T18:19:57.245728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471630592758024:2109] [node 6] ICS09 handshake done sender: [2:7577471630592758006:2118] self: [2:7577471630592758005:0] peer: [6:7577471634490591418:0] socket: 323 qp: -1
2025-11-27T18:19:57.219536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471632396969967:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:57.223645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:57.248305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471630592758024:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:57.219577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:57.218982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:57.217307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471626266687612:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(77051285ns) {HandleHandshakeStatus:329(77048033ns) {DropIncomingHandshake:442(15521ns) DropOutgoingHandshake:457(559ns) SwitchToState:179(723ns) ProcessPendingSessionEvents:495(62186295ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 62025663ns max# 9152411ns}}}
2025-11-27T18:19:57.245095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471630195624029:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20224632ns) {HandleHandshakeStatus:329(20222783ns) {DropIncomingHandshake:442(255ns) DropOutgoingHandshake:457(149ns) SwitchToState:179(182ns) ProcessPendingSessionEvents:495(822ns)}}
2025-11-27T18:19:57.219392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:57.250012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471632396969944:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.251232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:57.219669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471633523338728:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:57.255752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471630195624091:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471634490591412:2119] held: no
2025-11-27T18:19:57.219698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:57.250048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471632396969944:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.258615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471634490591423:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-27T18:19:57.256555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471630592758000:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:57.219790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471629228371404:2108] [node 4] ICP22 created new session: [5:7577471633523338730:2108]
2025-11-27T18:19:57.261344Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471626266687679:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(53145737ns) {HandleHandshakeStatus:329(53143358ns) {DropIncomingHandshake:442(18124ns) DropOutgoingHandshake:457(683ns) SwitchToState:179(129ns) ProcessPendingSessionEvents:495(43773011ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 43613512ns max# 10199856ns}}}
2025-11-27T18:19:57.261216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471632396969944:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:57.261160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471634490591412:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:57.222102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471633523338721:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.257896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471626297790690:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(50729958ns) {HandleHandshakeStatus:329(49852509ns) {DropIncomingHandshake:442(1530ns) DropOutgoingHandshake:457(713ns) SwitchToState:179(798ns) ProcessPendingSessionEvents:495(29047294ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28593447ns max# 7681984ns}}}
2025-11-27T18:19:57.269543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471628626488990:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(52781193ns) {HandleHandshakeStatus:329(52778233ns) {DropIncomingHandshake:442(174ns) DropOutgoingHandshake:457(289ns) SwitchToState:179(495ns) ProcessPendingSessionEvents:495(48392123ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 47855719ns max# 5398318ns}}}
2025-11-27T18:19:57.274904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471626266687678:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(40818420ns) {HandleHandshakeStatus:329(40816506ns) {DropIncomingHandshake:442(24587ns) DropOutgoingHandshake:457(1848ns) SwitchToState:179(471ns) ProcessPendingSessionEvents:495(37323866ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 37176265ns max# 9738317ns}}}
2025-11-27T18:19:57.282948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471628626488926:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.258180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471626297790692:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.274725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471628102002635:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.222655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471629228371342:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.274297Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471630195624030:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12320) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471634490591414:2121] held: no
2025-11-27T18:19:57.222716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471629228371341:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:57.258234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:57.274807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:57.280176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471628102002635:2108] [node 5] ICP22 created new session: [3:7577471632396969970:2108]
2025-11-27T18:19:57.305590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471630561654983:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:57.284161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471632396969970:2108] [node 5] ICS09 handshake done sender: [3:7577471632396969944:2114] self: [3:7577471632396969939:0] peer: [5:7577471633523338732:0] socket: 327 qp: -1
2025-11-27T18:19:57.292788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471632396969970:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:57.305643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471630561654983:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:57.299740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:57.283190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471630195624091:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.259104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471626297790692:2108] [node 5] ICP22 created new session: [2:7577471630592758026:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.297729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471634490591414:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:57.223172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471633523338730:2108] [node 4] ICS09 handshake done sender: [5:7577471629228371408:2112] self: [5:7577471633523338718:0] peer: [4:7577471628626488991:0] socket: 269 qp: -1
2025-11-27T18:19:57.301354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:57.223487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471633523338730:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:57.301413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471630195624091:2106] [node 5] ICP22 created new session: [6:7577471634490591425:2106]
2025-11-27T18:19:57.237467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:57.301460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471634490591425:2106] [node 5] ICS09 handshake done sender: [6:7577471634490591412:2119] self: [6:7577471634490591416:0] peer: [5:7577471633523338707:0] socket: 320 qp: -1
2025-11-27T18:19:57.260539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471633523338735:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20854
2025-11-27T18:19:57.301495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471634490591425:2106] [node 5] ICS10 traffic start
2025-11-27T18:19:57.301555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:57.263537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471633523338721:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:57.301762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471630195624091:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(21182366ns) {HandleHandshakeStatus:329(18503869ns) {DropIncomingHandshake:442(866ns) DropOutgoingHandshake:457(209ns) SwitchToState:179(1099ns) ProcessPendingSessionEvents:495(1192ns)}}
2025-11-27T18:19:57.293015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471629228371341:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.308601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:57.314296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471629228371341:2049] [node 2] ICP22 created new session: [5:7577471633523338737:2049]
2025-11-27T18:19:57.333514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471633523338737:2049] [node 2] ICS09 handshake done sender: [5:7577471633523338721:2119] self: [5:7577471633523338733:0] peer: [2:7577471630592757996:0] socket: 295 qp: -1
2025-11-27T18:19:57.335932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471633523338737:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:57.335997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:57.336114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471633523338736:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20854
2025-11-27T18:19:57.260540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471630592758026:2108] [node 5] ICS09 handshake done sender: [2:7577471630592758000:2116] self: [2:7577471630592757996:0] peer: [5:7577471633523338733:0] socket: 322 qp: -1
2025-11-27T18:19:57.260691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471630592758026:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:57.337779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471630561654983:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:57.260731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:57.339015Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471630195624030:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(51674324ns) {HandleHandshakeStatus:395(51670992ns) {LogHandshakeFail:472(15143ns) {UpdateErrorStateLog:545(4578ns)} DropOutgoingHandshake:457(385ns) ProcessPendingSessionEvents:495(51582106ns) {unordered DropSessionEvent:512 num# 100 sum# 51514189ns max# 9527180ns}}}
2025-11-27T18:19:57.339239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471626266687677:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:57.333523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471626297790628:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(52709740ns) {HandleHandshakeStatus:329(52707363ns) {DropIncomingHandshake:442(244ns) DropOutgoingHandshake:457(253ns) SwitchToState:179(295ns) ProcessPendingSessionEvents:495(47113721ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 41163004ns max# 8220023ns}}}
2025-11-27T18:19:57.339955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:57.339803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471630195624030:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.340101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471626266687677:2107] [node 4] ICP22 created new session: [1:7577471630561655011:2107]
2025-11-27T18:19:57.336531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471626297790628:2049] [node 3] ICP18 handshake (actor [2:7577471630592758015:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:57.340582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471629228371342:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5584) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471633523338724:2122] held: no
2025-11-27T18:19:57.345417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471630561655011:2107] [node 4] ICS09 handshake done sender: [1:7577471630561654983:2115] self: [1:7577471630561654978:0] peer: [4:7577471632921456319:0] socket: 313 qp: -1
2025-11-27T18:19:57.340337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:57.345223Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471628102002571:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(91261265ns) {HandleHandshakeStatus:329(91258542ns) {DropIncomingHandshake:442(773ns) DropOutgoingHandshake:457(4942ns) SwitchToState:179(804ns) ProcessPendingSessionEvents:495(77914531ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 77760309ns max# 11218506ns}}}
2025-11-27T18:19:57.345472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471630561655011:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:57.340709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471629228371404:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(108161133ns) {HandleHandshakeStatus:329(108158874ns) {DropIncomingHandshake:442(1983ns) DropOutgoingHandshake:457(6094ns) SwitchToState:179(453ns) ProcessPendingSessionEvents:495(80405989ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 73379182ns max# 6040881ns}}}
2025-11-27T18:19:57.340680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471628626488926:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23552) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471632921456309:2118] held: no
2025-11-27T18:19:57.340492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471630195624030:2049] [node 2] ICP22 created new session: [6:7577471634490591427:2049]
2025-11-27T18:19:57.345528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:57.345592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471632921456320:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8291
2025-11-27T18:19:57.346913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471626297790693:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(91688370ns) {HandleHandshakeStatus:329(91684839ns) {DropIncomingHandshake:442(12176ns) DropOutgoingHandshake:457(325ns) SwitchToState:179(304ns) ProcessPendingSessionEvents:495(79799992ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 77306956ns max# 8624409ns}}}
2025-11-27T18:19:57.343421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471634490591427:2049] [node 2] ICS09 handshake done sender: [6:7577471634490591414:2121] self: [6:7577471634490591418:0] peer: [2:7577471630592758005:0] socket: 303 qp: -1
2025-11-27T18:19:57.345609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471634490591427:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:57.346985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471626297790692:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(78679447ns) {HandleHandshakeStatus:329(78676100ns) {DropIncomingHandshake:442(13759ns) DropOutgoingHandshake:457(284ns) SwitchToState:179(371ns) ProcessPendingSessionEvents:495(75546697ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 75421741ns max# 11613997ns}}}
2025-11-27T18:19:57.349959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471628102002635:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(62255534ns) {HandleHandshakeStatus:329(62249821ns) {DropIncomingHandshake:442(22426ns) DropOutgoingHandshake:457(395ns) SwitchToState:179(562ns) ProcessPendingSessionEvents:495(23491774ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23421922ns max# 12766518ns}}}
2025-11-27T18:19:57.345773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:57.351339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471632921456309:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:57.351653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471628626488926:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.351922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:57.351977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471628626488926:2048] [node 1] ICP22 created new session: [4:7577471632921456321:2048]
2025-11-27T18:19:57.352142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471633523338724:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:57.352381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471632921456321:2048] [node 1] ICS09 handshake done sender: [4:7577471632921456309:2118] self: [4:7577471632921456319:0] peer: [1:7577471630561654978:0] socket: 314 qp: -1
2025-11-27T18:19:57.352519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471632921456321:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:57.352629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:57.352655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471629228371341:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(61631554ns) {HandleHandshakeStatus:329(61627158ns) {DropIncomingHandshake:442(2438ns) DropOutgoingHandshake:457(8411ns) SwitchToState:179(933ns) ProcessPendingSessionEvents:495(14122049ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14028916ns max# 3699458ns}}}
2025-11-27T18:19:57.352662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471629228371342:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:57.352719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:57.352732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471629228371341:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:57.352768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471629228371342:2050] [node 3] ICP22 created new session: [5:7577471633523338739:2050]
2025-11-27T18:19:57.352883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471633523338739:2050] [node 3] ICS09 handshake done sender: [5:7577471633523338724:2122] self: [5:7577471633523338732:0] peer: [3:7577471632396969939:0] socket: 328 qp: -1
2025-11-27T18:19:57.352951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471633523338739:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:57.352998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:57.397773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:57.398027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471630592758009:2048] [node 1] ICS01 socket: 279 reason# EndOfStream
2025-11-27T18:19:57.398070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471626297790627:2048] [node 1] ICP30 unregister session Session# [2:7577471630592758009:2048] VirtualId# [2:7577471630592758002:0]
2025-11-27T18:19:57.398104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471630592758009:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.398189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:57.398248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:57.399119Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:57.399323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471632396969964:2048] [node 1] ICS01 socket: 291 reason# EndOfStream
2025-11-27T18:19:57.399378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471628102002570:2048] [node 1] ICP30 unregister session Session# [3:7577471632396969964:2048] VirtualId# [3:7577471632396969947:0]
2025-11-27T18:19:57.399406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471632396969964:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.399476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:57.399525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:57.402278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:57.402506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471632921456321:2048] [node 1] ICS01 socket: 314 reason# EndOfStream
2025-11-27T18:19:57.402568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471628626488926:2048] [node 1] ICP30 unregister session Session# [4:7577471632921456321:2048] VirtualId# [4:7577471632921456319:0]
2025-11-27T18:19:57.402596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471632921456321:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.402663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:57.402711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:57.403446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:57.403660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471633523338727:2048] [node 1] ICS01 socket: 301 reason# EndOfStream
2025-11-27T18:19:57.403705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471629228371340:2048] [node 1] ICP30 unregister session Session# [5:7577471633523338727:2048] VirtualId# [5:7577471633523338720:0]
2025-11-27T18:19:57.403732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471633523338727:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.403791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:57.403844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:57.404549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:57.404731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471634490591421:2048] [node 1] ICS01 socket: 321 reason# EndOfStream
2025-11-27T18:19:57.404778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471630195624029:2048] [node 1] ICP30 unregister session Session# [6:7577471634490591421:2048] VirtualId# [6:7577471634490591415:0]
2025-11-27T18:19:57.404805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471634490591421:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.404871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:57.404911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:57.473249Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:57.473537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471632396969967:2049] [node 2] ICS07 socket disconnect 304 reason# EndOfStream
2025-11-27T18:19:57.473570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471632396969967:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.473643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:57.473696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471632396969967:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:57.476890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:57.477115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471632921456315:2049] [node 2] ICS01 socket: 297 reason# EndOfStream
2025-11-27T18:19:57.477167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471628626488927:2049] [node 2] ICP30 unregister session Session# [4:7577471632921456315:2049] VirtualId# [4:7577471632921456307:0]
2025-11-27T18:19:57.477201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471632921456315:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.477296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:57.477279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471628102002571:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12320) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:57.477326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471632396969967:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:57.477354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:57.478756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471628102002571:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12320) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:57.478922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471632396969967:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:57.478956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471628102002571:2049] [node 2] ICP30 unregister session Session# [3:7577471632396969967:2049] VirtualId# [3:7577471632396969960:0]
2025-11-27T18:19:57.479072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:57.479108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471628102002571:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12320) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:57.479134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12320) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:57.480086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:57.480286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471634490591427:2049] [node 2] ICS01 socket: 303 reason# EndOfStream
2025-11-27T18:19:57.480326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471630195624030:2049] [node 2] ICP30 unregister session Session# [6:7577471634490591427:2049] VirtualId# [6:7577471634490591418:0]
2025-11-27T18:19:57.480357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471634490591427:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.480426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:57.480486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:57.481385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:57.481577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471633523338737:2049] [node 2] ICS07 socket disconnect 295 reason# EndOfStream
2025-11-27T18:19:57.481608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471633523338737:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.481687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:57.481743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471633523338737:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:57.482549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471629228371341:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12320) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:57.482681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471633523338737:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:57.482750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471629228371341:2049] [node 2] ICP30 unregister session Session# [5:7577471633523338737:2049] VirtualId# [5:7577471633523338733:0]
2025-11-27T18:19:57.482916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:57.482983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471629228371341:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12320) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:57.483014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12320) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:57.489517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471628102002571:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:57.492547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471629228371341:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:57.546879Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:57.547125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471632921456310:2050] [node 3] ICS07 socket disconnect 277 reason# EndOfStream
2025-11-27T18:19:57.547159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471632921456310:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.547231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:57.547277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471632921456310:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:57.547969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471628626488928:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5584) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:57.547987Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:57.548017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471632921456310:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:57.548082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471628626488928:2050] [node 3] ICP30 unregister session Session# [4:7577471632921456310:2050] VirtualId# [4:7577471632921456306:0]
2025-11-27T18:19:57.548194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:57.548191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471633523338739:2050] [node 3] ICS01 socket: 328 reason# EndOfStream
2025-11-27T18:19:57.548222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471628626488928:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5584) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:57.548234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471629228371342:2050] [node 3] ICP30 unregister session Session# [5:7577471633523338739:2050] VirtualId# [5:7577471633523338732:0]
2025-11-27T18:19:57.548262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471633523338739:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.548241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5584) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:57.548319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:57.548372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:57.548961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:57.549131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471634490591420:2050] [node 3] ICS01 socket: 284 reason# EndOfStream
2025-11-27T18:19:57.549166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471630195624031:2050] [node 3] ICP30 unregister session Session# [6:7577471634490591420:2050] VirtualId# [6:7577471634490591410:0]
2025-11-27T18:19:57.549187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471634490591420:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.549241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:57.549315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:57.606438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:57.606695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471633523338730:2108] [node 4] ICS07 socket disconnect 269 reason# EndOfStream
2025-11-27T18:19:57.606746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471633523338730:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.606820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:57.606866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471633523338730:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:57.607591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:57.607684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471629228371404:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25376) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:57.607795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471634490591407:2109] [node 4] ICS01 socket: 271 reason# EndOfStream
2025-11-27T18:19:57.607843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471630195624093:2109] [node 4] ICP30 unregister session Session# [6:7577471634490591407:2109] VirtualId# [6:7577471634490591405:0]
2025-11-27T18:19:57.607882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471634490591407:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.607929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471633523338730:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:57.607965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:57.607988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471629228371404:2108] [node 4] ICP30 unregister session Session# [5:7577471633523338730:2108] VirtualId# [5:7577471633523338718:0]
2025-11-27T18:19:57.608015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:57.608091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:57.608160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471629228371404:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25376) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:57.608197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25376) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:57.617464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471629228371404:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:19:57.673347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:57.673582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471634490591425:2106] [node 5] ICS01 socket: 320 reason# EndOfStream
2025-11-27T18:19:57.673640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471630195624091:2106] [node 5] ICP30 unregister session Session# [6:7577471634490591425:2106] VirtualId# [6:7577471634490591416:0]
2025-11-27T18:19:57.673678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471634490591425:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:57.673749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:57.673791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:57.760044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471632345822657:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:57.760097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471632345822658:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:57.783637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471634217604478:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:57.783826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471634217604479:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:57.807489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471630714458298:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:57.808079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471630714458299:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:57.829051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471631063244810:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:57.829082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471631063244811:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:57.829211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471631063244812:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:57.851684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471633454270042:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:57.851691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471633454270043:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:57.851854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471633454270044:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:57.876270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471632908206443:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:57.876318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471632908206445:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:57.876438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471632908206444:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:57.952759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471632908206500:2101] [node 5] ICP01 ready to work
2025-11-27T18:19:57.975493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471632345822721:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:58.017251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471635358212171:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:58.016909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471634217604541:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:58.047634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471636640790018:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:58.053085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471637749237402:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:58.053086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471636640790019:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:58.057609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471637749237401:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:58.050405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471635009425658:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:58.039565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471638512571838:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:58.050413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471635009425657:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:58.053035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471637203173802:2106] [node 4] ICP01 ready to work
2025-11-27T18:19:58.058165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471635009425659:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:58.050548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471638512571839:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:58.061451Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471632908206500:2101] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(105668965ns) {Configure:104(105664737ns) {ProcessConfigured:135(7491648ns) {StartInitialHandshake:157(7487609ns) {DropHandshakes:470(2528ns) {DropIncomingHandshake:442(490ns) DropOutgoingHandshake:457(341ns)} GenerateSessionVirtualId:988(974ns) PrepareNewSessionHandshake:477(2057ns) {SwitchToState:179(743ns)}}}}}
2025-11-27T18:19:58.061072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471635358212172:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:58.063971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471632345822657:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(110960069ns) {Configure:104(110919762ns) {ProcessConfigured:135(1423627ns) {StartInitialHandshake:157(1421020ns) {DropHandshakes:470(3121ns) {DropIncomingHandshake:442(719ns) DropOutgoingHandshake:457(519ns)} GenerateSessionVirtualId:988(940ns) PrepareNewSessionHandshake:477(2526ns) {SwitchToState:179(857ns)}}}}}
2025-11-27T18:19:58.069248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471632345822658:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(115200790ns) {Configure:104(115195928ns) {ProcessConfigured:135(821131ns) {StartInitialHandshake:157(819176ns) {DropHandshakes:470(3968ns) {DropIncomingHandshake:442(1286ns) DropOutgoingHandshake:457(462ns)} GenerateSessionVirtualId:988(884ns) PrepareNewSessionHandshake:477(5637ns) {SwitchToState:179(1153ns)}}}}}
2025-11-27T18:19:58.075662Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471632908206443:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(45360419ns) {Configure:104(45353794ns) {ProcessConfigured:135(3320301ns) {StartInitialHandshake:157(3317870ns) {DropHandshakes:470(774339ns) {DropIncomingHandshake:442(1922ns) DropOutgoingHandshake:457(367ns)} GenerateSessionVirtualId:988(1697ns) PrepareNewSessionHandshake:477(4373ns) {SwitchToState:179(2103ns)}}}}}
2025-11-27T18:19:58.078950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471633454270043:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48176844ns) {Configure:104(48164064ns) {ProcessConfigured:135(799181ns) {StartInitialHandshake:157(797392ns) {DropHandshakes:470(1251ns) {DropIncomingHandshake:442(322ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(643ns) PrepareNewSessionHandshake:477(1703ns) {SwitchToState:179(602ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.079998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471634217604478:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(58925610ns) {Configure:104(58919795ns) {ProcessConfigured:135(844038ns) {StartInitialHandshake:157(841653ns) {DropHandshakes:470(1088ns) {DropIncomingHandshake:442(344ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(579ns) PrepareNewSessionHandshake:477(1630ns) {SwitchToState:179(566ns)}}}}}
2025-11-27T18:19:58.079961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471631063244811:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18471880ns) {Configure:104(18466563ns) {ProcessConfigured:135(888147ns) {StartInitialHandshake:157(885911ns) {DropHandshakes:470(868ns) {DropIncomingHandshake:442(146ns) DropOutgoingHandshake:457(361ns)} GenerateSessionVirtualId:988(353ns) PrepareNewSessionHandshake:477(1778ns) {SwitchToState:179(830ns)}}}}}
2025-11-27T18:19:58.086282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471630714458298:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38608695ns) {Configure:104(38599482ns) {ProcessConfigured:135(5536764ns) {StartInitialHandshake:157(5532762ns) {DropHandshakes:470(1121ns) {DropIncomingHandshake:442(393ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(604ns) PrepareNewSessionHandshake:477(2088ns) {SwitchToState:179(922ns)}}}}}
2025-11-27T18:19:58.080269Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471633454270042:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19370932ns) {Configure:104(19365417ns) {ProcessConfigured:135(1304220ns) {StartInitialHandshake:157(1302665ns) {DropHandshakes:470(1172ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(409ns) PrepareNewSessionHandshake:477(495724ns) {SwitchToState:179(2426ns)}}}}}
2025-11-27T18:19:58.089014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471631063244810:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18704579ns) {Configure:104(18699765ns) {ProcessConfigured:135(1301700ns) {StartInitialHandshake:157(1299771ns) {DropHandshakes:470(1244ns) {DropIncomingHandshake:442(399ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(612ns) PrepareNewSessionHandshake:477(959ns) {SwitchToState:179(371ns)}}}}}
2025-11-27T18:19:58.081612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471634217604479:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17185045ns) {Configure:104(17171378ns) {ProcessConfigured:135(2222091ns) {StartInitialHandshake:157(2219577ns) {DropHandshakes:470(1313ns) {DropIncomingHandshake:442(410ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(717ns) PrepareNewSessionHandshake:477(2860ns) {SwitchToState:179(1080ns)}}}}}
2025-11-27T18:19:58.093435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471635358212171:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16363542ns) {Configure:104(16359962ns) {ProcessConfigured:135(3047311ns) {StartInitialHandshake:157(3045780ns) {DropHandshakes:470(538ns) {DropIncomingHandshake:442(151ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:477(815889ns) {SwitchToState:179(3793ns)}}}}}
2025-11-27T18:19:58.087058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471635009425669:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49183
2025-11-27T18:19:58.081635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471634217604541:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18210032ns) {Configure:104(17862375ns) {ProcessConfigured:135(1924246ns) {StartInitialHandshake:157(1922659ns) {DropHandshakes:470(581ns) {DropIncomingHandshake:442(179ns) DropOutgoingHandshake:457(144ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:477(1472ns) {SwitchToState:179(622ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.089055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471630714458299:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24007045ns) {Configure:104(24002021ns) {ProcessConfigured:135(5538052ns) {StartInitialHandshake:157(5536272ns) {DropHandshakes:470(1127ns) {DropIncomingHandshake:442(308ns) DropOutgoingHandshake:457(256ns)} GenerateSessionVirtualId:988(451ns) PrepareNewSessionHandshake:477(2272ns) {SwitchToState:179(837ns)}}}}}
2025-11-27T18:19:58.101829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471635009425658:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(28009270ns) {Configure:104(28002038ns) {ProcessConfigured:135(9247713ns) {StartInitialHandshake:157(9245085ns) {DropHandshakes:470(1306ns) {DropIncomingHandshake:442(338ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(637ns) PrepareNewSessionHandshake:477(2281ns) {SwitchToState:179(983ns)}}}}}
2025-11-27T18:19:58.111997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471637749237416:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18798
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.102470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471635009425659:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27686764ns) {Configure:104(27682113ns) {ProcessConfigured:135(9506218ns) {StartInitialHandshake:157(9320654ns) {DropHandshakes:470(1622ns) {DropIncomingHandshake:442(751ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(939ns) PrepareNewSessionHandshake:477(5659ns) {SwitchToState:179(1242ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.112297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471637749237417:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18798
2025-11-27T18:19:58.119109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471636640790036:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8052
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.104004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471635009425657:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27463448ns) {Configure:104(27458845ns) {ProcessConfigured:135(8991252ns) {StartInitialHandshake:157(8989511ns) {DropHandshakes:470(1183ns) {DropIncomingHandshake:442(355ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(554ns) PrepareNewSessionHandshake:477(4022ns) {SwitchToState:179(2121ns)}}}}}
2025-11-27T18:19:58.117779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471637749237417:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.119300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471635009425669:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:19:58.124673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471636640790033:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8052
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.127525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471633454270044:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.133941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471637203173819:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54290
2025-11-27T18:19:58.127511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471636640790023:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.129677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471637749237416:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.123129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471630714458298:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.136612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471638512571850:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
2025-11-27T18:19:58.136794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471638512571852:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
2025-11-27T18:19:58.127553Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471636640790023:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.133951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471637203173820:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54290
2025-11-27T18:19:58.135912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471635009425665:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.142751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471637203173820:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.134379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471637749237418:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18798
2025-11-27T18:19:58.135951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471635009425665:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.127665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471636640790034:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8052
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.146060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471638512571851:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
2025-11-27T18:19:58.149499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471635009425665:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:58.150874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471637749237418:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.147503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471638512571853:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
2025-11-27T18:19:58.142811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471637203173819:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.129534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471636640790036:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.153062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471637749237402:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:19:58.150866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471632908206500:2101] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:58.133634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471636640790034:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.153306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471637749237406:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.149718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471638512571850:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.150855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471634217604478:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.153179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471637203173821:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54290
2025-11-27T18:19:58.154154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471635358212181:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.154472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471635009425657:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.153341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471637749237406:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.134481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471636640790035:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8052
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.154609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471635009425675:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49183
2025-11-27T18:19:58.136217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471636640790033:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.153210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471632908206443:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:58.154192Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471635358212181:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.156601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:58.157766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471635009425657:2108] [node 5] ICP22 created new session: [3:7577471635009425678:2108]
2025-11-27T18:19:58.156613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471635358212181:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:58.144723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471636640790035:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.157860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471637749237406:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:58.150952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471638512571852:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.173369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471635358212172:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.165648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471637749237425:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18798
2025-11-27T18:19:58.144962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471636640790023:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:58.152543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471638512571853:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.167163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471630714458298:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29727) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471635009425669:2114] held: no
2025-11-27T18:19:58.173123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471637203173821:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.172064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471637749237421:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18798
2025-11-27T18:19:58.153192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471638512571854:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
2025-11-27T18:19:58.154175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471636640790029:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.171387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471635009425678:2108] [node 5] ICS09 handshake done sender: [3:7577471635009425665:2110] self: [3:7577471635009425661:0] peer: [5:7577471637749237419:0] socket: 279 qp: -1
2025-11-27T18:19:58.174082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:19:58.174806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471632908206444:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:58.153598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471638512571851:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.154207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471636640790029:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.174902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471635358212172:2108] [node 5] ICP22 created new session: [4:7577471635358212187:2108]
2025-11-27T18:19:58.173396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471637749237401:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.171628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471635009425678:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:58.175721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471632908206443:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29727) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471637203173819:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.173452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:58.154365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471638512571854:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.175784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471635358212187:2108] [node 5] ICS09 handshake done sender: [4:7577471635358212181:2115] self: [4:7577471635358212179:0] peer: [5:7577471637749237422:0] socket: 294 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.154258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471632345822658:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.171687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:58.179829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471637203173824:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54290
2025-11-27T18:19:58.175821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471635358212187:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:58.178773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471638512571843:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.173502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471637749237401:2109] [node 6] ICP22 created new session: [5:7577471637749237426:2109]
2025-11-27T18:19:58.172073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471635009425677:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49183
2025-11-27T18:19:58.178808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471638512571843:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.175972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471635009425675:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.173551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471637749237426:2109] [node 6] ICS09 handshake done sender: [5:7577471637749237406:2110] self: [5:7577471637749237404:0] peer: [6:7577471637203173822:0] socket: 288 qp: -1
2025-11-27T18:19:58.182054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471638512571843:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:58.179516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471630714458299:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:58.173578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471637749237426:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: No RDMA context found for GID: 2025-11-27T18:19:58.154320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471636640790021:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:19:58.173607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:58.183289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471637203173825:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54290
2025-11-27T18:19:58.183000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471638512571863:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63816
2025-11-27T18:19:58.178160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.183577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471638512571846:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.183607Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471638512571846:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.185137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471638512571850:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:58.185242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471638512571838:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.186825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:58.187945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471638512571838:2109] [node 6] ICP22 created new session: [2:7577471638512571865:2109]
2025-11-27T18:19:58.183580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471637203173826:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54290
2025-11-27T18:19:58.154347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471636640790021:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.175562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471637749237417:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.196608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471634217604478:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.156556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471636640790041:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8052
2025-11-27T18:19:58.186773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471635358212188:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56858
2025-11-27T18:19:58.198755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471635009425683:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49183
2025-11-27T18:19:58.177282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471637749237425:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.188053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471632908206500:2101] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28233) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471637203173820:2118] held: no
2025-11-27T18:19:58.156543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:58.186783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471635358212191:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56858
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.195151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471637203173826:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.202995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471635009425657:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(26322425ns) {HandleHandshakeStatus:329(26319739ns) {DropIncomingHandshake:442(1227ns) DropOutgoingHandshake:457(2325ns) SwitchToState:179(407ns) ProcessPendingSessionEvents:495(4996914ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4928530ns max# 1653669ns}}}
2025-11-27T18:19:58.199208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:58.199610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471635358212188:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.177336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471633454270044:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.159736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471632345822658:2049] [node 3] ICP22 created new session: [1:7577471636640790043:2049]
2025-11-27T18:19:58.178980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:58.198615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471632908206444:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18681) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471637203173821:2119] held: no
2025-11-27T18:19:58.199774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471634217604478:2048] [node 1] ICP22 created new session: [2:7577471638512571868:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.205467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471635009425683:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.202161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471635358212191:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.171226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471636640790029:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:58.202311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471637203173819:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:58.200321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471638512571865:2109] [node 6] ICS09 handshake done sender: [2:7577471638512571843:2111] self: [2:7577471638512571841:0] peer: [6:7577471637203173827:0] socket: 305 qp: -1
2025-11-27T18:19:58.179455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471637749237424:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18798
2025-11-27T18:19:58.202352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471635358212192:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56858
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.208947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471635009425666:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.171345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471636640790021:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.204841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471632908206445:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.179501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471633454270044:2050] [node 3] ICP22 created new session: [5:7577471637749237428:2050]
2025-11-27T18:19:58.200371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471638512571865:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:58.208983Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471635009425666:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.204892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471631063244812:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:58.171614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471636640790043:2049] [node 3] ICS09 handshake done sender: [1:7577471636640790023:2112] self: [1:7577471636640790022:0] peer: [3:7577471635009425674:0] socket: 271 qp: -1
2025-11-27T18:19:58.206854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471637203173828:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54290
2025-11-27T18:19:58.200458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471638512571868:2048] [node 1] ICS09 handshake done sender: [2:7577471638512571850:2115] self: [2:7577471638512571858:0] peer: [1:7577471636640790020:0] socket: 274 qp: -1
2025-11-27T18:19:58.181062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471637749237428:2050] [node 3] ICS09 handshake done sender: [5:7577471637749237417:2116] self: [5:7577471637749237419:0] peer: [3:7577471635009425661:0] socket: 287 qp: -1
2025-11-27T18:19:58.205496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471631063244810:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:58.216418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471635358212193:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56858
2025-11-27T18:19:58.225194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471635358212196:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56858
2025-11-27T18:19:58.212551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471632908206443:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.209193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471630714458298:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21916231ns) {HandleHandshakeStatus:395(21912915ns) {LogHandshakeFail:472(14729ns) {UpdateErrorStateLog:545(6304ns)} DropOutgoingHandshake:457(2309ns) ProcessPendingSessionEvents:495(12679830ns) {unordered DropSessionEvent:512 num# 100 sum# 2789693ns max# 1309482ns}}}
2025-11-27T18:19:58.171661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471636640790043:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.200494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471638512571868:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:58.213572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:58.181106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471637749237428:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:58.226618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471635358212193:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.214040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471635009425673:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.213630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471632908206443:2048] [node 1] ICP22 created new session: [6:7577471637203173832:2048]
2025-11-27T18:19:58.227015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471635358212196:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.182026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:58.171719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:58.201567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471638512571846:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:58.214083Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471635009425673:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.202148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:58.182653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471633454270042:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:58.219173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471637203173832:2048] [node 1] ICS09 handshake done sender: [6:7577471637203173819:2117] self: [6:7577471637203173823:0] peer: [1:7577471636640790025:0] socket: 293 qp: -1
2025-11-27T18:19:58.217266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471635009425669:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:58.223929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471630714458298:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.223982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:58.224595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471630714458298:2048] [node 1] ICP22 created new session: [3:7577471635009425689:2048]
2025-11-27T18:19:58.225109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471635009425685:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49183
2025-11-27T18:19:58.225906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471635009425689:2048] [node 1] ICS09 handshake done sender: [3:7577471635009425669:2114] self: [3:7577471635009425674:0] peer: [1:7577471636640790022:0] socket: 278 qp: -1
2025-11-27T18:19:58.225946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471635009425689:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:58.226009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:58.226408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471635009425684:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49183
2025-11-27T18:19:58.226779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471635009425666:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:58.227002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471635009425673:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:58.237710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471635009425658:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.238351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471630714458299:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18681) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471635009425675:2116] held: no
2025-11-27T18:19:58.238615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:58.238943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471635009425659:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.239710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:58.239896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471635009425659:2109] [node 6] ICP22 created new session: [3:7577471635009425692:2109]
2025-11-27T18:19:58.172485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471636640790041:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.228779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471631063244811:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:58.172535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471636640790018:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.202894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:58.223542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471637203173833:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54290
2025-11-27T18:19:58.223761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471637203173832:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:58.224033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:58.229160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471632908206443:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17281604ns) {HandleHandshakeStatus:329(17278438ns) {DropIncomingHandshake:442(2733ns) DropOutgoingHandshake:457(447ns) SwitchToState:179(519ns) ProcessPendingSessionEvents:495(1150ns)}}
2025-11-27T18:19:58.236080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471637203173826:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:58.238678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471632908206445:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.238723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:58.241097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471632908206445:2050] [node 3] ICP22 created new session: [6:7577471637203173835:2050]
2025-11-27T18:19:58.241351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471632908206500:2101] [node 5] ICP35 event processing took too much time PendingConnection:279(29623624ns) {HandleHandshakeStatus:395(29619643ns) {LogHandshakeFail:472(16669ns) {UpdateErrorStateLog:545(5527ns)} DropOutgoingHandshake:457(1026ns) ProcessPendingSessionEvents:495(28346139ns) {unordered DropSessionEvent:512 num# 100 sum# 28280022ns max# 4280127ns}}}
2025-11-27T18:19:58.241802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471637203173835:2050] [node 3] ICS09 handshake done sender: [6:7577471637203173826:2122] self: [6:7577471637203173831:0] peer: [3:7577471635009425663:0] socket: 326 qp: -1
2025-11-27T18:19:58.241904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471637203173820:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:58.242255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471637203173835:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:58.242601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:58.236131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471635358212197:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56858
2025-11-27T18:19:58.240744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471635009425658:2107] [node 4] ICP22 created new session: [3:7577471635009425691:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.182983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471637749237418:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:58.173343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471632345822657:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.241206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471635009425692:2109] [node 6] ICS09 handshake done sender: [3:7577471635009425673:2115] self: [3:7577471635009425663:0] peer: [6:7577471637203173831:0] socket: 325 qp: -1
2025-11-27T18:19:58.207393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471634217604479:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.184024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471637749237402:2108] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.240689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471635358212188:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:58.174045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:58.207454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:58.251026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471632908206444:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(27264646ns) {HandleHandshakeStatus:395(27261738ns) {LogHandshakeFail:472(13283ns) {UpdateErrorStateLog:545(5291ns)} DropOutgoingHandshake:457(1642ns) ProcessPendingSessionEvents:495(25193366ns) {unordered DropSessionEvent:512 num# 100 sum# 25135883ns max# 3120350ns}}}
2025-11-27T18:19:58.244586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471631063244812:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.241786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471635009425691:2107] [node 4] ICS09 handshake done sender: [3:7577471635009425666:2111] self: [3:7577471635009425662:0] peer: [4:7577471635358212194:0] socket: 281 qp: -1
2025-11-27T18:19:58.185364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:58.174911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471636640790018:2110] [node 6] ICP22 created new session: [1:7577471636640790048:2110]
2025-11-27T18:19:58.241794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471635009425685:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.187920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471637749237402:2108] [node 4] ICP22 created new session: [5:7577471637749237432:2108]
2025-11-27T18:19:58.175970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:58.244659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:58.251045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471632908206500:2101] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.176030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471636640790048:2110] [node 6] ICS09 handshake done sender: [1:7577471636640790029:2114] self: [1:7577471636640790025:0] peer: [6:7577471637203173823:0] socket: 273 qp: -1
2025-11-27T18:19:58.207501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471634217604479:2049] [node 3] ICP22 created new session: [2:7577471638512571871:2049]
2025-11-27T18:19:58.245301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471631063244812:2050] [node 3] ICP22 created new session: [4:7577471635358212201:2050]
2025-11-27T18:19:58.241823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471635009425691:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:58.249576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471635358212199:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56858
2025-11-27T18:19:58.207555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471638512571871:2049] [node 3] ICS09 handshake done sender: [2:7577471638512571846:2112] self: [2:7577471638512571844:0] peer: [3:7577471635009425682:0] socket: 309 qp: -1
2025-11-27T18:19:58.245237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471635009425692:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:58.249929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471635358212201:2050] [node 3] ICS09 handshake done sender: [4:7577471635358212188:2117] self: [4:7577471635358212194:0] peer: [3:7577471635009425662:0] socket: 290 qp: -1
2025-11-27T18:19:58.196424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471637749237432:2108] [node 4] ICS09 handshake done sender: [5:7577471637749237418:2117] self: [5:7577471637749237422:0] peer: [4:7577471635358212179:0] socket: 296 qp: -1
2025-11-27T18:19:58.251981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471637203173821:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:58.207588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471638512571871:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:58.245296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:58.177229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471636640790048:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:58.250792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471635358212201:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:58.207619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:58.251975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:58.245305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:58.199045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471637749237432:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:58.237634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471638512571847:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.177285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:58.251986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.253062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471632908206500:2101] [node 5] ICP22 created new session: [6:7577471637203173837:2101]
2025-11-27T18:19:58.199105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:58.256495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471635358212191:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:58.178741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471632345822657:2048] [node 2] ICP22 created new session: [1:7577471636640790049:2048]
2025-11-27T18:19:58.237671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471638512571847:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.254858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471637203173837:2101] [node 5] ICS09 handshake done sender: [6:7577471637203173820:2118] self: [6:7577471637203173822:0] peer: [5:7577471637749237404:0] socket: 301 qp: -1
2025-11-27T18:19:58.217620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471637749237434:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18798
2025-11-27T18:19:58.246454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471638512571847:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:58.254002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471634217604541:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.254882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:58.254928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471634217604541:2107] [node 4] ICP22 created new session: [2:7577471638512571875:2107]
2025-11-27T18:19:58.256451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471638512571875:2107] [node 4] ICS09 handshake done sender: [2:7577471638512571847:2113] self: [2:7577471638512571845:0] peer: [4:7577471635358212198:0] socket: 272 qp: -1
2025-11-27T18:19:58.254919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471632908206444:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.257140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471631063244810:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.256486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471638512571875:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:58.255389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471637203173837:2101] [node 5] ICS10 traffic start
2025-11-27T18:19:58.179477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471636640790049:2048] [node 2] ICS09 handshake done sender: [1:7577471636640790021:2111] self: [1:7577471636640790020:0] peer: [2:7577471638512571858:0] socket: 270 qp: -1
2025-11-27T18:19:58.256526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:58.256031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:58.225238Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471633454270042:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29727) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471637749237425:2120] held: no
2025-11-27T18:19:58.179883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471636640790049:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:58.260991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:58.265652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471638512571849:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.253451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471637749237435:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18798
2025-11-27T18:19:58.179930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:58.256044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:58.261435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471635358212172:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(67311004ns) {HandleHandshakeStatus:329(67308635ns) {DropIncomingHandshake:442(203ns) DropOutgoingHandshake:457(289ns) SwitchToState:179(358ns) ProcessPendingSessionEvents:495(59188830ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 59036640ns max# 3207465ns}}}
2025-11-27T18:19:58.265696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471638512571849:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.256631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471632908206444:2049] [node 2] ICP22 created new session: [6:7577471637203173838:2049]
2025-11-27T18:19:58.185145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471636640790027:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.290729Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471630714458299:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(18305289ns) {HandleHandshakeStatus:395(18302437ns) {LogHandshakeFail:472(1153228ns) {UpdateErrorStateLog:545(5637ns)} DropOutgoingHandshake:457(312ns) ProcessPendingSessionEvents:495(17077595ns) {unordered DropSessionEvent:512 num# 100 sum# 5855328ns max# 1570279ns}}}
2025-11-27T18:19:58.267103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471631063244810:2048] [node 1] ICP22 created new session: [4:7577471635358212205:2048]
2025-11-27T18:19:58.255219Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471637749237401:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(55446122ns) {HandleHandshakeStatus:329(55444024ns) {DropIncomingHandshake:442(10734ns) DropOutgoingHandshake:457(444ns) SwitchToState:179(404ns) ProcessPendingSessionEvents:495(54672731ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 54523227ns max# 4228158ns}}}
2025-11-27T18:19:58.283467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471638512571849:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:58.185179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471636640790027:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.273472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471635358212205:2048] [node 1] ICS09 handshake done sender: [4:7577471635358212191:2118] self: [4:7577471635358212195:0] peer: [1:7577471636640790030:0] socket: 283 qp: -1
2025-11-27T18:19:58.282982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471635358212202:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56858
2025-11-27T18:19:58.291164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471635009425675:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:58.256451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471637749237435:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.257412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471637203173838:2049] [node 2] ICS09 handshake done sender: [6:7577471637203173821:2119] self: [6:7577471637203173827:0] peer: [2:7577471638512571841:0] socket: 306 qp: -1
2025-11-27T18:19:58.285554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471638512571839:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.200010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471636640790027:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:58.283068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471635358212205:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:58.256575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471633454270043:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:58.257919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471637203173838:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:58.293747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471630714458299:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.204987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471636640790019:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.283178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:58.281551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471633454270042:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20031528ns) {HandleHandshakeStatus:395(20028343ns) {LogHandshakeFail:472(14303ns) {UpdateErrorStateLog:545(5360ns)} DropOutgoingHandshake:457(1354ns) ProcessPendingSessionEvents:495(19297232ns) {unordered DropSessionEvent:512 num# 100 sum# 4569463ns max# 1680941ns}}}
2025-11-27T18:19:58.294266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:58.261101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:58.290026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471631063244811:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18681) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471635358212193:2120] held: no
2025-11-27T18:19:58.283369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471633454270044:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(72909558ns) {HandleHandshakeStatus:329(72906904ns) {DropIncomingHandshake:442(491ns) DropOutgoingHandshake:457(4717ns) SwitchToState:179(1793ns) ProcessPendingSessionEvents:495(66562934ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 63113455ns max# 7208847ns}}}
2025-11-27T18:19:58.294664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471630714458299:2049] [node 2] ICP22 created new session: [3:7577471635009425695:2049]
2025-11-27T18:19:58.208815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:58.283840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471632908206444:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(25932238ns) {HandleHandshakeStatus:329(25929617ns) {DropIncomingHandshake:442(1059ns) DropOutgoingHandshake:457(214ns) SwitchToState:179(417ns) ProcessPendingSessionEvents:495(2193ns)}}
2025-11-27T18:19:58.296454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471635009425695:2049] [node 2] ICS09 handshake done sender: [3:7577471635009425675:2116] self: [3:7577471635009425682:0] peer: [2:7577471638512571844:0] socket: 310 qp: -1
2025-11-27T18:19:58.288552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471637749237425:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:58.296928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471635009425695:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:58.288450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471637203173840:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54290
2025-11-27T18:19:58.212239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471636640790019:2108] [node 5] ICP22 created new session: [1:7577471636640790054:2108]
2025-11-27T18:19:58.298527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:58.290347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:58.291045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471637749237402:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(72540294ns) {HandleHandshakeStatus:329(72536831ns) {DropIncomingHandshake:442(442ns) DropOutgoingHandshake:457(3795ns) SwitchToState:179(363ns) ProcessPendingSessionEvents:495(51074318ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 49954168ns max# 5589319ns}}}
2025-11-27T18:19:58.293489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471638512571839:2108] [node 5] ICP22 created new session: [2:7577471638512571878:2108]
2025-11-27T18:19:58.299753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471631063244812:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(41191497ns) {HandleHandshakeStatus:329(41189544ns) {DropIncomingHandshake:442(2423ns) DropOutgoingHandshake:457(16851ns) SwitchToState:179(526ns) ProcessPendingSessionEvents:495(28195758ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28110659ns max# 6344598ns}}}
2025-11-27T18:19:58.212906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471636640790031:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.291137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471633454270042:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.299344Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471635009425659:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(55585640ns) {HandleHandshakeStatus:329(54864106ns) {DropIncomingHandshake:442(1064ns) DropOutgoingHandshake:457(615ns) SwitchToState:179(281ns) ProcessPendingSessionEvents:495(41493688ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 41380063ns max# 6327474ns}}}
2025-11-27T18:19:58.293884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471638512571878:2108] [node 5] ICS09 handshake done sender: [2:7577471638512571849:2114] self: [2:7577471638512571848:0] peer: [5:7577471637749237436:0] socket: 340 qp: -1
2025-11-27T18:19:58.301647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471635009425659:2109] [node 6] ICP18 handshake (actor [3:7577471635009425685:2120]) is too old
2025-11-27T18:19:58.301653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471632908206445:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(44987487ns) {HandleHandshakeStatus:329(44985027ns) {DropIncomingHandshake:442(487ns) DropOutgoingHandshake:457(4736ns) SwitchToState:179(307ns) ProcessPendingSessionEvents:495(37673633ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32059456ns max# 4402173ns}}}
2025-11-27T18:19:58.294358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471638512571878:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:58.294176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:58.300571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471635358212193:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:58.294858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:58.298706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471638512571838:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(92625031ns) {HandleHandshakeStatus:329(92621434ns) {DropIncomingHandshake:442(26726ns) DropOutgoingHandshake:457(238ns) SwitchToState:179(404ns) ProcessPendingSessionEvents:495(71457822ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 71290844ns max# 7120298ns}}}
2025-11-27T18:19:58.212933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471636640790031:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.294999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471633454270043:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18681) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471637749237435:2122] held: no
2025-11-27T18:19:58.301121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471635358212172:2108] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.300377Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471634217604478:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(92519273ns) {HandleHandshakeStatus:329(92515947ns) {DropIncomingHandshake:442(2145ns) DropOutgoingHandshake:457(12362ns) SwitchToState:179(849ns) ProcessPendingSessionEvents:495(77503336ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 77326713ns max# 7693771ns}}}
2025-11-27T18:19:58.301733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471635009425658:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(50798871ns) {HandleHandshakeStatus:329(50795236ns) {DropIncomingHandshake:442(15063ns) DropOutgoingHandshake:457(398ns) SwitchToState:179(629ns) ProcessPendingSessionEvents:495(35023949ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32645275ns max# 7500163ns}}}
2025-11-27T18:19:58.302489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471631063244811:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.216274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471636640790054:2108] [node 5] ICS09 handshake done sender: [1:7577471636640790027:2113] self: [1:7577471636640790026:0] peer: [5:7577471637749237431:0] socket: 319 qp: -1
2025-11-27T18:19:58.295222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471633454270042:2048] [node 1] ICP22 created new session: [5:7577471637749237438:2048]
2025-11-27T18:19:58.296240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471637749237438:2048] [node 1] ICS09 handshake done sender: [5:7577471637749237425:2120] self: [5:7577471637749237431:0] peer: [1:7577471636640790026:0] socket: 320 qp: -1
2025-11-27T18:19:58.298713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471637749237438:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:58.298756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:58.302714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:58.303797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471637749237437:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18798
2025-11-27T18:19:58.303592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471631063244811:2049] [node 2] ICP22 created new session: [4:7577471635358212207:2049]
2025-11-27T18:19:58.216331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471636640790054:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:58.305189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471635358212207:2049] [node 2] ICS09 handshake done sender: [4:7577471635358212193:2120] self: [4:7577471635358212198:0] peer: [2:7577471638512571845:0] socket: 300 qp: -1
2025-11-27T18:19:58.305371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471635358212207:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:58.217558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:58.224726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471636640790031:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:58.237071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471632345822721:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.305785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471637749237435:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:58.237656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:58.238942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471632345822721:2107] [node 4] ICP22 created new session: [1:7577471636640790057:2107]
2025-11-27T18:19:58.239660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471636640790057:2107] [node 4] ICS09 handshake done sender: [1:7577471636640790031:2115] self: [1:7577471636640790030:0] peer: [4:7577471635358212195:0] socket: 328 qp: -1
2025-11-27T18:19:58.307307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471637203173840:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.241115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471636640790057:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:58.241161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:58.305619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:58.256116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471636640790018:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(54542638ns) {HandleHandshakeStatus:329(54539509ns) {DropIncomingHandshake:442(16296ns) DropOutgoingHandshake:457(532ns) SwitchToState:179(694ns) ProcessPendingSessionEvents:495(45032211ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 44144986ns max# 4402321ns}}}
2025-11-27T18:19:58.257695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471632345822658:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(81821630ns) {HandleHandshakeStatus:329(81818709ns) {DropIncomingHandshake:442(13255ns) DropOutgoingHandshake:457(599ns) SwitchToState:179(328ns) ProcessPendingSessionEvents:495(61253595ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 61087760ns max# 4242879ns}}}
2025-11-27T18:19:58.269761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471632345822657:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(69223829ns) {HandleHandshakeStatus:329(69221697ns) {DropIncomingHandshake:442(206ns) DropOutgoingHandshake:457(227ns) SwitchToState:179(2325ns) ProcessPendingSessionEvents:495(59232310ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 55914607ns max# 4235868ns}}}
2025-11-27T18:19:58.308241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471637203173802:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:58.308198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471633454270043:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.273593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471632345822657:2048] [node 2] ICP18 handshake (actor [1:7577471636640790041:2120]) is too old
2025-11-27T18:19:58.299225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471636640790019:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(78548309ns) {HandleHandshakeStatus:329(78544876ns) {DropIncomingHandshake:442(15848ns) DropOutgoingHandshake:457(361ns) SwitchToState:179(799ns) ProcessPendingSessionEvents:495(58212035ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 58099277ns max# 9592670ns}}}
2025-11-27T18:19:58.309417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471635358212182:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:58.308266Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471632345822721:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(64655323ns) {HandleHandshakeStatus:329(64652027ns) {DropIncomingHandshake:442(16725ns) DropOutgoingHandshake:457(726ns) SwitchToState:179(634ns) ProcessPendingSessionEvents:495(55296800ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 55152524ns max# 6678440ns}}}
2025-11-27T18:19:58.309455Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471635358212182:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:58.309764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:58.309830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471631063244810:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(47066287ns) {HandleHandshakeStatus:329(47064117ns) {DropIncomingHandshake:442(1643ns) DropOutgoingHandshake:457(4299ns) SwitchToState:179(436ns) ProcessPendingSessionEvents:495(15327097ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14726362ns max# 2455734ns}}}
2025-11-27T18:19:58.310084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471633454270043:2049] [node 2] ICP22 created new session: [5:7577471637749237440:2049]
2025-11-27T18:19:58.310143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471637749237440:2049] [node 2] ICS09 handshake done sender: [5:7577471637749237435:2122] self: [5:7577471637749237436:0] peer: [2:7577471638512571848:0] socket: 341 qp: -1
2025-11-27T18:19:58.310179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471637749237440:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:58.310221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:58.310378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471631063244810:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:58.311004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471638512571839:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(21724674ns) {HandleHandshakeStatus:329(21721254ns) {DropIncomingHandshake:442(24758ns) DropOutgoingHandshake:457(615ns) SwitchToState:179(1136ns) ProcessPendingSessionEvents:495(8627148ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8531129ns max# 1376566ns}}}
2025-11-27T18:19:58.311577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471635358212182:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:58.311887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471635358212171:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:58.311840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471637203173843:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54290
2025-11-27T18:19:58.312028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:58.312095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471635358212171:2110] [node 6] ICP22 created new session: [4:7577471635358212210:2110]
2025-11-27T18:19:58.312121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471637203173840:2125] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:58.312140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471635358212210:2110] [node 6] ICS09 handshake done sender: [4:7577471635358212182:2116] self: [4:7577471635358212180:0] peer: [6:7577471637203173842:0] socket: 286 qp: -1
2025-11-27T18:19:58.312178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471635358212210:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:58.312214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:58.312515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471637203173802:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:58.312584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:58.312658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471637203173802:2106] [node 4] ICP22 created new session: [6:7577471637203173844:2106]
2025-11-27T18:19:58.312711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471637203173844:2106] [node 4] ICS09 handshake done sender: [6:7577471637203173840:2125] self: [6:7577471637203173842:0] peer: [4:7577471635358212180:0] socket: 297 qp: -1
2025-11-27T18:19:58.312748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471637203173844:2106] [node 4] ICS10 traffic start
2025-11-27T18:19:58.312827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:58.313765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471637203173802:2106] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:58.377665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:58.377946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471638512571868:2048] [node 1] ICS07 socket disconnect 274 reason# EndOfStream
2025-11-27T18:19:58.377987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471638512571868:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.378083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:58.378136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471638512571868:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:58.379075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:58.379267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471635009425689:2048] [node 1] ICS01 socket: 278 reason# EndOfStream
2025-11-27T18:19:58.379324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471630714458298:2048] [node 1] ICP30 unregister session Session# [3:7577471635009425689:2048] VirtualId# [3:7577471635009425674:0]
2025-11-27T18:19:58.379367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471635009425689:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.379445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:58.379506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:58.380588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471634217604478:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29727) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:58.380645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471638512571868:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:58.382655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471634217604478:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29727) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:58.382762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471638512571868:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:58.382844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471634217604478:2048] [node 1] ICP30 unregister session Session# [2:7577471638512571868:2048] VirtualId# [2:7577471638512571858:0]
2025-11-27T18:19:58.382776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:58.382973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471635358212205:2048] [node 1] ICS07 socket disconnect 283 reason# EndOfStream
2025-11-27T18:19:58.382972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:58.383011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471635358212205:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.383023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471634217604478:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.383076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:58.383039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.383119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471635358212205:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:58.383800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471631063244810:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29727) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:58.383866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471635358212205:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:58.383904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471631063244810:2048] [node 1] ICP30 unregister session Session# [4:7577471635358212205:2048] VirtualId# [4:7577471635358212195:0]
2025-11-27T18:19:58.384179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:58.384230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471631063244810:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.384179Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:58.384248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.384407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471637749237438:2048] [node 1] ICS01 socket: 320 reason# EndOfStream
2025-11-27T18:19:58.384457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471633454270042:2048] [node 1] ICP30 unregister session Session# [5:7577471637749237438:2048] VirtualId# [5:7577471637749237431:0]
2025-11-27T18:19:58.384508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471637749237438:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.384589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:58.384634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:58.385332Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:58.385549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471637203173832:2048] [node 1] ICS01 socket: 293 reason# EndOfStream
2025-11-27T18:19:58.385591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471632908206443:2048] [node 1] ICP30 unregister session Session# [6:7577471637203173832:2048] VirtualId# [6:7577471637203173823:0]
2025-11-27T18:19:58.385615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471637203173832:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.385667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:58.385703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:58.393772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471631063244810:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:58.453833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:58.454100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471635009425695:2049] [node 2] ICS01 socket: 310 reason# EndOfStream
2025-11-27T18:19:58.454168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471630714458299:2049] [node 2] ICP30 unregister session Session# [3:7577471635009425695:2049] VirtualId# [3:7577471635009425682:0]
2025-11-27T18:19:58.454220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471635009425695:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.454299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:58.454345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:58.456064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:58.456265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471635358212207:2049] [node 2] ICS01 socket: 300 reason# EndOfStream
2025-11-27T18:19:58.456306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471631063244811:2049] [node 2] ICP30 unregister session Session# [4:7577471635358212207:2049] VirtualId# [4:7577471635358212198:0]
2025-11-27T18:19:58.456371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471635358212207:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.456433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:58.456484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:58.457161Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:58.457375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471637749237440:2049] [node 2] ICS01 socket: 341 reason# EndOfStream
2025-11-27T18:19:58.457420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471633454270043:2049] [node 2] ICP30 unregister session Session# [5:7577471637749237440:2049] VirtualId# [5:7577471637749237436:0]
2025-11-27T18:19:58.457450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471637749237440:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.457524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:58.457566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:58.458214Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:58.458364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471637203173838:2049] [node 2] ICS01 socket: 306 reason# EndOfStream
2025-11-27T18:19:58.458411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471632908206444:2049] [node 2] ICP30 unregister session Session# [6:7577471637203173838:2049] VirtualId# [6:7577471637203173827:0]
2025-11-27T18:19:58.458451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471637203173838:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.458519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:58.458560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:58.522393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:58.522576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471635358212201:2050] [node 3] ICS07 socket disconnect 290 reason# EndOfStream
2025-11-27T18:19:58.522621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471635358212201:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.522704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:58.522754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471635358212201:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:58.523575Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:58.523588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471631063244812:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8128) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:58.523656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471635358212201:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:58.523712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471631063244812:2050] [node 3] ICP30 unregister session Session# [4:7577471635358212201:2050] VirtualId# [4:7577471635358212194:0]
2025-11-27T18:19:58.523824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:58.523812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471637749237428:2050] [node 3] ICS07 socket disconnect 287 reason# EndOfStream
2025-11-27T18:19:58.523865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471631063244812:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.523846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471637749237428:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.523881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.523910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:58.523959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471637749237428:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:58.524505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:58.524727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471637203173835:2050] [node 3] ICS07 socket disconnect 326 reason# EndOfStream
2025-11-27T18:19:58.524764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471637203173835:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.524755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471633454270044:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8128) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:58.524851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:58.524853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471637749237428:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:58.524899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471637203173835:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:58.524899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471633454270044:2050] [node 3] ICP30 unregister session Session# [5:7577471637749237428:2050] VirtualId# [5:7577471637749237419:0]
2025-11-27T18:19:58.524990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:58.525031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471633454270044:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.525048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.525558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471632908206445:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8128) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:58.525612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471637203173835:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:58.525655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471632908206445:2050] [node 3] ICP30 unregister session Session# [6:7577471637203173835:2050] VirtualId# [6:7577471637203173831:0]
2025-11-27T18:19:58.525754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:58.525818Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471632908206445:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.525837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8128) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.534066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471631063244812:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:58.534929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471633454270044:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:58.535951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471632908206445:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:58.597690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:58.597874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471637749237432:2108] [node 4] ICS07 socket disconnect 296 reason# EndOfStream
2025-11-27T18:19:58.597900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471637749237432:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.597971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:58.598004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471637749237432:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:58.598738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:58.598721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471637749237402:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6878) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:58.598799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471637749237432:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:58.598855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471637749237402:2108] [node 4] ICP30 unregister session Session# [5:7577471637749237432:2108] VirtualId# [5:7577471637749237422:0]
2025-11-27T18:19:58.598912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:58.598953Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471637749237402:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6878) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.598969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6878) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.598989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471637203173844:2106] [node 4] ICS07 socket disconnect 297 reason# EndOfStream
2025-11-27T18:19:58.599022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471637203173844:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.599101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:58.599145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471637203173844:2106] [node 4] ICS15 start handshake
2025-11-27T18:19:58.599826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471637203173802:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6878) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:58.599926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471637203173844:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:58.599969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471637203173802:2106] [node 4] ICP30 unregister session Session# [6:7577471637203173844:2106] VirtualId# [6:7577471637203173842:0]
2025-11-27T18:19:58.600090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:58.600130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471637203173802:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6878) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.600147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6878) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:58.609714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471637203173802:2106] [node 4] ICP33 wake up from error state
2025-11-27T18:19:58.658534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:58.658776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471637203173837:2101] [node 5] ICS01 socket: 301 reason# EndOfStream
2025-11-27T18:19:58.658823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471632908206500:2101] [node 5] ICP30 unregister session Session# [6:7577471637203173837:2101] VirtualId# [6:7577471637203173822:0]
2025-11-27T18:19:58.658860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471637203173837:2101] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:58.658947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:58.659003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:58.793311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471635590666707:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:58.793378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471635590666708:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:58.812968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471636244958725:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:58.813036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471636244958726:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:58.832941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471634961368408:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:58.832980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471634961368409:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:58.852442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471637643110141:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:58.852498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471637643110142:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:58.852579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471637643110143:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:58.875393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471635332335850:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:58.875451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471635332335851:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:58.875513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471635332335852:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:58.896907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471637680606867:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:58.896968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471637680606868:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:58.897001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471637680606869:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:58.995862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471637680606929:2107] [node 5] ICP01 ready to work
2025-11-27T18:19:59.077376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471640539926084:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:59.087301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471639256335767:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:59.108303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471639627303210:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:59.105426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471641938077502:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:59.111150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471639256335768:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:59.113976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471640539926085:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:59.114897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471639627303211:2110] [node 6] ICP01 ready to work
2025-11-27T18:19:59.115144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471639256335769:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:59.115194Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471635590666707:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(110804532ns) {Configure:104(110696405ns) {ProcessConfigured:135(35593212ns) {StartInitialHandshake:157(35589201ns) {DropHandshakes:470(3388ns) {DropIncomingHandshake:442(738ns) DropOutgoingHandshake:457(355ns)} GenerateSessionVirtualId:988(1106ns) PrepareNewSessionHandshake:477(4871ns) {SwitchToState:179(2050ns)}}}}}
2025-11-27T18:19:59.116120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471640539926086:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:59.114861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471641938077503:2111] [node 6] ICP01 ready to work
2025-11-27T18:19:59.116929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471639885634068:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:59.117448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471639885634069:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:59.118348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471639885634070:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:59.118493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471641975574229:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:59.140063Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471637643110141:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21589218ns) {Configure:104(21582289ns) {ProcessConfigured:135(85702ns) {StartInitialHandshake:157(81839ns) {DropHandshakes:470(1600ns) {DropIncomingHandshake:442(749ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(786ns) PrepareNewSessionHandshake:477(796ns) {SwitchToState:179(237ns)}}}}}
2025-11-27T18:19:59.141062Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471635332335850:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22462898ns) {Configure:104(22458509ns) {ProcessConfigured:135(69053ns) {StartInitialHandshake:157(65882ns) {DropHandshakes:470(7605ns) {DropIncomingHandshake:442(580ns) DropOutgoingHandshake:457(201ns)} GenerateSessionVirtualId:988(778ns) PrepareNewSessionHandshake:477(4549ns) {SwitchToState:179(3792ns)}}}}}
2025-11-27T18:19:59.141801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471634961368408:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20418734ns) {Configure:104(20414887ns) {ProcessConfigured:135(197078ns) {StartInitialHandshake:157(194244ns) {DropHandshakes:470(1062ns) {DropIncomingHandshake:442(256ns) DropOutgoingHandshake:457(154ns)} GenerateSessionVirtualId:988(815ns) PrepareNewSessionHandshake:477(1795ns) {SwitchToState:179(841ns)}}}}}
2025-11-27T18:19:59.167040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471639885634070:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19104569ns) {Configure:104(19100208ns) {ProcessConfigured:135(7117487ns) {StartInitialHandshake:157(7114316ns) {DropHandshakes:470(1280ns) {DropIncomingHandshake:442(342ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(527ns) PrepareNewSessionHandshake:477(5043ns) {SwitchToState:179(2908ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.171355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471640539926097:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55085
2025-11-27T18:19:59.176428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471635332335852:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(22119942ns) {Configure:104(22115853ns) {ProcessConfigured:135(66510ns) {StartInitialHandshake:157(63053ns) {DropHandshakes:470(5795ns) {DropIncomingHandshake:442(619ns) DropOutgoingHandshake:457(285ns)} GenerateSessionVirtualId:988(1347ns) PrepareNewSessionHandshake:477(1300ns) {SwitchToState:179(388ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.182723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471639256335783:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62481
2025-11-27T18:19:59.181644Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471641938077502:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18927834ns) {Configure:104(18922395ns) {ProcessConfigured:135(64589ns) {StartInitialHandshake:157(61245ns) {DropHandshakes:470(2043ns) {DropIncomingHandshake:442(507ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(1243ns) PrepareNewSessionHandshake:477(1308ns) {SwitchToState:179(427ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.186303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471639885634079:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54296
2025-11-27T18:19:59.186458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471639885634080:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54296
2025-11-27T18:19:59.186547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471639885634081:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54296
2025-11-27T18:19:59.191229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639885634080:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.191406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639885634079:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.194146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471639627303222:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18783
2025-11-27T18:19:59.194992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471639627303222:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.195472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471640539926097:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.195718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471640539926101:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55085
2025-11-27T18:19:59.195852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471640539926102:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55085
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.192606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471639256335784:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62481
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.198410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471641938077514:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55613
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.196584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471639256335783:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.199076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471640539926090:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.199117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471640539926090:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.200333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471640539926090:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.198920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471641938077515:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55613
2025-11-27T18:19:59.196858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471639256335785:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62481
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.200976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471641938077516:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55613
2025-11-27T18:19:59.196929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471634961368409:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.201351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471641938077517:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55613
2025-11-27T18:19:59.201115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471636244958726:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.201184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:59.204265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639885634081:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.201234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471636244958726:2049] [node 3] ICP22 created new session: [2:7577471640539926105:2049]
2025-11-27T18:19:59.201363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471640539926105:2049] [node 3] ICS09 handshake done sender: [2:7577471640539926090:2111] self: [2:7577471640539926089:0] peer: [3:7577471639256335786:0] socket: 286 qp: -1
2025-11-27T18:19:59.201394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471640539926105:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:59.201457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:59.204576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471641938077514:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.204651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471641938077515:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.204928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471639256335787:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62481
2025-11-27T18:19:59.205034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471637643110143:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.205754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471639256335785:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.205871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471639256335784:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.205469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471639885634085:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54296
2025-11-27T18:19:59.205668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471639885634086:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54296
2025-11-27T18:19:59.205881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639885634085:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.207756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471639256335775:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.207797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471639256335775:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.208267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471641938077517:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.210464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471640539926101:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.210457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471639256335783:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.212217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471640539926107:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55085
2025-11-27T18:19:59.212422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471636244958725:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.212595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471640539926102:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:19:59.213646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471639627303227:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18783
::ffff:127.1.0.0
2025-11-27T18:19:59.214118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471639256335788:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62481
2025-11-27T18:19:59.214069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639885634067:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.214108Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471639885634067:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.215806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471640539926111:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55085
2025-11-27T18:19:59.215210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471634961368409:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.216179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471639627303227:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.216448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:59.216274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471639885634067:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:59.217809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471640539926101:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:59.216745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471634961368409:2049] [node 2] ICP22 created new session: [3:7577471639256335790:2049]
2025-11-27T18:19:59.216807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471641938077516:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.217991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471635332335851:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.218781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471636244958725:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.216806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471639256335790:2049] [node 2] ICS09 handshake done sender: [3:7577471639256335783:2115] self: [3:7577471639256335786:0] peer: [2:7577471640539926089:0] socket: 289 qp: -1
2025-11-27T18:19:59.216847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471639256335790:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:59.216887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:59.218945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:59.219103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471636244958725:2048] [node 1] ICP22 created new session: [2:7577471640539926112:2048]
2025-11-27T18:19:59.219013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471635590666707:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.219149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471640539926112:2048] [node 1] ICS09 handshake done sender: [2:7577471640539926101:2116] self: [2:7577471640539926109:0] peer: [1:7577471639885634066:0] socket: 302 qp: -1
2025-11-27T18:19:59.219073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:59.219180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471640539926112:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:59.219237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:59.219454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471637643110141:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.219567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471637643110142:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.219720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471634961368409:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.219860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471641938077523:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55613
2025-11-27T18:19:59.220900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471634961368408:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.219116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471635590666707:2048] [node 2] ICP22 created new session: [1:7577471639885634092:2048]
2025-11-27T18:19:59.221280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471640539926094:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.219161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471639885634092:2048] [node 2] ICS09 handshake done sender: [1:7577471639885634067:2110] self: [1:7577471639885634066:0] peer: [2:7577471640539926109:0] socket: 293 qp: -1
2025-11-27T18:19:59.221323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471640539926094:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.219190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471639885634092:2048] [node 2] ICS10 traffic start
2025-11-27T18:19:59.219244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:59.221280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639885634086:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.222014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471639256335775:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:59.222078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471639627303229:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18783
2025-11-27T18:19:59.222047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639885634074:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.222075Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471639885634074:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.223362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471639256335788:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.223468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471639256335767:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.223534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:59.223517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471641938077515:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:59.223570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471639256335767:2107] [node 4] ICP22 created new session: [3:7577471639256335795:2107]
2025-11-27T18:19:59.223620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471639256335795:2107] [node 4] ICS09 handshake done sender: [3:7577471639256335775:2112] self: [3:7577471639256335774:0] peer: [4:7577471641938077522:0] socket: 282 qp: -1
2025-11-27T18:19:59.223659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471639256335795:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:59.223618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471640539926107:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.223775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:59.224587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639885634072:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.224265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471637643110143:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.224613Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471639885634072:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.224397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:59.224440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471637643110143:2050] [node 3] ICP22 created new session: [4:7577471641938077526:2050]
2025-11-27T18:19:59.224486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471641938077526:2050] [node 3] ICS09 handshake done sender: [4:7577471641938077515:2118] self: [4:7577471641938077522:0] peer: [3:7577471639256335774:0] socket: 298 qp: -1
2025-11-27T18:19:59.224513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471641938077526:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:59.224537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:59.226386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471639885634074:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.225484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471640539926092:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.226516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471639627303230:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18783
2025-11-27T18:19:59.225523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471640539926092:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.226908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471639256335767:2107] [node 4] ICP18 handshake (actor [3:7577471639256335788:2119]) is too old
2025-11-27T18:19:59.227782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471639885634072:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:59.228127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471639885634068:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.228195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:59.228244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471639885634068:2107] [node 4] ICP22 created new session: [1:7577471639885634098:2107]
2025-11-27T18:19:59.228293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471639885634098:2107] [node 4] ICS09 handshake done sender: [1:7577471639885634074:2112] self: [1:7577471639885634073:0] peer: [4:7577471641938077524:0] socket: 295 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.228326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471639885634098:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:59.228381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:59.228402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471641938077528:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55613
2025-11-27T18:19:59.228446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471640539926094:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:59.229211Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471640539926092:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.229727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471639627303231:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18783
2025-11-27T18:19:59.229841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471640539926085:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.229817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471641938077517:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:59.229906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:59.230252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471640539926085:2108] [node 5] ICP22 created new session: [2:7577471640539926118:2108]
2025-11-27T18:19:59.230410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471637643110141:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.230491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:59.230538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471637643110141:2048] [node 1] ICP22 created new session: [4:7577471641938077531:2048]
2025-11-27T18:19:59.230442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471640539926118:2108] [node 5] ICS09 handshake done sender: [2:7577471640539926094:2113] self: [2:7577471640539926093:0] peer: [5:7577471639627303228:0] socket: 319 qp: -1
2025-11-27T18:19:59.230580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471641938077531:2048] [node 1] ICS09 handshake done sender: [4:7577471641938077517:2120] self: [4:7577471641938077524:0] peer: [1:7577471639885634073:0] socket: 301 qp: -1
2025-11-27T18:19:59.230608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471641938077531:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:59.230489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471640539926118:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:59.230847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:59.230524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: 2025-11-27T18:19:59.232977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471639627303227:2117] [node 2] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:19:59.233047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471635590666708:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.233108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:59.233375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471635590666708:2049] [node 3] ICP22 created new session: [1:7577471639885634100:2049]
2025-11-27T18:19:59.233422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471639885634100:2049] [node 3] ICS09 handshake done sender: [1:7577471639885634072:2111] self: [1:7577471639885634071:0] peer: [3:7577471639256335793:0] socket: 294 qp: -1
2025-11-27T18:19:59.233456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471639885634100:2049] [node 3] ICS10 traffic start
2025-11-27T18:19:59.233493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.237059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471641975574241:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44396
2025-11-27T18:19:59.237960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471639256335798:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62481
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.240340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471639627303232:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18783
2025-11-27T18:19:59.240538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471639627303229:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.240605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471635332335851:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.240673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:59.240730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471635332335851:2049] [node 2] ICP22 created new session: [5:7577471639627303233:2049]
2025-11-27T18:19:59.240342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471640539926084:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.240779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471639627303233:2049] [node 2] ICS09 handshake done sender: [5:7577471639627303227:2117] self: [5:7577471639627303228:0] peer: [2:7577471640539926093:0] socket: 320 qp: -1
2025-11-27T18:19:59.240470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471641975574242:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44396
2025-11-27T18:19:59.240482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:59.240531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471640539926084:2107] [node 4] ICP22 created new session: [2:7577471640539926120:2107]
2025-11-27T18:19:59.240581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471640539926120:2107] [node 4] ICS09 handshake done sender: [2:7577471640539926092:2112] self: [2:7577471640539926091:0] peer: [4:7577471641938077525:0] socket: 287 qp: -1
2025-11-27T18:19:59.240616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471640539926120:2107] [node 4] ICS10 traffic start
2025-11-27T18:19:59.240663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.241236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471639627303233:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:59.241294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:59.241923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471641938077530:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55613
2025-11-27T18:19:59.244451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471639256335784:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:59.246193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471634961368408:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.246579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:59.246871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471641975574242:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.246645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471634961368408:2048] [node 1] ICP22 created new session: [3:7577471639256335799:2048]
2025-11-27T18:19:59.247075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471637680606869:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.246689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471639256335799:2048] [node 1] ICS09 handshake done sender: [3:7577471639256335784:2116] self: [3:7577471639256335793:0] peer: [1:7577471639885634071:0] socket: 300 qp: -1
2025-11-27T18:19:59.246728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471639256335799:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:59.246766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:59.249816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471635332335852:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.254469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471641975574241:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.254642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471637680606868:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.254695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471641938077516:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:59.254961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471639627303230:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.255119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471635332335850:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.255060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471637643110142:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11735) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471641938077516:2119] held: no
2025-11-27T18:19:59.255865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471640539926096:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.255954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471637643110142:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.255902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471640539926096:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.256372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:59.257490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471637643110142:2049] [node 2] ICP22 created new session: [4:7577471641938077534:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.257445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639885634076:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.258442Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471640539926096:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:59.257545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471641938077534:2049] [node 2] ICS09 handshake done sender: [4:7577471641938077516:2119] self: [4:7577471641938077525:0] peer: [2:7577471640539926091:0] socket: 299 qp: -1
2025-11-27T18:19:59.258512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471641938077534:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:59.258562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:59.257489Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471639885634076:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.259113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471641975574245:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44396
2025-11-27T18:19:59.259714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471639256335779:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.259754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471639256335779:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.260345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471639256335777:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.260374Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471639256335777:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.261694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471639627303238:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18783
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.262065Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471634961368408:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:19:59.262092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471641975574246:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44396
2025-11-27T18:19:59.262167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471640539926086:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.262276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:59.262328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471640539926086:2109] [node 6] ICP22 created new session: [2:7577471640539926123:2109]
2025-11-27T18:19:59.262375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471640539926123:2109] [node 6] ICS09 handshake done sender: [2:7577471640539926096:2114] self: [2:7577471640539926095:0] peer: [6:7577471641975574244:0] socket: 334 qp: -1
2025-11-27T18:19:59.262412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471640539926123:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:59.262752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471639256335779:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:59.262897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471641975574241:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:19:59.262452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:59.262280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471639885634076:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:59.263936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471639627303230:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:59.263697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471639885634069:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.263765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:59.264222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471639885634069:2108] [node 5] ICP22 created new session: [1:7577471639885634104:2108]
2025-11-27T18:19:59.264500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471635332335850:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.264484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471639256335777:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:59.264580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:59.264636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471635332335850:2048] [node 1] ICP22 created new session: [5:7577471639627303240:2048]
2025-11-27T18:19:59.264680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471639627303240:2048] [node 1] ICS09 handshake done sender: [5:7577471639627303230:2119] self: [5:7577471639627303237:0] peer: [1:7577471639885634075:0] socket: 327 qp: -1
2025-11-27T18:19:59.264582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471639885634104:2108] [node 5] ICS09 handshake done sender: [1:7577471639885634076:2113] self: [1:7577471639885634075:0] peer: [5:7577471639627303237:0] socket: 325 qp: -1
2025-11-27T18:19:59.264709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471639627303240:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:59.264637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471639885634104:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:59.264672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:59.264405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471637680606868:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.264739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:59.264983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:19:59.265109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471637680606868:2049] [node 2] ICP22 created new session: [6:7577471641975574248:2049]
2025-11-27T18:19:59.265221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471641975574248:2049] [node 2] ICS09 handshake done sender: [6:7577471641975574241:2115] self: [6:7577471641975574244:0] peer: [2:7577471640539926095:0] socket: 335 qp: -1
2025-11-27T18:19:59.265288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471641975574248:2049] [node 2] ICS10 traffic start
2025-11-27T18:19:59.265321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:19:59.265315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471639256335769:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.265356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:59.265398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471639256335769:2109] [node 6] ICP22 created new session: [3:7577471639256335803:2109]
2025-11-27T18:19:59.265438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471639256335803:2109] [node 6] ICS09 handshake done sender: [3:7577471639256335779:2114] self: [3:7577471639256335778:0] peer: [6:7577471641975574243:0] socket: 317 qp: -1
2025-11-27T18:19:59.265490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471639256335803:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:59.265520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.265847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471641938077533:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55613
2025-11-27T18:19:59.275563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471639627303232:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.266422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471639256335768:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.273597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:59.273669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471639256335768:2108] [node 5] ICP22 created new session: [3:7577471639256335805:2108]
2025-11-27T18:19:59.273718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471639256335805:2108] [node 5] ICS09 handshake done sender: [3:7577471639256335777:2113] self: [3:7577471639256335776:0] peer: [5:7577471639627303235:0] socket: 292 qp: -1
2025-11-27T18:19:59.273768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471639256335805:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:59.273808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:59.278072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471641975574245:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.282364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471641975574247:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44396
2025-11-27T18:19:59.283421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471641938077533:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.284058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471641975574242:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.284561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471639627303239:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18783
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.284963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471641975574250:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44396
2025-11-27T18:19:59.285195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471637680606867:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.285276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471637680606869:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.285322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:59.285367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471637680606869:2050] [node 3] ICP22 created new session: [6:7577471641975574253:2050]
2025-11-27T18:19:59.285411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471641975574253:2050] [node 3] ICS09 handshake done sender: [6:7577471641975574242:2116] self: [6:7577471641975574243:0] peer: [3:7577471639256335778:0] socket: 337 qp: -1
2025-11-27T18:19:59.285467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471641975574253:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:59.285546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:59.287882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471641975574250:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.287970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639885634078:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.288004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471639885634078:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.290779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471639885634078:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:59.290878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471641975574229:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:19:59.291108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471639885634070:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.291169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:59.291221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471639885634070:2109] [node 6] ICP22 created new session: [1:7577471639885634107:2109]
2025-11-27T18:19:59.291298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471639885634107:2109] [node 6] ICS09 handshake done sender: [1:7577471639885634078:2114] self: [1:7577471639885634077:0] peer: [6:7577471641975574252:0] socket: 305 qp: -1
2025-11-27T18:19:59.291341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471639885634107:2109] [node 6] ICS10 traffic start
2025-11-27T18:19:59.291308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471641938077513:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.291378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:59.291236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471641975574255:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44396
2025-11-27T18:19:59.291345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471641938077513:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.292508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471635332335852:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4596) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471639627303229:2118] held: no
2025-11-27T18:19:59.292335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471639256335807:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62481
2025-11-27T18:19:59.293334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471641938077513:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:19:59.293463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471639627303210:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.293554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471639627303229:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:19:59.293774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471641938077503:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.293873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:59.293921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471641938077503:2111] [node 6] ICP22 created new session: [4:7577471641938077539:2111]
2025-11-27T18:19:59.293968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471641938077539:2111] [node 6] ICS09 handshake done sender: [4:7577471641938077513:2116] self: [4:7577471641938077512:0] peer: [6:7577471641975574256:0] socket: 342 qp: -1
2025-11-27T18:19:59.294009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471641938077539:2111] [node 6] ICS10 traffic start
2025-11-27T18:19:59.294041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:59.295597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471641975574245:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:19:59.295486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471635332335852:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.295544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:19:59.295599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471635332335852:2050] [node 3] ICP22 created new session: [5:7577471639627303244:2050]
2025-11-27T18:19:59.295646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471639627303244:2050] [node 3] ICS09 handshake done sender: [5:7577471639627303229:2118] self: [5:7577471639627303235:0] peer: [3:7577471639256335776:0] socket: 324 qp: -1
2025-11-27T18:19:59.295675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471639627303244:2050] [node 3] ICS10 traffic start
2025-11-27T18:19:59.296890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471637680606867:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471641975574245:2117] held: no
2025-11-27T18:19:59.295709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:19:59.299343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471637680606867:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.298097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471641938077511:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.299411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:19:59.299463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471637680606867:2048] [node 1] ICP22 created new session: [6:7577471641975574258:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.298131Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471641938077511:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.304909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471639627303246:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18783
2025-11-27T18:19:59.299508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471641975574258:2048] [node 1] ICS09 handshake done sender: [6:7577471641975574245:2117] self: [6:7577471641975574252:0] peer: [1:7577471639885634077:0] socket: 308 qp: -1
2025-11-27T18:19:59.299546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471641975574258:2048] [node 1] ICS10 traffic start
2025-11-27T18:19:59.299459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471641938077511:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:59.299914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:19:59.299717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471641938077502:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.303023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471641975574257:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44396
2025-11-27T18:19:59.302265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:59.302341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471641938077502:2108] [node 5] ICP22 created new session: [4:7577471641938077542:2108]
2025-11-27T18:19:59.305073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471641938077542:2108] [node 5] ICS09 handshake done sender: [4:7577471641938077511:2115] self: [4:7577471641938077510:0] peer: [5:7577471639627303242:0] socket: 331 qp: -1
2025-11-27T18:19:59.305123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471641938077542:2108] [node 5] ICS10 traffic start
2025-11-27T18:19:59.305171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:59.305446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471641975574229:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15833) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471641975574250:2120] held: no
2025-11-27T18:19:59.305683Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471639627303232:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:19:59.305890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471639627303210:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.305950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:59.305999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471639627303210:2108] [node 4] ICP22 created new session: [5:7577471639627303247:2108]
2025-11-27T18:19:59.306039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471639627303247:2108] [node 4] ICS09 handshake done sender: [5:7577471639627303232:2121] self: [5:7577471639627303242:0] peer: [4:7577471641938077510:0] socket: 332 qp: -1
2025-11-27T18:19:59.306067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471639627303247:2108] [node 4] ICS10 traffic start
2025-11-27T18:19:59.306105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:59.306492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471641975574250:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.306877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471641975574260:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44396
2025-11-27T18:19:59.307043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471641975574229:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.307099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:19:59.307146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471641975574229:2110] [node 4] ICP22 created new session: [6:7577471641975574261:2110]
2025-11-27T18:19:59.307222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471641975574261:2110] [node 4] ICS09 handshake done sender: [6:7577471641975574250:2120] self: [6:7577471641975574256:0] peer: [4:7577471641938077512:0] socket: 307 qp: -1
2025-11-27T18:19:59.307254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471641975574261:2110] [node 4] ICS10 traffic start
2025-11-27T18:19:59.307296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:19:59.310338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471639627303221:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.310372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471639627303221:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:19:59.311604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471639627303221:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:19:59.311979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471639627303211:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:19:59.312037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:19:59.312078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471639627303211:2110] [node 6] ICP22 created new session: [5:7577471639627303250:2110]
2025-11-27T18:19:59.312116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471639627303250:2110] [node 6] ICS09 handshake done sender: [5:7577471639627303221:2115] self: [5:7577471639627303220:0] peer: [6:7577471641975574263:0] socket: 277 qp: -1
2025-11-27T18:19:59.312144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471639627303250:2110] [node 6] ICS10 traffic start
2025-11-27T18:19:59.312178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:19:59.309836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471641975574260:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:19:59.309969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471637680606929:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:19:59.311894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471641975574264:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44396
2025-11-27T18:19:59.313569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471641975574260:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:19:59.313826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471637680606929:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:19:59.313900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:19:59.313948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471637680606929:2107] [node 5] ICP22 created new session: [6:7577471641975574265:2107]
2025-11-27T18:19:59.313995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471641975574265:2107] [node 5] ICS09 handshake done sender: [6:7577471641975574260:2123] self: [6:7577471641975574263:0] peer: [5:7577471639627303220:0] socket: 314 qp: -1
2025-11-27T18:19:59.314027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471641975574265:2107] [node 5] ICS10 traffic start
2025-11-27T18:19:59.314053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:19:59.336303Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:19:59.336546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471640539926112:2048] [node 1] ICS07 socket disconnect 302 reason# EndOfStream
2025-11-27T18:19:59.336581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471640539926112:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.336657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:59.336716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471640539926112:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:59.338594Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:19:59.338803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471639256335799:2048] [node 1] ICS01 socket: 300 reason# EndOfStream
2025-11-27T18:19:59.338844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471634961368408:2048] [node 1] ICP30 unregister session Session# [3:7577471639256335799:2048] VirtualId# [3:7577471639256335793:0]
2025-11-27T18:19:59.338874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471639256335799:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.338960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:59.339018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:59.340701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471636244958725:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6356) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:19:59.340745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471640539926112:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:59.340898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:19:59.341104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471641938077531:2048] [node 1] ICS07 socket disconnect 301 reason# EndOfStream
2025-11-27T18:19:59.341138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471641938077531:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.341204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:59.341290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471641938077531:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:59.342533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471636244958725:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6356) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:59.342593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471640539926112:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:59.342774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471636244958725:2048] [node 1] ICP30 unregister session Session# [2:7577471640539926112:2048] VirtualId# [2:7577471640539926109:0]
2025-11-27T18:19:59.343004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:19:59.343197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471639627303240:2048] [node 1] ICS07 socket disconnect 327 reason# EndOfStream
2025-11-27T18:19:59.342888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:59.342927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471636244958725:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.343411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471637643110141:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6356) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:59.343262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471639627303240:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.342942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.343329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:59.343375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471639627303240:2048] [node 1] ICS15 start handshake
2025-11-27T18:19:59.344941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471635332335850:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6356) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:59.344988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471639627303240:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:59.345019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471635332335850:2048] [node 1] ICP30 unregister session Session# [5:7577471639627303240:2048] VirtualId# [5:7577471639627303237:0]
2025-11-27T18:19:59.345094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:59.345126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471635332335850:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.345139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.344982Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:19:59.345185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471641975574258:2048] [node 1] ICS01 socket: 308 reason# EndOfStream
2025-11-27T18:19:59.345225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471637680606867:2048] [node 1] ICP30 unregister session Session# [6:7577471641975574258:2048] VirtualId# [6:7577471641975574252:0]
2025-11-27T18:19:59.345269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471641975574258:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.345330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:19:59.345381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:59.343542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471641938077531:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:59.344166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471637643110141:2048] [node 1] ICP30 unregister session Session# [4:7577471641938077531:2048] VirtualId# [4:7577471641938077524:0]
2025-11-27T18:19:59.344379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:19:59.344431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471637643110141:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.344452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.353571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471636244958725:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:59.356701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471635332335850:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:59.354190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471637643110141:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:19:59.397508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:19:59.397744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471639256335790:2049] [node 2] ICS07 socket disconnect 289 reason# EndOfStream
2025-11-27T18:19:59.397774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471639256335790:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.397890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:59.397934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471639256335790:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:59.405529Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:19:59.405771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471641975574248:2049] [node 2] ICS07 socket disconnect 335 reason# EndOfStream
2025-11-27T18:19:59.405805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471641975574248:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.400143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:19:59.405887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:59.405486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471641938077534:2049] [node 2] ICS01 socket: 299 reason# EndOfStream
2025-11-27T18:19:59.405541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471637643110142:2049] [node 2] ICP30 unregister session Session# [4:7577471641938077534:2049] VirtualId# [4:7577471641938077525:0]
2025-11-27T18:19:59.405576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471641938077534:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.405674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:59.405723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:59.405829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:19:59.405931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471641975574248:2049] [node 2] ICS15 start handshake
2025-11-27T18:19:59.406023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471639627303233:2049] [node 2] ICS01 socket: 320 reason# EndOfStream
2025-11-27T18:19:59.406058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471635332335851:2049] [node 2] ICP30 unregister session Session# [5:7577471639627303233:2049] VirtualId# [5:7577471639627303228:0]
2025-11-27T18:19:59.406078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471639627303233:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.406129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:19:59.406157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:59.422570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471637680606868:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11735) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:59.429657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471641975574248:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:59.429723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471637680606868:2049] [node 2] ICP30 unregister session Session# [6:7577471641975574248:2049] VirtualId# [6:7577471641975574244:0]
2025-11-27T18:19:59.429846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:19:59.429890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471637680606868:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.429906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.439844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471637680606868:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:19:59.451341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:19:59.451568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471641938077526:2050] [node 3] ICS07 socket disconnect 298 reason# EndOfStream
2025-11-27T18:19:59.451596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471641938077526:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.451668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:59.451710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471641938077526:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:59.452551Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:19:59.452744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471639627303244:2050] [node 3] ICS01 socket: 324 reason# EndOfStream
2025-11-27T18:19:59.452790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471635332335852:2050] [node 3] ICP30 unregister session Session# [5:7577471639627303244:2050] VirtualId# [5:7577471639627303235:0]
2025-11-27T18:19:59.452826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471639627303244:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.452900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:59.452946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:59.455643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471637643110143:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4596) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:59.455707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471641938077526:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:59.455744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471637643110143:2050] [node 3] ICP30 unregister session Session# [4:7577471641938077526:2050] VirtualId# [4:7577471641938077522:0]
2025-11-27T18:19:59.455861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:59.455895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471637643110143:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4596) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.455910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4596) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.462893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471637643110143:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:59.464271Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:19:59.464533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471641975574253:2050] [node 3] ICS07 socket disconnect 337 reason# EndOfStream
2025-11-27T18:19:59.464563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471641975574253:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.464691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:19:59.464739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471641975574253:2050] [node 3] ICS15 start handshake
2025-11-27T18:19:59.477838Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471637680606869:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4596) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:59.477913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471641975574253:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:59.477945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471637680606869:2050] [node 3] ICP30 unregister session Session# [6:7577471641975574253:2050] VirtualId# [6:7577471641975574243:0]
2025-11-27T18:19:59.478003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:19:59.478038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471637680606869:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4596) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.478053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4596) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.488057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471637680606869:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:19:59.521033Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:19:59.521272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471641975574261:2110] [node 4] ICS01 socket: 307 reason# EndOfStream
2025-11-27T18:19:59.521316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471641975574229:2110] [node 4] ICP30 unregister session Session# [6:7577471641975574261:2110] VirtualId# [6:7577471641975574256:0]
2025-11-27T18:19:59.521341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471641975574261:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.521405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:59.521443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:59.521562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:19:59.521762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471639627303247:2108] [node 4] ICS07 socket disconnect 332 reason# EndOfStream
2025-11-27T18:19:59.521795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471639627303247:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.521863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:19:59.521900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471639627303247:2108] [node 4] ICS15 start handshake
2025-11-27T18:19:59.524095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471639627303210:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15833) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:59.524155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471639627303247:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:59.524201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471639627303210:2108] [node 4] ICP30 unregister session Session# [5:7577471639627303247:2108] VirtualId# [5:7577471639627303242:0]
2025-11-27T18:19:59.524300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:19:59.524330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471639627303210:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15833) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.524345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15833) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.565426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:19:59.565667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471641975574265:2107] [node 5] ICS07 socket disconnect 314 reason# EndOfStream
2025-11-27T18:19:59.565702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471641975574265:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:19:59.565795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:19:59.565839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471641975574265:2107] [node 5] ICS15 start handshake
2025-11-27T18:19:59.567394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471637680606929:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24393) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:19:59.567529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471641975574265:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:19:59.567630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471637680606929:2107] [node 5] ICP30 unregister session Session# [6:7577471641975574265:2107] VirtualId# [6:7577471641975574263:0]
2025-11-27T18:19:59.567841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:19:59.567935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471637680606929:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.567983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:19:59.577406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471637680606929:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:19:59.676589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471639627174265:2048] [node 2] ICP01 ready to work
2025-11-27T18:19:59.676748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471639627174266:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:59.700543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471642503958005:2049] [node 3] ICP01 ready to work
2025-11-27T18:19:59.701281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471642503958004:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:59.728059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471642908913930:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:59.728113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471642908913931:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:59.757163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471640381968042:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:59.757239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471640381968044:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:59.757244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471640381968043:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:59.781898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471640729938529:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:59.781978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471640729938530:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:59.782080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471640729938531:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:59.804109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471643369912101:2048] [node 1] ICP01 ready to work
2025-11-27T18:19:59.804168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471643369912103:2050] [node 3] ICP01 ready to work
2025-11-27T18:19:59.804214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471643369912102:2049] [node 2] ICP01 ready to work
2025-11-27T18:19:59.885892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471643369912161:2104] [node 5] ICP01 ready to work
2025-11-27T18:19:59.935736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471642503958085:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:59.964452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471642908913993:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:59.973980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471640729938593:2108] [node 4] ICP01 ready to work
2025-11-27T18:19:59.975205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471640381968105:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:59.974190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471642908913995:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:59.975364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471640729938594:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:59.975663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471640381968106:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:59.975598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471643369912168:2110] [node 4] ICP01 ready to work
2025-11-27T18:19:59.974266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471642908913994:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:59.986958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471639627174265:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(102169239ns) {Configure:104(102160602ns) {ProcessConfigured:135(350907ns) {StartInitialHandshake:157(349282ns) {DropHandshakes:470(304384ns) {DropIncomingHandshake:442(1102ns) DropOutgoingHandshake:457(337ns)} GenerateSessionVirtualId:988(764ns) PrepareNewSessionHandshake:477(1010ns) {SwitchToState:179(699ns)}}}}}
2025-11-27T18:19:59.987339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471639627174330:2107] [node 4] ICP01 ready to work
2025-11-27T18:19:59.987869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471639627174331:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:59.986599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471642503958086:2108] [node 5] ICP01 ready to work
2025-11-27T18:19:59.988867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471639627174266:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(102549126ns) {Configure:104(101607207ns) {ProcessConfigured:135(139209ns) {StartInitialHandshake:157(137008ns) {DropHandshakes:470(1949ns) {DropIncomingHandshake:442(325ns) DropOutgoingHandshake:457(653ns)} GenerateSessionVirtualId:988(419ns) PrepareNewSessionHandshake:477(1083ns) {SwitchToState:179(452ns)}}}}}
2025-11-27T18:19:59.989869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471639627174334:2109] [node 6] ICP01 ready to work
2025-11-27T18:19:59.991214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471642503958088:2118] [node 6] ICP01 ready to work
2025-11-27T18:19:59.991458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471642503958004:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(82403073ns) {Configure:104(82391519ns) {ProcessConfigured:135(495113ns) {StartInitialHandshake:157(493384ns) {DropHandshakes:470(1957ns) {DropIncomingHandshake:442(724ns) DropOutgoingHandshake:457(456ns)} GenerateSessionVirtualId:988(576ns) PrepareNewSessionHandshake:477(987ns) {SwitchToState:179(329ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.008150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471642908914009:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39014
2025-11-27T18:20:00.010856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471642908914009:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:00.012570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471642908913930:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.013424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639627174333:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.013278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471640729938592:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31248
2025-11-27T18:20:00.013285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471647203881306:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39014
2025-11-27T18:20:00.013475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471639627174333:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.021383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471647203881306:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.021767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471646798925394:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56619
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.021826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471646798925395:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56619
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.021753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471640729938529:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16107702ns) {Configure:104(16099620ns) {ProcessConfigured:135(6765307ns) {StartInitialHandshake:157(6762201ns) {DropHandshakes:470(2364ns) {DropIncomingHandshake:442(931ns) DropOutgoingHandshake:457(475ns)} GenerateSessionVirtualId:988(521ns) PrepareNewSessionHandshake:477(4186ns) {SwitchToState:179(1875ns)}}}}}
2025-11-27T18:20:00.021967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471640729938592:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:00.022063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471644676935411:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25095
::ffff:127.1.0.0
2025-11-27T18:20:00.022086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471639627174333:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:00.022138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471644676935412:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25095
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.022235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471646798925396:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56619
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.022352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471647203881309:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39014
2025-11-27T18:20:00.022446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471646798925395:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.022439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471645024905899:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31248
2025-11-27T18:20:00.022538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471639627174266:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.022605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:00.022468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471646798925394:2133] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.022874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471644676935412:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.022612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471642503958004:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.022922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471639627174266:2049] [node 3] ICP22 created new session: [1:7577471643922141642:2049]
2025-11-27T18:20:00.022883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471644676935411:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.023279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471643922141642:2049] [node 3] ICS09 handshake done sender: [1:7577471639627174333:2111] self: [1:7577471639627174332:0] peer: [3:7577471647203881307:0] socket: 289 qp: -1
2025-11-27T18:20:00.023340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471643922141642:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:00.023436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:00.023545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471646798925396:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.024130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471645024905899:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:00.024146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471642908914009:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.024256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471640381968042:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.024917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471647203881310:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39014
2025-11-27T18:20:00.024226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639627174329:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.024899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471645024905905:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31248
2025-11-27T18:20:00.024268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471639627174329:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.025312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471642908913930:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.024713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471640381968044:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.025653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471644676935418:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25095
2025-11-27T18:20:00.025569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:00.025913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471642908913930:2048] [node 1] ICP22 created new session: [3:7577471647203881312:2048]
2025-11-27T18:20:00.026922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471645024905907:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31248
2025-11-27T18:20:00.027102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471644676935419:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25095
2025-11-27T18:20:00.027049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471647203881312:2048] [node 1] ICS09 handshake done sender: [3:7577471642908914009:2115] self: [3:7577471647203881307:0] peer: [1:7577471639627174332:0] socket: 292 qp: -1
2025-11-27T18:20:00.027190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471640729938531:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.027155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471647203881312:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:00.027627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639627174337:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.027661Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471639627174337:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.027209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.027223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471647203881311:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39014
2025-11-27T18:20:00.031490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471646798925402:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56619
2025-11-27T18:20:00.031579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471642908913999:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.031566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471643922141641:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16762
2025-11-27T18:20:00.031620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471642908913999:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.031619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471646798925403:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56619
2025-11-27T18:20:00.032134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471645024905905:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.032265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471639627174329:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:00.032326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471642908914001:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.032336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471646798925403:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.032371Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471642908914001:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.032789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471640729938530:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.032800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471646798925402:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.032792Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471639627174337:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:00.033549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471644676935418:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.033695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471644676935419:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.033683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471643922141641:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.033675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471646798925392:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.033713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471646798925392:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.033767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471645024905909:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31248
2025-11-27T18:20:00.034538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471645024905907:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.033885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471644676935422:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25095
2025-11-27T18:20:00.033717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471646798925405:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56619
2025-11-27T18:20:00.034112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471640381968043:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.033748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471639627174265:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.033790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:00.033941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471639627174265:2048] [node 2] ICP22 created new session: [1:7577471643922141647:2048]
2025-11-27T18:20:00.034120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471643922141647:2048] [node 2] ICS09 handshake done sender: [1:7577471639627174329:2110] self: [1:7577471639627174328:0] peer: [2:7577471646798925400:0] socket: 287 qp: -1
2025-11-27T18:20:00.034561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471643922141647:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:00.034856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471640729938593:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.035502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471647203881311:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.034958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471644676935412:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:00.035765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471642503958093:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.035644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471647203881310:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.035775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471640381968112:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.034764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:00.035795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471642503958093:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.035800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471640381968112:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.035865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471646798925395:2134] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:00.035916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471640381968042:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.038275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471642908913999:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:00.036278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:00.038220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471642503958004:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.036733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471640381968042:2048] [node 1] ICP22 created new session: [4:7577471644676935426:2048]
2025-11-27T18:20:00.038285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.037844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471644676935426:2048] [node 1] ICS09 handshake done sender: [4:7577471644676935412:2115] self: [4:7577471644676935420:0] peer: [1:7577471639627174335:0] socket: 296 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.038331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471642503958004:2048] [node 1] ICP22 created new session: [2:7577471646798925410:2048]
2025-11-27T18:20:00.038229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471644676935426:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:00.038384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471646798925410:2048] [node 1] ICS09 handshake done sender: [2:7577471646798925395:2134] self: [2:7577471646798925400:0] peer: [1:7577471639627174328:0] socket: 298 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.038388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:00.039825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471645024905912:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31248
2025-11-27T18:20:00.040262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471647664879475:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54811
2025-11-27T18:20:00.040443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471642908914001:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:00.040704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471645024905913:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31248
2025-11-27T18:20:00.038424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471646798925410:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.038454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:00.041993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471644676935425:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25095
2025-11-27T18:20:00.041827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471642908913993:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.042067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471645024905899:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:00.042000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:00.043092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471646798925392:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:00.042064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471642908913993:2107] [node 4] ICP22 created new session: [3:7577471647203881319:2107]
2025-11-27T18:20:00.043249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471647203881319:2107] [node 4] ICS09 handshake done sender: [3:7577471642908913999:2111] self: [3:7577471642908913998:0] peer: [4:7577471644676935421:0] socket: 285 qp: -1
2025-11-27T18:20:00.043288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471647203881319:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:00.043454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:00.044062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471642908913994:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.044111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:00.044135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471639627174330:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.044022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471642503958086:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.043581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471640729938531:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.044162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471642908913994:2108] [node 5] ICP22 created new session: [3:7577471647203881321:2108]
2025-11-27T18:20:00.044223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:00.044077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:00.044214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471647203881321:2108] [node 5] ICS09 handshake done sender: [3:7577471642908914001:2112] self: [3:7577471642908914000:0] peer: [5:7577471645024905908:0] socket: 286 qp: -1
2025-11-27T18:20:00.044627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471639627174330:2107] [node 4] ICP22 created new session: [1:7577471643922141649:2107]
2025-11-27T18:20:00.043645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:00.044549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471647203881321:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:00.044124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471642503958086:2108] [node 5] ICP22 created new session: [2:7577471646798925412:2108]
2025-11-27T18:20:00.044181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471646798925412:2108] [node 5] ICS09 handshake done sender: [2:7577471646798925392:2131] self: [2:7577471646798925390:0] peer: [5:7577471645024905910:0] socket: 307 qp: -1
2025-11-27T18:20:00.044725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471643922141649:2107] [node 4] ICS09 handshake done sender: [1:7577471639627174337:2112] self: [1:7577471639627174335:0] peer: [4:7577471644676935420:0] socket: 290 qp: -1
2025-11-27T18:20:00.044679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:00.044225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471646798925412:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:00.043958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471640729938531:2050] [node 3] ICP22 created new session: [5:7577471645024905915:2050]
2025-11-27T18:20:00.044762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471643922141649:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:00.044790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:00.044537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:00.044929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471640381968112:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:00.044007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471645024905915:2050] [node 3] ICS09 handshake done sender: [5:7577471645024905899:2115] self: [5:7577471645024905908:0] peer: [3:7577471642908914000:0] socket: 300 qp: -1
2025-11-27T18:20:00.044045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471645024905915:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:00.044077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.045725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471644676935411:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:00.045785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471644676935429:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25095
2025-11-27T18:20:00.046253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471640381968105:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.046478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471640381968044:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.046595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:00.046621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.046755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471640381968044:2050] [node 3] ICP22 created new session: [4:7577471644676935432:2050]
2025-11-27T18:20:00.046761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471640381968105:2108] [node 5] ICP22 created new session: [4:7577471644676935431:2108]
2025-11-27T18:20:00.046802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471644676935431:2108] [node 5] ICS09 handshake done sender: [4:7577471640381968112:2112] self: [4:7577471640381968111:0] peer: [5:7577471645024905911:0] socket: 313 qp: -1
2025-11-27T18:20:00.046811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471644676935432:2050] [node 3] ICS09 handshake done sender: [4:7577471644676935411:2114] self: [4:7577471644676935421:0] peer: [3:7577471642908913998:0] socket: 294 qp: -1
2025-11-27T18:20:00.046833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471644676935431:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:00.046849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471644676935432:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:00.046870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:00.046890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.046918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471647664879476:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54811
2025-11-27T18:20:00.048167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471645024905914:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31248
2025-11-27T18:20:00.048507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471642503958093:2130] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:00.049316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471639627174265:2048] [node 2] ICP18 handshake (actor [1:7577471643922141641:2115]) is too old
2025-11-27T18:20:00.048886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471645024905909:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.049325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471645024905905:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:00.050116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471640729938530:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.050313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471642503958085:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.050385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:00.050385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:00.050562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471640729938530:2049] [node 2] ICP22 created new session: [5:7577471645024905917:2049]
2025-11-27T18:20:00.050425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471642503958085:2107] [node 4] ICP22 created new session: [2:7577471646798925416:2107]
2025-11-27T18:20:00.050987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471645024905917:2049] [node 2] ICS09 handshake done sender: [5:7577471645024905905:2117] self: [5:7577471645024905910:0] peer: [2:7577471646798925390:0] socket: 309 qp: -1
2025-11-27T18:20:00.051083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471646798925416:2107] [node 4] ICS09 handshake done sender: [2:7577471642503958093:2130] self: [2:7577471642503958092:0] peer: [4:7577471644676935423:0] socket: 306 qp: -1
2025-11-27T18:20:00.051378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471645024905917:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:00.051657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:00.051673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471646798925416:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.059054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471645024905907:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:00.051710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:00.059949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471647664879477:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54811
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.060302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471647664879478:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54811
2025-11-27T18:20:00.060898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471647664879475:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.061132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471643369912103:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.062553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471640729938593:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.062662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471647203881323:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39014
2025-11-27T18:20:00.062661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:00.062709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471640729938593:2108] [node 4] ICP22 created new session: [5:7577471645024905920:2108]
2025-11-27T18:20:00.063022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471642908914008:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.063025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471647664879478:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.063053Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471642908914008:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.063599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471643369912102:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.062974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471640729938529:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.063891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471647203881323:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.063877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471645024905920:2108] [node 4] ICS09 handshake done sender: [5:7577471645024905907:2118] self: [5:7577471645024905911:0] peer: [4:7577471640381968111:0] socket: 320 qp: -1
2025-11-27T18:20:00.064407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471642908913931:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.064384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471646798925393:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.063957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471645024905920:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:00.064875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471647664879477:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.063999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:00.065035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471642908914008:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:00.064422Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471646798925393:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.064367Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471640729938530:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:00.065754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471642908913995:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.065839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:00.066021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471642503958091:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.065892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471642908913995:2109] [node 6] ICP22 created new session: [3:7577471647203881326:2109]
2025-11-27T18:20:00.065998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471647203881326:2109] [node 6] ICS09 handshake done sender: [3:7577471642908914008:2114] self: [3:7577471642908914007:0] peer: [6:7577471647664879479:0] socket: 338 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.066058Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471642503958091:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.066099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471643369912101:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.066049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471647203881326:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.066087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:00.066240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471647664879481:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54811
2025-11-27T18:20:00.066653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471647664879482:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54811
2025-11-27T18:20:00.066730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639627174343:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.066861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471646798925393:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:00.066777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471639627174343:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.067030Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471640381968043:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11229) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471644676935418:2117] held: no
2025-11-27T18:20:00.068251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471643369912103:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26264) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471647664879475:2115] held: no
2025-11-27T18:20:00.068274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471640729938593:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:00.068310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471642503958088:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.068378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:00.068429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471642503958088:2118] [node 6] ICP22 created new session: [2:7577471646798925420:2118]
2025-11-27T18:20:00.068304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471639627174343:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:00.068658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471639627174334:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.068480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471646798925420:2118] [node 6] ICS09 handshake done sender: [2:7577471646798925393:2132] self: [2:7577471646798925391:0] peer: [6:7577471647664879480:0] socket: 316 qp: -1
2025-11-27T18:20:00.069068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:00.069139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471644676935418:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:00.068585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471646798925420:2118] [node 6] ICS10 traffic start
2025-11-27T18:20:00.068627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:00.069115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471639627174334:2109] [node 6] ICP22 created new session: [1:7577471643922141653:2109]
2025-11-27T18:20:00.069461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471643922141653:2109] [node 6] ICS09 handshake done sender: [1:7577471639627174343:2114] self: [1:7577471639627174342:0] peer: [6:7577471647664879483:0] socket: 310 qp: -1
2025-11-27T18:20:00.069626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471643922141653:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:00.069672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:00.069847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471647664879475:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.070472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471639627174338:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.070437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471647203881328:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39014
2025-11-27T18:20:00.070502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471639627174338:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.070642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471647664879484:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54811
2025-11-27T18:20:00.070732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471643369912103:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.070796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:00.070867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471643369912103:2050] [node 3] ICP22 created new session: [6:7577471647664879486:2050]
2025-11-27T18:20:00.070929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471647664879486:2050] [node 3] ICS09 handshake done sender: [6:7577471647664879475:2115] self: [6:7577471647664879479:0] peer: [3:7577471642908914007:0] socket: 339 qp: -1
2025-11-27T18:20:00.070968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471647664879486:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:00.071011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:00.072440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471640381968043:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.071089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471642503958091:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:00.072486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:00.073615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471647203881323:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:00.072761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471640381968043:2049] [node 2] ICP22 created new session: [4:7577471644676935435:2049]
2025-11-27T18:20:00.073576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471644676935435:2049] [node 2] ICS09 handshake done sender: [4:7577471644676935418:2117] self: [4:7577471644676935423:0] peer: [2:7577471642503958092:0] socket: 314 qp: -1
2025-11-27T18:20:00.074003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471642503958005:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.073811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471644676935435:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:00.073857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:00.074074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.074107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471642908913931:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.074747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:00.074139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471642503958005:2049] [node 3] ICP22 created new session: [2:7577471646798925422:2049]
2025-11-27T18:20:00.074789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471646798925422:2049] [node 3] ICS09 handshake done sender: [2:7577471642503958091:2129] self: [2:7577471642503958090:0] peer: [3:7577471647203881324:0] socket: 321 qp: -1
2025-11-27T18:20:00.075142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471642908913931:2049] [node 2] ICP22 created new session: [3:7577471647203881329:2049]
2025-11-27T18:20:00.075119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471646798925422:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:00.075601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:00.075207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471647203881329:2049] [node 2] ICS09 handshake done sender: [3:7577471647203881323:2120] self: [3:7577471647203881324:0] peer: [2:7577471642503958090:0] socket: 319 qp: -1
2025-11-27T18:20:00.075151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471643922141655:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16762
2025-11-27T18:20:00.075645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471647203881329:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:00.075792Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471639627174338:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:00.075679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.076681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471643922141655:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.076886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471645024905924:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31248
2025-11-27T18:20:00.076911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471639627174331:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.076950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:00.077003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471639627174331:2108] [node 5] ICP22 created new session: [1:7577471643922141657:2108]
2025-11-27T18:20:00.077134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471643369912102:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11229) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471647664879478:2118] held: no
2025-11-27T18:20:00.077232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471645024905909:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:00.077150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471643922141657:2108] [node 5] ICS09 handshake done sender: [1:7577471639627174338:2113] self: [1:7577471639627174336:0] peer: [5:7577471645024905921:0] socket: 327 qp: -1
2025-11-27T18:20:00.077193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471643922141657:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:00.077663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471640729938529:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.077251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:00.077872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:00.077936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471640729938529:2048] [node 1] ICP22 created new session: [5:7577471645024905925:2048]
2025-11-27T18:20:00.078034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471645024905925:2048] [node 1] ICS09 handshake done sender: [5:7577471645024905909:2119] self: [5:7577471645024905921:0] peer: [1:7577471639627174336:0] socket: 331 qp: -1
No RDMA context found for GID: 2025-11-27T18:20:00.078090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471645024905925:2048] [node 1] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:20:00.078125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:00.078215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471647664879478:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:00.078232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471647664879485:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54811
2025-11-27T18:20:00.079535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471647664879476:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.079556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471643369912102:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.079596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:00.079636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471643369912102:2049] [node 2] ICP22 created new session: [6:7577471647664879490:2049]
2025-11-27T18:20:00.079680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471647664879490:2049] [node 2] ICS09 handshake done sender: [6:7577471647664879478:2118] self: [6:7577471647664879480:0] peer: [2:7577471646798925391:0] socket: 312 qp: -1
2025-11-27T18:20:00.079717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471647664879490:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:00.079731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471647664879481:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.079762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:00.079786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471647664879477:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:00.079638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471639627174331:2108] [node 5] ICP18 handshake (actor [1:7577471643922141655:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.080425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471643369912101:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.080502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:00.080548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471643369912101:2048] [node 1] ICP22 created new session: [6:7577471647664879492:2048]
2025-11-27T18:20:00.080579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471643922141659:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16762
2025-11-27T18:20:00.080600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471647664879492:2048] [node 1] ICS09 handshake done sender: [6:7577471647664879477:2117] self: [6:7577471647664879483:0] peer: [1:7577471639627174342:0] socket: 315 qp: -1
2025-11-27T18:20:00.080627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471647664879492:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.080662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:00.080807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471644676935437:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25095
2025-11-27T18:20:00.081388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471643369912168:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.082142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471643369912161:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.082256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471640381968114:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.082292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471640381968114:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.082767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471645024905898:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.082799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471645024905898:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.083387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471640381968114:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.083742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471640381968106:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.083885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471645024905898:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:00.084094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471647664879496:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54811
2025-11-27T18:20:00.084218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471640729938594:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.084370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:00.084332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:00.084425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471640381968106:2109] [node 6] ICP22 created new session: [4:7577471644676935439:2109]
2025-11-27T18:20:00.084422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471640729938594:2109] [node 6] ICP22 created new session: [5:7577471645024905928:2109]
2025-11-27T18:20:00.084508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471644676935439:2109] [node 6] ICS09 handshake done sender: [4:7577471640381968114:2113] self: [4:7577471640381968113:0] peer: [6:7577471647664879494:0] socket: 343 qp: -1
2025-11-27T18:20:00.084504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471645024905928:2109] [node 6] ICS09 handshake done sender: [5:7577471645024905898:2114] self: [5:7577471645024905896:0] peer: [6:7577471647664879495:0] socket: 311 qp: -1
2025-11-27T18:20:00.084546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471644676935439:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:00.084538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471645024905928:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:00.084617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:00.084610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.084685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471647664879476:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:00.084719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471647664879497:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54811
2025-11-27T18:20:00.084793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471644676935437:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.085298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471643369912168:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.085350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:00.085409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471643369912168:2110] [node 4] ICP22 created new session: [6:7577471647664879498:2110]
2025-11-27T18:20:00.085458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471647664879498:2110] [node 4] ICS09 handshake done sender: [6:7577471647664879476:2116] self: [6:7577471647664879494:0] peer: [4:7577471640381968113:0] socket: 344 qp: -1
2025-11-27T18:20:00.085494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471647664879498:2110] [node 4] ICS10 traffic start
2025-11-27T18:20:00.085521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:00.086342Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471643369912161:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4218) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471647664879481:2119] held: no
2025-11-27T18:20:00.086457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471640381968106:2109] [node 6] ICP18 handshake (actor [4:7577471644676935437:2122]) is too old
2025-11-27T18:20:00.087859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471647664879481:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:00.088304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471643369912161:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.088358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:00.088401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471643369912161:2104] [node 5] ICP22 created new session: [6:7577471647664879500:2104]
2025-11-27T18:20:00.088447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471647664879500:2104] [node 5] ICS09 handshake done sender: [6:7577471647664879481:2119] self: [6:7577471647664879495:0] peer: [5:7577471645024905896:0] socket: 317 qp: -1
2025-11-27T18:20:00.088486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471647664879500:2104] [node 5] ICS10 traffic start
2025-11-27T18:20:00.088514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:00.133100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:00.133383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471646798925410:2048] [node 1] ICS07 socket disconnect 298 reason# EndOfStream
2025-11-27T18:20:00.133421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471646798925410:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.133504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:00.133556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471646798925410:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:00.134430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:00.134649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471647203881312:2048] [node 1] ICS07 socket disconnect 292 reason# EndOfStream
2025-11-27T18:20:00.134686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471647203881312:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.134761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:00.134802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471647203881312:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:00.136264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471642503958004:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:00.136315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471646798925410:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:00.137898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471642908913930:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:00.137951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471647203881312:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:00.138867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471642503958004:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:00.138933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471646798925410:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:00.139027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471642503958004:2048] [node 1] ICP30 unregister session Session# [2:7577471646798925410:2048] VirtualId# [2:7577471646798925400:0]
2025-11-27T18:20:00.139083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:00.139120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471642503958004:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.139911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:00.139146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.139979Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471642908913930:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:00.140038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471647203881312:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:00.140078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471642908913930:2048] [node 1] ICP30 unregister session Session# [3:7577471647203881312:2048] VirtualId# [3:7577471647203881307:0]
2025-11-27T18:20:00.140160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471644676935426:2048] [node 1] ICS01 socket: 296 reason# EndOfStream
2025-11-27T18:20:00.140216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:00.140225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471640381968042:2048] [node 1] ICP30 unregister session Session# [4:7577471644676935426:2048] VirtualId# [4:7577471644676935420:0]
2025-11-27T18:20:00.140265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471642908913930:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.140263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471644676935426:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.140285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.140336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:00.140399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:00.141331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:00.141557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471645024905925:2048] [node 1] ICS07 socket disconnect 331 reason# EndOfStream
2025-11-27T18:20:00.141599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471645024905925:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.141673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:00.141734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471645024905925:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:00.142452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471640729938529:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:00.142607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471645024905925:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:00.142649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471640729938529:2048] [node 1] ICP30 unregister session Session# [5:7577471645024905925:2048] VirtualId# [5:7577471645024905921:0]
2025-11-27T18:20:00.142715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:00.142770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471640729938529:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.142790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.142046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:00.142267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471647664879492:2048] [node 1] ICS07 socket disconnect 315 reason# EndOfStream
2025-11-27T18:20:00.142297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471647664879492:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.142368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:00.142417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471647664879492:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:00.143281Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471643369912101:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:00.143326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471647664879492:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:00.143421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471643369912101:2048] [node 1] ICP30 unregister session Session# [6:7577471647664879492:2048] VirtualId# [6:7577471647664879483:0]
2025-11-27T18:20:00.143558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:00.143587Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471643369912101:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.143602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31297) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.147764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471642503958004:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:00.149825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471642908913930:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:00.152908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471640729938529:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:00.153927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471643369912101:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:00.227226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:00.227488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471647203881329:2049] [node 2] ICS01 socket: 319 reason# EndOfStream
2025-11-27T18:20:00.227546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471642908913931:2049] [node 2] ICP30 unregister session Session# [3:7577471647203881329:2049] VirtualId# [3:7577471647203881324:0]
2025-11-27T18:20:00.227589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471647203881329:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.227672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:00.227900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:00.231915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:00.232189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471644676935435:2049] [node 2] ICS01 socket: 314 reason# EndOfStream
2025-11-27T18:20:00.232246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471640381968043:2049] [node 2] ICP30 unregister session Session# [4:7577471644676935435:2049] VirtualId# [4:7577471644676935423:0]
2025-11-27T18:20:00.232302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471644676935435:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.232386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:00.232446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:00.233144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:00.233357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471645024905917:2049] [node 2] ICS07 socket disconnect 309 reason# EndOfStream
2025-11-27T18:20:00.233384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471645024905917:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.233443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:00.233474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471645024905917:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:00.234201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471640729938530:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11229) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:00.234296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471645024905917:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:00.234336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471640729938530:2049] [node 2] ICP30 unregister session Session# [5:7577471645024905917:2049] VirtualId# [5:7577471645024905910:0]
2025-11-27T18:20:00.234375Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:00.234468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:00.234511Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471640729938530:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11229) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.234523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11229) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.234607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471647664879490:2049] [node 2] ICS01 socket: 312 reason# EndOfStream
2025-11-27T18:20:00.234652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471643369912102:2049] [node 2] ICP30 unregister session Session# [6:7577471647664879490:2049] VirtualId# [6:7577471647664879480:0]
2025-11-27T18:20:00.234677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471647664879490:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.234752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:00.234815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:00.244049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471640729938530:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:00.287158Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:00.287431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471644676935432:2050] [node 3] ICS07 socket disconnect 294 reason# EndOfStream
2025-11-27T18:20:00.287470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471644676935432:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.287570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:00.287631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471644676935432:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:00.288321Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:00.288561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471645024905915:2050] [node 3] ICS07 socket disconnect 300 reason# EndOfStream
2025-11-27T18:20:00.288599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471645024905915:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.288688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:00.288741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471645024905915:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:00.288864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471640381968044:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26264) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:00.289046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471644676935432:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:00.289095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471640381968044:2050] [node 3] ICP30 unregister session Session# [4:7577471644676935432:2050] VirtualId# [4:7577471644676935421:0]
2025-11-27T18:20:00.289212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:00.289246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471640381968044:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.289282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.289694Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471640729938531:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26264) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:00.289711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:00.289744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471645024905915:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:00.289844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471640729938531:2050] [node 3] ICP30 unregister session Session# [5:7577471645024905915:2050] VirtualId# [5:7577471645024905908:0]
2025-11-27T18:20:00.289890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:00.289927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471640729938531:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.289929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471647664879486:2050] [node 3] ICS01 socket: 339 reason# EndOfStream
2025-11-27T18:20:00.289941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26264) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.289972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471643369912103:2050] [node 3] ICP30 unregister session Session# [6:7577471647664879486:2050] VirtualId# [6:7577471647664879479:0]
2025-11-27T18:20:00.290003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471647664879486:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.290070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:00.290113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:00.299333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471640729938531:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:00.299395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471640381968044:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:00.337278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:00.337532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471645024905920:2108] [node 4] ICS07 socket disconnect 320 reason# EndOfStream
2025-11-27T18:20:00.337563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471645024905920:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.337652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:00.337721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471645024905920:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:00.339273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471640729938593:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1890) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:00.339368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471645024905920:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:00.339414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471640729938593:2108] [node 4] ICP30 unregister session Session# [5:7577471645024905920:2108] VirtualId# [5:7577471645024905911:0]
2025-11-27T18:20:00.339468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:00.339512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471640729938593:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1890) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.339529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1890) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.341441Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:00.341807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471647664879498:2110] [node 4] ICS07 socket disconnect 344 reason# EndOfStream
2025-11-27T18:20:00.341874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471647664879498:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.342032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:00.342135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471647664879498:2110] [node 4] ICS15 start handshake
2025-11-27T18:20:00.352826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471640729938593:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:00.356221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471643369912168:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1890) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:00.356293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471647664879498:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:00.356344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471643369912168:2110] [node 4] ICP30 unregister session Session# [6:7577471647664879498:2110] VirtualId# [6:7577471647664879494:0]
2025-11-27T18:20:00.356687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:00.356729Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471643369912168:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1890) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.356747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1890) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:00.366942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471643369912168:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:20:00.429612Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:00.429983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471647664879500:2104] [node 5] ICS01 socket: 317 reason# EndOfStream
2025-11-27T18:20:00.430107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471643369912161:2104] [node 5] ICP30 unregister session Session# [6:7577471647664879500:2104] VirtualId# [6:7577471647664879495:0]
2025-11-27T18:20:00.430194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471647664879500:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:00.430308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:00.430363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:00.534804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471646301877539:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:00.534866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471646301877540:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:00.557298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471643952625298:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:00.557721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471643952625299:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:00.580950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471647403003487:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:00.580957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471647403003488:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:00.609925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471645312725307:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:00.609971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471645312725308:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:00.617343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471645312725306:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:00.629412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471646601954374:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:00.629470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471646601954375:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:00.629927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471646601954373:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:00.651522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471646458735325:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:00.651564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471646458735326:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:00.651593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471646458735327:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:00.732636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471646458735384:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:00.793792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471643952625361:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:00.810598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471646301877602:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:00.818318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471645312725370:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:00.825584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471646601954436:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:00.829694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471647403003551:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:00.825329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471643952625362:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:00.810666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471646301877603:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:00.829965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471647403003552:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:00.818326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471645312725371:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:00.834966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471646458735390:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:00.833467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471647403003550:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:00.829771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471646301877605:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:00.835247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471646301877540:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(97527500ns) {Configure:104(97509521ns) {ProcessConfigured:135(8218944ns) {StartInitialHandshake:157(8215915ns) {DropHandshakes:470(2587ns) {DropIncomingHandshake:442(595ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(914ns) PrepareNewSessionHandshake:477(2442ns) {SwitchToState:179(1115ns)}}}}}
2025-11-27T18:20:00.825339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471643952625363:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:00.836237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471646458735384:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(102654042ns) {Configure:104(102650301ns) {ProcessConfigured:135(559312ns) {StartInitialHandshake:157(556525ns) {DropHandshakes:470(2307ns) {DropIncomingHandshake:442(659ns) DropOutgoingHandshake:457(430ns)} GenerateSessionVirtualId:988(984ns) PrepareNewSessionHandshake:477(3941ns) {SwitchToState:179(711ns)}}}}}
2025-11-27T18:20:00.835042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471646601954437:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:00.851245Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471643952625298:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(51398523ns) {Configure:104(51393559ns) {ProcessConfigured:135(1670953ns) {StartInitialHandshake:157(1669581ns) {DropHandshakes:470(1148ns) {DropIncomingHandshake:442(260ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(648ns) PrepareNewSessionHandshake:477(1552ns) {SwitchToState:179(866ns)}}}}}
2025-11-27T18:20:00.852929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471647403003488:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16283079ns) {Configure:104(16118888ns) {ProcessConfigured:135(6054137ns) {StartInitialHandshake:157(5863978ns) {DropHandshakes:470(1968ns) {DropIncomingHandshake:442(313ns) DropOutgoingHandshake:457(185ns)} GenerateSessionVirtualId:988(1805ns) PrepareNewSessionHandshake:477(1730ns) {SwitchToState:179(528ns)}}}}}
2025-11-27T18:20:00.853893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471647403003487:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17170370ns) {Configure:104(17164254ns) {ProcessConfigured:135(745930ns) {StartInitialHandshake:157(743772ns) {DropHandshakes:470(1059ns) {DropIncomingHandshake:442(287ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(560ns) PrepareNewSessionHandshake:477(1869ns) {SwitchToState:179(652ns)}}}}}
2025-11-27T18:20:00.854250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471647403003550:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16161367ns) {Configure:104(16156837ns) {ProcessConfigured:135(809206ns) {StartInitialHandshake:157(806844ns) {DropHandshakes:470(1205ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(958ns) PrepareNewSessionHandshake:477(2284ns) {SwitchToState:179(1131ns)}}}}}
2025-11-27T18:20:00.855046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471646601954374:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18564747ns) {Configure:104(18558549ns) {ProcessConfigured:135(2289957ns) {StartInitialHandshake:157(2287774ns) {DropHandshakes:470(288535ns) {DropIncomingHandshake:442(350ns) DropOutgoingHandshake:457(1140ns)} GenerateSessionVirtualId:988(1297ns) PrepareNewSessionHandshake:477(2134ns) {SwitchToState:179(916ns)}}}}}
2025-11-27T18:20:00.855093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471645312725307:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18886605ns) {Configure:104(18880356ns) {ProcessConfigured:135(721515ns) {StartInitialHandshake:157(719257ns) {DropHandshakes:470(1170ns) {DropIncomingHandshake:442(379ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(405ns) PrepareNewSessionHandshake:477(1716ns) {SwitchToState:179(516ns)}}}}}
2025-11-27T18:20:00.855166Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471645312725308:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18598606ns) {Configure:104(18594679ns) {ProcessConfigured:135(621959ns) {StartInitialHandshake:157(620491ns) {DropHandshakes:470(1256ns) {DropIncomingHandshake:442(319ns) DropOutgoingHandshake:457(185ns)} GenerateSessionVirtualId:988(354ns) PrepareNewSessionHandshake:477(1107ns) {SwitchToState:179(506ns)}}}}}
2025-11-27T18:20:00.856457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471646601954436:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19269447ns) {Configure:104(19266092ns) {ProcessConfigured:135(712750ns) {StartInitialHandshake:157(710205ns) {DropHandshakes:470(1148ns) {DropIncomingHandshake:442(301ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(399ns) PrepareNewSessionHandshake:477(1198ns) {SwitchToState:179(335ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.870801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471647403003563:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37971
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.883766Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471643952625362:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28142586ns) {Configure:104(28137959ns) {ProcessConfigured:135(56237ns) {StartInitialHandshake:157(53402ns) {DropHandshakes:470(1875ns) {DropIncomingHandshake:442(493ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(622ns) PrepareNewSessionHandshake:477(4328ns) {SwitchToState:179(513ns)}}}}}
2025-11-27T18:20:00.881349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471646301877613:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28204
2025-11-27T18:20:00.885971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471646458735390:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29128899ns) {Configure:104(29124369ns) {ProcessConfigured:135(6653440ns) {StartInitialHandshake:157(6650893ns) {DropHandshakes:470(1952ns) {DropIncomingHandshake:442(856ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(677ns) PrepareNewSessionHandshake:477(3047ns) {SwitchToState:179(1372ns)}}}}}
2025-11-27T18:20:00.886897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471643952625299:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33121345ns) {Configure:104(33116383ns) {ProcessConfigured:135(3604494ns) {StartInitialHandshake:157(3602348ns) {DropHandshakes:470(2491ns) {DropIncomingHandshake:442(528ns) DropOutgoingHandshake:457(426ns)} GenerateSessionVirtualId:988(941ns) PrepareNewSessionHandshake:477(2621ns) {SwitchToState:179(1549ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.888527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471646458735399:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23836
2025-11-27T18:20:00.888915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471646458735401:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23836
2025-11-27T18:20:00.889116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471646601954373:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32725181ns) {Configure:104(32712893ns) {ProcessConfigured:135(1762665ns) {StartInitialHandshake:157(1760941ns) {DropHandshakes:470(2471ns) {DropIncomingHandshake:442(488ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(472ns) PrepareNewSessionHandshake:477(270158ns) {SwitchToState:179(2093ns)}}}}}
2025-11-27T18:20:00.889450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471646301877613:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.889421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471646458735399:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.889864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471646458735401:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.890600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471646458735327:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.891507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471647403003563:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.891047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471646458735390:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.891545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471647403003560:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.891177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471646301877605:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16838422ns) {Configure:104(16831630ns) {ProcessConfigured:135(194602ns) {StartInitialHandshake:157(192274ns) {DropHandshakes:470(1463ns) {DropIncomingHandshake:442(345ns) DropOutgoingHandshake:457(291ns)} GenerateSessionVirtualId:988(788ns) PrepareNewSessionHandshake:477(1407ns) {SwitchToState:179(467ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.898463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471646458735408:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23836
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.899080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471646458735409:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23836
2025-11-27T18:20:00.899422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471646458735401:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.891571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471647403003560:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.892846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471646301877617:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28204
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.892847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471645312725377:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.899811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471643952625374:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24150
2025-11-27T18:20:00.906303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471646601954444:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26998
2025-11-27T18:20:00.892883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471645312725377:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.915440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471646458735399:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:00.892765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471647403003487:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.898718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471646301877617:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.909294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471643952625375:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24150
2025-11-27T18:20:00.910601Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471646601954437:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18927719ns) {Configure:104(18920961ns) {ProcessConfigured:135(64449ns) {StartInitialHandshake:157(62136ns) {DropHandshakes:470(7690ns) {DropIncomingHandshake:442(578ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(1018ns) PrepareNewSessionHandshake:477(1405ns) {SwitchToState:179(369ns)}}}}}
2025-11-27T18:20:00.897357Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471645312725377:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.893340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471647403003560:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:00.921458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471646458735327:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.930576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471643952625374:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.905719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471646301877606:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.910709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471646601954447:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26998
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.921725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471645312725371:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.921523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:00.921748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471646458735327:2050] [node 3] ICP22 created new session: [6:7577471646458735410:2050]
2025-11-27T18:20:00.905784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471646301877606:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.932587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471647403003551:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.911460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471646601954449:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26998
2025-11-27T18:20:00.921767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:00.933552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471643952625376:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24150
2025-11-27T18:20:00.914389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471646301877606:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:00.933501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:00.931184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471646458735410:2050] [node 3] ICS09 handshake done sender: [6:7577471646458735401:2116] self: [6:7577471646458735407:0] peer: [3:7577471647403003559:0] socket: 304 qp: -1
2025-11-27T18:20:00.925481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471645312725371:2109] [node 6] ICP22 created new session: [4:7577471645312725386:2109]
2025-11-27T18:20:00.932467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471646301877623:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28204
2025-11-27T18:20:00.933623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471647403003551:2109] [node 6] ICP22 created new session: [3:7577471647403003572:2109]
2025-11-27T18:20:00.914664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471646601954446:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26998
2025-11-27T18:20:00.931950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471646601954447:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.934737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471643952625298:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.931579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471646458735410:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.930834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471645312725386:2109] [node 6] ICS09 handshake done sender: [4:7577471645312725377:2113] self: [4:7577471645312725376:0] peer: [6:7577471646458735406:0] socket: 290 qp: -1
2025-11-27T18:20:00.935075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471643952625375:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.932916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471646301877540:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.934750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471647403003572:2109] [node 6] ICS09 handshake done sender: [3:7577471647403003560:2113] self: [3:7577471647403003559:0] peer: [6:7577471646458735407:0] socket: 299 qp: -1
2025-11-27T18:20:00.931974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471646601954444:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.931973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:00.931920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471645312725386:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.933378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:00.935188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471647403003572:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:00.932311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:00.934315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471645312725373:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.932602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471646458735390:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.932171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471646601954446:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.935229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:00.933563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471646301877540:2049] [node 3] ICP22 created new session: [1:7577471646301877624:2049]
2025-11-27T18:20:00.932967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471646601954437:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.934346Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471645312725373:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.932761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:00.940065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471643952625380:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24150
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.937860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471645312725387:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48676
2025-11-27T18:20:00.933046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471646601954373:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.933095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471646458735390:2109] [node 4] ICP22 created new session: [6:7577471646458735414:2109]
2025-11-27T18:20:00.938986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471647403003562:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.937873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471645312725373:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.933412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471646601954375:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.933160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471646458735414:2109] [node 4] ICS09 handshake done sender: [6:7577471646458735399:2115] self: [6:7577471646458735406:0] peer: [4:7577471645312725376:0] socket: 302 qp: -1
2025-11-27T18:20:00.934754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471646601954449:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.934125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471646301877624:2049] [node 3] ICS09 handshake done sender: [1:7577471646301877606:2110] self: [1:7577471646301877604:0] peer: [3:7577471647403003567:0] socket: 293 qp: -1
2025-11-27T18:20:00.933203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471646458735414:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:00.933243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:00.942073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471645312725370:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.942421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471643952625380:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.939015Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471647403003562:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:00.934727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471646301877624:2049] [node 3] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:20:00.942405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471646601954459:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26998
2025-11-27T18:20:00.934759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:00.942206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471647403003569:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37971
2025-11-27T18:20:00.935032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471646301877610:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.935093Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471646301877610:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.943072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471646601954460:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26998
2025-11-27T18:20:00.942544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471647403003570:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37971
2025-11-27T18:20:00.939022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471646301877608:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.943155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:00.939051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471646301877608:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.943491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471647403003571:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37971
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.943209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471645312725370:2108] [node 5] ICP22 created new session: [4:7577471645312725392:2108]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:00.943757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471643952625376:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.942009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471646301877610:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:00.944298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471647403003569:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.943509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471645312725387:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.942582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471646301877623:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.943547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471645312725392:2108] [node 5] ICS09 handshake done sender: [4:7577471645312725373:2111] self: [4:7577471645312725372:0] peer: [5:7577471646601954457:0] socket: 288 qp: -1
2025-11-27T18:20:00.944741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471647403003562:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.944964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471646601954446:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:00.943396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471646301877602:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.943767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471645312725392:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.945486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471647403003563:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:00.943815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:00.944989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471646601954444:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.945657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471646458735412:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23836
2025-11-27T18:20:00.943529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:00.946025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471643952625384:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24150
2025-11-27T18:20:00.945605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471646601954437:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.945007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471645312725391:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48676
2025-11-27T18:20:00.944330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471646301877602:2108] [node 5] ICP22 created new session: [1:7577471646301877630:2108]
2025-11-27T18:20:00.945648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:00.945987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471646601954461:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26998
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.948826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471643952625374:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:00.946031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471646601954437:2108] [node 4] ICP22 created new session: [5:7577471646601954462:2108]
2025-11-27T18:20:00.945326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471646301877630:2108] [node 5] ICS09 handshake done sender: [1:7577471646301877610:2112] self: [1:7577471646301877609:0] peer: [5:7577471646601954456:0] socket: 296 qp: -1
2025-11-27T18:20:00.946969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471646458735413:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23836
2025-11-27T18:20:00.947269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471645312725307:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.946897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471647403003552:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.945372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471646301877630:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:00.948509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471646458735413:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.946688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471646601954373:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.947666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471645312725391:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.947211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471647403003571:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.949319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471646458735326:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.945820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:00.947246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:00.949379Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471646458735327:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(21643633ns) {HandleHandshakeStatus:329(21639767ns) {DropIncomingHandshake:442(812ns) DropOutgoingHandshake:457(6454ns) SwitchToState:179(802ns) ProcessPendingSessionEvents:495(9467861ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9223253ns max# 1945370ns}}}
2025-11-27T18:20:00.947261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471647403003487:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.947299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:00.946922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471646601954462:2108] [node 4] ICS09 handshake done sender: [5:7577471646601954444:2113] self: [5:7577471646601954457:0] peer: [4:7577471645312725372:0] socket: 294 qp: -1
2025-11-27T18:20:00.947359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471646301877608:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:00.950194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471643952625298:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:00.950035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471645312725306:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.947195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471646601954462:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:00.947653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471647403003552:2108] [node 5] ICP22 created new session: [3:7577471647403003576:2108]
2025-11-27T18:20:00.957737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471646458735412:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.957892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471646458735384:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:00.949328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471646301877629:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28204
2025-11-27T18:20:00.947265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:00.953461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471645312725371:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(21451072ns) {HandleHandshakeStatus:329(21447245ns) {DropIncomingHandshake:442(281ns) DropOutgoingHandshake:457(230ns) SwitchToState:179(166ns) ProcessPendingSessionEvents:495(9457613ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8465174ns max# 1189615ns}}}
2025-11-27T18:20:00.952138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471643952625367:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.948503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471647403003487:2048] [node 1] ICP22 created new session: [3:7577471647403003577:2048]
2025-11-27T18:20:00.947380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:00.956485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471646301877539:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:00.952176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471643952625367:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.948783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471647403003576:2108] [node 5] ICS09 handshake done sender: [3:7577471647403003562:2114] self: [3:7577471647403003561:0] peer: [5:7577471646601954455:0] socket: 300 qp: -1
2025-11-27T18:20:00.948808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471647403003577:2048] [node 1] ICS09 handshake done sender: [3:7577471647403003563:2115] self: [3:7577471647403003567:0] peer: [1:7577471646301877604:0] socket: 297 qp: -1
2025-11-27T18:20:00.947700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471646601954373:2048] [node 1] ICP22 created new session: [5:7577471646601954463:2048]
2025-11-27T18:20:00.952398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:00.956572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:00.949069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471647403003576:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:00.953699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471643952625298:2048] [node 1] ICP22 created new session: [2:7577471643952625387:2048]
2025-11-27T18:20:00.957416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471646301877539:2048] [node 2] ICP22 created new session: [1:7577471646301877634:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.948788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471646601954463:2048] [node 1] ICS09 handshake done sender: [5:7577471646601954446:2114] self: [5:7577471646601954456:0] peer: [1:7577471646301877609:0] socket: 305 qp: -1
2025-11-27T18:20:00.949108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:00.956484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471643952625369:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.949418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471647403003577:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:00.964674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471646301877634:2048] [node 2] ICS09 handshake done sender: [1:7577471646301877608:2111] self: [1:7577471646301877607:0] peer: [2:7577471643952625381:0] socket: 295 qp: -1
2025-11-27T18:20:00.948948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471646601954463:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:00.949830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:00.956521Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471643952625369:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.964945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471646301877634:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:00.957534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471643952625387:2048] [node 1] ICS09 handshake done sender: [2:7577471643952625374:2115] self: [2:7577471643952625381:0] peer: [1:7577471646301877607:0] socket: 306 qp: -1
2025-11-27T18:20:00.959582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471643952625387:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:00.959628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:00.972870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471643952625386:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24150
2025-11-27T18:20:00.949174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:00.965682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:00.985533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471646301877612:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.964978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471646601954443:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:00.989409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471646458735419:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23836
2025-11-27T18:20:00.985563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471646301877612:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.965013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471646601954443:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:00.989649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471643952625369:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:00.990016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471646458735418:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23836
2025-11-27T18:20:00.989733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471643952625367:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:00.992740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471646458735413:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:01.002559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471646458735326:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:01.003753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:01.009480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471646601954447:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:01.005126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471646458735326:2049] [node 2] ICP22 created new session: [6:7577471650753702716:2049]
2025-11-27T18:20:01.009590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471643952625363:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.009812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471650753702716:2049] [node 2] ICS09 handshake done sender: [6:7577471646458735413:2121] self: [6:7577471646458735416:0] peer: [2:7577471643952625368:0] socket: 335 qp: -1
2025-11-27T18:20:01.009685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:01.009861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471650753702716:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:01.009706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471643952625386:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.013623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471645312725397:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48676
2025-11-27T18:20:01.014330Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471646301877602:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(42156363ns) {HandleHandshakeStatus:329(42153724ns) {DropIncomingHandshake:442(347ns) DropOutgoingHandshake:457(241ns) SwitchToState:179(623ns) ProcessPendingSessionEvents:495(38310331ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 33980031ns max# 6451743ns}}}
2025-11-27T18:20:01.014515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471646601954437:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(40092687ns) {HandleHandshakeStatus:329(40089653ns) {DropIncomingHandshake:442(515ns) DropOutgoingHandshake:457(5144ns) SwitchToState:179(389ns) ProcessPendingSessionEvents:495(36253545ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36145234ns max# 6351440ns}}}
2025-11-27T18:20:01.010740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471643952625361:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:01.015112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471647403003552:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(42523183ns) {HandleHandshakeStatus:329(42520974ns) {DropIncomingHandshake:442(236ns) DropOutgoingHandshake:457(174ns) SwitchToState:179(282ns) ProcessPendingSessionEvents:495(39353128ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39245946ns max# 12389762ns}}}
2025-11-27T18:20:01.014761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471646301877540:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:01.014073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471645312725370:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(43677387ns) {HandleHandshakeStatus:329(43442829ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(422ns) SwitchToState:179(332ns) ProcessPendingSessionEvents:495(40507758ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39075392ns max# 6335778ns}}}
2025-11-27T18:20:01.015003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471646601954375:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:01.011164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:01.010865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471643952625363:2109] [node 6] ICP22 created new session: [2:7577471648247592688:2109]
2025-11-27T18:20:01.015025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471645312725387:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:01.011198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:01.015047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:01.011584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471643952625361:2107] [node 4] ICP22 created new session: [2:7577471648247592689:2107]
2025-11-27T18:20:01.015390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471646601954375:2050] [node 3] ICP22 created new session: [5:7577471650896921764:2050]
2025-11-27T18:20:01.011630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471648247592688:2109] [node 6] ICS09 handshake done sender: [2:7577471643952625369:2112] self: [2:7577471643952625368:0] peer: [6:7577471646458735416:0] socket: 330 qp: -1
2025-11-27T18:20:01.017466Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471647403003487:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(53927881ns) {HandleHandshakeStatus:329(53925756ns) {DropIncomingHandshake:442(493ns) DropOutgoingHandshake:457(3166ns) SwitchToState:179(332ns) ProcessPendingSessionEvents:495(43645009ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 43551851ns max# 11785860ns}}}
2025-11-27T18:20:01.016292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471650896921764:2050] [node 3] ICS09 handshake done sender: [5:7577471646601954447:2115] self: [5:7577471646601954455:0] peer: [3:7577471647403003561:0] socket: 307 qp: -1
2025-11-27T18:20:01.017557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471647403003552:2108] [node 5] ICP18 handshake (actor [3:7577471647403003569:2116]) is too old
2025-11-27T18:20:01.016803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471650896921764:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:01.016844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:01.013016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471648247592689:2107] [node 4] ICS09 handshake done sender: [2:7577471643952625367:2111] self: [2:7577471643952625366:0] peer: [4:7577471645312725395:0] socket: 327 qp: -1
2025-11-27T18:20:01.017003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471646601954373:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(53854338ns) {HandleHandshakeStatus:329(53851826ns) {DropIncomingHandshake:442(400ns) DropOutgoingHandshake:457(4105ns) SwitchToState:179(1478ns) ProcessPendingSessionEvents:495(47719666ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 47595256ns max# 6048428ns}}}
2025-11-27T18:20:01.013032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471648247592688:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:01.018645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471645312725307:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:01.013132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471648247592689:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:01.021548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471646301877612:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:01.018693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:01.013193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:01.020062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471645312725307:2049] [node 2] ICP22 created new session: [4:7577471649607692696:2049]
2025-11-27T18:20:01.032125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471646301877603:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:01.013349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:01.024107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471649607692696:2049] [node 2] ICS09 handshake done sender: [4:7577471645312725387:2116] self: [4:7577471645312725395:0] peer: [2:7577471643952625366:0] socket: 328 qp: -1
2025-11-27T18:20:01.015705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471643952625298:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(50299691ns) {HandleHandshakeStatus:329(50297565ns) {DropIncomingHandshake:442(500ns) DropOutgoingHandshake:457(9417ns) SwitchToState:179(348ns) ProcessPendingSessionEvents:495(29321254ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26083719ns max# 6363509ns}}}
2025-11-27T18:20:01.032217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:01.025491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471649607692696:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:01.032022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:01.032489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471646301877603:2107] [node 4] ICP22 created new session: [1:7577471650596844933:2107]
2025-11-27T18:20:01.033062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471650596844933:2107] [node 4] ICS09 handshake done sender: [1:7577471646301877612:2113] self: [1:7577471646301877611:0] peer: [4:7577471645312725396:0] socket: 339 qp: -1
2025-11-27T18:20:01.033190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471650596844933:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:01.033243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:01.033399Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471646301877539:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(64612550ns) {HandleHandshakeStatus:329(64610250ns) {DropIncomingHandshake:442(19689ns) DropOutgoingHandshake:457(785ns) SwitchToState:179(453ns) ProcessPendingSessionEvents:495(46347035ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 42160353ns max# 6237249ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.035988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471650753702718:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23836
2025-11-27T18:20:01.036942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471646601954443:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:01.036562Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471646458735326:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(30164630ns) {HandleHandshakeStatus:329(30162182ns) {DropIncomingHandshake:442(1118ns) DropOutgoingHandshake:457(6021ns) SwitchToState:179(390ns) ProcessPendingSessionEvents:495(19566037ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19470825ns max# 4418540ns}}}
2025-11-27T18:20:01.037772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471646458735412:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:01.038471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471646601954375:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(18245929ns) {HandleHandshakeStatus:329(17997026ns) {DropIncomingHandshake:442(573ns) DropOutgoingHandshake:457(6089ns) SwitchToState:179(460ns) ProcessPendingSessionEvents:495(15972019ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15886521ns max# 4508202ns}}}
2025-11-27T18:20:01.038215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471643952625361:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(23431382ns) {HandleHandshakeStatus:329(23428443ns) {DropIncomingHandshake:442(1296ns) DropOutgoingHandshake:457(646ns) SwitchToState:179(2104ns) ProcessPendingSessionEvents:495(18578226ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17766553ns max# 8063721ns}}}
2025-11-27T18:20:01.039200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471646458735384:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:01.039375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:01.038504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471643952625363:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(27345277ns) {HandleHandshakeStatus:329(27341515ns) {DropIncomingHandshake:442(19803ns) DropOutgoingHandshake:457(438ns) SwitchToState:179(468ns) ProcessPendingSessionEvents:495(17604290ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17514709ns max# 4150508ns}}}
2025-11-27T18:20:01.039588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471646601954436:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.039737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471646458735384:2105] [node 5] ICP22 created new session: [6:7577471650753702719:2105]
2025-11-27T18:20:01.039645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:01.040998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471651697970878:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37971
No RDMA context found for GID: 2025-11-27T18:20:01.040470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471650753702719:2105] [node 5] ICS09 handshake done sender: [6:7577471646458735412:2120] self: [6:7577471646458735417:0] peer: [5:7577471646601954442:0] socket: 333 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:01.039934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471646601954436:2109] [node 6] ICP22 created new session: [5:7577471650896921766:2109]
2025-11-27T18:20:01.041076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471650753702719:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:01.040492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471650896921766:2109] [node 6] ICS09 handshake done sender: [5:7577471646601954443:2112] self: [5:7577471646601954442:0] peer: [6:7577471646458735417:0] socket: 323 qp: -1
2025-11-27T18:20:01.043140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471643952625361:2107] [node 4] ICP18 handshake (actor [2:7577471643952625386:2120]) is too old
2025-11-27T18:20:01.041339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471650896921766:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:01.041111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.041384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.044293Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471645312725307:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(18685177ns) {HandleHandshakeStatus:329(18683627ns) {DropIncomingHandshake:442(1172ns) DropOutgoingHandshake:457(5023ns) SwitchToState:179(301ns) ProcessPendingSessionEvents:495(4515380ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4442929ns max# 1100163ns}}}
2025-11-27T18:20:01.044936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471649607692695:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48676
2025-11-27T18:20:01.045124Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471645312725306:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11374) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471645312725391:2117] held: no
2025-11-27T18:20:01.042970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471650896921763:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26998
2025-11-27T18:20:01.045386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471649607692694:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48676
2025-11-27T18:20:01.050607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471651697970878:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.050811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471647403003488:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:01.051051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471649607692694:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.051128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471645312725391:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:01.051135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471646458735418:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.051075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471643952625373:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.051426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471650896921763:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.051247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471645312725308:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:01.051116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471643952625373:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:01.051440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471645312725306:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:01.051628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:01.051941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471645312725306:2048] [node 1] ICP22 created new session: [4:7577471649607692699:2048]
2025-11-27T18:20:01.052376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471649607692699:2048] [node 1] ICS09 handshake done sender: [4:7577471645312725391:2117] self: [4:7577471645312725396:0] peer: [1:7577471646301877611:0] socket: 341 qp: -1
2025-11-27T18:20:01.052762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471649607692699:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:01.052805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:01.052802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471646458735325:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:01.052851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471646601954374:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:01.053207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471647403003558:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.053249Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471647403003558:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:01.053701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471646301877619:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.054381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471643952625372:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.053735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471646301877619:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.054416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471643952625372:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:01.055692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471649607692701:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48676
2025-11-27T18:20:01.055869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471647403003558:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:01.056265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471649607692694:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:01.056806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471643952625373:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:01.056956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471647403003550:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:01.056810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471645312725308:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:01.057014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:01.057363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471647403003550:2107] [node 4] ICP22 created new session: [3:7577471651697970881:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.057578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471646301877619:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:01.057419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471651697970881:2107] [node 4] ICS09 handshake done sender: [3:7577471647403003558:2112] self: [3:7577471647403003555:0] peer: [4:7577471649607692698:0] socket: 357 qp: -1
2025-11-27T18:20:01.057796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471643952625299:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:01.058171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471651697970881:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:01.058564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:01.058633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:01.059463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471646301877605:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.058590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471651697970882:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37971
2025-11-27T18:20:01.058799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471643952625299:2049] [node 3] ICP22 created new session: [2:7577471648247592695:2049]
2025-11-27T18:20:01.060101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:01.059521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:01.058777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471645312725308:2050] [node 3] ICP22 created new session: [4:7577471649607692702:2050]
2025-11-27T18:20:01.060977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471650896921769:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26998
2025-11-27T18:20:01.059865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471649607692702:2050] [node 3] ICS09 handshake done sender: [4:7577471649607692694:2119] self: [4:7577471649607692698:0] peer: [3:7577471647403003555:0] socket: 314 qp: -1
2025-11-27T18:20:01.060248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471646301877605:2109] [node 6] ICP22 created new session: [1:7577471650596844936:2109]
2025-11-27T18:20:01.059451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471648247592695:2049] [node 3] ICS09 handshake done sender: [2:7577471643952625373:2114] self: [2:7577471643952625370:0] peer: [3:7577471651697970879:0] socket: 292 qp: -1
2025-11-27T18:20:01.060462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471651697970878:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:01.066141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471650896921763:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:01.060045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471648247592695:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:01.060091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471649607692702:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:01.060624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471650596844936:2109] [node 6] ICS09 handshake done sender: [1:7577471646301877619:2116] self: [1:7577471646301877618:0] peer: [6:7577471650753702721:0] socket: 320 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.070706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471647403003488:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:01.077441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471646458735325:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11374) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471646458735418:2122] held: no
2025-11-27T18:20:01.060494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:01.060814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471650596844936:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:01.082313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471646601954374:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:01.082232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:01.060836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471643952625372:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:01.083557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471650753702722:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23836
2025-11-27T18:20:01.082371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:01.060503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:01.083960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471647403003488:2049] [node 2] ICP22 created new session: [3:7577471651697970884:2049]
2025-11-27T18:20:01.060851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:01.062702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471643952625362:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:01.084255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471646601954374:2049] [node 2] ICP22 created new session: [5:7577471650896921770:2049]
2025-11-27T18:20:01.084642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471646458735418:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:01.087272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471646458735325:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:01.087326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:01.084472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471651697970884:2049] [node 2] ICS09 handshake done sender: [3:7577471651697970878:2119] self: [3:7577471651697970879:0] peer: [2:7577471643952625370:0] socket: 324 qp: -1
2025-11-27T18:20:01.070668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:01.073818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471643952625362:2108] [node 5] ICP22 created new session: [2:7577471648247592697:2108]
2025-11-27T18:20:01.084511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471651697970884:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:01.082437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471648247592697:2108] [node 5] ICS09 handshake done sender: [2:7577471643952625372:2113] self: [2:7577471643952625371:0] peer: [5:7577471650896921768:0] socket: 355 qp: -1
2025-11-27T18:20:01.087371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471646458735325:2048] [node 1] ICP22 created new session: [6:7577471650753702723:2048]
2025-11-27T18:20:01.084942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:01.082483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471648247592697:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:01.084940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471650896921770:2049] [node 2] ICS09 handshake done sender: [5:7577471650896921763:2122] self: [5:7577471650896921768:0] peer: [2:7577471643952625371:0] socket: 346 qp: -1
2025-11-27T18:20:01.084511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:01.089665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471646301877605:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(29966995ns) {HandleHandshakeStatus:329(29964308ns) {DropIncomingHandshake:442(13552ns) DropOutgoingHandshake:457(247ns) SwitchToState:179(319ns) ProcessPendingSessionEvents:495(26387755ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25480728ns max# 4160847ns}}}
2025-11-27T18:20:01.087759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471650753702723:2048] [node 1] ICS09 handshake done sender: [6:7577471646458735418:2122] self: [6:7577471650753702721:0] peer: [1:7577471646301877618:0] socket: 344 qp: -1
2025-11-27T18:20:01.087976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471650753702723:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:01.090313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:01.084972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471650896921770:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:01.087011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:01.091193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471646601954374:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:01.091679Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471647403003550:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(34043757ns) {HandleHandshakeStatus:329(34041062ns) {DropIncomingHandshake:442(15561ns) DropOutgoingHandshake:457(252ns) SwitchToState:179(396ns) ProcessPendingSessionEvents:495(30394798ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27376444ns max# 7551464ns}}}
2025-11-27T18:20:01.091924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471643952625362:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(28286569ns) {HandleHandshakeStatus:329(28284096ns) {DropIncomingHandshake:442(11839ns) DropOutgoingHandshake:457(4094ns) SwitchToState:179(542ns) ProcessPendingSessionEvents:495(6066842ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5903336ns max# 2408204ns}}}
2025-11-27T18:20:01.091945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471643952625299:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(33410487ns) {HandleHandshakeStatus:329(33408369ns) {DropIncomingHandshake:442(13626ns) DropOutgoingHandshake:457(289ns) SwitchToState:179(438ns) ProcessPendingSessionEvents:495(29350423ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29250305ns max# 4904036ns}}}
2025-11-27T18:20:01.091681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471645312725308:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(33969823ns) {HandleHandshakeStatus:329(33967822ns) {DropIncomingHandshake:442(756ns) DropOutgoingHandshake:457(4953ns) SwitchToState:179(582ns) ProcessPendingSessionEvents:495(29917405ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29807601ns max# 10821039ns}}}
2025-11-27T18:20:01.093101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471647403003488:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(21581400ns) {HandleHandshakeStatus:329(21576223ns) {DropIncomingHandshake:442(1197ns) DropOutgoingHandshake:457(8345ns) SwitchToState:179(441ns) ProcessPendingSessionEvents:495(5539412ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5475994ns max# 2333828ns}}}
2025-11-27T18:20:01.093185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471647403003488:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:01.152752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:01.152971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471643952625387:2048] [node 1] ICS01 socket: 306 reason# EndOfStream
2025-11-27T18:20:01.153007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471643952625298:2048] [node 1] ICP30 unregister session Session# [2:7577471643952625387:2048] VirtualId# [2:7577471643952625381:0]
2025-11-27T18:20:01.153036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471643952625387:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.153128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:01.153196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:01.153991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:01.154222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471647403003577:2048] [node 1] ICS07 socket disconnect 297 reason# EndOfStream
2025-11-27T18:20:01.154254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471647403003577:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.154319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:01.154364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471647403003577:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:01.156611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:01.156837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471649607692699:2048] [node 1] ICS01 socket: 341 reason# EndOfStream
2025-11-27T18:20:01.156887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471645312725306:2048] [node 1] ICP30 unregister session Session# [4:7577471649607692699:2048] VirtualId# [4:7577471645312725396:0]
2025-11-27T18:20:01.156771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471647403003487:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11374) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:01.156917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471649607692699:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.156815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471647403003577:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:01.156998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:01.157224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:01.157978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471647403003487:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11374) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:01.158108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471647403003577:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:01.158165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471647403003487:2048] [node 1] ICP30 unregister session Session# [3:7577471647403003577:2048] VirtualId# [3:7577471647403003567:0]
2025-11-27T18:20:01.158127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:01.158267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:01.158291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471646601954463:2048] [node 1] ICS07 socket disconnect 305 reason# EndOfStream
2025-11-27T18:20:01.158303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471647403003487:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11374) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.158325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471646601954463:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.158322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11374) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.158405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:01.158452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471646601954463:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:01.159000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:01.159063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471646601954373:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11374) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:01.159131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471646601954463:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:01.159217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471650753702723:2048] [node 1] ICS01 socket: 344 reason# EndOfStream
2025-11-27T18:20:01.159276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471646458735325:2048] [node 1] ICP30 unregister session Session# [6:7577471650753702723:2048] VirtualId# [6:7577471650753702721:0]
2025-11-27T18:20:01.159169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471646601954373:2048] [node 1] ICP30 unregister session Session# [5:7577471646601954463:2048] VirtualId# [5:7577471646601954456:0]
2025-11-27T18:20:01.159306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471650753702723:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.159349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:01.159381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471646601954373:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11374) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.159394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11374) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.159392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:01.159435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:01.167691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471647403003487:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:01.173348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471646601954373:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:01.198343Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:01.198596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471651697970884:2049] [node 2] ICS07 socket disconnect 324 reason# EndOfStream
2025-11-27T18:20:01.198632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471651697970884:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.198722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:01.198777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471651697970884:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:01.201455Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:01.201573Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471647403003488:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22110) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:01.201626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471651697970884:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:01.203850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:01.201743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471649607692696:2049] [node 2] ICS07 socket disconnect 328 reason# EndOfStream
2025-11-27T18:20:01.204073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471650753702716:2049] [node 2] ICS07 socket disconnect 335 reason# EndOfStream
2025-11-27T18:20:01.201787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471649607692696:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.204112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471650753702716:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.204109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:01.204190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:01.204193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:01.204243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471650753702716:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:01.204260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471649607692696:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:01.204354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471650896921770:2049] [node 2] ICS01 socket: 346 reason# EndOfStream
2025-11-27T18:20:01.204403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471646601954374:2049] [node 2] ICP30 unregister session Session# [5:7577471650896921770:2049] VirtualId# [5:7577471650896921768:0]
2025-11-27T18:20:01.204444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471650896921770:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.204529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:01.204590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:01.204736Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471647403003488:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22110) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:01.204811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471651697970884:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:01.204863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471647403003488:2049] [node 2] ICP30 unregister session Session# [3:7577471651697970884:2049] VirtualId# [3:7577471651697970879:0]
2025-11-27T18:20:01.205098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:01.205145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471647403003488:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.205155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.205365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471645312725307:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22110) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:01.205435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471646458735326:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22110) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:01.205905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471649607692696:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:01.209415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471650753702716:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:01.209491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471646458735326:2049] [node 2] ICP30 unregister session Session# [6:7577471650753702716:2049] VirtualId# [6:7577471646458735416:0]
2025-11-27T18:20:01.209456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471645312725307:2049] [node 2] ICP30 unregister session Session# [4:7577471649607692696:2049] VirtualId# [4:7577471645312725395:0]
2025-11-27T18:20:01.209597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:01.209646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:01.209681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471645312725307:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.209648Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471646458735326:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.209696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.209672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22110) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.219908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471646458735326:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:01.219908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471645312725307:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:01.254826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:01.255075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471649607692702:2050] [node 3] ICS07 socket disconnect 314 reason# EndOfStream
2025-11-27T18:20:01.255114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471649607692702:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.255204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:01.255250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471649607692702:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:01.255966Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:01.256025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471645312725308:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21396) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:01.256171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471649607692702:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:01.256185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471650896921764:2050] [node 3] ICS07 socket disconnect 307 reason# EndOfStream
2025-11-27T18:20:01.256220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471650896921764:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.256290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:01.256332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471650896921764:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:01.256412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471645312725308:2050] [node 3] ICP30 unregister session Session# [4:7577471649607692702:2050] VirtualId# [4:7577471649607692698:0]
2025-11-27T18:20:01.256478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:01.256519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471645312725308:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.256530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.257114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471646601954375:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21396) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:01.257203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471650896921764:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:01.257248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471646601954375:2050] [node 3] ICP30 unregister session Session# [5:7577471650896921764:2050] VirtualId# [5:7577471646601954455:0]
2025-11-27T18:20:01.257398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:01.257432Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471646601954375:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.257447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.257439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:01.257647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471646458735410:2050] [node 3] ICS07 socket disconnect 304 reason# EndOfStream
2025-11-27T18:20:01.257687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471646458735410:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.257782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:01.257851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471646458735410:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:01.258551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471646458735327:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21396) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:01.258594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471646458735410:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:01.258654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471646458735327:2050] [node 3] ICP30 unregister session Session# [6:7577471646458735410:2050] VirtualId# [6:7577471646458735407:0]
2025-11-27T18:20:01.258766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:01.258790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471646458735327:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.258801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:01.265980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471645312725308:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:01.266970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471646601954375:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:01.268174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471646458735327:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:01.325434Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:01.325677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471646601954462:2108] [node 4] ICS01 socket: 294 reason# EndOfStream
2025-11-27T18:20:01.325838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471646601954437:2108] [node 4] ICP30 unregister session Session# [5:7577471646601954462:2108] VirtualId# [5:7577471646601954457:0]
2025-11-27T18:20:01.325879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471646601954462:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.325965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:01.326011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:01.334553Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:01.334832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471646458735414:2109] [node 4] ICS01 socket: 302 reason# EndOfStream
2025-11-27T18:20:01.334888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471646458735390:2109] [node 4] ICP30 unregister session Session# [6:7577471646458735414:2109] VirtualId# [6:7577471646458735406:0]
2025-11-27T18:20:01.334928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471646458735414:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.335025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:01.335083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:01.366339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:01.366598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471650753702719:2105] [node 5] ICS01 socket: 333 reason# EndOfStream
2025-11-27T18:20:01.366647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471646458735384:2105] [node 5] ICP30 unregister session Session# [6:7577471650753702719:2105] VirtualId# [6:7577471646458735417:0]
2025-11-27T18:20:01.366687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471650753702719:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:01.366777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:01.366861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:01.480136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471647931998806:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:01.480508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471647931998807:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:01.501583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471651167081444:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:01.501647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471651167081445:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:01.522054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471648613799954:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:01.522284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471648613799953:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:01.546141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471648142623511:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:01.546206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471648142623509:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:01.546217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471648142623510:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:01.566725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471651123089291:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:01.566804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471651123089292:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:01.566846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471651123089293:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:01.588933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471649603483761:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:01.588980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471649603483762:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:01.589012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471649603483763:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:01.675142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471651167081505:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:01.662384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471647931998868:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:01.727809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471647931998871:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:01.765628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471651167081508:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:01.782161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471648142623572:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:01.758933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471649603483822:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:01.746638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471647931998870:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:01.768619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471648613800019:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:01.797478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471651123089354:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:01.782245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471649603483823:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:01.782163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471648613800021:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:01.795600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471648142623573:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:01.800267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471651167081509:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:01.797490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471651123089355:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:01.800037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471648613800020:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:01.846012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471647931998870:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(56827079ns) {Configure:104(56821565ns) {ProcessConfigured:135(69086ns) {StartInitialHandshake:157(67221ns) {DropHandshakes:470(1947ns) {DropIncomingHandshake:442(457ns) DropOutgoingHandshake:457(272ns)} GenerateSessionVirtualId:988(549ns) PrepareNewSessionHandshake:477(4750ns) {SwitchToState:179(663ns)}}}}}
2025-11-27T18:20:01.847464Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471648142623511:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(46485675ns) {Configure:104(46480711ns) {ProcessConfigured:135(2551495ns) {StartInitialHandshake:157(2549111ns) {DropHandshakes:470(1415ns) {DropIncomingHandshake:442(363ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(697ns) PrepareNewSessionHandshake:477(6277ns) {SwitchToState:179(2338ns)}}}}}
2025-11-27T18:20:01.862981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471647931998807:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(187132312ns) {Configure:104(187108469ns) {ProcessConfigured:135(3675102ns) {StartInitialHandshake:157(3673172ns) {DropHandshakes:470(3275ns) {DropIncomingHandshake:442(783ns) DropOutgoingHandshake:457(414ns)} GenerateSessionVirtualId:988(560ns) PrepareNewSessionHandshake:477(2496ns) {SwitchToState:179(1407ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.864234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471648613800019:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(68397649ns) {Configure:104(68391194ns) {ProcessConfigured:135(13482852ns) {StartInitialHandshake:157(13479633ns) {DropHandshakes:470(1291ns) {DropIncomingHandshake:442(392ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(1391ns) PrepareNewSessionHandshake:477(2539ns) {SwitchToState:179(1409ns)}}}}}
2025-11-27T18:20:01.864200Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471647931998868:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(199627929ns) {Configure:104(199624262ns) {ProcessConfigured:135(8566078ns) {StartInitialHandshake:157(3990825ns) {DropHandshakes:470(4331ns) {DropIncomingHandshake:442(573ns) DropOutgoingHandshake:457(438ns)} GenerateSessionVirtualId:988(2071ns) PrepareNewSessionHandshake:477(3293ns) {SwitchToState:179(1615ns)}}}}}
2025-11-27T18:20:01.865952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471651123089291:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(61029876ns) {Configure:104(61023836ns) {ProcessConfigured:135(13905477ns) {StartInitialHandshake:157(13903367ns) {DropHandshakes:470(1726ns) {DropIncomingHandshake:442(445ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(448ns) PrepareNewSessionHandshake:477(4457ns) {SwitchToState:179(2310ns)}}}}}
2025-11-27T18:20:01.866156Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471648142623509:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(161947817ns) {Configure:104(161941784ns) {ProcessConfigured:135(4143666ns) {StartInitialHandshake:157(4141891ns) {DropHandshakes:470(1699ns) {DropIncomingHandshake:442(386ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(607ns) PrepareNewSessionHandshake:477(3408ns) {SwitchToState:179(1699ns)}}}}}
2025-11-27T18:20:01.864591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471648613799954:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(174831281ns) {Configure:104(174825100ns) {ProcessConfigured:135(1491654ns) {StartInitialHandshake:157(1489243ns) {DropHandshakes:470(2297ns) {DropIncomingHandshake:442(544ns) DropOutgoingHandshake:457(490ns)} GenerateSessionVirtualId:988(472ns) PrepareNewSessionHandshake:477(5057ns) {SwitchToState:179(560ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.866748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471647931998872:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37718
2025-11-27T18:20:01.867269Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471651123089355:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22380101ns) {Configure:104(22374226ns) {ProcessConfigured:135(2612376ns) {StartInitialHandshake:157(2609913ns) {DropHandshakes:470(1578ns) {DropIncomingHandshake:442(482ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(482ns) PrepareNewSessionHandshake:477(468432ns) {SwitchToState:179(3750ns)}}}}}
2025-11-27T18:20:01.871724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471651167081505:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(48055663ns) {Configure:104(48051278ns) {ProcessConfigured:135(2560147ns) {StartInitialHandshake:157(2558161ns) {DropHandshakes:470(132581ns) {DropIncomingHandshake:442(1095ns) DropOutgoingHandshake:457(244ns)} GenerateSessionVirtualId:988(864ns) PrepareNewSessionHandshake:477(2305ns) {SwitchToState:179(722ns)}}}}}
2025-11-27T18:20:01.873438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471648142623510:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(76202177ns) {Configure:104(76194602ns) {ProcessConfigured:135(2196753ns) {StartInitialHandshake:157(2194799ns) {DropHandshakes:470(1348ns) {DropIncomingHandshake:442(455ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(387ns) PrepareNewSessionHandshake:477(2101ns) {SwitchToState:179(842ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.871216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471651123089360:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12326
2025-11-27T18:20:01.874804Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471651167081444:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(67411279ns) {Configure:104(67401231ns) {ProcessConfigured:135(1474977ns) {StartInitialHandshake:157(1052445ns) {DropHandshakes:470(1607ns) {DropIncomingHandshake:442(519ns) DropOutgoingHandshake:457(242ns)} GenerateSessionVirtualId:988(721ns) PrepareNewSessionHandshake:477(1613ns) {SwitchToState:179(688ns)}}}}}
2025-11-27T18:20:01.872023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471651123089292:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(53413017ns) {Configure:104(53404024ns) {ProcessConfigured:135(5392006ns) {StartInitialHandshake:157(5389795ns) {DropHandshakes:470(1066ns) {DropIncomingHandshake:442(339ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(343ns) PrepareNewSessionHandshake:477(3327874ns) {SwitchToState:179(4867ns)}}}}}
2025-11-27T18:20:01.876720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471651123089360:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.875420Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471649603483762:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23561805ns) {Configure:104(23557521ns) {ProcessConfigured:135(1637064ns) {StartInitialHandshake:157(1635259ns) {DropHandshakes:470(1505ns) {DropIncomingHandshake:442(349ns) DropOutgoingHandshake:457(483ns)} GenerateSessionVirtualId:988(529ns) PrepareNewSessionHandshake:477(2108ns) {SwitchToState:179(938ns)}}}}}
2025-11-27T18:20:01.875457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471647931998871:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25578366ns) {Configure:104(25574065ns) {ProcessConfigured:135(3579186ns) {StartInitialHandshake:157(3577138ns) {DropHandshakes:470(891ns) {DropIncomingHandshake:442(166ns) DropOutgoingHandshake:457(104ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:477(2064ns) {SwitchToState:179(785ns)}}}}}
2025-11-27T18:20:01.879111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471648142623572:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27698578ns) {Configure:104(27690858ns) {ProcessConfigured:135(7337811ns) {StartInitialHandshake:157(7335845ns) {DropHandshakes:470(928ns) {DropIncomingHandshake:442(263ns) DropOutgoingHandshake:457(149ns)} GenerateSessionVirtualId:988(1026ns) PrepareNewSessionHandshake:477(2462ns) {SwitchToState:179(1101ns)}}}}}
2025-11-27T18:20:01.877631Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471649603483763:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(73565993ns) {Configure:104(73556959ns) {ProcessConfigured:135(6801033ns) {StartInitialHandshake:157(6799365ns) {DropHandshakes:470(1027ns) {DropIncomingHandshake:442(254ns) DropOutgoingHandshake:457(162ns)} GenerateSessionVirtualId:988(615ns) PrepareNewSessionHandshake:477(2208ns) {SwitchToState:179(865ns)}}}}}
2025-11-27T18:20:01.885484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471647931998872:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.886429Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471651167081445:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(54526325ns) {Configure:104(54511242ns) {ProcessConfigured:135(2349689ns) {StartInitialHandshake:157(2347943ns) {DropHandshakes:470(1412ns) {DropIncomingHandshake:442(430ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(572ns) PrepareNewSessionHandshake:477(2424ns) {SwitchToState:179(831ns)}}}}}
2025-11-27T18:20:01.878082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471649603483826:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59957
2025-11-27T18:20:01.886733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471651167081508:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(34691143ns) {Configure:104(34686979ns) {ProcessConfigured:135(3372466ns) {StartInitialHandshake:157(3370525ns) {DropHandshakes:470(1505ns) {DropIncomingHandshake:442(491ns) DropOutgoingHandshake:457(267ns)} GenerateSessionVirtualId:988(533ns) PrepareNewSessionHandshake:477(1988ns) {SwitchToState:179(1114ns)}}}}}
2025-11-27T18:20:01.878547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471649603483833:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59957
2025-11-27T18:20:01.885554Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471649603483761:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31162338ns) {Configure:104(31155399ns) {ProcessConfigured:135(3804021ns) {StartInitialHandshake:157(3802141ns) {DropHandshakes:470(2082ns) {DropIncomingHandshake:442(472ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(585ns) PrepareNewSessionHandshake:477(13978ns) {SwitchToState:179(813ns)}}}}}
2025-11-27T18:20:01.887867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471647931998806:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47518403ns) {Configure:104(44100566ns) {ProcessConfigured:135(56711ns) {StartInitialHandshake:157(54099ns) {DropHandshakes:470(3345ns) {DropIncomingHandshake:442(735ns) DropOutgoingHandshake:457(551ns)} GenerateSessionVirtualId:988(659ns) PrepareNewSessionHandshake:477(1981ns) {SwitchToState:179(631ns)}}}}}
2025-11-27T18:20:01.886429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471649603483826:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.903356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471648613800023:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.888349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471649603483763:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.903410Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471648613800023:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.927201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471651167081519:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.927386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471651167081520:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
2025-11-27T18:20:01.927635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471651167081519:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.917602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471648613800032:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41020
2025-11-27T18:20:01.921633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471648613800033:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41020
2025-11-27T18:20:01.893678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471649603483833:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.941131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471651123089293:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(66148369ns) {Configure:104(66138236ns) {ProcessConfigured:135(7510542ns) {StartInitialHandshake:157(7507064ns) {DropHandshakes:470(3581ns) {DropIncomingHandshake:442(1022ns) DropOutgoingHandshake:457(676ns)} GenerateSessionVirtualId:988(392ns) PrepareNewSessionHandshake:477(5263ns) {SwitchToState:179(3013ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.946595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471647931998884:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37718
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.941369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471649603483822:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(43011523ns) {Configure:104(43002216ns) {ProcessConfigured:135(13068302ns) {StartInitialHandshake:157(13065123ns) {DropHandshakes:470(3198ns) {DropIncomingHandshake:442(1042ns) DropOutgoingHandshake:457(336ns)} GenerateSessionVirtualId:988(848ns) PrepareNewSessionHandshake:477(2556980ns) {SwitchToState:179(4825ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.940442Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471648613800023:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:01.941571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471651123089293:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:01.962449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471647931998883:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37718
2025-11-27T18:20:01.946847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471651167081518:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
2025-11-27T18:20:01.957920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471648142623586:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:20:01.962444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471651167081520:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.963601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471647931998889:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37718
2025-11-27T18:20:01.967468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471651167081518:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.967226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471648142623573:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(51567256ns) {Configure:104(51560763ns) {ProcessConfigured:135(2949406ns) {StartInitialHandshake:157(2946039ns) {DropHandshakes:470(3303ns) {DropIncomingHandshake:442(666ns) DropOutgoingHandshake:457(484ns)} GenerateSessionVirtualId:988(802ns) PrepareNewSessionHandshake:477(2840ns) {SwitchToState:179(1153ns)}}}}}
2025-11-27T18:20:01.964535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471647931998890:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37718
2025-11-27T18:20:01.965808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471647931998889:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.967726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471648613800025:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.968489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471651123089359:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.967778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471648613800025:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.968523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471651123089359:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:01.966067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471647931998883:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.968143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471649603483822:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.967225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471647931998884:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.968553Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471651123089354:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(80380124ns) {Configure:104(80011367ns) {ProcessConfigured:135(1268745ns) {StartInitialHandshake:157(1265425ns) {DropHandshakes:470(292469ns) {DropIncomingHandshake:442(825ns) DropOutgoingHandshake:457(366ns)} GenerateSessionVirtualId:988(695ns) PrepareNewSessionHandshake:477(1878ns) {SwitchToState:179(1097ns)}}}}}
2025-11-27T18:20:01.969289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471648613800021:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.967438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471647931998890:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.974750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471648142623585:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:20:01.975098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471651123089359:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:01.969346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:01.991236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471649603483842:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59957
2025-11-27T18:20:01.989632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471648142623585:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.969692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471648613800021:2110] [node 6] ICP22 created new session: [3:7577471648613800037:2110]
2025-11-27T18:20:01.993626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471648142623510:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:01.981371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471648613800035:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41020
2025-11-27T18:20:01.994098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471648142623586:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.981592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471648613800032:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.994889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471648142623509:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:01.981748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471648613800037:2110] [node 6] ICS09 handshake done sender: [3:7577471648613800023:2111] self: [3:7577471648613800022:0] peer: [6:7577471649603483837:0] socket: 297 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.995855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471651123089355:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:01.997848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471649603483826:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:01.999462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471651167081512:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.981812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471648613800037:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:01.999273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:01.999430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471649603483843:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59957
2025-11-27T18:20:01.982176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:01.999512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471651167081512:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:01.999682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471647931998875:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:01.999489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471651123089355:2109] [node 6] ICP22 created new session: [5:7577471651123089373:2109]
2025-11-27T18:20:01.999523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471649603483845:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59957
2025-11-27T18:20:01.985031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471648613800036:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41020
2025-11-27T18:20:01.999747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471647931998875:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:02.000320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471649603483763:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:01.999694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471651167081515:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.001527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471651123089373:2109] [node 6] ICS09 handshake done sender: [5:7577471651123089359:2111] self: [5:7577471651123089358:0] peer: [6:7577471649603483844:0] socket: 303 qp: -1
2025-11-27T18:20:02.000378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:02.000435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471649603483763:2050] [node 3] ICP22 created new session: [6:7577471653898451142:2050]
2025-11-27T18:20:02.002667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471651123089373:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:02.005366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471647931998875:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:01.989788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471648613800033:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.999727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471651167081529:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
2025-11-27T18:20:02.004110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: 2025-11-27T18:20:01.993745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471648613800025:2112] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:02.001519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471649603483833:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.001572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471653898451142:2050] [node 3] ICS09 handshake done sender: [6:7577471649603483826:2110] self: [6:7577471649603483837:0] peer: [3:7577471648613800022:0] socket: 298 qp: -1
2025-11-27T18:20:02.015615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471647931998870:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.005324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471651123089372:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12326
2025-11-27T18:20:01.999735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471651167081515:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:01.994175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471648613800035:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.015913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:02.002562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471653898451142:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:02.015543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471651123089360:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:02.002443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471651167081515:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:01.994905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471648613799954:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:02.016710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471652437590887:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:20:02.002675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:02.013432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471651167081512:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:02.016230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471647931998870:2107] [node 4] ICP22 created new session: [1:7577471652226966192:2107]
2025-11-27T18:20:02.016802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471651123089293:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:01.997009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471648613800020:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.025653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471648142623510:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24328) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471648142623585:2114] held: no
2025-11-27T18:20:02.003005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471649603483843:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.013487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471651167081529:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:01.999478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:02.015348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471651167081505:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.015406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:02.016522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471647931998880:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.016980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:01.999797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471648613800020:2109] [node 5] ICP22 created new session: [3:7577471648613800043:2109]
2025-11-27T18:20:02.003275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471649603483822:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.016554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471647931998880:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:02.015944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471651167081445:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.018521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471651123089293:2050] [node 3] ICP22 created new session: [5:7577471655418056672:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.015949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471651167081505:2107] [node 4] ICP22 created new session: [2:7577471655462048829:2107]
2025-11-27T18:20:02.000303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471648613800036:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.032610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471655418056672:2050] [node 3] ICS09 handshake done sender: [5:7577471651123089360:2112] self: [5:7577471651123089368:0] peer: [3:7577471648613800024:0] socket: 300 qp: -1
2025-11-27T18:20:02.016902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471647931998876:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.003313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.003015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471648613800043:2109] [node 5] ICS09 handshake done sender: [3:7577471648613800025:2112] self: [3:7577471648613800024:0] peer: [5:7577471651123089368:0] socket: 299 qp: -1
2025-11-27T18:20:02.016002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:02.032683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471655418056672:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:02.032749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.067037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471655418056674:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12326
2025-11-27T18:20:02.016931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471652226966192:2107] [node 4] ICS09 handshake done sender: [1:7577471647931998875:2111] self: [1:7577471647931998874:0] peer: [4:7577471648142623590:0] socket: 331 qp: -1
2025-11-27T18:20:02.003366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471649603483822:2106] [node 5] ICP22 created new session: [6:7577471653898451144:2106]
2025-11-27T18:20:02.003811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471648613800043:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:02.067116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471655418056675:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12326
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:02.004475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:02.003418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471653898451144:2106] [node 5] ICS09 handshake done sender: [6:7577471649603483833:2114] self: [6:7577471649603483844:0] peer: [5:7577471651123089358:0] socket: 304 qp: -1
2025-11-27T18:20:02.008749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471648613799953:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:02.003442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471653898451144:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:02.016549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471652908767341:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41020
2025-11-27T18:20:02.003491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:02.017107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471652908767342:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41020
2025-11-27T18:20:02.035184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471652908767341:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.013892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471649603483761:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:02.068878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471653898451149:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59957
2025-11-27T18:20:02.041864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471648613799954:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24328) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471648613800035:2117] held: no
2025-11-27T18:20:02.064556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471652437590888:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:20:02.016010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471655462048829:2107] [node 4] ICS09 handshake done sender: [2:7577471651167081512:2110] self: [2:7577471651167081510:0] peer: [4:7577471648142623589:0] socket: 321 qp: -1
2025-11-27T18:20:02.016354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471655462048829:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:02.049619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471648613800035:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:02.066899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471652437590889:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:20:02.016933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471647931998876:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:02.071510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471653898451148:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59957
2025-11-27T18:20:02.072224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471655418056675:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.017107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471652226966192:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:02.016391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471651167081445:2049] [node 3] ICP22 created new session: [2:7577471655462048830:2049]
2025-11-27T18:20:02.083595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471653898451149:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.083074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471651123089355:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(71245649ns) {HandleHandshakeStatus:329(71239749ns) {DropIncomingHandshake:442(227ns) DropOutgoingHandshake:457(585ns) SwitchToState:179(434ns) ProcessPendingSessionEvents:495(53613729ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 52142406ns max# 7857435ns}}}
2025-11-27T18:20:02.083810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471648142623510:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(49847987ns) {HandleHandshakeStatus:395(45338073ns) {LogHandshakeFail:472(4049029ns) {UpdateErrorStateLog:545(6623ns)} DropOutgoingHandshake:457(508ns) ProcessPendingSessionEvents:495(41216907ns) {unordered DropSessionEvent:512 num# 100 sum# 41148965ns max# 7488299ns}}}
2025-11-27T18:20:02.016581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471655462048830:2049] [node 3] ICS09 handshake done sender: [2:7577471651167081515:2112] self: [2:7577471651167081514:0] peer: [3:7577471648613800042:0] socket: 327 qp: -1
2025-11-27T18:20:02.017161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.085525Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471649603483843:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:02.084076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471648142623509:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22163) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471648142623586:2115] held: no
2025-11-27T18:20:02.083733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471651123089292:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:02.016665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:02.035420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471647931998880:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:02.086885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471652437590887:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.016750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471655462048830:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:02.035913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471647931998876:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:02.086827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471649603483761:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.100407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471652437590890:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.017074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:02.093381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.042660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471647931998807:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.121478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471651123089292:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24328) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471655418056675:2118] held: no
2025-11-27T18:20:02.042726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:02.043462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471647931998807:2049] [node 3] ICP22 created new session: [1:7577471652226966197:2049]
2025-11-27T18:20:02.043498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471647931998871:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.044282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:02.046227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471652226966197:2049] [node 3] ICS09 handshake done sender: [1:7577471647931998876:2112] self: [1:7577471647931998873:0] peer: [3:7577471652908767344:0] socket: 330 qp: -1
2025-11-27T18:20:02.046296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471647931998871:2109] [node 6] ICP22 created new session: [1:7577471652226966198:2109]
2025-11-27T18:20:02.050249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471652226966198:2109] [node 6] ICS09 handshake done sender: [1:7577471647931998880:2114] self: [1:7577471647931998879:0] peer: [6:7577471653898451146:0] socket: 333 qp: -1
2025-11-27T18:20:02.053807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471652226966198:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:02.057626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471652226966197:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:02.057657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:02.057679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:02.103261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471647931998870:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(51522718ns) {HandleHandshakeStatus:329(51520681ns) {DropIncomingHandshake:442(27584ns) DropOutgoingHandshake:457(315ns) SwitchToState:179(283ns) ProcessPendingSessionEvents:495(33967073ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 33883819ns max# 8642029ns}}}
2025-11-27T18:20:02.089685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471651167081522:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.101471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471648142623585:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:02.134186Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471648613800020:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(70134416ns) {HandleHandshakeStatus:329(70131998ns) {DropIncomingHandshake:442(226ns) DropOutgoingHandshake:457(391ns) SwitchToState:179(525ns) ProcessPendingSessionEvents:495(58336353ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 54175729ns max# 5948805ns}}}
2025-11-27T18:20:02.099677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471649603483761:2048] [node 1] ICP22 created new session: [6:7577471653898451150:2048]
2025-11-27T18:20:02.140958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471648613799953:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22163) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471648613800036:2118] held: no
2025-11-27T18:20:02.089722Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471651167081522:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:02.126067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471648142623511:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:02.100933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471653898451150:2048] [node 1] ICS09 handshake done sender: [6:7577471649603483843:2118] self: [6:7577471653898451146:0] peer: [1:7577471647931998879:0] socket: 337 qp: -1
2025-11-27T18:20:02.149130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471655418056678:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12326
2025-11-27T18:20:02.149273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471655418056679:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12326
2025-11-27T18:20:02.166646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471655418056678:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.106691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471651167081522:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:02.182313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471651123089292:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(60349234ns) {HandleHandshakeStatus:395(60345475ns) {LogHandshakeFail:472(20111ns) {UpdateErrorStateLog:545(5800ns)} DropOutgoingHandshake:457(1529ns) ProcessPendingSessionEvents:495(60232136ns) {unordered DropSessionEvent:512 num# 100 sum# 10717281ns max# 5070713ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.137717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471652437590890:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.102350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471653898451150:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:02.141982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471648613800020:2109] [node 5] ICP18 handshake (actor [3:7577471652908767341:2119]) is too old
2025-11-27T18:20:02.142263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471652908767345:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41020
2025-11-27T18:20:02.146994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471648613800029:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.147025Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471648613800029:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:02.160669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471648613799954:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(37386527ns) {HandleHandshakeStatus:395(37384543ns) {LogHandshakeFail:472(13927ns) {UpdateErrorStateLog:545(5987ns)} DropOutgoingHandshake:457(2074ns) ProcessPendingSessionEvents:495(33090015ns) {unordered DropSessionEvent:512 num# 100 sum# 8291829ns max# 3623306ns}}}
2025-11-27T18:20:02.173598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471648613799954:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.173674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:02.173734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471648613799954:2049] [node 2] ICP22 created new session: [3:7577471652908767346:2049]
2025-11-27T18:20:02.173788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471652908767346:2049] [node 2] ICS09 handshake done sender: [3:7577471648613800035:2117] self: [3:7577471648613800042:0] peer: [2:7577471651167081514:0] socket: 329 qp: -1
2025-11-27T18:20:02.173830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471652908767346:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:02.173868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:02.174248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471648613800036:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:02.177803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471648613800021:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(109158162ns) {HandleHandshakeStatus:329(109155472ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(627ns) SwitchToState:179(353ns) ProcessPendingSessionEvents:495(83913921ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 82490205ns max# 7706203ns}}}
2025-11-27T18:20:02.179932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471648613799953:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.186580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471648613800021:2110] [node 6] ICP18 handshake (actor [3:7577471648613800032:2115]) is too old
2025-11-27T18:20:02.195554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471651123089354:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:02.140833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471648142623510:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.102403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:02.191792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:02.196221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471648613800029:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:02.201921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471648613799953:2048] [node 1] ICP22 created new session: [3:7577471652908767349:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.195906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471655418056674:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.106934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471651167081509:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.155613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:02.210198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471652908767349:2048] [node 1] ICS09 handshake done sender: [3:7577471648613800036:2118] self: [3:7577471652908767344:0] peer: [1:7577471647931998873:0] socket: 334 qp: -1
2025-11-27T18:20:02.214247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471652908767349:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:02.197494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471655418056675:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:02.107020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:02.111132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471649603483762:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:02.221218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471648613800019:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.195995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471649603483763:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(140788754ns) {HandleHandshakeStatus:329(140786660ns) {DropIncomingHandshake:442(977ns) DropOutgoingHandshake:457(5172ns) SwitchToState:179(912ns) ProcessPendingSessionEvents:495(135862234ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 130033765ns max# 9908074ns}}}
2025-11-27T18:20:02.162865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471648142623510:2049] [node 2] ICP22 created new session: [4:7577471652437590896:2049]
2025-11-27T18:20:02.226184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471647931998878:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.210286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471651123089291:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:02.107101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471651167081509:2108] [node 5] ICP22 created new session: [2:7577471655462048836:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.221301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:02.107153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471655462048836:2108] [node 5] ICS09 handshake done sender: [2:7577471651167081522:2117] self: [2:7577471651167081521:0] peer: [5:7577471655418056677:0] socket: 357 qp: -1
2025-11-27T18:20:02.226224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471647931998878:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:02.181327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471652437590896:2049] [node 2] ICS09 handshake done sender: [4:7577471648142623585:2114] self: [4:7577471648142623589:0] peer: [2:7577471651167081510:0] socket: 326 qp: -1
2025-11-27T18:20:02.209664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471653898451153:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59957
2025-11-27T18:20:02.213686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471651123089292:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.221401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:02.109210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471651167081505:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(53447397ns) {HandleHandshakeStatus:329(53444655ns) {DropIncomingHandshake:442(14076ns) DropOutgoingHandshake:457(425ns) SwitchToState:179(377ns) ProcessPendingSessionEvents:495(43263147ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 41975807ns max# 7824423ns}}}
2025-11-27T18:20:02.192547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471652437590896:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:02.221191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:02.225203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471648613800019:2108] [node 4] ICP22 created new session: [3:7577471652908767350:2108]
2025-11-27T18:20:02.241401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471647931998882:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.237528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471652908767350:2108] [node 4] ICS09 handshake done sender: [3:7577471648613800029:2113] self: [3:7577471648613800028:0] peer: [4:7577471652437590894:0] socket: 348 qp: -1
2025-11-27T18:20:02.224991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471651123089293:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(83853392ns) {HandleHandshakeStatus:329(83849915ns) {DropIncomingHandshake:442(1374ns) DropOutgoingHandshake:457(5640ns) SwitchToState:179(268ns) ProcessPendingSessionEvents:495(66374182ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60326540ns max# 5172206ns}}}
2025-11-27T18:20:02.241438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471647931998882:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:02.237595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471652908767350:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:02.240581Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471648613799953:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(52900096ns) {HandleHandshakeStatus:329(52896426ns) {DropIncomingHandshake:442(1083ns) DropOutgoingHandshake:457(477ns) SwitchToState:179(662ns) ProcessPendingSessionEvents:495(1310ns)}}
2025-11-27T18:20:02.250982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471647931998871:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(152370437ns) {HandleHandshakeStatus:329(152367893ns) {DropIncomingHandshake:442(13999ns) DropOutgoingHandshake:457(203ns) SwitchToState:179(256ns) ProcessPendingSessionEvents:495(124150703ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 118137428ns max# 13339432ns}}}
2025-11-27T18:20:02.245950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:02.225708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471651123089292:2049] [node 2] ICP22 created new session: [5:7577471655418056682:2049]
2025-11-27T18:20:02.252109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471647931998878:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:02.238816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471655418056682:2049] [node 2] ICS09 handshake done sender: [5:7577471655418056675:2118] self: [5:7577471655418056677:0] peer: [2:7577471651167081521:0] socket: 344 qp: -1
2025-11-27T18:20:02.240194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471655418056682:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:02.240237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:02.245667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471655418056683:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12326
2025-11-27T18:20:02.251788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471651123089292:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(36393912ns) {HandleHandshakeStatus:329(36390675ns) {DropIncomingHandshake:442(1290ns) DropOutgoingHandshake:457(456ns) SwitchToState:179(831ns) ProcessPendingSessionEvents:495(999ns)}}
2025-11-27T18:20:02.109468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471655462048836:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:02.209867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471653898451154:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59957
2025-11-27T18:20:02.252791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471655418056678:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:02.252793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471647931998807:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(131979280ns) {HandleHandshakeStatus:329(131976120ns) {DropIncomingHandshake:442(14965ns) DropOutgoingHandshake:457(369ns) SwitchToState:179(741ns) ProcessPendingSessionEvents:495(107526517ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 99386188ns max# 15136991ns}}}
2025-11-27T18:20:02.255575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471651123089354:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.255629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:02.192604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:02.225037Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471649603483822:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(102358710ns) {HandleHandshakeStatus:329(102355714ns) {DropIncomingHandshake:442(739ns) DropOutgoingHandshake:457(4507ns) SwitchToState:179(324ns) ProcessPendingSessionEvents:495(96788681ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 86401669ns max# 7884966ns}}}
2025-11-27T18:20:02.109531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:02.226461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471653898451149:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:02.241008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471653898451153:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.117561Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471651167081445:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(52836795ns) {HandleHandshakeStatus:329(52833731ns) {DropIncomingHandshake:442(10394ns) DropOutgoingHandshake:457(312ns) SwitchToState:179(179ns) ProcessPendingSessionEvents:495(35662702ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 35244937ns max# 11473009ns}}}
2025-11-27T18:20:02.252366Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471649603483761:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(152377818ns) {HandleHandshakeStatus:329(152374399ns) {DropIncomingHandshake:442(1137ns) DropOutgoingHandshake:457(4094ns) SwitchToState:179(826ns) ProcessPendingSessionEvents:495(105139943ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 98085446ns max# 22125562ns}}}
2025-11-27T18:20:02.195457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471652437590897:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:20:02.262802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471647931998868:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.257999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471651123089354:2108] [node 4] ICP22 created new session: [5:7577471655418056686:2108]
2025-11-27T18:20:02.195521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471652437590891:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15396
2025-11-27T18:20:02.195864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471648142623581:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.252428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471649603483762:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.262884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:02.130872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471651167081509:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(23100466ns) {HandleHandshakeStatus:329(23097440ns) {DropIncomingHandshake:442(17705ns) DropOutgoingHandshake:457(1403ns) SwitchToState:179(656ns) ProcessPendingSessionEvents:495(570449ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 503170ns max# 74428ns}}}
2025-11-27T18:20:02.195889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471648142623581:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.262934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471647931998882:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:02.263095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471647931998868:2108] [node 5] ICP22 created new session: [1:7577471652226966204:2108]
2025-11-27T18:20:02.267601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471652226966204:2108] [node 5] ICS09 handshake done sender: [1:7577471647931998878:2113] self: [1:7577471647931998877:0] peer: [5:7577471655418056681:0] socket: 354 qp: -1
2025-11-27T18:20:02.252474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:02.267663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471652226966204:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:02.195958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471648142623510:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(61819448ns) {HandleHandshakeStatus:329(61816125ns) {DropIncomingHandshake:442(2535ns) DropOutgoingHandshake:457(744ns) SwitchToState:179(391ns) ProcessPendingSessionEvents:495(1482ns)}}
2025-11-27T18:20:02.264800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471655418056686:2108] [node 4] ICS09 handshake done sender: [5:7577471655418056678:2119] self: [5:7577471655418056680:0] peer: [4:7577471648142623580:0] socket: 311 qp: -1
2025-11-27T18:20:02.272365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471648613800019:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(48561030ns) {HandleHandshakeStatus:329(48558749ns) {DropIncomingHandshake:442(15836ns) DropOutgoingHandshake:457(575ns) SwitchToState:179(125ns) ProcessPendingSessionEvents:495(18888389ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18798901ns max# 6943634ns}}}
2025-11-27T18:20:02.131701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471655462048838:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
2025-11-27T18:20:02.268626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:02.252518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471649603483762:2049] [node 2] ICP22 created new session: [6:7577471653898451155:2049]
2025-11-27T18:20:02.268130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471655418056686:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:02.252570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471653898451155:2049] [node 2] ICS09 handshake done sender: [6:7577471653898451149:2121] self: [6:7577471653898451152:0] peer: [2:7577471651167081516:0] socket: 341 qp: -1
2025-11-27T18:20:02.221144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471652437590891:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.137303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471651167081517:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.269979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471647931998806:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.269746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:02.252598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471653898451155:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:02.137350Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471651167081517:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:02.187133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471651167081517:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:02.210125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471655462048838:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.224913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471652437590887:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:02.270041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:02.217387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471651167081508:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.233177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471648142623509:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(48438733ns) {HandleHandshakeStatus:395(48435370ns) {LogHandshakeFail:472(47360ns) {UpdateErrorStateLog:545(26173ns)} DropOutgoingHandshake:457(1568ns) ProcessPendingSessionEvents:495(41655288ns) {unordered DropSessionEvent:512 num# 100 sum# 8260152ns max# 6015465ns}}}
2025-11-27T18:20:02.246018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471648142623511:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.217489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:02.247244Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471648142623586:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:02.252618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471649603483761:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:02.249127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:02.249193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471648142623511:2050] [node 3] ICP22 created new session: [4:7577471652437590901:2050]
2025-11-27T18:20:02.252639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:02.271011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471647931998806:2048] [node 2] ICP22 created new session: [1:7577471652226966206:2048]
2025-11-27T18:20:02.282569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471649603483823:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:02.250364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471648142623581:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:02.217542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471651167081508:2109] [node 6] ICP22 created new session: [2:7577471655462048840:2109]
2025-11-27T18:20:02.274168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471652226966206:2048] [node 2] ICS09 handshake done sender: [1:7577471647931998882:2115] self: [1:7577471647931998881:0] peer: [2:7577471655462048842:0] socket: 362 qp: -1
2025-11-27T18:20:02.250423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471652437590901:2050] [node 3] ICS09 handshake done sender: [4:7577471652437590887:2117] self: [4:7577471652437590894:0] peer: [3:7577471648613800028:0] socket: 350 qp: -1
2025-11-27T18:20:02.295877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471649603483762:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(42556531ns) {HandleHandshakeStatus:329(42553266ns) {DropIncomingHandshake:442(821ns) DropOutgoingHandshake:457(4871ns) SwitchToState:179(892ns) ProcessPendingSessionEvents:495(39694327ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39012452ns max# 7519300ns}}}
2025-11-27T18:20:02.296172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471649603483762:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:02.217594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471655462048840:2109] [node 6] ICS09 handshake done sender: [2:7577471651167081517:2113] self: [2:7577471651167081516:0] peer: [6:7577471653898451152:0] socket: 315 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.275063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471652226966206:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:02.250845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471652437590901:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:02.217635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471655462048840:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:02.277113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:02.217673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:02.251724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:02.317920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471655418056685:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12326
2025-11-27T18:20:02.230733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471651167081444:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:02.321471Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471651123089291:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22163) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471655418056674:2117] held: no
2025-11-27T18:20:02.271147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471655462048843:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2143
2025-11-27T18:20:02.253020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471648142623509:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.332208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471647931998868:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(67434821ns) {HandleHandshakeStatus:329(67429264ns) {DropIncomingHandshake:442(17983ns) DropOutgoingHandshake:457(545ns) SwitchToState:179(704ns) ProcessPendingSessionEvents:495(55034440ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 54908136ns max# 10061834ns}}}
2025-11-27T18:20:02.254574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471648142623572:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.325245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471655418056674:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:02.275899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471651167081508:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(43170806ns) {HandleHandshakeStatus:329(39186969ns) {DropIncomingHandshake:442(19309ns) DropOutgoingHandshake:457(590ns) SwitchToState:179(725ns) ProcessPendingSessionEvents:495(27505078ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27430324ns max# 9324092ns}}}
2025-11-27T18:20:02.332876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471651123089291:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.254625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:02.277757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471655462048838:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:02.332925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:02.254680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471648142623572:2108] [node 5] ICP22 created new session: [4:7577471652437590903:2108]
2025-11-27T18:20:02.281168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471651167081444:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.337040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471651123089291:2048] [node 1] ICP22 created new session: [5:7577471655418056688:2048]
2025-11-27T18:20:02.281241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:02.254724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471652437590903:2108] [node 5] ICS09 handshake done sender: [4:7577471648142623581:2113] self: [4:7577471648142623580:0] peer: [5:7577471655418056680:0] socket: 309 qp: -1
2025-11-27T18:20:02.337468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471655418056688:2048] [node 1] ICS09 handshake done sender: [5:7577471655418056674:2117] self: [5:7577471655418056681:0] peer: [1:7577471647931998877:0] socket: 355 qp: -1
2025-11-27T18:20:02.254760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471652437590903:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:02.338014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471655418056688:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:02.254791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:02.257899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:02.345777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:02.262801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471648142623509:2048] [node 1] ICP22 created new session: [4:7577471652437590905:2048]
2025-11-27T18:20:02.263650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471652437590905:2048] [node 1] ICS09 handshake done sender: [4:7577471648142623586:2115] self: [4:7577471648142623590:0] peer: [1:7577471647931998874:0] socket: 332 qp: -1
2025-11-27T18:20:02.264953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471652437590905:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:02.265022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:02.274308Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471648142623509:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16649865ns) {HandleHandshakeStatus:329(16647611ns) {DropIncomingHandshake:442(238ns) DropOutgoingHandshake:457(271ns) SwitchToState:179(666ns) ProcessPendingSessionEvents:495(946ns)}}
2025-11-27T18:20:02.317936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471648142623588:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:02.317985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471648142623588:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:02.327710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471648142623588:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:02.358126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471648142623573:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:02.358257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:02.358312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471648142623573:2109] [node 6] ICP22 created new session: [4:7577471652437590908:2109]
2025-11-27T18:20:02.358360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471652437590908:2109] [node 6] ICS09 handshake done sender: [4:7577471648142623588:2116] self: [4:7577471648142623587:0] peer: [6:7577471653898451157:0] socket: 360 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:02.358401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471652437590908:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:02.281702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471651167081444:2048] [node 1] ICP22 created new session: [2:7577471655462048844:2048]
2025-11-27T18:20:02.361439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471653898451158:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59957
2025-11-27T18:20:02.358273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471651123089291:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25275113ns) {HandleHandshakeStatus:329(25271304ns) {DropIncomingHandshake:442(1065ns) DropOutgoingHandshake:457(560ns) SwitchToState:179(634ns) ProcessPendingSessionEvents:495(2219ns)}}
2025-11-27T18:20:02.361603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471651123089354:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(80854076ns) {HandleHandshakeStatus:329(80851118ns) {DropIncomingHandshake:442(385ns) DropOutgoingHandshake:457(4645ns) SwitchToState:179(337ns) ProcessPendingSessionEvents:495(66558333ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 66439297ns max# 14278933ns}}}
2025-11-27T18:20:02.285543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471655462048844:2048] [node 1] ICS09 handshake done sender: [2:7577471655462048838:2119] self: [2:7577471655462048842:0] peer: [1:7577471647931998881:0] socket: 363 qp: -1
2025-11-27T18:20:02.290892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471655462048844:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:02.293996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:02.362783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471647931998806:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(91947001ns) {HandleHandshakeStatus:329(91943968ns) {DropIncomingHandshake:442(15981ns) DropOutgoingHandshake:457(649ns) SwitchToState:179(636ns) ProcessPendingSessionEvents:495(83462571ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 83338811ns max# 13663492ns}}}
2025-11-27T18:20:02.358438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:02.361676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471648142623572:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(53392476ns) {HandleHandshakeStatus:329(53389151ns) {DropIncomingHandshake:442(11839ns) DropOutgoingHandshake:457(210ns) SwitchToState:179(413ns) ProcessPendingSessionEvents:495(53045433ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 52940416ns max# 13163310ns}}}
2025-11-27T18:20:02.362629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471648142623511:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(49257510ns) {HandleHandshakeStatus:329(49254073ns) {DropIncomingHandshake:442(2214ns) DropOutgoingHandshake:457(7232ns) SwitchToState:179(769ns) ProcessPendingSessionEvents:495(31930300ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 31826545ns max# 5336634ns}}}
2025-11-27T18:20:02.362822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471653898451153:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:02.364412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471649603483823:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:02.365358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:02.365650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471649603483823:2107] [node 4] ICP22 created new session: [6:7577471653898451159:2107]
2025-11-27T18:20:02.366265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471653898451159:2107] [node 4] ICS09 handshake done sender: [6:7577471653898451153:2122] self: [6:7577471653898451157:0] peer: [4:7577471648142623587:0] socket: 361 qp: -1
2025-11-27T18:20:02.366521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471653898451159:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:02.367389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:02.369084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471651167081444:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(87715896ns) {HandleHandshakeStatus:329(87712049ns) {DropIncomingHandshake:442(2318ns) DropOutgoingHandshake:457(7151ns) SwitchToState:179(847ns) ProcessPendingSessionEvents:495(69045294ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 64711024ns max# 16557634ns}}}
2025-11-27T18:20:02.414700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:02.414968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471655462048844:2048] [node 1] ICS07 socket disconnect 363 reason# EndOfStream
2025-11-27T18:20:02.415000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471655462048844:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.415078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:02.415118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471655462048844:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:02.415954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:02.416195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471652908767349:2048] [node 1] ICS01 socket: 334 reason# EndOfStream
2025-11-27T18:20:02.416244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471648613799953:2048] [node 1] ICP30 unregister session Session# [3:7577471652908767349:2048] VirtualId# [3:7577471652908767344:0]
2025-11-27T18:20:02.416291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471652908767349:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.416382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:02.416447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:02.418626Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:02.419002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471652437590905:2048] [node 1] ICS01 socket: 332 reason# EndOfStream
2025-11-27T18:20:02.419074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471648142623509:2048] [node 1] ICP30 unregister session Session# [4:7577471652437590905:2048] VirtualId# [4:7577471648142623590:0]
2025-11-27T18:20:02.419122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471652437590905:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.419195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:02.418960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471651167081444:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22163) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:02.419485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:02.419013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471655462048844:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:02.421113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:02.421360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471655418056688:2048] [node 1] ICS01 socket: 355 reason# EndOfStream
2025-11-27T18:20:02.421406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471651123089291:2048] [node 1] ICP30 unregister session Session# [5:7577471655418056688:2048] VirtualId# [5:7577471655418056681:0]
2025-11-27T18:20:02.421437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471655418056688:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.421505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:02.421545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:02.423150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:02.422278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471651167081444:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22163) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:02.431604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471653898451150:2048] [node 1] ICS07 socket disconnect 337 reason# EndOfStream
2025-11-27T18:20:02.431709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471653898451150:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.431804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:02.431843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471653898451150:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:02.450163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471649603483761:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22163) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:02.450349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471653898451150:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:02.450389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471649603483761:2048] [node 1] ICP30 unregister session Session# [6:7577471653898451150:2048] VirtualId# [6:7577471653898451146:0]
2025-11-27T18:20:02.450509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:02.450549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471649603483761:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22163) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.450577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22163) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.459957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471649603483761:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:02.498393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:02.498678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471652908767346:2049] [node 2] ICS01 socket: 329 reason# EndOfStream
2025-11-27T18:20:02.498750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471648613799954:2049] [node 2] ICP30 unregister session Session# [3:7577471652908767346:2049] VirtualId# [3:7577471648613800042:0]
2025-11-27T18:20:02.498805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471652908767346:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.498937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:02.499015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:02.501926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:02.502144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471652437590896:2049] [node 2] ICS01 socket: 326 reason# EndOfStream
2025-11-27T18:20:02.502200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471648142623510:2049] [node 2] ICP30 unregister session Session# [4:7577471652437590896:2049] VirtualId# [4:7577471648142623589:0]
2025-11-27T18:20:02.502237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471652437590896:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.502343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:02.502400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:02.503478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:02.503668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471655418056682:2049] [node 2] ICS01 socket: 344 reason# EndOfStream
2025-11-27T18:20:02.503699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471651123089292:2049] [node 2] ICP30 unregister session Session# [5:7577471655418056682:2049] VirtualId# [5:7577471655418056677:0]
2025-11-27T18:20:02.503718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471655418056682:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.503779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:02.503818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:02.504590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:02.504789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471653898451155:2049] [node 2] ICS07 socket disconnect 341 reason# EndOfStream
2025-11-27T18:20:02.504828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471653898451155:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.504896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:02.504943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471653898451155:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:02.506178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471649603483762:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24328) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:02.506249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471653898451155:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:02.506285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471649603483762:2049] [node 2] ICP30 unregister session Session# [6:7577471653898451155:2049] VirtualId# [6:7577471653898451152:0]
2025-11-27T18:20:02.506399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:02.506444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471649603483762:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24328) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.506458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24328) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.515277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471649603483762:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:02.609448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:02.609508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:02.609762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471652437590901:2050] [node 3] ICS07 socket disconnect 350 reason# EndOfStream
2025-11-27T18:20:02.609765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471655418056672:2050] [node 3] ICS07 socket disconnect 300 reason# EndOfStream
2025-11-27T18:20:02.609840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471652437590901:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.609825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471655418056672:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.609951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:02.609960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:02.610037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471655418056672:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:02.610036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471652437590901:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:02.611691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471651123089293:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15520) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:02.611768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471655418056672:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:02.611863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471651123089293:2050] [node 3] ICP30 unregister session Session# [5:7577471655418056672:2050] VirtualId# [5:7577471651123089368:0]
2025-11-27T18:20:02.617571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:02.617650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471651123089293:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.611903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471648142623511:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15520) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:02.609568Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:02.617717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471652437590901:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:02.625416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471648142623511:2050] [node 3] ICP30 unregister session Session# [4:7577471652437590901:2050] VirtualId# [4:7577471652437590894:0]
2025-11-27T18:20:02.609940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471653898451142:2050] [node 3] ICS07 socket disconnect 298 reason# EndOfStream
2025-11-27T18:20:02.626265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:02.625959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.609996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471653898451142:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.626351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471648142623511:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.626369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.610149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:02.610235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471653898451142:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:02.626002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471649603483763:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15520) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:02.626115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471653898451142:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:02.626499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471649603483763:2050] [node 3] ICP30 unregister session Session# [6:7577471653898451142:2050] VirtualId# [6:7577471649603483837:0]
2025-11-27T18:20:02.626599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:02.626628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471649603483763:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.626642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15520) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.636861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471651123089293:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:02.638880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471649603483763:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:02.685440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:02.685696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471655418056686:2108] [node 4] ICS01 socket: 311 reason# EndOfStream
2025-11-27T18:20:02.685772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471651123089354:2108] [node 4] ICP30 unregister session Session# [5:7577471655418056686:2108] VirtualId# [5:7577471655418056680:0]
2025-11-27T18:20:02.685832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471655418056686:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.685969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:02.686055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:02.685417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:02.685677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471653898451159:2107] [node 4] ICS07 socket disconnect 361 reason# EndOfStream
2025-11-27T18:20:02.685711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471653898451159:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.685820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:02.685914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471653898451159:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:02.692687Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471649603483823:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9276) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:02.692816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471653898451159:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:02.692867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471649603483823:2107] [node 4] ICP30 unregister session Session# [6:7577471653898451159:2107] VirtualId# [6:7577471653898451157:0]
2025-11-27T18:20:02.692966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:02.693012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471649603483823:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9276) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.693041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9276) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.703669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471649603483823:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:02.753535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:02.753762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471653898451144:2106] [node 5] ICS07 socket disconnect 304 reason# EndOfStream
2025-11-27T18:20:02.753801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471653898451144:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:02.753900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:02.753942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471653898451144:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:02.754967Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471649603483822:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9776) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:02.755029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471653898451144:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:02.755126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471649603483822:2106] [node 5] ICP30 unregister session Session# [6:7577471653898451144:2106] VirtualId# [6:7577471649603483844:0]
2025-11-27T18:20:02.755315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:02.755379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471649603483822:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:02.755420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:02.936545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471654852248226:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:02.936586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471654852248227:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:02.964906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471655062173915:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:02.964951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471655062173916:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:02.999096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471654979234592:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:02.999150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471654979234593:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:03.024143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471656576569459:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:03.024180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471656576569460:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:03.024200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471656576569461:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:03.048213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471657856431894:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:03.048256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471657856431895:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:03.048285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471657856431896:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:03.073496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471657771447280:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:03.073541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471657771447281:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:03.073579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471657771447282:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:03.301153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471659274201952:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:03.325945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471657856431956:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:03.281573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471657771447340:2102] [node 4] ICP01 ready to work
2025-11-27T18:20:03.370242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471659357141275:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:03.317418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471659274201950:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:03.302662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471657771447341:2100] [node 5] ICP01 ready to work
2025-11-27T18:20:03.369669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471656576569524:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:03.384328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471659357141274:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.384592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471659147215587:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:03.405707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471659357141279:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:03.435167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471657856431961:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:03.383921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471659147215585:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:03.434116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471656576569525:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:03.429586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471659274201953:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:03.443428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471656576569526:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23566
2025-11-27T18:20:03.405315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471659147215586:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:03.513566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471657771447281:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(159107625ns) {Configure:104(159102757ns) {ProcessConfigured:135(41044012ns) {StartInitialHandshake:157(41041359ns) {DropHandshakes:470(2779ns) {DropIncomingHandshake:442(654ns) DropOutgoingHandshake:457(345ns)} GenerateSessionVirtualId:988(1068ns) PrepareNewSessionHandshake:477(3066ns) {SwitchToState:179(1366ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.522233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471657856431965:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19280
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.530737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471657771447282:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25200654ns) {Configure:104(25190453ns) {ProcessConfigured:135(61295ns) {StartInitialHandshake:157(59311ns) {DropHandshakes:470(2250ns) {DropIncomingHandshake:442(425ns) DropOutgoingHandshake:457(300ns)} GenerateSessionVirtualId:988(489ns) PrepareNewSessionHandshake:477(2236ns) {SwitchToState:179(795ns)}}}}}
2025-11-27T18:20:03.545658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471657856431965:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.549323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471656576569526:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.546212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471659147215588:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26150
2025-11-27T18:20:03.569610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471659147215588:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.549869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471656576569460:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(209841911ns) {Configure:104(209838862ns) {ProcessConfigured:135(14198071ns) {StartInitialHandshake:157(14194866ns) {DropHandshakes:470(514ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(135ns)} GenerateSessionVirtualId:988(133ns) PrepareNewSessionHandshake:477(2671ns) {SwitchToState:179(1703ns)}}}}}
2025-11-27T18:20:03.573606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471657856431956:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.614276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471657771447280:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(199612213ns) {Configure:104(199603068ns) {ProcessConfigured:135(9010517ns) {StartInitialHandshake:157(9006703ns) {DropHandshakes:470(2451ns) {DropIncomingHandshake:442(697ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(1154ns) PrepareNewSessionHandshake:477(2139ns) {SwitchToState:179(610ns)}}}}}
2025-11-27T18:20:03.589705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471659147215590:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26150
2025-11-27T18:20:03.594846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471659274201952:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(71868525ns) {Configure:104(71862981ns) {ProcessConfigured:135(22622869ns) {StartInitialHandshake:157(22620670ns) {DropHandshakes:470(1811ns) {DropIncomingHandshake:442(463ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(474ns) PrepareNewSessionHandshake:477(2436ns) {SwitchToState:179(118ns)}}}}}
2025-11-27T18:20:03.593319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471656576569532:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.610105Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471659357141279:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(134421473ns) {Configure:104(134415173ns) {ProcessConfigured:135(7792048ns) {StartInitialHandshake:157(7788517ns) {DropHandshakes:470(2703ns) {DropIncomingHandshake:442(531ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(1061ns) PrepareNewSessionHandshake:477(3344ns) {SwitchToState:179(1613ns)}}}}}
2025-11-27T18:20:03.615433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471659147215590:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.638402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471657856431896:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(126948427ns) {Configure:104(126943340ns) {ProcessConfigured:135(17713021ns) {StartInitialHandshake:157(17709551ns) {DropHandshakes:470(1005ns) {DropIncomingHandshake:442(252ns) DropOutgoingHandshake:457(100ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:477(3442ns) {SwitchToState:179(1623ns)}}}}}
2025-11-27T18:20:03.593378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471656576569532:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.606867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471654979234593:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(78601419ns) {Configure:104(78594511ns) {ProcessConfigured:135(68791ns) {StartInitialHandshake:157(67016ns) {DropHandshakes:470(2061ns) {DropIncomingHandshake:442(382ns) DropOutgoingHandshake:457(298ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:477(20223ns) {SwitchToState:179(302ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.629505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471657771447341:2100] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(83474828ns) {Configure:104(83469500ns) {ProcessConfigured:135(22615163ns) {StartInitialHandshake:157(22612923ns) {DropHandshakes:470(735ns) {DropIncomingHandshake:442(120ns) DropOutgoingHandshake:457(94ns)} GenerateSessionVirtualId:988(580ns) PrepareNewSessionHandshake:477(3961715ns) {SwitchToState:179(4526ns)}}}}}
2025-11-27T18:20:03.618401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471655062173916:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(84726860ns) {Configure:104(84720980ns) {ProcessConfigured:135(62807ns) {StartInitialHandshake:157(60033ns) {DropHandshakes:470(907ns) {DropIncomingHandshake:442(115ns) DropOutgoingHandshake:457(150ns)} GenerateSessionVirtualId:988(353ns) PrepareNewSessionHandshake:477(1690ns) {SwitchToState:179(417ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.607004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471659274201954:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21564
2025-11-27T18:20:03.653368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471659147215589:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26150
2025-11-27T18:20:03.629658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471657771447340:2102] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(127730947ns) {Configure:104(127723624ns) {ProcessConfigured:135(18737861ns) {StartInitialHandshake:157(18734653ns) {DropHandshakes:470(4250ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(358ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:477(2820ns) {SwitchToState:179(1718ns)}}}}}
2025-11-27T18:20:03.614583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471656576569537:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23566
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.637082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471659357141274:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(45045462ns) {Configure:104(45036022ns) {ProcessConfigured:135(6748606ns) {StartInitialHandshake:157(6743005ns) {DropHandshakes:470(2099ns) {DropIncomingHandshake:442(511ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(1090ns) PrepareNewSessionHandshake:477(2837ns) {SwitchToState:179(537ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.633839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471659274201954:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.624327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471656576569537:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.657406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471657771447345:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56317
2025-11-27T18:20:03.672953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471654852248227:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(136372529ns) {Configure:104(136365701ns) {ProcessConfigured:135(24218732ns) {StartInitialHandshake:157(24214431ns) {DropHandshakes:470(2726ns) {DropIncomingHandshake:442(590ns) DropOutgoingHandshake:457(383ns)} GenerateSessionVirtualId:988(788ns) PrepareNewSessionHandshake:477(2805ns) {SwitchToState:179(1218ns)}}}}}
2025-11-27T18:20:03.647750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471659357141290:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36681
2025-11-27T18:20:03.676987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471657856431975:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19280
2025-11-27T18:20:03.664881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471657771447356:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56317
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.660439Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471654979234592:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(127088350ns) {Configure:104(127081400ns) {ProcessConfigured:135(17611277ns) {StartInitialHandshake:157(17608360ns) {DropHandshakes:470(7420998ns) {DropIncomingHandshake:442(1936ns) DropOutgoingHandshake:457(351ns)} GenerateSessionVirtualId:988(1040ns) PrepareNewSessionHandshake:477(3126ns) {SwitchToState:179(1856ns)}}}}}
2025-11-27T18:20:03.633779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471656576569532:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:03.682229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471657856431895:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(123123878ns) {Configure:104(123118759ns) {ProcessConfigured:135(15546999ns) {StartInitialHandshake:157(15544742ns) {DropHandshakes:470(2361ns) {DropIncomingHandshake:442(834ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(750ns) PrepareNewSessionHandshake:477(3088ns) {SwitchToState:179(2085ns)}}}}}
2025-11-27T18:20:03.648191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471659357141290:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.674426Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471654852248226:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(158702800ns) {Configure:104(158699192ns) {ProcessConfigured:135(4568765ns) {StartInitialHandshake:157(4565940ns) {DropHandshakes:470(2374ns) {DropIncomingHandshake:442(657ns) DropOutgoingHandshake:457(339ns)} GenerateSessionVirtualId:988(958ns) PrepareNewSessionHandshake:477(3204ns) {SwitchToState:179(1289ns)}}}}}
2025-11-27T18:20:03.642681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471656576569460:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.675319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471659274201950:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(140390936ns) {Configure:104(140385870ns) {ProcessConfigured:135(18374657ns) {StartInitialHandshake:157(18371047ns) {DropHandshakes:470(2838ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(745ns)} GenerateSessionVirtualId:988(564ns) PrepareNewSessionHandshake:477(5135273ns) {SwitchToState:179(4667ns)}}}}}
2025-11-27T18:20:03.682360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471656576569524:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.686315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:03.686382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471656576569524:2108] [node 5] ICP22 created new session: [4:7577471656576569541:2108]
2025-11-27T18:20:03.686448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471656576569541:2108] [node 5] ICS09 handshake done sender: [4:7577471656576569532:2113] self: [4:7577471656576569531:0] peer: [5:7577471657856431969:0] socket: 308 qp: -1
2025-11-27T18:20:03.686490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471656576569541:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:03.686900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:03.695527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471659274201956:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.665037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471657771447358:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56317
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.684258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471659357141284:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.683977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471657856431971:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19280
2025-11-27T18:20:03.677748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471659147215589:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.695571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471659274201956:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.665115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471657771447359:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56317
2025-11-27T18:20:03.703891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471659147215592:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.673663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471657771447359:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.717958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471659274201956:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:03.684295Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471659357141284:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.697976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471659357141292:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36681
2025-11-27T18:20:03.699526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471659357141284:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:03.707512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471659357141277:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.707558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471659357141277:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.703922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471659147215592:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.675114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471657771447345:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.749586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471659357141292:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.749746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471659357141277:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:03.750212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471659357141279:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.750261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:03.750312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471659357141279:2108] [node 4] ICP22 created new session: [2:7577471659357141296:2108]
2025-11-27T18:20:03.750362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471659357141296:2108] [node 4] ICS09 handshake done sender: [2:7577471659357141284:2112] self: [2:7577471659357141283:0] peer: [4:7577471656576569540:0] socket: 318 qp: -1
2025-11-27T18:20:03.750401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471659357141296:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:03.750438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:03.721322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471656576569538:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23566
2025-11-27T18:20:03.727262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471657856431956:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471657856431965:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.676820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471657771447356:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.733929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471659274201952:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.734431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471657856431966:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.747221Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471656576569525:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(170888427ns) {Configure:104(170880981ns) {ProcessConfigured:135(23301514ns) {StartInitialHandshake:157(23298426ns) {DropHandshakes:470(1476ns) {DropIncomingHandshake:442(466ns) DropOutgoingHandshake:457(201ns)} GenerateSessionVirtualId:988(469ns) PrepareNewSessionHandshake:477(2532ns) {SwitchToState:179(986ns)}}}}}
2025-11-27T18:20:03.709481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471659147215585:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(131253856ns) {Configure:104(131246174ns) {ProcessConfigured:135(12134744ns) {StartInitialHandshake:157(10136186ns) {DropHandshakes:470(3349ns) {DropIncomingHandshake:442(1668ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(1351ns) PrepareNewSessionHandshake:477(2133ns) {SwitchToState:179(1281ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.765589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471659357141275:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.733990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:03.754004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471656576569547:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23566
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.713421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471659147215592:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:03.681448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471657771447282:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.734475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471657856431966:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.761278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471656576569538:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.684119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471657771447281:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.765681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.746773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471659274201952:2109] [node 6] ICP22 created new session: [3:7577471659274201972:2109]
2025-11-27T18:20:03.753695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471659147215586:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.738128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471657856431975:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.689385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471657771447358:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.765735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471659357141275:2109] [node 6] ICP22 created new session: [2:7577471659357141300:2109]
2025-11-27T18:20:03.771428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471656576569459:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.753749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:03.741396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471657856431965:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:03.752378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471659274201969:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21564
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.765809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471659357141300:2109] [node 6] ICS09 handshake done sender: [2:7577471659357141277:2110] self: [2:7577471659357141276:0] peer: [6:7577471657771447362:0] socket: 301 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.754063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471659274201972:2109] [node 6] ICS09 handshake done sender: [3:7577471659274201956:2111] self: [3:7577471659274201955:0] peer: [6:7577471657771447363:0] socket: 324 qp: -1
2025-11-27T18:20:03.780883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471656576569543:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23566
2025-11-27T18:20:03.754324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471657856431956:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.697963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471657771447280:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.753805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471659147215586:2109] [node 6] ICP22 created new session: [1:7577471659147215609:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.699686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471657771447341:2100] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.758357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:03.758594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471659274201970:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21564
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.766146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471659357141300:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:03.787897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471656576569537:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.753882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471659147215609:2109] [node 6] ICS09 handshake done sender: [1:7577471659147215592:2113] self: [1:7577471659147215591:0] peer: [6:7577471657771447365:0] socket: 320 qp: -1
2025-11-27T18:20:03.761024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471657771447369:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56317
2025-11-27T18:20:03.777305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471657771447281:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18831) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471657771447345:2109] held: no
2025-11-27T18:20:03.797849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471657771447366:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56317
2025-11-27T18:20:03.799072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471657771447367:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56317
2025-11-27T18:20:03.801349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471657771447341:2100] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471657771447356:2116] held: no
2025-11-27T18:20:03.802009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471657771447356:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:03.804234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471657771447359:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:03.812813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471657771447341:2100] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.812864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:03.813033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471657771447341:2100] [node 5] ICP22 created new session: [6:7577471657771447371:2100]
2025-11-27T18:20:03.813411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471657771447358:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:03.813987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471657771447368:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56317
2025-11-27T18:20:03.814401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471657771447282:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.814446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:03.814492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471657771447282:2050] [node 3] ICP22 created new session: [6:7577471657771447372:2050]
2025-11-27T18:20:03.814535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471657771447372:2050] [node 3] ICS09 handshake done sender: [6:7577471657771447359:2118] self: [6:7577471657771447363:0] peer: [3:7577471659274201955:0] socket: 330 qp: -1
2025-11-27T18:20:03.814571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471657771447372:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:03.814607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:03.818813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471657771447371:2100] [node 5] ICS09 handshake done sender: [6:7577471657771447356:2116] self: [6:7577471657771447364:0] peer: [5:7577471657856431964:0] socket: 323 qp: -1
2025-11-27T18:20:03.818869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471657771447371:2100] [node 5] ICS10 traffic start
2025-11-27T18:20:03.827508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:03.832190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471657771447345:2109] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:03.837842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471657771447280:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.837902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:03.837960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471657771447280:2048] [node 1] ICP22 created new session: [6:7577471657771447376:2048]
2025-11-27T18:20:03.838000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471657771447376:2048] [node 1] ICS09 handshake done sender: [6:7577471657771447358:2117] self: [6:7577471657771447365:0] peer: [1:7577471659147215591:0] socket: 329 qp: -1
2025-11-27T18:20:03.838036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471657771447376:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:03.838068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:03.849678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471657771447281:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.849725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:03.849772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471657771447281:2049] [node 2] ICP22 created new session: [6:7577471657771447379:2049]
2025-11-27T18:20:03.849819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471657771447379:2049] [node 2] ICS09 handshake done sender: [6:7577471657771447345:2109] self: [6:7577471657771447362:0] peer: [2:7577471659357141276:0] socket: 307 qp: -1
2025-11-27T18:20:03.849852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471657771447379:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:03.849885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:03.850247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471657771447375:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56317
2025-11-27T18:20:03.793631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471656576569549:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23566
2025-11-27T18:20:03.768991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471659357141279:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(17926319ns) {HandleHandshakeStatus:329(17923841ns) {DropIncomingHandshake:442(214ns) DropOutgoingHandshake:457(383ns) SwitchToState:179(686ns) ProcessPendingSessionEvents:495(17509366ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17448495ns max# 17003043ns}}}
2025-11-27T18:20:03.761536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471657856431956:2108] [node 4] ICP22 created new session: [5:7577471657856431979:2108]
2025-11-27T18:20:03.753920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471659147215609:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.761158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471659274201972:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:03.753963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:03.761225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:03.860212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471657771447375:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.794201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471656576569538:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:03.777173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:03.765551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471657856431966:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:03.794886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471656576569543:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.787895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471659147215594:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.779958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471659357141302:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36681
2025-11-27T18:20:03.773350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471659274201969:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.797616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471656576569459:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.799677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471656576569461:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.799750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:03.804668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471656576569459:2048] [node 1] ICP22 created new session: [4:7577471656576569551:2048]
2025-11-27T18:20:03.809782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471656576569460:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.809889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471656576569551:2048] [node 1] ICS09 handshake done sender: [4:7577471656576569538:2116] self: [4:7577471656576569548:0] peer: [1:7577471659147215593:0] socket: 322 qp: -1
2025-11-27T18:20:03.809937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471656576569551:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:03.809991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:03.811652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471656576569459:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16347785ns) {HandleHandshakeStatus:329(16345007ns) {DropIncomingHandshake:442(2333ns) DropOutgoingHandshake:457(2085819ns) SwitchToState:179(180ns) ProcessPendingSessionEvents:495(595695ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 525747ns max# 53079ns}}}
2025-11-27T18:20:03.816105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:03.816159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471656576569460:2049] [node 2] ICP22 created new session: [4:7577471656576569553:2049]
2025-11-27T18:20:03.832054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471656576569553:2049] [node 2] ICS09 handshake done sender: [4:7577471656576569537:2115] self: [4:7577471656576569540:0] peer: [2:7577471659357141283:0] socket: 319 qp: -1
2025-11-27T18:20:03.832505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471656576569555:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23566
2025-11-27T18:20:03.835963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471656576569553:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:03.836024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:03.852377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471656576569557:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23566
2025-11-27T18:20:03.865452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471656576569557:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.873190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471656576569543:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:03.873671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471656576569545:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.873707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471656576569545:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.897592Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471656576569524:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(117202498ns) {HandleHandshakeStatus:329(117199200ns) {DropIncomingHandshake:442(13991ns) DropOutgoingHandshake:457(453ns) SwitchToState:179(442ns) ProcessPendingSessionEvents:495(100031152ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 95457779ns max# 16065698ns}}}
2025-11-27T18:20:03.860364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471657771447340:2102] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.765835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471657856431961:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.787932Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471659147215594:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.863816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471657771447341:2100] [node 5] ICP35 event processing took too much time PendingConnection:279(48386609ns) {HandleHandshakeStatus:329(48384033ns) {DropIncomingHandshake:442(1439ns) DropOutgoingHandshake:457(307ns) SwitchToState:179(572ns) ProcessPendingSessionEvents:495(840ns)}}
2025-11-27T18:20:03.776936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471659274201970:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.780029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471659357141304:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36681
2025-11-27T18:20:03.765906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:03.905896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471656576569460:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(105156616ns) {HandleHandshakeStatus:329(105154427ns) {DropIncomingHandshake:442(1989ns) DropOutgoingHandshake:457(9003ns) SwitchToState:179(502ns) ProcessPendingSessionEvents:495(36115120ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36051411ns max# 15033681ns}}}
2025-11-27T18:20:03.790917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471659147215586:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(36399095ns) {HandleHandshakeStatus:329(36387314ns) {DropIncomingHandshake:442(243ns) DropOutgoingHandshake:457(998ns) SwitchToState:179(335ns) ProcessPendingSessionEvents:495(590242ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 527388ns max# 80038ns}}}
2025-11-27T18:20:03.794770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471654979234592:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.787132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471659357141302:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.766827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471657856431961:2110] [node 6] ICP22 created new session: [5:7577471657856431980:2110]
2025-11-27T18:20:03.914705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471656576569461:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15444) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471656576569543:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.917453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471656576569545:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:03.776886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471657856431979:2108] [node 4] ICS09 handshake done sender: [5:7577471657856431965:2112] self: [5:7577471657856431969:0] peer: [4:7577471656576569531:0] socket: 309 qp: -1
2025-11-27T18:20:03.799872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471659274201977:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21564
2025-11-27T18:20:03.787272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471655062173915:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.794974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471659147215594:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.797533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471659147215601:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.938596Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471657771447340:2102] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471657771447375:2123] held: no
2025-11-27T18:20:03.918443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471656576569525:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.815943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471659274201973:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21564
2025-11-27T18:20:03.791748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471659357141304:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.780579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471657856431979:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.818746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471659274201961:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.797575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471659147215601:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.918491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:03.827650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471659357141299:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.787837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:03.833065Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471659357141279:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:03.918535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471656576569525:2110] [node 6] ICP22 created new session: [4:7577471656576569560:2110]
2025-11-27T18:20:03.808784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471659147215611:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26150
2025-11-27T18:20:03.818788Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471659274201961:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.797024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471657856431980:2110] [node 6] ICS09 handshake done sender: [5:7577471657856431966:2113] self: [5:7577471657856431964:0] peer: [6:7577471657771447364:0] socket: 317 qp: -1
2025-11-27T18:20:03.830522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471659274201977:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.918578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471656576569560:2110] [node 6] ICS09 handshake done sender: [4:7577471656576569545:2118] self: [4:7577471656576569544:0] peer: [6:7577471657771447381:0] socket: 369 qp: -1
2025-11-27T18:20:03.845653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471659357141307:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36681
2025-11-27T18:20:03.992457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471657771447382:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56317
2025-11-27T18:20:03.832822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471659274201961:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:03.813477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471659147215587:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.797082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471657856431980:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:03.840698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471654979234593:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.918611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471656576569560:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:03.864044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471659357141289:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.813530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:03.798353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:03.918644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:03.864090Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471659357141289:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.852770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471659274201973:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.815826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471659147215599:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:04.026847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471657771447375:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:03.807440Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471657856431961:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(40793097ns) {HandleHandshakeStatus:329(40790672ns) {DropIncomingHandshake:442(13829ns) DropOutgoingHandshake:457(281ns) SwitchToState:179(456ns) ProcessPendingSessionEvents:495(581702ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 517746ns max# 62258ns}}}
2025-11-27T18:20:03.861552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471659274201953:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.865462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471659357141302:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:03.956832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471656576569461:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(61242608ns) {HandleHandshakeStatus:395(57069979ns) {LogHandshakeFail:472(15596ns) {UpdateErrorStateLog:545(5068ns)} DropOutgoingHandshake:457(1570ns) ProcessPendingSessionEvents:495(39906476ns) {unordered DropSessionEvent:512 num# 100 sum# 39853003ns max# 8032690ns}}}
2025-11-27T18:20:03.823192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471657856431956:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(61683278ns) {HandleHandshakeStatus:329(61679775ns) {DropIncomingHandshake:442(549ns) DropOutgoingHandshake:457(1913ns) SwitchToState:179(2297ns) ProcessPendingSessionEvents:495(879ns)}}
2025-11-27T18:20:04.055946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471657771447340:2102] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.815863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471659147215599:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.880484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471655062173915:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.889184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:03.893394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471659357141289:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:03.893693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471655062173915:2048] [node 1] ICP22 created new session: [2:7577471659357141309:2048]
2025-11-27T18:20:03.902242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471659357141309:2048] [node 1] ICS09 handshake done sender: [2:7577471659357141302:2118] self: [2:7577471659357141305:0] peer: [1:7577471659147215600:0] socket: 361 qp: -1
2025-11-27T18:20:03.906985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471659357141309:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:03.915871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:03.917622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471655062173916:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.917698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:03.917758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471655062173916:2049] [node 3] ICP22 created new session: [2:7577471659357141311:2049]
2025-11-27T18:20:03.919822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471659357141311:2049] [node 3] ICS09 handshake done sender: [2:7577471659357141289:2114] self: [2:7577471659357141288:0] peer: [3:7577471659274201979:0] socket: 336 qp: -1
2025-11-27T18:20:03.919863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471659357141311:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:03.919884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471659357141287:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.919900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:03.919909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471659357141287:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.962424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471659357141287:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:03.974341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471659357141275:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(207967521ns) {HandleHandshakeStatus:329(207963474ns) {DropIncomingHandshake:442(17465ns) DropOutgoingHandshake:457(607ns) SwitchToState:179(486ns) ProcessPendingSessionEvents:495(174073619ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 173936226ns max# 10511008ns}}}
2025-11-27T18:20:03.989623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471659357141274:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.989692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:03.992126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471659357141274:2107] [node 5] ICP22 created new session: [2:7577471659357141314:2107]
2025-11-27T18:20:03.993574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471659357141314:2107] [node 5] ICS09 handshake done sender: [2:7577471659357141287:2113] self: [2:7577471659357141286:0] peer: [5:7577471657856431986:0] socket: 353 qp: -1
2025-11-27T18:20:04.005251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471659357141314:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:04.021375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:03.829892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471657856431985:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19280
2025-11-27T18:20:03.841165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471657856431982:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19280
2025-11-27T18:20:03.865667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471657856431982:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.868741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471657856431985:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.956896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471656576569461:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.873336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471657856431895:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.892749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471657856431894:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.956945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:03.957007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471656576569461:2050] [node 3] ICP22 created new session: [4:7577471656576569562:2050]
2025-11-27T18:20:03.902318Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471657856431894:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19463881ns) {IncomingHandshake:261(19461066ns) {DropIncomingHandshake:442(552ns) PrepareNewSessionHandshake:477(1876ns) {SwitchToState:179(645ns)} GenerateSessionVirtualId:988(1219ns) IssueIncomingHandshakeReply:193(19436537ns) {SwitchToState:179(1220ns)}}}
2025-11-27T18:20:03.957069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471656576569562:2050] [node 3] ICS09 handshake done sender: [4:7577471656576569543:2117] self: [4:7577471656576569550:0] peer: [3:7577471659274201960:0] socket: 341 qp: -1
2025-11-27T18:20:03.957110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471656576569562:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:03.957161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:03.950950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471657856431988:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19280
2025-11-27T18:20:03.961813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471657856431988:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.965569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471657856431896:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:03.969340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471657856431989:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19280
2025-11-27T18:20:03.987551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471657856431894:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9830) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471657856431985:2119] held: no
2025-11-27T18:20:03.988733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471657856431992:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19280
2025-11-27T18:20:03.992076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471657856431985:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:04.004771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471657856431894:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:04.004832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:04.004884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471657856431894:2048] [node 1] ICP22 created new session: [5:7577471662151399289:2048]
2025-11-27T18:20:04.004930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471662151399289:2048] [node 1] ICS09 handshake done sender: [5:7577471657856431985:2119] self: [5:7577471657856431987:0] peer: [1:7577471659147215604:0] socket: 366 qp: -1
2025-11-27T18:20:04.004966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471662151399289:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:04.005001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:04.021609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471657856431988:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:04.036773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471657856431895:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18831) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471657856431982:2118] held: no
2025-11-27T18:20:04.044955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471657856431896:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:04.045022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:04.050318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471657856431896:2050] [node 3] ICP22 created new session: [5:7577471662151399291:2050]
2025-11-27T18:20:04.056304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471662151399291:2050] [node 3] ICS09 handshake done sender: [5:7577471657856431988:2120] self: [5:7577471657856431990:0] peer: [3:7577471659274201962:0] socket: 312 qp: -1
2025-11-27T18:20:04.056360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471662151399291:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:04.064478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:04.075735Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471657856431895:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(52568196ns) {HandleHandshakeStatus:395(52565519ns) {LogHandshakeFail:472(4384647ns) {UpdateErrorStateLog:545(7912ns)} DropOutgoingHandshake:457(2582ns) ProcessPendingSessionEvents:495(38180801ns) {unordered DropSessionEvent:512 num# 100 sum# 12982477ns max# 6940033ns}}}
2025-11-27T18:20:04.074739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471655062173916:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(156239206ns) {HandleHandshakeStatus:329(156235630ns) {DropIncomingHandshake:442(15035ns) DropOutgoingHandshake:457(456ns) SwitchToState:179(697ns) ProcessPendingSessionEvents:495(153823223ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 150897809ns max# 21465085ns}}}
2025-11-27T18:20:04.077134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471659357141274:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(90894420ns) {HandleHandshakeStatus:329(90889800ns) {DropIncomingHandshake:442(17623ns) DropOutgoingHandshake:457(653ns) SwitchToState:179(972ns) ProcessPendingSessionEvents:495(39422956ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39351447ns max# 14188980ns}}}
2025-11-27T18:20:04.056021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:04.058904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471657771447340:2102] [node 4] ICP22 created new session: [6:7577471662066414679:2102]
2025-11-27T18:20:04.060019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471662066414679:2102] [node 4] ICS09 handshake done sender: [6:7577471657771447375:2123] self: [6:7577471657771447381:0] peer: [4:7577471656576569544:0] socket: 311 qp: -1
2025-11-27T18:20:04.070914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471662066414679:2102] [node 4] ICS10 traffic start
2025-11-27T18:20:04.070994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:04.074163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471657771447282:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(195589008ns) {HandleHandshakeStatus:329(195586781ns) {DropIncomingHandshake:442(375ns) DropOutgoingHandshake:457(4696ns) SwitchToState:179(437ns) ProcessPendingSessionEvents:495(195214814ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 195026038ns max# 10336812ns}}}
2025-11-27T18:20:03.864373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:04.079021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471657856431991:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19280
2025-11-27T18:20:03.822777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471659147215614:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26150
2025-11-27T18:20:03.868989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471659274201953:2107] [node 4] ICP22 created new session: [3:7577471659274201981:2107]
2025-11-27T18:20:04.080672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471657856431982:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:03.877577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471659274201980:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21564
2025-11-27T18:20:03.883953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471659274201981:2107] [node 4] ICS09 handshake done sender: [3:7577471659274201961:2113] self: [3:7577471659274201960:0] peer: [4:7577471656576569550:0] socket: 333 qp: -1
2025-11-27T18:20:04.077774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471655062173915:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(196358448ns) {HandleHandshakeStatus:329(196355091ns) {DropIncomingHandshake:442(780ns) DropOutgoingHandshake:457(26883ns) SwitchToState:179(2675ns) ProcessPendingSessionEvents:495(141501907ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 125219813ns max# 16158777ns}}}
2025-11-27T18:20:03.884005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471659274201981:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:03.823427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471659147215587:2107] [node 4] ICP22 created new session: [1:7577471659147215613:2107]
2025-11-27T18:20:03.893426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:03.909460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471659274201970:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:03.911498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471659274201985:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21564
2025-11-27T18:20:03.911843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471654979234592:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.911924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:03.911977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471654979234592:2048] [node 1] ICP22 created new session: [3:7577471659274201986:2048]
2025-11-27T18:20:03.912023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471659274201986:2048] [node 1] ICS09 handshake done sender: [3:7577471659274201970:2117] self: [3:7577471659274201976:0] peer: [1:7577471659147215598:0] socket: 343 qp: -1
2025-11-27T18:20:03.912057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471659274201986:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:03.912096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:03.919386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471654979234592:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:03.929914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471659274201977:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:03.940480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471659274201952:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(205708316ns) {HandleHandshakeStatus:329(205704743ns) {DropIncomingHandshake:442(909ns) DropOutgoingHandshake:457(852ns) SwitchToState:179(528ns) ProcessPendingSessionEvents:495(147856117ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 147689822ns max# 11456229ns}}}
2025-11-27T18:20:03.940709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471659274201952:2109] [node 6] ICP18 handshake (actor [3:7577471659274201969:2116]) is too old
2025-11-27T18:20:03.942002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471654979234593:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:03.945341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:03.949500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471654979234593:2049] [node 2] ICP22 created new session: [3:7577471659274201988:2049]
2025-11-27T18:20:03.959327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471659274201988:2049] [node 2] ICS09 handshake done sender: [3:7577471659274201977:2119] self: [3:7577471659274201979:0] peer: [2:7577471659357141288:0] socket: 339 qp: -1
2025-11-27T18:20:03.962962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471659274201988:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:03.965552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:03.986096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471659274201966:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.986132Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471659274201966:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.992113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471659274201966:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:04.009416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471659274201950:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:04.010195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:04.014103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471659274201953:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(134605810ns) {HandleHandshakeStatus:329(134603361ns) {DropIncomingHandshake:442(16536ns) DropOutgoingHandshake:457(329ns) SwitchToState:179(387ns) ProcessPendingSessionEvents:495(72743737ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 62975651ns max# 7312608ns}}}
2025-11-27T18:20:04.025579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471659274201950:2108] [node 5] ICP22 created new session: [3:7577471663569169287:2108]
2025-11-27T18:20:04.025650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471663569169287:2108] [node 5] ICS09 handshake done sender: [3:7577471659274201966:2114] self: [3:7577471659274201962:0] peer: [5:7577471657856431990:0] socket: 304 qp: -1
2025-11-27T18:20:04.025694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471663569169287:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:04.025744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:04.048867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471659274201950:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17036755ns) {HandleHandshakeStatus:329(17033685ns) {DropIncomingHandshake:442(17037ns) DropOutgoingHandshake:457(542ns) SwitchToState:179(877ns) ProcessPendingSessionEvents:495(577346ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 525010ns max# 47277ns}}}
2025-11-27T18:20:04.075424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471654979234593:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(132571876ns) {HandleHandshakeStatus:329(132569224ns) {DropIncomingHandshake:442(582ns) DropOutgoingHandshake:457(3279426ns) SwitchToState:179(886ns) ProcessPendingSessionEvents:495(93146871ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 93029226ns max# 12837569ns}}}
2025-11-27T18:20:04.093402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471657856431895:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:04.097428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471657771447340:2102] [node 4] ICP35 event processing took too much time PendingConnection:279(35382547ns) {HandleHandshakeStatus:329(35379322ns) {DropIncomingHandshake:442(3207ns) DropOutgoingHandshake:457(230ns) SwitchToState:179(765ns) ProcessPendingSessionEvents:495(1284ns)}}
2025-11-27T18:20:03.830319Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471659147215599:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:03.830471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471659147215613:2107] [node 4] ICS09 handshake done sender: [1:7577471659147215594:2114] self: [1:7577471659147215593:0] peer: [4:7577471656576569548:0] socket: 321 qp: -1
2025-11-27T18:20:03.835243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471659147215613:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:03.837369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471659147215611:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.837556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471659147215601:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:03.852723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:03.857931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471654852248227:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.857973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:03.858022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471654852248227:2049] [node 3] ICP22 created new session: [1:7577471659147215619:2049]
2025-11-27T18:20:03.858064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471659147215619:2049] [node 3] ICS09 handshake done sender: [1:7577471659147215599:2115] self: [1:7577471659147215598:0] peer: [3:7577471659274201976:0] socket: 342 qp: -1
2025-11-27T18:20:03.858101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471659147215619:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:03.858142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:03.859439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471654852248226:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.859489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:03.859547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471654852248226:2048] [node 2] ICP22 created new session: [1:7577471659147215621:2048]
2025-11-27T18:20:03.859585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471659147215621:2048] [node 2] ICS09 handshake done sender: [1:7577471659147215601:2116] self: [1:7577471659147215600:0] peer: [2:7577471659357141305:0] socket: 358 qp: -1
2025-11-27T18:20:03.859612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471659147215621:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:03.859642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:03.865650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471654852248227:2049] [node 3] ICP18 handshake (actor [1:7577471659147215611:2118]) is too old
2025-11-27T18:20:03.882277Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471659147215587:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(67928787ns) {HandleHandshakeStatus:329(67926344ns) {DropIncomingHandshake:442(15957ns) DropOutgoingHandshake:457(425ns) SwitchToState:179(324ns) ProcessPendingSessionEvents:495(7885815ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7827786ns max# 7350404ns}}}
2025-11-27T18:20:03.907038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471659147215605:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:03.907078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471659147215605:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:03.941630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471659147215605:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:03.969353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471659147215585:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:03.969438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:03.977778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471659147215585:2108] [node 5] ICP22 created new session: [1:7577471659147215624:2108]
2025-11-27T18:20:04.106620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471656576569525:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(168175849ns) {HandleHandshakeStatus:329(168173393ns) {DropIncomingHandshake:442(12262ns) DropOutgoingHandshake:457(197ns) SwitchToState:179(371ns) ProcessPendingSessionEvents:495(167851807ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 167231601ns max# 16512743ns}}}
2025-11-27T18:20:03.992541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471659147215624:2108] [node 5] ICS09 handshake done sender: [1:7577471659147215605:2117] self: [1:7577471659147215604:0] peer: [5:7577471657856431987:0] socket: 350 qp: -1
2025-11-27T18:20:03.992747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471659147215624:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:03.992797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:04.078980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471659147215585:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(112577841ns) {HandleHandshakeStatus:329(112574104ns) {DropIncomingHandshake:442(26295ns) DropOutgoingHandshake:457(436ns) SwitchToState:179(586ns) ProcessPendingSessionEvents:495(59698846ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 59607148ns max# 11188787ns}}}
2025-11-27T18:20:04.093898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:04.102620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471657856431895:2049] [node 2] ICP22 created new session: [5:7577471662151399293:2049]
2025-11-27T18:20:04.106667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471662151399293:2049] [node 2] ICS09 handshake done sender: [5:7577471657856431982:2118] self: [5:7577471657856431986:0] peer: [2:7577471659357141286:0] socket: 357 qp: -1
2025-11-27T18:20:04.106724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471662151399293:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:04.106779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:04.109021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471657856431896:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(63272634ns) {HandleHandshakeStatus:329(63268506ns) {DropIncomingHandshake:442(867ns) DropOutgoingHandshake:457(8038ns) SwitchToState:179(1734ns) ProcessPendingSessionEvents:495(27974019ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27900778ns max# 8846580ns}}}
2025-11-27T18:20:04.109103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471657856431896:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:04.157148Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:04.157386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471659357141309:2048] [node 1] ICS07 socket disconnect 361 reason# EndOfStream
2025-11-27T18:20:04.157433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471659357141309:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.157515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:04.157558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471659357141309:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:04.162039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:04.162283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471659274201986:2048] [node 1] ICS07 socket disconnect 343 reason# EndOfStream
2025-11-27T18:20:04.162313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471659274201986:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.162401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:04.162459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471659274201986:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:04.162836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471655062173915:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9830) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:04.162981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471659357141309:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:04.163665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471654979234592:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9830) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:04.164447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471659274201986:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:04.164494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471654979234592:2048] [node 1] ICP30 unregister session Session# [3:7577471659274201986:2048] VirtualId# [3:7577471659274201976:0]
2025-11-27T18:20:04.164537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:04.164568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471654979234592:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9830) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:04.164581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9830) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:04.165664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471655062173915:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9830) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:04.168501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:04.168792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:04.170135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471662151399289:2048] [node 1] ICS01 socket: 366 reason# EndOfStream
2025-11-27T18:20:04.172083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471654979234592:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:04.165847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471659357141309:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:04.170329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471656576569551:2048] [node 1] ICS01 socket: 322 reason# EndOfStream
2025-11-27T18:20:04.176083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471657856431894:2048] [node 1] ICP30 unregister session Session# [5:7577471662151399289:2048] VirtualId# [5:7577471657856431987:0]
2025-11-27T18:20:04.176139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471662151399289:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.177962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:04.170373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471656576569459:2048] [node 1] ICP30 unregister session Session# [4:7577471656576569551:2048] VirtualId# [4:7577471656576569548:0]
2025-11-27T18:20:04.181568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471657771447376:2048] [node 1] ICS01 socket: 329 reason# EndOfStream
2025-11-27T18:20:04.176291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471656576569551:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.176224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:04.177053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:04.177130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:04.181610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471657771447280:2048] [node 1] ICP30 unregister session Session# [6:7577471657771447376:2048] VirtualId# [6:7577471657771447365:0]
2025-11-27T18:20:04.181645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471657771447376:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.177175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:04.181965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:04.184530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:04.250848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:04.251087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471659274201988:2049] [node 2] ICS07 socket disconnect 339 reason# EndOfStream
2025-11-27T18:20:04.251133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471659274201988:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.251220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:04.251281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471659274201988:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:04.252704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471654979234593:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18831) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:04.252776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471659274201988:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:04.252809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471654979234593:2049] [node 2] ICP30 unregister session Session# [3:7577471659274201988:2049] VirtualId# [3:7577471659274201979:0]
2025-11-27T18:20:04.252912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:04.252952Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471654979234593:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18831) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:04.252967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18831) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:04.259868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:04.259934Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:04.260111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471662151399293:2049] [node 2] ICS01 socket: 357 reason# EndOfStream
2025-11-27T18:20:04.260165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471657856431895:2049] [node 2] ICP30 unregister session Session# [5:7577471662151399293:2049] VirtualId# [5:7577471657856431986:0]
2025-11-27T18:20:04.260156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471656576569553:2049] [node 2] ICS07 socket disconnect 319 reason# EndOfStream
2025-11-27T18:20:04.260199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471662151399293:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.260197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471656576569553:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.260272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:04.260299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:04.260317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471656576569553:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:04.260460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:04.261369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471656576569460:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18831) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:04.263431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471654979234593:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:04.261436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471656576569553:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:04.261614Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:04.261477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471656576569460:2049] [node 2] ICP30 unregister session Session# [4:7577471656576569553:2049] VirtualId# [4:7577471656576569540:0]
2025-11-27T18:20:04.265386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:04.265440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471656576569460:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18831) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:04.265457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18831) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:04.273654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471656576569460:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:04.265686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471657771447379:2049] [node 2] ICS01 socket: 307 reason# EndOfStream
2025-11-27T18:20:04.265740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471657771447281:2049] [node 2] ICP30 unregister session Session# [6:7577471657771447379:2049] VirtualId# [6:7577471657771447362:0]
2025-11-27T18:20:04.265955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471657771447379:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.266056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:04.273477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:04.351600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:04.351851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471656576569562:2050] [node 3] ICS01 socket: 341 reason# EndOfStream
2025-11-27T18:20:04.351899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471656576569461:2050] [node 3] ICP30 unregister session Session# [4:7577471656576569562:2050] VirtualId# [4:7577471656576569550:0]
2025-11-27T18:20:04.351931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471656576569562:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.352007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:04.352047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:04.354155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:04.354382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471662151399291:2050] [node 3] ICS07 socket disconnect 312 reason# EndOfStream
2025-11-27T18:20:04.354423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471662151399291:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.354490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:04.354538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471662151399291:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:04.355503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471657856431896:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15444) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:04.355513Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:04.355748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471657771447372:2050] [node 3] ICS07 socket disconnect 330 reason# EndOfStream
2025-11-27T18:20:04.355586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471662151399291:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:04.355787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471657771447372:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.355621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471657856431896:2050] [node 3] ICP30 unregister session Session# [5:7577471662151399291:2050] VirtualId# [5:7577471657856431990:0]
2025-11-27T18:20:04.355855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:04.355847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:04.355893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471657856431896:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15444) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:04.355894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471657771447372:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:04.355930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15444) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:04.356741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471657771447282:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15444) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:04.356789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471657771447372:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:04.356816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471657771447282:2050] [node 3] ICP30 unregister session Session# [6:7577471657771447372:2050] VirtualId# [6:7577471657771447363:0]
2025-11-27T18:20:04.356902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:04.356932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471657771447282:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15444) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:04.356958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15444) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:04.365668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471657856431896:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:04.366633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471657771447282:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:04.435248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:04.435277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:04.435511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471657856431979:2108] [node 4] ICS01 socket: 309 reason# EndOfStream
2025-11-27T18:20:04.435559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471657856431956:2108] [node 4] ICP30 unregister session Session# [5:7577471657856431979:2108] VirtualId# [5:7577471657856431969:0]
2025-11-27T18:20:04.435590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471657856431979:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.435671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:04.435734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:04.435481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471662066414679:2102] [node 4] ICS01 socket: 311 reason# EndOfStream
2025-11-27T18:20:04.435542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471657771447340:2102] [node 4] ICP30 unregister session Session# [6:7577471662066414679:2102] VirtualId# [6:7577471657771447381:0]
2025-11-27T18:20:04.435579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471662066414679:2102] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.435656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:04.435727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:04.515122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:04.515390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471657771447371:2100] [node 5] ICS01 socket: 323 reason# EndOfStream
2025-11-27T18:20:04.515459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471657771447341:2100] [node 5] ICP30 unregister session Session# [6:7577471657771447371:2100] VirtualId# [6:7577471657771447364:0]
2025-11-27T18:20:04.515494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471657771447371:2100] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:04.515580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:04.515643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:04.638026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471662199149489:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:04.638092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471662199149490:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:04.662737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471661971161759:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:04.662824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471661971161760:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:04.686093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471664699266607:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:04.686135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471664699266608:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:04.708679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471664275371138:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:04.708724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471664275371139:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:04.708758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471664275371140:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:04.731013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471660845710694:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:04.731059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471660845710695:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:04.731101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471660845710696:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:04.756357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471664083751559:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:04.756402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471664083751560:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:04.756443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471664083751561:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:04.853451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471664083751618:2104] [node 5] ICP01 ready to work
2025-11-27T18:20:04.949358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471661971161822:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:04.952748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471660845710757:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:04.949880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471661971161823:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:04.950453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471661971161824:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:04.952730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471664275371201:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:04.954379Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471662199149489:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(100571339ns) {Configure:104(100562416ns) {ProcessConfigured:135(61602782ns) {StartInitialHandshake:157(61598809ns) {DropHandshakes:470(3115ns) {DropIncomingHandshake:442(693ns) DropOutgoingHandshake:457(416ns)} GenerateSessionVirtualId:988(933ns) PrepareNewSessionHandshake:477(4188ns) {SwitchToState:179(1852ns)}}}}}
2025-11-27T18:20:04.961442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471664275371202:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:04.952623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471664699266671:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:04.961537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471664083751623:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:04.954556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471664699266672:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:04.961743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471660845710758:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:04.969401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471662199149554:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:04.970063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471662199149555:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:04.970571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471662199149556:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:04.962393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471664699266673:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:04.981789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471661971161759:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30442110ns) {Configure:104(30435974ns) {ProcessConfigured:135(5773392ns) {StartInitialHandshake:157(5769768ns) {DropHandshakes:470(2139ns) {DropIncomingHandshake:442(676ns) DropOutgoingHandshake:457(327ns)} GenerateSessionVirtualId:988(832ns) PrepareNewSessionHandshake:477(3959ns) {SwitchToState:179(2160ns)}}}}}
2025-11-27T18:20:04.983657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471664275371138:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19808966ns) {Configure:104(19803401ns) {ProcessConfigured:135(238182ns) {StartInitialHandshake:157(236532ns) {DropHandshakes:470(1150ns) {DropIncomingHandshake:442(417ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(470ns) PrepareNewSessionHandshake:477(1491ns) {SwitchToState:179(568ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.001550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471666266129131:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15365
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.002852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471666266129132:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15365
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.012268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668994233980:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-27T18:20:05.012446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668994233981:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-27T18:20:05.012521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668994233982:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.019066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471666494116864:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8200
2025-11-27T18:20:05.022163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668994233981:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.021654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471666494116865:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8200
2025-11-27T18:20:05.022523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668994233980:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.023441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471666494116866:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8200
2025-11-27T18:20:05.024789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471666494116864:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.025149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471662199149558:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.025178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471662199149558:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.022927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471664699266607:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.023016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668994233986:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-27T18:20:05.025956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471666266129131:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.026489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471668570338509:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33876
2025-11-27T18:20:05.026621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471668570338510:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33876
2025-11-27T18:20:05.026697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471668570338511:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33876
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.027214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668994233986:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.027327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668994233982:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.027572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471666266129132:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:05.027696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471666266129136:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15365
2025-11-27T18:20:05.026248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471666494116867:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8200
2025-11-27T18:20:05.027311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471666494116866:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.028586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471668570338511:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.027622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471664699266608:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.028056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471666266129137:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15365
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.029411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668994233991:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-11-27T18:20:05.029480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668994233992:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.028445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471661971161759:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.028703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471668570338510:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.028871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471661971161828:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.028895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471661971161828:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.027404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471666494116865:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.029036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471668570338509:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.029398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471664275371140:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.030901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471664699266679:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.029458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471664275371139:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.027526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471666494116867:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.032292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471666494116869:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8200
2025-11-27T18:20:05.030934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471664699266679:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.029505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471664275371138:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.033369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471662199149560:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.033397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471662199149560:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.033402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471664699266679:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:05.033953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471662199149558:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:05.034911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471666266129136:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.035227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471666266129137:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.035374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471661971161830:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.035546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471664699266671:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.035655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:05.035722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471664699266671:2107] [node 4] ICP22 created new session: [3:7577471668994233994:2107]
2025-11-27T18:20:05.035687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471662199149553:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.035719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471662199149553:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.036285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471662199149490:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.036352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:05.035418Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471661971161830:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.036409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471662199149490:2049] [node 3] ICP22 created new session: [1:7577471666494116875:2049]
2025-11-27T18:20:05.036459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471666494116875:2049] [node 3] ICS09 handshake done sender: [1:7577471662199149558:2111] self: [1:7577471662199149557:0] peer: [3:7577471668994233988:0] socket: 307 qp: -1
2025-11-27T18:20:05.038319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471668994233994:2107] [node 4] ICS09 handshake done sender: [3:7577471664699266679:2113] self: [3:7577471664699266678:0] peer: [4:7577471668570338515:0] socket: 325 qp: -1
2025-11-27T18:20:05.036495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471666494116875:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:05.039092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471668994233994:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:05.036526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:05.039132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:05.038849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471662199149560:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:05.040353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471662199149554:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.040395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:05.040443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471662199149554:2107] [node 4] ICP22 created new session: [1:7577471666494116879:2107]
2025-11-27T18:20:05.040636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471666494116879:2107] [node 4] ICS09 handshake done sender: [1:7577471662199149560:2112] self: [1:7577471662199149559:0] peer: [4:7577471668570338517:0] socket: 308 qp: -1
2025-11-27T18:20:05.040663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471666494116879:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:05.040693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.042089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471666266129139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15365
2025-11-27T18:20:05.042287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471661971161828:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:05.043144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471661971161830:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:05.043674Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471662199149553:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.045553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471666494116869:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.045592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471661971161760:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.045678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471662199149489:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.045668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:05.045680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471665140678068:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44850
2025-11-27T18:20:05.045718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471661971161760:2049] [node 3] ICP22 created new session: [2:7577471666266129146:2049]
2025-11-27T18:20:05.045734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:05.045801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471662199149489:2048] [node 2] ICP22 created new session: [1:7577471666494116881:2048]
2025-11-27T18:20:05.045780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471666266129146:2049] [node 3] ICS09 handshake done sender: [2:7577471661971161828:2111] self: [2:7577471661971161827:0] peer: [3:7577471668994233990:0] socket: 317 qp: -1
2025-11-27T18:20:05.045839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471666494116881:2048] [node 2] ICS09 handshake done sender: [1:7577471662199149553:2110] self: [1:7577471662199149552:0] peer: [2:7577471666266129138:0] socket: 306 qp: -1
2025-11-27T18:20:05.045866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471666494116881:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:05.045816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471666266129146:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:05.045854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:05.045894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:05.045831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471668570338518:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33876
2025-11-27T18:20:05.045949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471668994233980:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:05.046397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471664699266607:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471668994233980:2116] held: no
2025-11-27T18:20:05.047023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471664699266607:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.047052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.047217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471668570338519:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33876
2025-11-27T18:20:05.047296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471664699266607:2048] [node 1] ICP22 created new session: [3:7577471668994233997:2048]
2025-11-27T18:20:05.047350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471668994233997:2048] [node 1] ICS09 handshake done sender: [3:7577471668994233980:2116] self: [3:7577471668994233988:0] peer: [1:7577471662199149557:0] socket: 320 qp: -1
2025-11-27T18:20:05.047383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471668994233997:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:05.047425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.048845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471668570338520:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33876
2025-11-27T18:20:05.049379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471666266129139:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.049611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471661971161822:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.049776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:05.049825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471661971161822:2107] [node 4] ICP22 created new session: [2:7577471666266129148:2107]
2025-11-27T18:20:05.049872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471666266129148:2107] [node 4] ICS09 handshake done sender: [2:7577471661971161830:2112] self: [2:7577471661971161829:0] peer: [4:7577471668570338516:0] socket: 318 qp: -1
2025-11-27T18:20:05.049906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471666266129148:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:05.049937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.050926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668994233996:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.051438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471662199149554:2107] [node 4] ICP18 handshake (actor [1:7577471666494116869:2119]) is too old
2025-11-27T18:20:05.051550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471666266129143:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15365
2025-11-27T18:20:05.052515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471668994233982:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.053226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471665140678068:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.053331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668994233991:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.053396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471664699266608:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.053359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471665140678069:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44850
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.053476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471660845710694:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.053439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:05.053510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471664699266608:2049] [node 2] ICP22 created new session: [3:7577471668994234001:2049]
2025-11-27T18:20:05.053458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668378718933:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3091
2025-11-27T18:20:05.053768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471668994234001:2049] [node 2] ICS09 handshake done sender: [3:7577471668994233982:2118] self: [3:7577471668994233990:0] peer: [2:7577471661971161827:0] socket: 322 qp: -1
2025-11-27T18:20:05.053768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668378718934:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3091
2025-11-27T18:20:05.053830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471668994234001:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:05.053865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:05.054354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471662199149562:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.054389Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471662199149562:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.054561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471668570338511:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:05.055447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471668570338509:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:05.055889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471662199149562:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:05.055700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471668378718933:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.055834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471664083751559:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.056350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471668570338510:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:05.056466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471661971161759:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471666266129131:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.056773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471662199149564:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.056806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471664275371138:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471668570338509:2115] held: no
2025-11-27T18:20:05.056805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471662199149564:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.056786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471665140678071:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44850
2025-11-27T18:20:05.057815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471665140678068:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:05.057873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471664275371138:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.057923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:05.057967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471664275371138:2048] [node 1] ICP22 created new session: [4:7577471668570338521:2048]
2025-11-27T18:20:05.058030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471668570338521:2048] [node 1] ICS09 handshake done sender: [4:7577471668570338509:2115] self: [4:7577471668570338517:0] peer: [1:7577471662199149559:0] socket: 331 qp: -1
2025-11-27T18:20:05.058072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471668570338521:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:05.057721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471662199149555:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.058110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:05.057788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:05.061090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471660845710694:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.057833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471662199149555:2108] [node 5] ICP22 created new session: [1:7577471666494116885:2108]
2025-11-27T18:20:05.057874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471666494116885:2108] [node 5] ICS09 handshake done sender: [1:7577471662199149562:2113] self: [1:7577471662199149561:0] peer: [5:7577471665140678070:0] socket: 358 qp: -1
2025-11-27T18:20:05.061232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:05.061386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471660845710694:2048] [node 1] ICP22 created new session: [5:7577471665140678072:2048]
2025-11-27T18:20:05.057910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471666494116885:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:05.061468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471665140678072:2048] [node 1] ICS09 handshake done sender: [5:7577471665140678068:2115] self: [5:7577471665140678070:0] peer: [1:7577471662199149561:0] socket: 359 qp: -1
2025-11-27T18:20:05.057954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:05.061514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471665140678072:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:05.061577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:05.062421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471666266129131:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:05.062307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471664275371140:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.062089Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471664699266608:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:05.062353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.062392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471664275371140:2050] [node 3] ICP22 created new session: [4:7577471668570338523:2050]
2025-11-27T18:20:05.062438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471668570338523:2050] [node 3] ICS09 handshake done sender: [4:7577471668570338511:2117] self: [4:7577471668570338515:0] peer: [3:7577471664699266678:0] socket: 333 qp: -1
2025-11-27T18:20:05.062466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471668570338523:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:05.062505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:05.063364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668378718936:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3091
2025-11-27T18:20:05.064730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471668378718934:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.065365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471664083751561:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.067067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471664275371139:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.067117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:05.065429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471664699266671:2107] [node 4] ICP18 handshake (actor [3:7577471668994233991:2120]) is too old
2025-11-27T18:20:05.065420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471661971161822:2107] [node 4] ICP18 handshake (actor [2:7577471666266129139:2119]) is too old
2025-11-27T18:20:05.068184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471664275371139:2049] [node 2] ICP22 created new session: [4:7577471668570338525:2049]
2025-11-27T18:20:05.068632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471668570338525:2049] [node 2] ICS09 handshake done sender: [4:7577471668570338510:2116] self: [4:7577471668570338516:0] peer: [2:7577471661971161829:0] socket: 332 qp: -1
2025-11-27T18:20:05.068687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471668570338525:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:05.065527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471661971161759:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.068731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:05.065568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:05.065622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471661971161759:2048] [node 1] ICP22 created new session: [2:7577471666266129152:2048]
2025-11-27T18:20:05.065663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471666266129152:2048] [node 1] ICS09 handshake done sender: [2:7577471666266129131:2115] self: [2:7577471666266129138:0] peer: [1:7577471662199149552:0] socket: 310 qp: -1
2025-11-27T18:20:05.065697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471666266129152:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:05.068912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668378718938:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3091
2025-11-27T18:20:05.065727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:05.069009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668994233977:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.069060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471668994233977:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.073443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471665140678074:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44850
2025-11-27T18:20:05.074242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471662199149564:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:05.075390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471662199149556:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.075461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:05.075508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471662199149556:2109] [node 6] ICP22 created new session: [1:7577471666494116888:2109]
2025-11-27T18:20:05.075683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471666494116888:2109] [node 6] ICS09 handshake done sender: [1:7577471662199149564:2114] self: [1:7577471662199149563:0] peer: [6:7577471668378718935:0] socket: 328 qp: -1
2025-11-27T18:20:05.075745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471666494116888:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:05.075787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:05.075850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471668994233977:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:05.078450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471668378718933:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:05.078622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471665140678069:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.078994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471660845710696:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.078455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471664699266673:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.078519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:05.078577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471664699266673:2110] [node 6] ICP22 created new session: [3:7577471668994234004:2110]
2025-11-27T18:20:05.080478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471664083751559:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.080538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:05.080602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471664083751559:2048] [node 1] ICP22 created new session: [6:7577471668378718940:2048]
2025-11-27T18:20:05.080820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471668378718940:2048] [node 1] ICS09 handshake done sender: [6:7577471668378718933:2115] self: [6:7577471668378718935:0] peer: [1:7577471662199149563:0] socket: 361 qp: -1
2025-11-27T18:20:05.080860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471668378718940:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:05.080889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:05.078627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471668994234004:2110] [node 6] ICS09 handshake done sender: [3:7577471668994233977:2114] self: [3:7577471668994233976:0] peer: [6:7577471668378718937:0] socket: 360 qp: -1
2025-11-27T18:20:05.078658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471668994234004:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:05.078695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:05.083290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668994233979:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.083319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471668994233979:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.084292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471668994233979:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:05.084518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471665140678077:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44850
2025-11-27T18:20:05.086706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471664699266672:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.086762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.087521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471668570338527:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33876
2025-11-27T18:20:05.085892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471665140678069:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:05.087782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471664699266672:2109] [node 5] ICP22 created new session: [3:7577471668994234007:2109]
2025-11-27T18:20:05.087848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471668994234007:2109] [node 5] ICS09 handshake done sender: [3:7577471668994233979:2115] self: [3:7577471668994233978:0] peer: [5:7577471665140678076:0] socket: 342 qp: -1
2025-11-27T18:20:05.086223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471660845710696:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.088340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471668994234007:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:05.088376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:05.086253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:05.086292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471660845710696:2050] [node 3] ICP22 created new session: [5:7577471665140678078:2050]
2025-11-27T18:20:05.086540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471665140678078:2050] [node 3] ICS09 handshake done sender: [5:7577471665140678069:2116] self: [5:7577471665140678076:0] peer: [3:7577471668994233978:0] socket: 364 qp: -1
2025-11-27T18:20:05.086661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471665140678078:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:05.086684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:05.090825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471665140678074:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.087256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668378718939:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3091
2025-11-27T18:20:05.092219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471668570338527:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.092308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471660845710695:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.092475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471665140678080:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44850
2025-11-27T18:20:05.093060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471661971161832:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.093094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471661971161832:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.095358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471661971161832:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:05.095352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471665140678080:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.096125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471661971161823:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.096192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:05.096240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471661971161823:2108] [node 5] ICP22 created new session: [2:7577471666266129155:2108]
2025-11-27T18:20:05.096286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471666266129155:2108] [node 5] ICS09 handshake done sender: [2:7577471661971161832:2113] self: [2:7577471661971161831:0] peer: [5:7577471665140678082:0] socket: 324 qp: -1
2025-11-27T18:20:05.096339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471666266129155:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:05.097929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471664083751561:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471668378718934:2116] held: no
2025-11-27T18:20:05.096375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:05.099191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471668378718934:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:05.099688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471668378718936:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.100334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471664083751561:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.100375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:05.100535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471664083751561:2050] [node 3] ICP22 created new session: [6:7577471668378718944:2050]
2025-11-27T18:20:05.100849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471668378718944:2050] [node 3] ICS09 handshake done sender: [6:7577471668378718934:2116] self: [6:7577471668378718937:0] peer: [3:7577471668994233976:0] socket: 362 qp: -1
2025-11-27T18:20:05.100930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471668378718944:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:05.100974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:05.101019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471660845710757:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.101121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471660845710695:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1340) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471665140678074:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.101803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471665140678083:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44850
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.102673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668378718942:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3091
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.102724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471665140678074:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:05.102804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668378718943:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3091
2025-11-27T18:20:05.102967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471664083751560:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.102437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471668570338506:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.102476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471668570338506:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.103287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471661971161834:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.103324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471661971161834:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.104264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471661971161834:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:05.104999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471668570338506:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:05.105166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471660845710695:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.105243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:05.105347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471660845710695:2049] [node 2] ICP22 created new session: [5:7577471665140678087:2049]
2025-11-27T18:20:05.105454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471665140678087:2049] [node 2] ICS09 handshake done sender: [5:7577471665140678074:2118] self: [5:7577471665140678082:0] peer: [2:7577471661971161831:0] socket: 330 qp: -1
2025-11-27T18:20:05.105500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471665140678087:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:05.105535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:05.105520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471664275371201:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.104832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471661971161824:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.105576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:05.104913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:05.105615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471664275371201:2108] [node 5] ICP22 created new session: [4:7577471668570338533:2108]
2025-11-27T18:20:05.104969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471661971161824:2109] [node 6] ICP22 created new session: [2:7577471666266129158:2109]
2025-11-27T18:20:05.105661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471668570338533:2108] [node 5] ICS09 handshake done sender: [4:7577471668570338506:2113] self: [4:7577471668570338505:0] peer: [5:7577471665140678084:0] socket: 344 qp: -1
2025-11-27T18:20:05.105025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471666266129158:2109] [node 6] ICS09 handshake done sender: [2:7577471661971161834:2114] self: [2:7577471661971161833:0] peer: [6:7577471668378718948:0] socket: 341 qp: -1
2025-11-27T18:20:05.105061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471666266129158:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:05.105102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:05.106498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471668378718942:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.106627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471668378718943:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.105689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471668570338533:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:05.105721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.108878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471665140678085:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44850
2025-11-27T18:20:05.109019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471665140678086:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44850
2025-11-27T18:20:05.108907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471668570338532:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33876
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.109438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668378718949:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3091
2025-11-27T18:20:05.109869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471664083751560:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1340) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471668378718936:2117] held: no
2025-11-27T18:20:05.110776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471664083751623:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.110858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471664083751618:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.111262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471668378718936:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:05.111552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471668570338532:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.111762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471664083751560:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.111819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:05.111870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471664083751560:2049] [node 2] ICP22 created new session: [6:7577471668378718952:2049]
2025-11-27T18:20:05.111871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471668570338508:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.111914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471668378718952:2049] [node 2] ICS09 handshake done sender: [6:7577471668378718936:2117] self: [6:7577471668378718948:0] peer: [2:7577471661971161833:0] socket: 363 qp: -1
2025-11-27T18:20:05.111947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471668378718952:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:05.111902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471668570338508:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.112112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:05.112777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471668570338508:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:05.113104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471664275371202:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.113153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:05.113201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471664275371202:2109] [node 6] ICP22 created new session: [4:7577471668570338537:2109]
2025-11-27T18:20:05.113245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471668570338537:2109] [node 6] ICS09 handshake done sender: [4:7577471668570338508:2114] self: [4:7577471668570338507:0] peer: [6:7577471668378718950:0] socket: 367 qp: -1
2025-11-27T18:20:05.113355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471668570338537:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:05.113395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.113981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668378718954:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3091
2025-11-27T18:20:05.115195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471665140678085:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.115312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471665140678064:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.115345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471665140678064:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.116293Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471660845710757:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21636) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471665140678080:2120] held: no
2025-11-27T18:20:05.116317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471668378718942:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:05.116993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471665140678080:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:05.116561Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471664083751623:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21636) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471668378718942:2120] held: no
2025-11-27T18:20:05.117216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471664083751623:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.117272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:05.117319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471664083751623:2109] [node 4] ICP22 created new session: [6:7577471668378718955:2109]
2025-11-27T18:20:05.117364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471668378718955:2109] [node 4] ICS09 handshake done sender: [6:7577471668378718942:2120] self: [6:7577471668378718950:0] peer: [4:7577471668570338507:0] socket: 338 qp: -1
2025-11-27T18:20:05.117401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471668378718955:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:05.117432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:05.117715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471660845710757:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.117755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:05.117792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471660845710757:2108] [node 4] ICP22 created new session: [5:7577471665140678090:2108]
2025-11-27T18:20:05.117840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471665140678090:2108] [node 4] ICS09 handshake done sender: [5:7577471665140678080:2120] self: [5:7577471665140678084:0] peer: [4:7577471668570338505:0] socket: 335 qp: -1
2025-11-27T18:20:05.117874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471665140678090:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:05.117901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.119149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471665140678064:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:05.119247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668378718957:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3091
2025-11-27T18:20:05.119617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471660845710758:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.119698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:05.119755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471660845710758:2109] [node 6] ICP22 created new session: [5:7577471665140678093:2109]
2025-11-27T18:20:05.119789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471668378718943:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:05.119800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471665140678093:2109] [node 6] ICS09 handshake done sender: [5:7577471665140678064:2114] self: [5:7577471665140678063:0] peer: [6:7577471668378718951:0] socket: 368 qp: -1
2025-11-27T18:20:05.119835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471665140678093:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:05.119862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:05.120048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471664083751618:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.120092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:05.120128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471664083751618:2104] [node 5] ICP22 created new session: [6:7577471668378718958:2104]
2025-11-27T18:20:05.120169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471668378718958:2104] [node 5] ICS09 handshake done sender: [6:7577471668378718943:2121] self: [6:7577471668378718951:0] peer: [5:7577471665140678063:0] socket: 340 qp: -1
2025-11-27T18:20:05.120205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471668378718958:2104] [node 5] ICS10 traffic start
2025-11-27T18:20:05.120233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:05.143869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:05.144122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471666266129152:2048] [node 1] ICS01 socket: 310 reason# EndOfStream
2025-11-27T18:20:05.144173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471661971161759:2048] [node 1] ICP30 unregister session Session# [2:7577471666266129152:2048] VirtualId# [2:7577471666266129138:0]
2025-11-27T18:20:05.144209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471666266129152:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.144331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:05.144391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:05.145285Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:05.145505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471668994233997:2048] [node 1] ICS01 socket: 320 reason# EndOfStream
2025-11-27T18:20:05.145553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471664699266607:2048] [node 1] ICP30 unregister session Session# [3:7577471668994233997:2048] VirtualId# [3:7577471668994233988:0]
2025-11-27T18:20:05.145580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471668994233997:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.145649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:05.145700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:05.147769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:05.147928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471668570338521:2048] [node 1] ICS01 socket: 331 reason# EndOfStream
2025-11-27T18:20:05.147966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471664275371138:2048] [node 1] ICP30 unregister session Session# [4:7577471668570338521:2048] VirtualId# [4:7577471668570338517:0]
2025-11-27T18:20:05.148008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471668570338521:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.148072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:05.148129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:05.148708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:05.148900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471665140678072:2048] [node 1] ICS07 socket disconnect 359 reason# EndOfStream
2025-11-27T18:20:05.148940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471665140678072:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.149009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:05.149076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471665140678072:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:05.149720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:05.149863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471668378718940:2048] [node 1] ICS07 socket disconnect 361 reason# EndOfStream
2025-11-27T18:20:05.149885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471668378718940:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.149862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471660845710694:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2080) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.149914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471665140678072:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.149941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:05.149949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471660845710694:2048] [node 1] ICP30 unregister session Session# [5:7577471665140678072:2048] VirtualId# [5:7577471665140678070:0]
2025-11-27T18:20:05.149974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471668378718940:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:05.150038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:05.150079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471660845710694:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2080) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.150094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2080) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.150614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471664083751559:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2080) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.150651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471668378718940:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.150670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471664083751559:2048] [node 1] ICP30 unregister session Session# [6:7577471668378718940:2048] VirtualId# [6:7577471668378718935:0]
2025-11-27T18:20:05.150732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:05.150754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471664083751559:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2080) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.150762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2080) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.160256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471660845710694:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:05.160256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471664083751559:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:05.196690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:05.196929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471668994234001:2049] [node 2] ICS01 socket: 322 reason# EndOfStream
2025-11-27T18:20:05.196981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471664699266608:2049] [node 2] ICP30 unregister session Session# [3:7577471668994234001:2049] VirtualId# [3:7577471668994233990:0]
2025-11-27T18:20:05.197029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471668994234001:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.197122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:05.197186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:05.199156Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:05.199373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471668570338525:2049] [node 2] ICS07 socket disconnect 332 reason# EndOfStream
2025-11-27T18:20:05.199416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471668570338525:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.199503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:05.199561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471668570338525:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:05.200110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:05.200337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471665140678087:2049] [node 2] ICS01 socket: 330 reason# EndOfStream
2025-11-27T18:20:05.200391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471660845710695:2049] [node 2] ICP30 unregister session Session# [5:7577471665140678087:2049] VirtualId# [5:7577471665140678082:0]
2025-11-27T18:20:05.200393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471664275371139:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1340) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.200426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471665140678087:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.200448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471668570338525:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.200482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471664275371139:2049] [node 2] ICP30 unregister session Session# [4:7577471668570338525:2049] VirtualId# [4:7577471668570338516:0]
2025-11-27T18:20:05.200493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:05.200524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:05.200559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471664275371139:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.200556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:05.200582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.201059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:05.201241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471668378718952:2049] [node 2] ICS01 socket: 363 reason# EndOfStream
2025-11-27T18:20:05.201293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471664083751560:2049] [node 2] ICP30 unregister session Session# [6:7577471668378718952:2049] VirtualId# [6:7577471668378718948:0]
2025-11-27T18:20:05.201321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471668378718952:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.201403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:05.201453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:05.210432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471664275371139:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:05.230749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:05.230956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471668570338523:2050] [node 3] ICS07 socket disconnect 333 reason# EndOfStream
2025-11-27T18:20:05.231007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471668570338523:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.231090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:05.231140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471668570338523:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:05.231801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:05.231974Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471664275371140:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.232013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471665140678078:2050] [node 3] ICS07 socket disconnect 364 reason# EndOfStream
2025-11-27T18:20:05.232046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471665140678078:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.232020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471668570338523:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.232045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471664275371140:2050] [node 3] ICP30 unregister session Session# [4:7577471668570338523:2050] VirtualId# [4:7577471668570338515:0]
2025-11-27T18:20:05.232107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:05.232127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:05.232149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471665140678078:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:05.232151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471664275371140:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.232161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.232858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471660845710696:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.232866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:05.232906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471665140678078:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.232936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471660845710696:2050] [node 3] ICP30 unregister session Session# [5:7577471665140678078:2050] VirtualId# [5:7577471665140678076:0]
2025-11-27T18:20:05.232970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:05.232995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471660845710696:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.233009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.233058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471668378718944:2050] [node 3] ICS01 socket: 362 reason# EndOfStream
2025-11-27T18:20:05.233106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471664083751561:2050] [node 3] ICP30 unregister session Session# [6:7577471668378718944:2050] VirtualId# [6:7577471668378718937:0]
2025-11-27T18:20:05.233138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471668378718944:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.233227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:05.233319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:05.243206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471660845710696:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:05.260217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:05.260490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471665140678090:2108] [node 4] ICS01 socket: 335 reason# EndOfStream
2025-11-27T18:20:05.260540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471660845710757:2108] [node 4] ICP30 unregister session Session# [5:7577471665140678090:2108] VirtualId# [5:7577471665140678084:0]
2025-11-27T18:20:05.260594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471665140678090:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.260670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:05.260717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:05.261325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:05.261516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471668378718955:2109] [node 4] ICS01 socket: 338 reason# EndOfStream
2025-11-27T18:20:05.261554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471664083751623:2109] [node 4] ICP30 unregister session Session# [6:7577471668378718955:2109] VirtualId# [6:7577471668378718950:0]
2025-11-27T18:20:05.261587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471668378718955:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.261642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:05.261673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:05.311998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:05.312237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471668378718958:2104] [node 5] ICS07 socket disconnect 340 reason# EndOfStream
2025-11-27T18:20:05.312277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471668378718958:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.312362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:05.312419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471668378718958:2104] [node 5] ICS15 start handshake
2025-11-27T18:20:05.313502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471664083751618:2104] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12975) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.313556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471668378718958:2104] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.313596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471664083751618:2104] [node 5] ICP30 unregister session Session# [6:7577471668378718958:2104] VirtualId# [6:7577471668378718951:0]
2025-11-27T18:20:05.313701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:05.313738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471664083751618:2104] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12975) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.313755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12975) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.324109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471664083751618:2104] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:05.380970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471668161178405:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:05.381001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471668161178406:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:05.400388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471668074831131:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:05.400732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471668074831132:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:05.420867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471668324445449:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:05.420934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471668324445450:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:05.442191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471666972070945:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:05.442241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471666972070947:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:05.442347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471666972070946:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:05.464215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471668883177973:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:05.464311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471668883177975:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:05.464521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471668883177974:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:05.485614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471668466693809:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:05.485647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471668466693810:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:05.485813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471668466693811:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:05.558966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471668466693869:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:05.579158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471668324445513:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:05.580553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471668883178038:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:05.580467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471668074831195:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:05.581077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471666972071008:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:05.581491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471668161178469:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:05.581667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471668466693871:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:05.581024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471668324445512:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:05.581700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471668883178037:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:05.581660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471668161178468:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:05.581086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471668074831196:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:05.582212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471668324445514:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:05.582239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471668161178470:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:05.582276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471666972071009:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:05.582158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471668074831197:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.594566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471668161178482:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.597293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471666972071018:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-11-27T18:20:05.597941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471668074831206:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7534
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.598009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471668883178047:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55850
2025-11-27T18:20:05.598382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471668074831207:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7534
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.599007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471668883178047:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.600436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471666972071024:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.601787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471668074831213:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7534
2025-11-27T18:20:05.601843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471668074831214:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7534
2025-11-27T18:20:05.602342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668324445523:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46932
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.603240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471668074831206:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.602954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668324445529:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46932
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.605084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471668161178485:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-27T18:20:05.605314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668324445530:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46932
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.605860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471668161178482:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.606025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471668074831215:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7534
2025-11-27T18:20:05.606113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471668161178487:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-27T18:20:05.605357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668324445531:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46932
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.606577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471668883178054:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55850
2025-11-27T18:20:05.606496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471668074831207:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.607102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471668161178486:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-27T18:20:05.607639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471666972071018:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.607202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471668161178487:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.608598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471668161178488:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44356
2025-11-27T18:20:05.608499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471666972070945:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.609950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668324445531:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.609949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471668161178476:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.610784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668324445523:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.609977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471668161178476:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.610559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471668161178486:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.611293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471668074831214:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.611396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471668074831213:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.611497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471668074831131:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.611548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471668161178476:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:05.611534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471668324445449:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.611586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471668161178488:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.611701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471668161178485:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.611729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471666972071026:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-11-27T18:20:05.611747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471666972071024:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.611925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471666972070946:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.611934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471668161178474:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.611962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471668161178474:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.612018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668324445530:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.612132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668324445529:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.612147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471668074831203:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.613952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471668324445450:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.613840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471668883178054:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.612196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471666972071018:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:05.614102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471668883177975:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.612155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471668074831215:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.612190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471668074831203:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.613885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471668161178472:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.613925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471668161178469:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.614635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668324445520:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.613931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471668161178472:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.614073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471666972070945:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.614097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:05.614669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471668324445520:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.614892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471668074831203:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:05.614119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:05.616439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471668074831201:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.614301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471668161178469:2107] [node 4] ICP22 created new session: [1:7577471668161178493:2107]
2025-11-27T18:20:05.614307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471666972070945:2048] [node 1] ICP22 created new session: [4:7577471666972071028:2048]
2025-11-27T18:20:05.616469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471668074831201:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.614647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471668161178493:2107] [node 4] ICS09 handshake done sender: [1:7577471668161178476:2112] self: [1:7577471668161178475:0] peer: [4:7577471666972071025:0] socket: 317 qp: -1
2025-11-27T18:20:05.614688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471668161178493:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:05.614631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471666972071028:2048] [node 1] ICS09 handshake done sender: [4:7577471666972071018:2114] self: [4:7577471666972071025:0] peer: [1:7577471668161178475:0] socket: 324 qp: -1
2025-11-27T18:20:05.618755Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471668324445520:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:05.614679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471666972071028:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:05.617912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471668074831195:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.614721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:05.614731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:05.618113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.618407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471668074831195:2107] [node 4] ICP22 created new session: [2:7577471668074831222:2107]
2025-11-27T18:20:05.618862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471668074831222:2107] [node 4] ICS09 handshake done sender: [2:7577471668074831203:2113] self: [2:7577471668074831202:0] peer: [4:7577471666972071027:0] socket: 331 qp: -1
2025-11-27T18:20:05.619012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471668074831222:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:05.622071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471668883178056:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55850
2025-11-27T18:20:05.622067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471668161178474:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:05.619056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.620972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471668074831201:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:05.621630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471668074831132:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.622001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:05.622441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471668324445513:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.622133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471668074831132:2049] [node 3] ICP22 created new session: [2:7577471668074831227:2049]
2025-11-27T18:20:05.622683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471666972071030:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-11-27T18:20:05.622505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:05.622183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471668074831227:2049] [node 3] ICS09 handshake done sender: [2:7577471668074831201:2112] self: [2:7577471668074831200:0] peer: [3:7577471668324445535:0] socket: 329 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.622743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471668324445513:2108] [node 5] ICP22 created new session: [3:7577471668324445539:2108]
2025-11-27T18:20:05.622868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471668324445539:2108] [node 5] ICS09 handshake done sender: [3:7577471668324445520:2112] self: [3:7577471668324445519:0] peer: [5:7577471668883178055:0] socket: 348 qp: -1
2025-11-27T18:20:05.622695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471668074831227:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:05.622903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471668324445539:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:05.622733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:05.623486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471668883178057:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55850
2025-11-27T18:20:05.622938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.623555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471668161178472:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:05.623963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471668161178406:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.623937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471668883178054:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:05.624047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:05.624087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471668883178058:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55850
2025-11-27T18:20:05.624313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471668883177975:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.624371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.624165Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471666972070946:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28189) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471666972071024:2116] held: no
2025-11-27T18:20:05.624582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471668883177975:2050] [node 3] ICP22 created new session: [5:7577471668883178059:2050]
2025-11-27T18:20:05.624201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471668161178406:2049] [node 3] ICP22 created new session: [1:7577471668161178500:2049]
2025-11-27T18:20:05.624653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471668161178500:2049] [node 3] ICS09 handshake done sender: [1:7577471668161178474:2111] self: [1:7577471668161178473:0] peer: [3:7577471668324445533:0] socket: 316 qp: -1
2025-11-27T18:20:05.625153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668324445537:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46932
2025-11-27T18:20:05.624693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471668161178500:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:05.624911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471668883178059:2050] [node 3] ICS09 handshake done sender: [5:7577471668883178054:2117] self: [5:7577471668883178055:0] peer: [3:7577471668324445519:0] socket: 349 qp: -1
2025-11-27T18:20:05.624726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:05.624942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471668883178059:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:05.625072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.626950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668324445538:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46932
2025-11-27T18:20:05.627528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471666972071024:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.627810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471668883178061:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55850
2025-11-27T18:20:05.627918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471668074831225:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7534
2025-11-27T18:20:05.628003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471668324445523:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:05.628068Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471668324445529:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:05.628759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471666972070946:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.628871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:05.628858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471668324445449:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.628920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471666972070946:2049] [node 2] ICP22 created new session: [4:7577471666972071033:2049]
2025-11-27T18:20:05.628920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:05.628977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471668324445449:2048] [node 1] ICP22 created new session: [3:7577471668324445542:2048]
2025-11-27T18:20:05.629017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471668324445542:2048] [node 1] ICS09 handshake done sender: [3:7577471668324445523:2114] self: [3:7577471668324445533:0] peer: [1:7577471668161178473:0] socket: 323 qp: -1
2025-11-27T18:20:05.628993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471666972071033:2049] [node 2] ICS09 handshake done sender: [4:7577471666972071024:2116] self: [4:7577471666972071027:0] peer: [2:7577471668074831202:0] socket: 341 qp: -1
2025-11-27T18:20:05.629027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471666972071033:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:05.629047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471668324445542:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:05.629057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:05.629048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471668074831206:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:05.629215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:05.629048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471668161178405:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.629231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:05.629487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471668161178405:2048] [node 2] ICP22 created new session: [1:7577471668161178502:2048]
2025-11-27T18:20:05.629988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471668161178502:2048] [node 2] ICS09 handshake done sender: [1:7577471668161178472:2110] self: [1:7577471668161178471:0] peer: [2:7577471668074831216:0] socket: 314 qp: -1
2025-11-27T18:20:05.629980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471668074831131:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.630166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471668161178502:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:05.630203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:05.630184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:05.630226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471668074831131:2048] [node 1] ICP22 created new session: [2:7577471668074831230:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.630333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471668074831230:2048] [node 1] ICS09 handshake done sender: [2:7577471668074831206:2115] self: [2:7577471668074831216:0] peer: [1:7577471668161178471:0] socket: 321 qp: -1
2025-11-27T18:20:05.630358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471668074831230:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:05.630576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:05.630977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471668324445450:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.631027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:05.630901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668466693887:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1342
2025-11-27T18:20:05.631074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471668324445450:2049] [node 2] ICP22 created new session: [3:7577471668324445544:2049]
2025-11-27T18:20:05.631122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471668324445544:2049] [node 2] ICS09 handshake done sender: [3:7577471668324445529:2116] self: [3:7577471668324445535:0] peer: [2:7577471668074831200:0] socket: 337 qp: -1
2025-11-27T18:20:05.631178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471668324445544:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:05.631205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.631315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668466693888:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1342
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.632136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471666972071035:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-11-27T18:20:05.632218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471666972071036:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-11-27T18:20:05.634556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471666972071036:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.635120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471668466693888:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.635306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471668883178061:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.635393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471668466693871:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.635481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471668883178038:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.635603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471668883178057:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.635655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471668324445450:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:05.635604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471666972071023:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.635634Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471666972071023:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.635710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471666972071017:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.635742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471668883177974:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.635732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471666972071017:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.636408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471666972071023:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:05.636747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471668883178058:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.636451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471668074831205:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.638593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471666972071009:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.636480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471668074831205:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.638870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:05.638916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471666972071009:2109] [node 6] ICP22 created new session: [4:7577471666972071040:2109]
2025-11-27T18:20:05.638636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471668883177973:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.639029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668466693890:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1342
2025-11-27T18:20:05.639055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471666972071040:2109] [node 6] ICS09 handshake done sender: [4:7577471666972071023:2115] self: [4:7577471666972071022:0] peer: [6:7577471668466693889:0] socket: 370 qp: -1
2025-11-27T18:20:05.639637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471668161178478:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.639287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471668074831205:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:05.639625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668466693891:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1342
2025-11-27T18:20:05.639674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471668161178478:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.639188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471666972071040:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:05.639441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:05.640272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471668074831196:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.640426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471668466693888:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:05.639844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471666972071017:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:05.640450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.641178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471668466693891:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.640932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471668074831196:2108] [node 5] ICP22 created new session: [2:7577471668074831233:2108]
2025-11-27T18:20:05.641237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471668466693871:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.641623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471666972071008:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.641615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471668883178067:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55850
2025-11-27T18:20:05.641291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:05.640965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471668074831233:2108] [node 5] ICS09 handshake done sender: [2:7577471668074831205:2114] self: [2:7577471668074831204:0] peer: [5:7577471668883178064:0] socket: 362 qp: -1
2025-11-27T18:20:05.641636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471668161178478:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.641518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471668466693871:2107] [node 4] ICP22 created new session: [6:7577471668466693892:2107]
2025-11-27T18:20:05.641302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471668074831233:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:05.643201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471668161178468:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.641686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:05.641782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471668466693892:2107] [node 4] ICS09 handshake done sender: [6:7577471668466693888:2116] self: [6:7577471668466693889:0] peer: [4:7577471666972071022:0] socket: 371 qp: -1
2025-11-27T18:20:05.641504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:05.643261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471668883178066:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55850
2025-11-27T18:20:05.641898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471666972071008:2108] [node 5] ICP22 created new session: [4:7577471666972071042:2108]
2025-11-27T18:20:05.641913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471668466693892:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:05.643770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:05.642135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:05.643823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471668161178468:2108] [node 5] ICP22 created new session: [1:7577471668161178506:2108]
2025-11-27T18:20:05.643179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471666972071042:2108] [node 5] ICS09 handshake done sender: [4:7577471666972071017:2113] self: [4:7577471666972071016:0] peer: [5:7577471668883178063:0] socket: 328 qp: -1
2025-11-27T18:20:05.643228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471666972071042:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:05.643963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471668161178506:2108] [node 5] ICS09 handshake done sender: [1:7577471668161178478:2113] self: [1:7577471668161178477:0] peer: [5:7577471668883178065:0] socket: 365 qp: -1
2025-11-27T18:20:05.643637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:05.644162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471668466693809:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.644350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471668883178057:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:05.643998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471668161178506:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:05.644084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:05.647011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471668883177974:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28189) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471668883178057:2119] held: no
2025-11-27T18:20:05.647311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471668466693887:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.647384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471668161178484:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.647417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471668161178484:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.649389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668466693894:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1342
2025-11-27T18:20:05.649423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471668883177974:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.649473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:05.649634Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471668161178484:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:05.649592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471668883177974:2049] [node 2] ICP22 created new session: [5:7577471668883178069:2049]
2025-11-27T18:20:05.649696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471668466693810:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.649722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471668883178069:2049] [node 2] ICS09 handshake done sender: [5:7577471668883178057:2119] self: [5:7577471668883178064:0] peer: [2:7577471668074831204:0] socket: 363 qp: -1
2025-11-27T18:20:05.649746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471668324445547:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46932
2025-11-27T18:20:05.649768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471668883178069:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:05.649802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:05.650196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471668883178061:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:05.650377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471668161178470:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.650413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471668466693894:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.650405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471668074831212:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.650445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.650495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471668161178470:2109] [node 6] ICP22 created new session: [1:7577471668161178509:2109]
2025-11-27T18:20:05.650439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471668074831212:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.650574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471668161178509:2109] [node 6] ICS09 handshake done sender: [1:7577471668161178484:2115] self: [1:7577471668161178483:0] peer: [6:7577471668466693895:0] socket: 327 qp: -1
2025-11-27T18:20:05.650552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471666972071035:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.650585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471668883178068:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55850
2025-11-27T18:20:05.650628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471668161178509:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:05.650827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:05.651075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471666972070947:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.651557Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471668883177973:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471668883178058:2120] held: no
2025-11-27T18:20:05.651729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668324445522:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.651711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471668074831212:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:05.651759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471668324445522:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.652453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471668074831197:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.652663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:05.653561Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471668466693809:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471668466693891:2118] held: no
2025-11-27T18:20:05.653799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471668324445522:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:05.653452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471668074831197:2110] [node 6] ICP22 created new session: [2:7577471668074831236:2110]
2025-11-27T18:20:05.653729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471668074831236:2110] [node 6] ICS09 handshake done sender: [2:7577471668074831212:2117] self: [2:7577471668074831211:0] peer: [6:7577471668466693897:0] socket: 350 qp: -1
2025-11-27T18:20:05.654175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471668883178038:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.654229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:05.653936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471668074831236:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:05.654191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:05.654412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471668324445512:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.654508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471668883178038:2108] [node 4] ICP22 created new session: [5:7577471668883178071:2108]
2025-11-27T18:20:05.654490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:05.654611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471668324445512:2107] [node 4] ICP22 created new session: [3:7577471668324445549:2107]
No RDMA context found for GID: 2025-11-27T18:20:05.654637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471668883178071:2108] [node 4] ICS09 handshake done sender: [5:7577471668883178061:2121] self: [5:7577471668883178063:0] peer: [4:7577471666972071016:0] socket: 340 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:05.654924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471668324445549:2107] [node 4] ICS09 handshake done sender: [3:7577471668324445522:2113] self: [3:7577471668324445521:0] peer: [4:7577471666972071044:0] socket: 344 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.654992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471668324445549:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:05.654672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471668883178071:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:05.655036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:05.655046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471666972071045:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-11-27T18:20:05.654710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:05.655099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668466693898:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1342
2025-11-27T18:20:05.655361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471668466693811:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.655387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471666972071035:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:05.655883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471666972070947:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.656024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:05.656145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471666972070947:2050] [node 3] ICP22 created new session: [4:7577471666972071046:2050]
2025-11-27T18:20:05.656293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471666972071046:2050] [node 3] ICS09 handshake done sender: [4:7577471666972071035:2119] self: [4:7577471666972071044:0] peer: [3:7577471668324445521:0] socket: 368 qp: -1
2025-11-27T18:20:05.656353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471666972071046:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:05.656389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:05.656452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471668883178058:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:05.656493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668324445547:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.656455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471668883178038:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:05.658729Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471668466693810:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28189) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471668466693887:2115] held: no
2025-11-27T18:20:05.658741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471668324445525:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.658784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471668883177973:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.658837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:05.658776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471668324445525:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.659066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471668883177973:2048] [node 1] ICP22 created new session: [5:7577471668883178074:2048]
2025-11-27T18:20:05.659124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471668883178074:2048] [node 1] ICS09 handshake done sender: [5:7577471668883178058:2120] self: [5:7577471668883178065:0] peer: [1:7577471668161178477:0] socket: 366 qp: -1
2025-11-27T18:20:05.659051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471668466693891:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:05.659161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471668883178074:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:05.660079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:05.660275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471666972070947:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.660440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668466693899:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1342
2025-11-27T18:20:05.660449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471668324445525:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:05.660459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471668466693809:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.660502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:05.660539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471668466693809:2048] [node 1] ICP22 created new session: [6:7577471668466693901:2048]
2025-11-27T18:20:05.660578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471668466693901:2048] [node 1] ICS09 handshake done sender: [6:7577471668466693891:2118] self: [6:7577471668466693895:0] peer: [1:7577471668161178483:0] socket: 372 qp: -1
2025-11-27T18:20:05.660613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471668466693901:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:05.660648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:05.662296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471668324445514:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.662373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:05.662415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471668324445514:2109] [node 6] ICP22 created new session: [3:7577471668324445553:2109]
2025-11-27T18:20:05.662476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471668324445553:2109] [node 6] ICS09 handshake done sender: [3:7577471668324445525:2115] self: [3:7577471668324445524:0] peer: [6:7577471668466693900:0] socket: 378 qp: -1
2025-11-27T18:20:05.662514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471668324445553:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:05.662546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:05.663310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471668466693887:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.663561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668466693903:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1342
2025-11-27T18:20:05.663660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471668466693810:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.663700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:05.663822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471668466693810:2049] [node 2] ICP22 created new session: [6:7577471668466693905:2049]
2025-11-27T18:20:05.664076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471668466693905:2049] [node 2] ICS09 handshake done sender: [6:7577471668466693887:2115] self: [6:7577471668466693897:0] peer: [2:7577471668074831211:0] socket: 352 qp: -1
2025-11-27T18:20:05.664103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471668466693905:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:05.664143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.664420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668466693904:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1342
2025-11-27T18:20:05.665216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471668466693811:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21687) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471668466693894:2119] held: no
2025-11-27T18:20:05.665929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471668466693904:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.665996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471668466693894:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:05.666316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471668466693811:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.666353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:05.666394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471668466693869:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:05.666396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471668466693811:2050] [node 3] ICP22 created new session: [6:7577471668466693907:2050]
2025-11-27T18:20:05.666434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471668466693907:2050] [node 3] ICS09 handshake done sender: [6:7577471668466693894:2119] self: [6:7577471668466693900:0] peer: [3:7577471668324445524:0] socket: 338 qp: -1
2025-11-27T18:20:05.666462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471668466693907:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:05.666491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:05.666639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471668883178052:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:05.666686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471668883178052:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:05.667528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471668883178052:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:05.667675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471668466693910:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1342
2025-11-27T18:20:05.667778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471668883178037:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:05.667857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:05.667911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471668466693904:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:05.667920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471668883178037:2110] [node 6] ICP22 created new session: [5:7577471668883178077:2110]
2025-11-27T18:20:05.668032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471668883178077:2110] [node 6] ICS09 handshake done sender: [5:7577471668883178052:2116] self: [5:7577471668883178048:0] peer: [6:7577471668466693908:0] socket: 330 qp: -1
2025-11-27T18:20:05.668168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471668466693869:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:05.668177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471668883178077:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:05.668225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:05.668304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471668466693869:2105] [node 5] ICP22 created new session: [6:7577471668466693911:2105]
2025-11-27T18:20:05.668218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:05.668375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471668466693911:2105] [node 5] ICS09 handshake done sender: [6:7577471668466693904:2123] self: [6:7577471668466693908:0] peer: [5:7577471668883178048:0] socket: 367 qp: -1
2025-11-27T18:20:05.668410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471668466693911:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:05.668439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:05.669060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471668466693869:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:05.713307Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:05.713569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471668074831230:2048] [node 1] ICS07 socket disconnect 321 reason# EndOfStream
2025-11-27T18:20:05.713612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471668074831230:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.713692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:05.713746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471668074831230:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:05.714900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:05.715102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471668324445542:2048] [node 1] ICS07 socket disconnect 323 reason# EndOfStream
2025-11-27T18:20:05.715136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471668324445542:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.715209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:05.715264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471668324445542:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:05.716619Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471668074831131:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:05.716688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471668074831230:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:05.717946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471668324445449:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:05.717990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471668324445542:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:05.719074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471668074831131:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.719088Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:05.719181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471668074831230:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.719357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471666972071028:2048] [node 1] ICS07 socket disconnect 324 reason# EndOfStream
2025-11-27T18:20:05.719394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471666972071028:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.719399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471668074831131:2048] [node 1] ICP30 unregister session Session# [2:7577471668074831230:2048] VirtualId# [2:7577471668074831216:0]
2025-11-27T18:20:05.719464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:05.719466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:05.719507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471668074831131:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.719522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.719511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471666972071028:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:05.720100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471666972070945:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.720157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471666972071028:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.720277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471666972070945:2048] [node 1] ICP30 unregister session Session# [4:7577471666972071028:2048] VirtualId# [4:7577471666972071025:0]
2025-11-27T18:20:05.720329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:05.720495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471666972070945:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.720511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.720618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471668324445449:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.720677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471668324445542:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.720714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471668324445449:2048] [node 1] ICP30 unregister session Session# [3:7577471668324445542:2048] VirtualId# [3:7577471668324445533:0]
2025-11-27T18:20:05.720882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:05.720917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471668324445449:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.720931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17581) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.722017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:05.722233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471668883178074:2048] [node 1] ICS01 socket: 366 reason# EndOfStream
2025-11-27T18:20:05.722282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471668883177973:2048] [node 1] ICP30 unregister session Session# [5:7577471668883178074:2048] VirtualId# [5:7577471668883178065:0]
2025-11-27T18:20:05.722319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471668883178074:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.722398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:05.722463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:05.723090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:05.723272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471668466693901:2048] [node 1] ICS01 socket: 372 reason# EndOfStream
2025-11-27T18:20:05.723315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471668466693809:2048] [node 1] ICP30 unregister session Session# [6:7577471668466693901:2048] VirtualId# [6:7577471668466693895:0]
2025-11-27T18:20:05.723342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471668466693901:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.723404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:05.723448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:05.729560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471668074831131:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:05.730606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471666972070945:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:05.731623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471668324445449:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:05.795113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:05.795354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471668324445544:2049] [node 2] ICS07 socket disconnect 337 reason# EndOfStream
2025-11-27T18:20:05.795392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471668324445544:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.795480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:05.795533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471668324445544:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:05.797647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471668324445450:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28189) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:05.797690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471668324445544:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:05.798886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:05.799095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471666972071033:2049] [node 2] ICS01 socket: 341 reason# EndOfStream
2025-11-27T18:20:05.799162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471666972070946:2049] [node 2] ICP30 unregister session Session# [4:7577471666972071033:2049] VirtualId# [4:7577471666972071027:0]
2025-11-27T18:20:05.799206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471666972071033:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.799279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:05.799329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:05.799711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471668324445450:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28189) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.799814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471668324445544:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.799856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471668324445450:2049] [node 2] ICP30 unregister session Session# [3:7577471668324445544:2049] VirtualId# [3:7577471668324445535:0]
2025-11-27T18:20:05.799897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:05.799928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471668324445450:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28189) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.799946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28189) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.800443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:05.800648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471668883178069:2049] [node 2] ICS01 socket: 363 reason# EndOfStream
2025-11-27T18:20:05.800699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471668883177974:2049] [node 2] ICP30 unregister session Session# [5:7577471668883178069:2049] VirtualId# [5:7577471668883178064:0]
2025-11-27T18:20:05.800725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471668883178069:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.800785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:05.800833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:05.801431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:05.801634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471668466693905:2049] [node 2] ICS01 socket: 352 reason# EndOfStream
2025-11-27T18:20:05.801674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471668466693810:2049] [node 2] ICP30 unregister session Session# [6:7577471668466693905:2049] VirtualId# [6:7577471668466693897:0]
2025-11-27T18:20:05.801707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471668466693905:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.801760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:05.801797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:05.825542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:05.825725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471666972071046:2050] [node 3] ICS07 socket disconnect 368 reason# EndOfStream
2025-11-27T18:20:05.825773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471666972071046:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.825862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:05.825903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471666972071046:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:05.826585Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471666972070947:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21687) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.826566Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:05.826641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471666972071046:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.826680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471666972070947:2050] [node 3] ICP30 unregister session Session# [4:7577471666972071046:2050] VirtualId# [4:7577471666972071044:0]
2025-11-27T18:20:05.826775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:05.826812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471666972070947:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21687) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.826783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471668883178059:2050] [node 3] ICS07 socket disconnect 349 reason# EndOfStream
2025-11-27T18:20:05.826821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21687) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.826819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471668883178059:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.826894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:05.826953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471668883178059:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:05.827429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:05.827644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471668466693907:2050] [node 3] ICS01 socket: 338 reason# EndOfStream
2025-11-27T18:20:05.827664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471668883177975:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21687) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.827692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471668466693811:2050] [node 3] ICP30 unregister session Session# [6:7577471668466693907:2050] VirtualId# [6:7577471668466693900:0]
2025-11-27T18:20:05.827861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471668883178059:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.827826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471668466693907:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.827894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:05.827949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:05.828182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471668883177975:2050] [node 3] ICP30 unregister session Session# [5:7577471668883178059:2050] VirtualId# [5:7577471668883178055:0]
2025-11-27T18:20:05.828294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:05.828333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471668883177975:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21687) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.828346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21687) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.837102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471666972070947:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:05.838140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471668883177975:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:05.899491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:05.899733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471668883178071:2108] [node 4] ICS07 socket disconnect 340 reason# EndOfStream
2025-11-27T18:20:05.899777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471668883178071:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.899855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:05.899903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471668883178071:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:05.900570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:05.900656Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471668883178038:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8744) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.900720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471668883178071:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.900774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471668883178038:2108] [node 4] ICP30 unregister session Session# [5:7577471668883178071:2108] VirtualId# [5:7577471668883178063:0]
2025-11-27T18:20:05.900761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471668466693892:2107] [node 4] ICS07 socket disconnect 371 reason# EndOfStream
2025-11-27T18:20:05.900815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471668466693892:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.900886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:05.900896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:05.900925Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471668883178038:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8744) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.900947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8744) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.900936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471668466693892:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:05.901730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471668466693871:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8744) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.901816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471668466693892:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.901867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471668466693871:2107] [node 4] ICP30 unregister session Session# [6:7577471668466693892:2107] VirtualId# [6:7577471668466693889:0]
2025-11-27T18:20:05.901903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:05.901934Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471668466693871:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8744) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.901947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8744) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.911914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471668466693871:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:05.946296Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:05.946568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471668466693911:2105] [node 5] ICS07 socket disconnect 367 reason# EndOfStream
2025-11-27T18:20:05.946612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471668466693911:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:05.946720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:05.946795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471668466693911:2105] [node 5] ICS15 start handshake
2025-11-27T18:20:05.947568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471668466693869:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10970) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:05.947664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471668466693911:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:05.947723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471668466693869:2105] [node 5] ICP30 unregister session Session# [6:7577471668466693911:2105] VirtualId# [6:7577471668466693908:0]
2025-11-27T18:20:05.947779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:05.947823Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471668466693869:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10970) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.947854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10970) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:05.957976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471668466693869:2105] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:06.026826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471669198603962:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:06.026840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471669198603961:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:06.045996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471670983632370:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:06.046205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471670983632371:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:06.068455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471671118288775:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:06.069330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471671118288774:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:06.091197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471670191124554:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:06.091250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471670191124555:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:06.091349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471670191124556:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:06.113339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471673157239747:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:06.113400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471673157239749:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:06.113577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471673157239748:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:06.133858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471669576658155:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:06.133922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471669576658156:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:06.134043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471669576658157:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:06.203799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471669576658216:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:06.238430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471669198604024:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:06.246860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471673157239810:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:06.246884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471670983632434:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:06.246691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471669198604025:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:06.255185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471673157239811:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:06.252155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471669198604026:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:06.255734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471671118288837:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:06.252061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471669576658217:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:06.255742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471671118288838:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:06.255561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471670191124617:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:06.255729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471670191124618:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:06.256917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471670983632435:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:06.257084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471671118288839:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:06.261648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471670983632436:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:06.269727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471669198603962:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(76704382ns) {Configure:104(76696195ns) {ProcessConfigured:135(101973ns) {StartInitialHandshake:157(99138ns) {DropHandshakes:470(2468ns) {DropIncomingHandshake:442(502ns) DropOutgoingHandshake:457(479ns)} GenerateSessionVirtualId:988(1337ns) PrepareNewSessionHandshake:477(1694ns) {SwitchToState:179(427ns)}}}}}
2025-11-27T18:20:06.273828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471669576658156:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16895105ns) {Configure:104(16887081ns) {ProcessConfigured:135(2018515ns) {StartInitialHandshake:157(2016623ns) {DropHandshakes:470(1081855ns) {DropIncomingHandshake:442(718ns) DropOutgoingHandshake:457(1503ns)} GenerateSessionVirtualId:988(2308ns) PrepareNewSessionHandshake:477(1952ns) {SwitchToState:179(923ns)}}}}}
2025-11-27T18:20:06.274267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471669576658216:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16481577ns) {Configure:104(16475265ns) {ProcessConfigured:135(1474795ns) {StartInitialHandshake:157(473527ns) {DropHandshakes:470(2223ns) {DropIncomingHandshake:442(883ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(10577ns) PrepareNewSessionHandshake:477(2350ns) {SwitchToState:179(1192ns)}}}}}
2025-11-27T18:20:06.276258Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471673157239748:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18734166ns) {Configure:104(18712765ns) {ProcessConfigured:135(934291ns) {StartInitialHandshake:157(931847ns) {DropHandshakes:470(1213ns) {DropIncomingHandshake:442(355ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(665ns) PrepareNewSessionHandshake:477(2442ns) {SwitchToState:179(1239ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.278632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471670191124625:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6399
2025-11-27T18:20:06.279579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471669198604040:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44333
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.282946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471670983632443:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61515
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.292480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471669198604041:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44333
2025-11-27T18:20:06.293018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471670983632443:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.293044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471670191124625:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.293958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471671118288846:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.293666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471670191124554:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.294926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471669198604032:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.295208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471671118288847:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.296395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471670983632449:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61515
2025-11-27T18:20:06.294948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471669198604040:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.296728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471671118288846:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.297304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471670983632370:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.297481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471671118288847:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.298003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471670983632450:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61515
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.294963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471669198604032:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.297865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471671118288774:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.298305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471669576658233:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56438
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.297641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471669198604032:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:06.298371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471670983632449:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.298409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471673157239823:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46441
2025-11-27T18:20:06.299259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471671118288855:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-27T18:20:06.299864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471670983632450:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.299649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471673157239821:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46441
2025-11-27T18:20:06.300146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471670191124634:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6399
2025-11-27T18:20:06.300890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471669198604030:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.299905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471670983632453:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61515
2025-11-27T18:20:06.300896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471669198604031:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.300693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471673157239821:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.301996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471669576658234:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56438
2025-11-27T18:20:06.301914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471670191124625:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.308054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471670983632455:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61515
2025-11-27T18:20:06.300924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471669198604031:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.309163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471670191124554:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.300936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471669198604030:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.310658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471669576658234:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.309285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:06.310760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471670983632455:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.311720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471671118288855:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.311728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471673157239823:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.301875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471669198604042:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44333
2025-11-27T18:20:06.310933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471669576658233:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.311958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471670983632453:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.303389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471669198604041:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.312034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471673157239748:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.312018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471669576658157:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.309348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471670191124554:2048] [node 1] ICP22 created new session: [4:7577471670191124635:2048]
2025-11-27T18:20:06.311948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471671118288858:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-27T18:20:06.303444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471669198604024:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.313274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471670983632441:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.312289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471669576658156:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.309392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471670191124635:2048] [node 1] ICS09 handshake done sender: [4:7577471670191124625:2113] self: [4:7577471670191124633:0] peer: [1:7577471669198604029:0] socket: 321 qp: -1
2025-11-27T18:20:06.312731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471671118288775:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.313308Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471670983632441:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.304342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:06.310472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471670191124635:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.313867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471670983632442:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.313274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471671118288845:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.304597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471669198604024:2107] [node 4] ICP22 created new session: [1:7577471669198604046:2107]
2025-11-27T18:20:06.310532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:06.313873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471670983632440:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.313293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471671118288858:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.310438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471669198604046:2107] [node 4] ICS09 handshake done sender: [1:7577471669198604032:2112] self: [1:7577471669198604029:0] peer: [4:7577471670191124633:0] socket: 320 qp: -1
2025-11-27T18:20:06.310678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471669198604046:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:06.313903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471670983632442:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.313305Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471671118288845:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.310718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.317545Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471671118288845:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:06.313951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471670983632440:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.314401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471669198604031:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:06.315535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471669198604045:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44333
2025-11-27T18:20:06.317444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471670983632442:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:06.316512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471669198604030:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:06.319627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471673157239829:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46441
2025-11-27T18:20:06.319731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471671118288861:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-27T18:20:06.319791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471669576658237:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56438
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.317281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471669198604042:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.318107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471670983632441:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:06.317538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471669198603961:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.320303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471670983632371:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.320398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471671118288837:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.318428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:06.320629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471669576658238:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56438
2025-11-27T18:20:06.320439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471670983632434:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.320455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:06.321960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471669576658237:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.320597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:06.320636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471671118288837:2109] [node 6] ICP22 created new session: [3:7577471671118288864:2109]
2025-11-27T18:20:06.322607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471669576658217:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.320653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471670983632434:2108] [node 5] ICP22 created new session: [2:7577471670983632465:2108]
2025-11-27T18:20:06.322513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471673157239830:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46441
2025-11-27T18:20:06.322526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471671118288864:2109] [node 6] ICS09 handshake done sender: [3:7577471671118288845:2112] self: [3:7577471671118288844:0] peer: [6:7577471669576658235:0] socket: 347 qp: -1
2025-11-27T18:20:06.318493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471669198603961:2048] [node 2] ICP22 created new session: [1:7577471669198604052:2048]
2025-11-27T18:20:06.321950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:06.322891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471669576658234:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:06.322646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471671118288864:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:06.322187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471670983632371:2049] [node 3] ICP22 created new session: [2:7577471670983632464:2049]
2025-11-27T18:20:06.318974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471669198604052:2048] [node 2] ICS09 handshake done sender: [1:7577471669198604031:2111] self: [1:7577471669198604028:0] peer: [2:7577471670983632454:0] socket: 318 qp: -1
2025-11-27T18:20:06.323699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471670191124630:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.324721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471669576658157:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.323751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471670191124630:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.319204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471669198604052:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:06.322249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471670983632464:2049] [node 3] ICS09 handshake done sender: [2:7577471670983632442:2113] self: [2:7577471670983632439:0] peer: [3:7577471671118288859:0] socket: 340 qp: -1
2025-11-27T18:20:06.322680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471671118288863:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9503
2025-11-27T18:20:06.326749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471673157239823:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:06.319362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471669198604045:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.325372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:06.325834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471669576658157:2050] [node 3] ICP22 created new session: [6:7577471669576658241:2050]
2025-11-27T18:20:06.319634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:06.330447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471670191124630:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:06.322840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:06.322252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471670983632465:2108] [node 5] ICS09 handshake done sender: [2:7577471670983632441:2112] self: [2:7577471670983632438:0] peer: [5:7577471673157239828:0] socket: 341 qp: -1
2025-11-27T18:20:06.329747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471669576658241:2050] [node 3] ICS09 handshake done sender: [6:7577471669576658234:2116] self: [6:7577471669576658235:0] peer: [3:7577471671118288844:0] socket: 350 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.324718Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471671118288846:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:06.319946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471669198603962:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.331911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471673157239748:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.320302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:06.322278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471670983632464:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:06.330464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471671118288855:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:06.334284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471670191124618:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.339673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471671118288837:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18776796ns) {HandleHandshakeStatus:329(18426445ns) {DropIncomingHandshake:442(258ns) DropOutgoingHandshake:457(350ns) SwitchToState:179(414ns) ProcessPendingSessionEvents:495(14865095ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14774648ns max# 4755622ns}}}
2025-11-27T18:20:06.322291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471670983632465:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:06.339772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471671118288774:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.329921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471669576658241:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:06.337843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:06.320612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471669198603962:2049] [node 3] ICP22 created new session: [1:7577471669198604054:2049]
2025-11-27T18:20:06.322321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:06.331969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:06.322107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471669198604054:2049] [node 3] ICS09 handshake done sender: [1:7577471669198604030:2110] self: [1:7577471669198604027:0] peer: [3:7577471671118288856:0] socket: 319 qp: -1
2025-11-27T18:20:06.340273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471670191124618:2109] [node 6] ICP22 created new session: [4:7577471670191124639:2109]
2025-11-27T18:20:06.331194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471669576658240:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56438
2025-11-27T18:20:06.340241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:06.322334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:06.333101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471673157239748:2049] [node 2] ICP22 created new session: [5:7577471673157239832:2049]
2025-11-27T18:20:06.322147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471669198604054:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:06.341153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471671118288775:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471671118288855:2117] held: no
2025-11-27T18:20:06.331306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:06.341147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471670191124639:2109] [node 6] ICS09 handshake done sender: [4:7577471670191124630:2114] self: [4:7577471670191124629:0] peer: [6:7577471669576658239:0] socket: 324 qp: -1
2025-11-27T18:20:06.330467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471670983632462:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61515
2025-11-27T18:20:06.322564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:06.341662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471670983632440:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:06.340182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471673157239832:2049] [node 2] ICS09 handshake done sender: [5:7577471673157239823:2116] self: [5:7577471673157239828:0] peer: [2:7577471670983632438:0] socket: 343 qp: -1
2025-11-27T18:20:06.341463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471670191124639:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:06.343711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471669576658233:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:06.340370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471673157239831:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46441
2025-11-27T18:20:06.341289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471671118288774:2048] [node 1] ICP22 created new session: [3:7577471671118288867:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.341501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:06.353315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471669198604024:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(21629581ns) {HandleHandshakeStatus:329(21626924ns) {DropIncomingHandshake:442(12201ns) DropOutgoingHandshake:457(543ns) SwitchToState:179(266ns) ProcessPendingSessionEvents:495(14226071ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14078787ns max# 1316775ns}}}
2025-11-27T18:20:06.343905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471669576658156:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.340444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471673157239832:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:06.353688Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471670191124554:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22331490ns) {HandleHandshakeStatus:329(22329201ns) {DropIncomingHandshake:442(3092ns) DropOutgoingHandshake:457(23448ns) SwitchToState:179(726ns) ProcessPendingSessionEvents:495(14535296ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14380381ns max# 1281376ns}}}
2025-11-27T18:20:06.341599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471671118288867:2048] [node 1] ICS09 handshake done sender: [3:7577471671118288846:2113] self: [3:7577471671118288856:0] peer: [1:7577471669198604027:0] socket: 328 qp: -1
2025-11-27T18:20:06.343951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:06.354684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471670983632443:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:06.354843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471670191124640:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6399
2025-11-27T18:20:06.341637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471671118288867:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:06.340489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:06.343994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471669576658156:2049] [node 2] ICP22 created new session: [6:7577471669576658245:2049]
2025-11-27T18:20:06.341669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:06.344039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471669576658245:2049] [node 2] ICS09 handshake done sender: [6:7577471669576658233:2115] self: [6:7577471669576658236:0] peer: [2:7577471670983632437:0] socket: 339 qp: -1
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.344072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471669576658245:2049] [node 2] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:20:06.357788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471670983632435:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.344102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:06.358468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471669576658244:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56438
2025-11-27T18:20:06.358017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:06.358066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471670983632435:2110] [node 6] ICP22 created new session: [2:7577471670983632470:2110]
2025-11-27T18:20:06.358454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471670983632470:2110] [node 6] ICS09 handshake done sender: [2:7577471670983632440:2111] self: [2:7577471670983632437:0] peer: [6:7577471669576658236:0] socket: 338 qp: -1
2025-11-27T18:20:06.358602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471670983632470:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:06.358652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:06.358788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471673157239835:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46441
2025-11-27T18:20:06.358830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471671118288775:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.358849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471670191124640:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.358881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:06.358899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471669576658237:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:06.358937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471669198603961:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(29589739ns) {HandleHandshakeStatus:329(29587204ns) {DropIncomingHandshake:442(782ns) DropOutgoingHandshake:457(433ns) SwitchToState:179(463ns) ProcessPendingSessionEvents:495(27014961ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26896084ns max# 4071563ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.359693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471671118288775:2049] [node 2] ICP22 created new session: [3:7577471671118288869:2049]
2025-11-27T18:20:06.359912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471671118288869:2049] [node 2] ICS09 handshake done sender: [3:7577471671118288855:2117] self: [3:7577471671118288859:0] peer: [2:7577471670983632439:0] socket: 345 qp: -1
2025-11-27T18:20:06.360373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471670983632371:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(36184372ns) {HandleHandshakeStatus:329(36182608ns) {DropIncomingHandshake:442(25374ns) DropOutgoingHandshake:457(195ns) SwitchToState:179(1416ns) ProcessPendingSessionEvents:495(33783683ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26907470ns max# 4174903ns}}}
2025-11-27T18:20:06.360178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471670191124643:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6399
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.360102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471671118288869:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:06.360148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:06.360626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471670191124644:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6399
2025-11-27T18:20:06.360644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471670983632370:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.360671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471670983632434:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(37431804ns) {HandleHandshakeStatus:329(37429575ns) {DropIncomingHandshake:442(13043ns) DropOutgoingHandshake:457(246ns) SwitchToState:179(315ns) ProcessPendingSessionEvents:495(34838200ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 34719749ns max# 6457468ns}}}
2025-11-27T18:20:06.360661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471669576658217:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.360697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:06.360736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:06.360760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471670983632370:2048] [node 1] ICP22 created new session: [2:7577471670983632472:2048]
2025-11-27T18:20:06.360779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471669576658217:2107] [node 4] ICP22 created new session: [6:7577471669576658247:2107]
2025-11-27T18:20:06.360818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471670983632472:2048] [node 1] ICS09 handshake done sender: [2:7577471670983632443:2114] self: [2:7577471670983632454:0] peer: [1:7577471669198604028:0] socket: 323 qp: -1
2025-11-27T18:20:06.360840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471669576658247:2107] [node 4] ICS09 handshake done sender: [6:7577471669576658237:2117] self: [6:7577471669576658239:0] peer: [4:7577471670191124629:0] socket: 366 qp: -1
2025-11-27T18:20:06.360849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471670983632472:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:06.360874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471669576658247:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:06.360892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:06.360911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:06.360957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471669576658157:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(34020506ns) {HandleHandshakeStatus:329(34017959ns) {DropIncomingHandshake:442(1226ns) DropOutgoingHandshake:457(11478ns) SwitchToState:179(371ns) ProcessPendingSessionEvents:495(24944555ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24821577ns max# 6101525ns}}}
2025-11-27T18:20:06.361083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471671118288774:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19035020ns) {HandleHandshakeStatus:329(19031789ns) {DropIncomingHandshake:442(673ns) DropOutgoingHandshake:457(5211ns) SwitchToState:179(513ns) ProcessPendingSessionEvents:495(16496371ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16391599ns max# 5826160ns}}}
2025-11-27T18:20:06.361100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471673157239748:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(26963189ns) {HandleHandshakeStatus:329(26960069ns) {DropIncomingHandshake:442(806ns) DropOutgoingHandshake:457(4854ns) SwitchToState:179(844ns) ProcessPendingSessionEvents:495(16524234ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16204271ns max# 5709493ns}}}
2025-11-27T18:20:06.361198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471669198603962:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(37873187ns) {HandleHandshakeStatus:329(37870741ns) {DropIncomingHandshake:442(726ns) DropOutgoingHandshake:457(470ns) SwitchToState:179(325ns) ProcessPendingSessionEvents:495(35016650ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 34879194ns max# 6286869ns}}}
2025-11-27T18:20:06.361196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471673157239748:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:06.361225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471670191124618:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(24613840ns) {HandleHandshakeStatus:329(24610747ns) {DropIncomingHandshake:442(449ns) DropOutgoingHandshake:457(372ns) SwitchToState:179(439ns) ProcessPendingSessionEvents:495(12622524ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12509470ns max# 6466750ns}}}
2025-11-27T18:20:06.363196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471670191124643:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.363686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471673157239831:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.365549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471673157239749:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.366679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471671118288849:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.366759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471673157239829:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.367188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471673157239747:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.366744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471671118288849:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.367586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471669198604056:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44333
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.368940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471673157239835:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.369091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471670191124645:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6399
2025-11-27T18:20:06.369308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471670191124618:2109] [node 6] ICP18 handshake (actor [4:7577471670191124640:2117]) is too old
2025-11-27T18:20:06.369659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471669198604039:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.369286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471673157239810:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.369671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471670191124632:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.369695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471669198604039:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.369696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471670191124632:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.371996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471670191124632:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:06.372051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471670191124645:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.372052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471673157239839:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46441
2025-11-27T18:20:06.373722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471670191124617:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.373800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.373837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471670191124617:2108] [node 5] ICP22 created new session: [4:7577471670191124648:2108]
2025-11-27T18:20:06.373871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471670191124648:2108] [node 5] ICS09 handshake done sender: [4:7577471670191124632:2115] self: [4:7577471670191124631:0] peer: [5:7577471673157239838:0] socket: 383 qp: -1
2025-11-27T18:20:06.373581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471671118288849:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:06.373917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471673157239840:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46441
2025-11-27T18:20:06.373897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471670191124648:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:06.374666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471671118288838:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.375231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471669198603962:2049] [node 3] ICP18 handshake (actor [1:7577471669198604042:2117]) is too old
2025-11-27T18:20:06.373920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:06.375224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:06.374644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471670191124556:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.375367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471671118288838:2108] [node 5] ICP22 created new session: [3:7577471671118288873:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.375923Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471673157239831:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:06.376679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471671118288873:2108] [node 5] ICS09 handshake done sender: [3:7577471671118288849:2115] self: [3:7577471671118288848:0] peer: [5:7577471673157239836:0] socket: 346 qp: -1
2025-11-27T18:20:06.376732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471671118288873:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:06.377569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471673157239835:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:06.377234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471669576658249:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56438
2025-11-27T18:20:06.376996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:06.380064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471673157239810:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65304) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471673157239835:2120] held: no
2025-11-27T18:20:06.379510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471671118288851:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.380937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471669198604039:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:06.380951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471669576658249:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.379559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471671118288851:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.381514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471669576658155:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.382758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471673157239749:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.383346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471669198604025:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.382806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:06.383412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:06.384276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471669198604037:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.383179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471673157239749:2050] [node 3] ICP22 created new session: [5:7577471673157239842:2050]
2025-11-27T18:20:06.384311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471669198604037:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.384587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471669198604025:2108] [node 5] ICP22 created new session: [1:7577471669198604060:2108]
2025-11-27T18:20:06.384594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471673157239842:2050] [node 3] ICS09 handshake done sender: [5:7577471673157239831:2119] self: [5:7577471673157239836:0] peer: [3:7577471671118288848:0] socket: 376 qp: -1
2025-11-27T18:20:06.384798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471669198604060:2108] [node 5] ICS09 handshake done sender: [1:7577471669198604039:2114] self: [1:7577471669198604038:0] peer: [5:7577471673157239837:0] socket: 364 qp: -1
2025-11-27T18:20:06.384836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471669198604060:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:06.385392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471673157239810:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.385435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471673157239842:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:06.385526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:06.385573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:06.385586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:06.385785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471673157239810:2108] [node 4] ICP22 created new session: [5:7577471673157239843:2108]
2025-11-27T18:20:06.386423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471669198604037:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:06.386765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471671118288851:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:06.385941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471673157239843:2108] [node 4] ICS09 handshake done sender: [5:7577471673157239835:2120] self: [5:7577471673157239838:0] peer: [4:7577471670191124631:0] socket: 384 qp: -1
2025-11-27T18:20:06.386150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471673157239843:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.387061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471670191124644:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.386185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:06.389046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471671118288839:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.389134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471669198604026:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.389188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:06.389226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471669198604026:2109] [node 6] ICP22 created new session: [1:7577471669198604063:2109]
2025-11-27T18:20:06.389324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:06.388997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471669576658251:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56438
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.389390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471669198604063:2109] [node 6] ICS09 handshake done sender: [1:7577471669198604037:2113] self: [1:7577471669198604036:0] peer: [6:7577471669576658250:0] socket: 385 qp: -1
2025-11-27T18:20:06.389389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471671118288839:2107] [node 4] ICP22 created new session: [3:7577471671118288876:2107]
2025-11-27T18:20:06.389422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471669198604063:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:06.389502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471671118288876:2107] [node 4] ICS09 handshake done sender: [3:7577471671118288851:2116] self: [3:7577471671118288850:0] peer: [4:7577471670191124650:0] socket: 348 qp: -1
2025-11-27T18:20:06.389560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:06.389544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471671118288876:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:06.389591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471670191124651:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6399
2025-11-27T18:20:06.389647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471669576658249:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:06.389602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:06.389637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471670191124555:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.389737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471673157239749:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:06.390203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471670191124645:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:06.391756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471670983632452:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.392104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471670191124556:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7973) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471670191124645:2120] held: no
2025-11-27T18:20:06.391814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471670983632452:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.392885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471669576658155:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11693) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471669576658249:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.395747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471669576658155:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.395801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:06.395852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471669576658155:2048] [node 1] ICP22 created new session: [6:7577471669576658252:2048]
2025-11-27T18:20:06.395899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471669576658252:2048] [node 1] ICS09 handshake done sender: [6:7577471669576658249:2121] self: [6:7577471669576658250:0] peer: [1:7577471669198604036:0] socket: 344 qp: -1
2025-11-27T18:20:06.395939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471669576658252:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:06.395755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471670191124556:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.396478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471670983632452:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:06.395806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:06.395623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471673157239747:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11693) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471673157239829:2117] held: no
2025-11-27T18:20:06.395859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471670191124556:2050] [node 3] ICP22 created new session: [4:7577471670191124653:2050]
2025-11-27T18:20:06.395999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:06.395634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471673157239841:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46441
2025-11-27T18:20:06.395921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471670191124653:2050] [node 3] ICS09 handshake done sender: [4:7577471670191124645:2120] self: [4:7577471670191124650:0] peer: [3:7577471671118288850:0] socket: 335 qp: -1
2025-11-27T18:20:06.396667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471670983632436:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.395957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471670191124653:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:06.395990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.396831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:06.396883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471670983632436:2107] [node 4] ICP22 created new session: [2:7577471670983632475:2107]
2025-11-27T18:20:06.396934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471670191124655:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6399
2025-11-27T18:20:06.396924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471670983632475:2107] [node 4] ICS09 handshake done sender: [2:7577471670983632452:2118] self: [2:7577471670983632451:0] peer: [4:7577471670191124652:0] socket: 381 qp: -1
2025-11-27T18:20:06.396954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471670983632475:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:06.397409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471673157239829:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:06.396991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:06.397881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471670191124555:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471670191124644:2119] held: no
2025-11-27T18:20:06.398669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471673157239747:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.398703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:06.398732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471673157239747:2048] [node 1] ICP22 created new session: [5:7577471673157239847:2048]
2025-11-27T18:20:06.398769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471673157239847:2048] [node 1] ICS09 handshake done sender: [5:7577471673157239829:2117] self: [5:7577471673157239837:0] peer: [1:7577471669198604038:0] socket: 365 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.398807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471673157239847:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:06.398834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:06.398929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471669576658254:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56438
2025-11-27T18:20:06.401900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471670191124644:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:06.402126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471670191124555:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.402176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:06.402227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471670191124555:2049] [node 2] ICP22 created new session: [4:7577471670191124656:2049]
2025-11-27T18:20:06.402288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471670191124656:2049] [node 2] ICS09 handshake done sender: [4:7577471670191124644:2119] self: [4:7577471670191124652:0] peer: [2:7577471670983632451:0] socket: 382 qp: -1
2025-11-27T18:20:06.402336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471670191124656:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:06.402376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:06.402663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471669576658254:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.402817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471669576658216:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:06.403070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471673157239822:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:06.403106Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471673157239822:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:06.403985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471673157239822:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:06.404104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471669576658256:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56438
2025-11-27T18:20:06.404163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471673157239811:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:06.404283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:06.404354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471669576658254:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:06.404358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471673157239811:2109] [node 6] ICP22 created new session: [5:7577471673157239850:2109]
2025-11-27T18:20:06.404409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471673157239850:2109] [node 6] ICS09 handshake done sender: [5:7577471673157239822:2115] self: [5:7577471673157239820:0] peer: [6:7577471669576658255:0] socket: 330 qp: -1
2025-11-27T18:20:06.404436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471673157239850:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:06.404647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471669576658216:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:06.404572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:06.404723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:06.404826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471669576658216:2105] [node 5] ICP22 created new session: [6:7577471669576658257:2105]
2025-11-27T18:20:06.404913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471669576658257:2105] [node 5] ICS09 handshake done sender: [6:7577471669576658254:2123] self: [6:7577471669576658255:0] peer: [5:7577471673157239820:0] socket: 332 qp: -1
2025-11-27T18:20:06.404966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471669576658257:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:06.405034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:06.447292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:06.447479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471670983632472:2048] [node 1] ICS07 socket disconnect 323 reason# EndOfStream
2025-11-27T18:20:06.447506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471670983632472:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.447568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:06.447603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471670983632472:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:06.448795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:06.449033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471671118288867:2048] [node 1] ICS01 socket: 328 reason# EndOfStream
2025-11-27T18:20:06.449087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471671118288774:2048] [node 1] ICP30 unregister session Session# [3:7577471671118288867:2048] VirtualId# [3:7577471671118288856:0]
2025-11-27T18:20:06.449122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471671118288867:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.449192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:06.449242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:06.450380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471670983632370:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11693) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:06.450431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471670983632472:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:06.452266Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471670983632370:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11693) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:06.452365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471670983632472:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:06.452411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471670983632370:2048] [node 1] ICP30 unregister session Session# [2:7577471670983632472:2048] VirtualId# [2:7577471670983632454:0]
2025-11-27T18:20:06.452533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:06.452572Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471670983632370:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:06.452588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:06.452700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:06.452939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471670191124635:2048] [node 1] ICS07 socket disconnect 321 reason# EndOfStream
2025-11-27T18:20:06.452979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471670191124635:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.453054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:06.453093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471670191124635:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:06.453831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471670191124554:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11693) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:06.453944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:06.454077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471670191124635:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:06.454120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471670191124554:2048] [node 1] ICP30 unregister session Session# [4:7577471670191124635:2048] VirtualId# [4:7577471670191124633:0]
2025-11-27T18:20:06.454231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:06.454253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471673157239847:2048] [node 1] ICS01 socket: 365 reason# EndOfStream
2025-11-27T18:20:06.454286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471670191124554:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:06.454319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471673157239747:2048] [node 1] ICP30 unregister session Session# [5:7577471673157239847:2048] VirtualId# [5:7577471673157239837:0]
2025-11-27T18:20:06.454304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:06.454354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471673157239847:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.454412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:06.454458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:06.455257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:06.455462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471669576658252:2048] [node 1] ICS01 socket: 344 reason# EndOfStream
2025-11-27T18:20:06.455519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471669576658155:2048] [node 1] ICP30 unregister session Session# [6:7577471669576658252:2048] VirtualId# [6:7577471669576658250:0]
2025-11-27T18:20:06.455548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471669576658252:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.455619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:06.455661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:06.463825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471670191124554:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:06.499275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:06.499449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471671118288869:2049] [node 2] ICS01 socket: 345 reason# EndOfStream
2025-11-27T18:20:06.499484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471671118288775:2049] [node 2] ICP30 unregister session Session# [3:7577471671118288869:2049] VirtualId# [3:7577471671118288859:0]
2025-11-27T18:20:06.499515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471671118288869:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.499593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:06.499646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:06.501729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:06.501875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471670191124656:2049] [node 2] ICS01 socket: 382 reason# EndOfStream
2025-11-27T18:20:06.501918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471670191124555:2049] [node 2] ICP30 unregister session Session# [4:7577471670191124656:2049] VirtualId# [4:7577471670191124652:0]
2025-11-27T18:20:06.501936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471670191124656:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.501994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:06.502049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:06.502750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:06.502987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471673157239832:2049] [node 2] ICS01 socket: 343 reason# EndOfStream
2025-11-27T18:20:06.503026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471673157239748:2049] [node 2] ICP30 unregister session Session# [5:7577471673157239832:2049] VirtualId# [5:7577471673157239828:0]
2025-11-27T18:20:06.503052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471673157239832:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.503112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:06.503156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:06.503824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:06.504022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471669576658245:2049] [node 2] ICS01 socket: 339 reason# EndOfStream
2025-11-27T18:20:06.504076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471669576658156:2049] [node 2] ICP30 unregister session Session# [6:7577471669576658245:2049] VirtualId# [6:7577471669576658236:0]
2025-11-27T18:20:06.504104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471669576658245:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.504180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:06.504225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:06.575507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:06.575675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471670191124653:2050] [node 3] ICS01 socket: 335 reason# EndOfStream
2025-11-27T18:20:06.575717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471670191124556:2050] [node 3] ICP30 unregister session Session# [4:7577471670191124653:2050] VirtualId# [4:7577471670191124650:0]
2025-11-27T18:20:06.575747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471670191124653:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.575835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:06.575883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:06.576678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:06.576898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471673157239842:2050] [node 3] ICS07 socket disconnect 376 reason# EndOfStream
2025-11-27T18:20:06.576948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471673157239842:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.577020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:06.577076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471673157239842:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:06.577716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:06.577898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471669576658241:2050] [node 3] ICS07 socket disconnect 350 reason# EndOfStream
2025-11-27T18:20:06.577853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471673157239749:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7973) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:06.577940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471669576658241:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.577914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471673157239842:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:06.577951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471673157239749:2050] [node 3] ICP30 unregister session Session# [5:7577471673157239842:2050] VirtualId# [5:7577471673157239836:0]
2025-11-27T18:20:06.578090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:06.578125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471673157239749:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7973) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:06.578143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7973) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:06.578029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:06.578083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471669576658241:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:06.578848Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471669576658157:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7973) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:06.578968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471669576658241:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:06.579008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471669576658157:2050] [node 3] ICP30 unregister session Session# [6:7577471669576658241:2050] VirtualId# [6:7577471669576658235:0]
2025-11-27T18:20:06.579103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:06.579142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471669576658157:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7973) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:06.579156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7973) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:06.587745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471673157239749:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:06.588717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471669576658157:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:06.629726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:06.629974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471673157239843:2108] [node 4] ICS01 socket: 384 reason# EndOfStream
2025-11-27T18:20:06.630035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471673157239810:2108] [node 4] ICP30 unregister session Session# [5:7577471673157239843:2108] VirtualId# [5:7577471673157239838:0]
2025-11-27T18:20:06.630076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471673157239843:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.630148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:06.630205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:06.630920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:06.631140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471669576658247:2107] [node 4] ICS07 socket disconnect 366 reason# EndOfStream
2025-11-27T18:20:06.631181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471669576658247:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.631258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:06.631310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471669576658247:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:06.632121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471669576658217:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65304) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:06.632235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471669576658247:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:06.632277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471669576658217:2107] [node 4] ICP30 unregister session Session# [6:7577471669576658247:2107] VirtualId# [6:7577471669576658239:0]
2025-11-27T18:20:06.632410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:06.632450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471669576658217:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65304) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:06.632479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65304) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:06.642007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471669576658217:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:06.699136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:06.699401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471669576658257:2105] [node 5] ICS07 socket disconnect 332 reason# EndOfStream
2025-11-27T18:20:06.699439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471669576658257:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:06.699519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:06.699580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471669576658257:2105] [node 5] ICS15 start handshake
2025-11-27T18:20:06.700298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471669576658216:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27061) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:06.700377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471669576658257:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:06.700445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471669576658216:2105] [node 5] ICP30 unregister session Session# [6:7577471669576658257:2105] VirtualId# [6:7577471669576658255:0]
2025-11-27T18:20:06.700491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:06.700524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471669576658216:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27061) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:06.700541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27061) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:06.811115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471672550652924:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:06.811163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471672550652925:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:06.830752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471670424861359:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:06.830789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471670424861360:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:06.851063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471670039678786:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:06.851320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471670039678787:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:06.872726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471670454743268:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:06.872776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471670454743269:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:06.872781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471670454743270:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:06.893372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471671041772732:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:06.893420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471671041772734:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:06.893538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471671041772733:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:06.915267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471669689525278:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:06.915348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471669689525279:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:06.915392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471669689525280:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:06.984692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471669689525328:2092] [node 4] ICP01 ready to work
2025-11-27T18:20:06.991742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471669689525326:2096] [node 5] ICP01 ready to work
2025-11-27T18:20:07.001563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471672550652987:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:07.028938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471674334646147:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:07.035414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471676845620284:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:07.042469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471674749710629:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:07.040346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471674334646148:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:07.046451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471675336740091:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:07.048924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471674719828720:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:07.053455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471674749710630:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:07.053926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471676845620285:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:07.040680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471674334646149:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:07.048892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471675336740092:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:07.053436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471674719828718:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:07.053942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471674719828719:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:07.068228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471669689525280:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(67334537ns) {Configure:104(67319765ns) {ProcessConfigured:135(65420ns) {StartInitialHandshake:157(62914ns) {DropHandshakes:470(2031ns) {DropIncomingHandshake:442(556ns) DropOutgoingHandshake:457(318ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:477(1695ns) {SwitchToState:179(581ns)}}}}}
2025-11-27T18:20:07.067691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471672550652924:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(78764363ns) {Configure:104(78759415ns) {ProcessConfigured:135(3606137ns) {StartInitialHandshake:157(3603136ns) {DropHandshakes:470(2662ns) {DropIncomingHandshake:442(507ns) DropOutgoingHandshake:457(257ns)} GenerateSessionVirtualId:988(859ns) PrepareNewSessionHandshake:477(2938ns) {SwitchToState:179(1476ns)}}}}}
2025-11-27T18:20:07.085983Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471670454743268:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19157047ns) {Configure:104(19151786ns) {ProcessConfigured:135(5701903ns) {StartInitialHandshake:157(5698821ns) {DropHandshakes:470(3049ns) {DropIncomingHandshake:442(650ns) DropOutgoingHandshake:457(655ns)} GenerateSessionVirtualId:988(907ns) PrepareNewSessionHandshake:477(3202ns) {SwitchToState:179(1481ns)}}}}}
2025-11-27T18:20:07.092807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471676845620284:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23442137ns) {Configure:104(23438081ns) {ProcessConfigured:135(8318550ns) {StartInitialHandshake:157(8314288ns) {DropHandshakes:470(2502ns) {DropIncomingHandshake:442(390ns) DropOutgoingHandshake:457(419ns)} GenerateSessionVirtualId:988(777ns) PrepareNewSessionHandshake:477(5632ns) {SwitchToState:179(2472ns)}}}}}
2025-11-27T18:20:07.093908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471670039678786:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(36464914ns) {Configure:104(36456112ns) {ProcessConfigured:135(17223037ns) {StartInitialHandshake:157(17220291ns) {DropHandshakes:470(1182ns) {DropIncomingHandshake:442(317ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(565ns) PrepareNewSessionHandshake:477(4479ns) {SwitchToState:179(2518ns)}}}}}
2025-11-27T18:20:07.096150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471674719828719:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(34569723ns) {Configure:104(34566171ns) {ProcessConfigured:135(2525976ns) {StartInitialHandshake:157(2524937ns) {DropHandshakes:470(754ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(445ns) PrepareNewSessionHandshake:477(1530ns) {SwitchToState:179(311ns)}}}}}
2025-11-27T18:20:07.094330Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471669689525328:2092] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36073961ns) {Configure:104(36066709ns) {ProcessConfigured:135(5295402ns) {StartInitialHandshake:157(5292689ns) {DropHandshakes:470(3232ns) {DropIncomingHandshake:442(1061ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(1103ns) PrepareNewSessionHandshake:477(2721ns) {SwitchToState:179(1541ns)}}}}}
2025-11-27T18:20:07.094176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471670039678787:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(61561025ns) {Configure:104(56343508ns) {ProcessConfigured:135(15852968ns) {StartInitialHandshake:157(15850582ns) {DropHandshakes:470(3833887ns) {DropIncomingHandshake:442(2184ns) DropOutgoingHandshake:457(304ns)} GenerateSessionVirtualId:988(3234ns) PrepareNewSessionHandshake:477(3161ns) {SwitchToState:179(1689ns)}}}}}
2025-11-27T18:20:07.096090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471672550652925:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35402205ns) {Configure:104(35396906ns) {ProcessConfigured:135(1428739ns) {StartInitialHandshake:157(1425958ns) {DropHandshakes:470(2292ns) {DropIncomingHandshake:442(505ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(907ns) PrepareNewSessionHandshake:477(1187ns) {SwitchToState:179(126ns)}}}}}
2025-11-27T18:20:07.109418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471674719828720:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(35138853ns) {Configure:104(35134283ns) {ProcessConfigured:135(866869ns) {StartInitialHandshake:157(864675ns) {DropHandshakes:470(1150ns) {DropIncomingHandshake:442(267ns) DropOutgoingHandshake:457(134ns)} GenerateSessionVirtualId:988(413ns) PrepareNewSessionHandshake:477(1572ns) {SwitchToState:179(741ns)}}}}}
2025-11-27T18:20:07.105556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471671041772733:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(37077540ns) {Configure:104(37073462ns) {ProcessConfigured:135(1382998ns) {StartInitialHandshake:157(1380318ns) {DropHandshakes:470(939ns) {DropIncomingHandshake:442(272ns) DropOutgoingHandshake:457(174ns)} GenerateSessionVirtualId:988(782ns) PrepareNewSessionHandshake:477(1492ns) {SwitchToState:179(107ns)}}}}}
2025-11-27T18:20:07.095448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471669689525278:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34776607ns) {Configure:104(34771424ns) {ProcessConfigured:135(2084220ns) {StartInitialHandshake:157(2075797ns) {DropHandshakes:470(2405ns) {DropIncomingHandshake:442(995ns) DropOutgoingHandshake:457(304ns)} GenerateSessionVirtualId:988(1416ns) PrepareNewSessionHandshake:477(2355ns) {SwitchToState:179(186ns)}}}}}
2025-11-27T18:20:07.109437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471674749710630:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(36273776ns) {Configure:104(36266431ns) {ProcessConfigured:135(1065736ns) {StartInitialHandshake:157(1064216ns) {DropHandshakes:470(1318ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(512ns) PrepareNewSessionHandshake:477(1772ns) {SwitchToState:179(1192ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.098822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471672550652987:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36088002ns) {Configure:104(36083083ns) {ProcessConfigured:135(1205601ns) {StartInitialHandshake:157(1203858ns) {DropHandshakes:470(1490ns) {DropIncomingHandshake:442(554ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(602ns) PrepareNewSessionHandshake:477(1797ns) {SwitchToState:179(806ns)}}}}}
2025-11-27T18:20:07.109778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471674334646149:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(36832758ns) {Configure:104(36828057ns) {ProcessConfigured:135(438414ns) {StartInitialHandshake:157(436139ns) {DropHandshakes:470(673ns) {DropIncomingHandshake:442(221ns) DropOutgoingHandshake:457(181ns)} GenerateSessionVirtualId:988(450ns) PrepareNewSessionHandshake:477(285335ns) {SwitchToState:179(1160ns)}}}}}
2025-11-27T18:20:07.125050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471674719828718:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36563747ns) {Configure:104(36559173ns) {ProcessConfigured:135(1498773ns) {StartInitialHandshake:157(1496697ns) {DropHandshakes:470(364669ns) {DropIncomingHandshake:442(278ns) DropOutgoingHandshake:457(654ns)} GenerateSessionVirtualId:988(915ns) PrepareNewSessionHandshake:477(1406ns) {SwitchToState:179(534ns)}}}}}
2025-11-27T18:20:07.113495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471669689525279:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36752384ns) {Configure:104(36738988ns) {ProcessConfigured:135(1631432ns) {StartInitialHandshake:157(1628899ns) {DropHandshakes:470(1255ns) {DropIncomingHandshake:442(272ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(774965ns) {SwitchToState:179(1585ns)}}}}}
2025-11-27T18:20:07.121557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471670454743269:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(37519919ns) {Configure:104(37513484ns) {ProcessConfigured:135(1605025ns) {StartInitialHandshake:157(1603088ns) {DropHandshakes:470(399992ns) {DropIncomingHandshake:442(205ns) DropOutgoingHandshake:457(1220ns)} GenerateSessionVirtualId:988(1331ns) PrepareNewSessionHandshake:477(1267ns) {SwitchToState:179(679ns)}}}}}
2025-11-27T18:20:07.113980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471675336740092:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(53998281ns) {Configure:104(53992496ns) {ProcessConfigured:135(18005036ns) {StartInitialHandshake:157(18002391ns) {DropHandshakes:470(1032ns) {DropIncomingHandshake:442(287ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(286ns) PrepareNewSessionHandshake:477(5355ns) {SwitchToState:179(3184ns)}}}}}
2025-11-27T18:20:07.137328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471674749710629:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(37042609ns) {Configure:104(37036789ns) {ProcessConfigured:135(719081ns) {StartInitialHandshake:157(717600ns) {DropHandshakes:470(993ns) {DropIncomingHandshake:442(290ns) DropOutgoingHandshake:457(190ns)} GenerateSessionVirtualId:988(370ns) PrepareNewSessionHandshake:477(2244ns) {SwitchToState:179(1219ns)}}}}}
2025-11-27T18:20:07.128762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471674334646160:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6229
2025-11-27T18:20:07.125747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471675336740091:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36337104ns) {Configure:104(36330051ns) {ProcessConfigured:135(1812571ns) {StartInitialHandshake:157(1811046ns) {DropHandshakes:470(1097ns) {DropIncomingHandshake:442(310ns) DropOutgoingHandshake:457(186ns)} GenerateSessionVirtualId:988(322ns) PrepareNewSessionHandshake:477(1342ns) {SwitchToState:179(127ns)}}}}}
2025-11-27T18:20:07.141802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471670424861360:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(61411542ns) {Configure:104(61404763ns) {ProcessConfigured:135(22459570ns) {StartInitialHandshake:157(22456643ns) {DropHandshakes:470(1088ns) {DropIncomingHandshake:442(433ns) DropOutgoingHandshake:457(251ns)} GenerateSessionVirtualId:988(464ns) PrepareNewSessionHandshake:477(3946ns) {SwitchToState:179(2027ns)}}}}}
2025-11-27T18:20:07.129671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471674334646148:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(37536565ns) {Configure:104(37531668ns) {ProcessConfigured:135(1862334ns) {StartInitialHandshake:157(1860207ns) {DropHandshakes:470(884ns) {DropIncomingHandshake:442(447ns) DropOutgoingHandshake:457(104ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:477(672455ns) {SwitchToState:179(3066ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:07.148272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471669689525326:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(74640566ns) {Configure:104(74632123ns) {ProcessConfigured:135(1886404ns) {StartInitialHandshake:157(1883255ns) {DropHandshakes:470(2370ns) {DropIncomingHandshake:442(579ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(582ns) PrepareNewSessionHandshake:477(2679ns) {SwitchToState:179(1421ns)}}}}}
::ffff:127.1.0.0
2025-11-27T18:20:07.125774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471671041772734:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(36115784ns) {Configure:104(36111203ns) {ProcessConfigured:135(527594ns) {StartInitialHandshake:157(526175ns) {DropHandshakes:470(851ns) {DropIncomingHandshake:442(268ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:477(1162ns) {SwitchToState:179(563ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.151215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471676845620298:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31214
2025-11-27T18:20:07.150640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471675336740104:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36362
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.153593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471674719828734:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60239
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.158727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471674719828735:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60239
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.161276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471676845620297:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31214
2025-11-27T18:20:07.165622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471675336740105:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36362
2025-11-27T18:20:07.166358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471674334646164:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6229
2025-11-27T18:20:07.166866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471675336740104:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.166835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471674719828735:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.167267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471674334646160:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.166889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471674719828734:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.167536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471671041772733:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.167851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471670424861359:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.168117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471673984492650:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35144
2025-11-27T18:20:07.168383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471676845620297:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.167463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471674749710639:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56303
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.168391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471673984492648:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35144
2025-11-27T18:20:07.168590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471671041772732:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53884000ns) {Configure:104(53875653ns) {ProcessConfigured:135(563029ns) {StartInitialHandshake:157(559078ns) {DropHandshakes:470(3947ns) {DropIncomingHandshake:442(871ns) DropOutgoingHandshake:457(677ns)} GenerateSessionVirtualId:988(945ns) PrepareNewSessionHandshake:477(1711ns) {SwitchToState:179(673ns)}}}}}
2025-11-27T18:20:07.168382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471674719828722:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.168477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471674334646164:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.168445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471676845620287:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.168242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471670454743270:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(70983257ns) {Configure:104(70976759ns) {ProcessConfigured:135(5363238ns) {StartInitialHandshake:157(5359581ns) {DropHandshakes:470(2222ns) {DropIncomingHandshake:442(564ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(1085ns) PrepareNewSessionHandshake:477(3013ns) {SwitchToState:179(1468ns)}}}}}
2025-11-27T18:20:07.168469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471676845620287:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.178207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471670039678786:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.168411Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471674719828722:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.182181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471673984492650:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.173389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471675336740106:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36362
2025-11-27T18:20:07.185466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471673984492648:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.178253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471674749710640:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56303
2025-11-27T18:20:07.187054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471673984492651:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35144
2025-11-27T18:20:07.185937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471676845620285:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19171779ns) {Configure:104(19163051ns) {ProcessConfigured:135(369531ns) {StartInitialHandshake:157(367230ns) {DropHandshakes:470(1473ns) {DropIncomingHandshake:442(292ns) DropOutgoingHandshake:457(319ns)} GenerateSessionVirtualId:988(605ns) PrepareNewSessionHandshake:477(1033ns) {SwitchToState:179(362ns)}}}}}
2025-11-27T18:20:07.187050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471674334646156:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.188247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471669689525280:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.187031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471674719828736:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60239
2025-11-27T18:20:07.187082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471674334646156:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.173710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471675336740105:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.186676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471676845620293:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.188342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471674719828722:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:07.182158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471671041772734:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.185504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471674749710639:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:07.188263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471669689525279:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.190588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471674334646157:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.190746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471675336740106:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.186705Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471676845620293:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.187070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471674749710640:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.187076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471670454743269:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.190625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471674334646156:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:07.191847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471674719828725:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.188349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471676845620287:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:07.190891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471675336740091:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.190633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471674334646157:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.191882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471674719828725:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.193047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471674749710635:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.190298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471676845620298:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.195459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471673984492651:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.191875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471674334646167:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6229
2025-11-27T18:20:07.193082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471674749710635:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.190698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471676845620302:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31214
2025-11-27T18:20:07.191932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471674719828741:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60239
2025-11-27T18:20:07.196933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471669689525278:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.192753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471674334646149:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.191952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471674719828736:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.198769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471674749710635:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:07.191996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471676845620293:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:07.194629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471674334646157:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:07.200816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471671041772734:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21784) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471675336740105:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.192823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471674719828719:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.193027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471672550652924:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.194633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:07.193039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:07.193068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:07.194613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471674719828726:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.195479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471672550652924:2048] [node 2] ICP22 created new session: [1:7577471676845620306:2048]
2025-11-27T18:20:07.194647Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471674719828726:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.195479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471674719828719:2108] [node 5] ICP22 created new session: [2:7577471674719828742:2108]
2025-11-27T18:20:07.196518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471676845620306:2048] [node 2] ICS09 handshake done sender: [1:7577471676845620287:2110] self: [1:7577471676845620286:0] peer: [2:7577471674719828737:0] socket: 337 qp: -1
2025-11-27T18:20:07.197057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471674719828742:2108] [node 5] ICS09 handshake done sender: [2:7577471674719828722:2110] self: [2:7577471674719828721:0] peer: [5:7577471675336740109:0] socket: 324 qp: -1
2025-11-27T18:20:07.196574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471672550652925:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.204419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471674749710630:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.194700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471674334646149:2108] [node 5] ICP22 created new session: [3:7577471674334646170:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.204347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471675336740111:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36362
2025-11-27T18:20:07.205484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471673984492657:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35144
2025-11-27T18:20:07.196633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:07.204496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:07.197086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471674719828742:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:07.205216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471675336740113:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36362
2025-11-27T18:20:07.196479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471674334646170:2108] [node 5] ICS09 handshake done sender: [3:7577471674334646156:2114] self: [3:7577471674334646155:0] peer: [5:7577471675336740110:0] socket: 333 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.196981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471672550652925:2049] [node 3] ICP22 created new session: [1:7577471676845620307:2049]
2025-11-27T18:20:07.217313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471673984492659:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35144
2025-11-27T18:20:07.197381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:07.209134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471674749710630:2108] [node 5] ICP22 created new session: [4:7577471674749710651:2108]
2025-11-27T18:20:07.196528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471674334646170:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:07.224707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471675336740114:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36362
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.198701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471676845620306:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:07.217334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471674749710651:2108] [node 5] ICS09 handshake done sender: [4:7577471674749710635:2113] self: [4:7577471674749710633:0] peer: [5:7577471675336740112:0] socket: 352 qp: -1
2025-11-27T18:20:07.200772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471674719828740:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60239
2025-11-27T18:20:07.196569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.227739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471673984492648:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:07.198842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471676845620307:2049] [node 3] ICS09 handshake done sender: [1:7577471676845620293:2113] self: [1:7577471676845620290:0] peer: [3:7577471674334646166:0] socket: 342 qp: -1
2025-11-27T18:20:07.200826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471674334646148:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.201506Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471674719828735:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:07.225344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471674749710651:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:07.227869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471671041772733:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20459) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471675336740104:2114] held: no
2025-11-27T18:20:07.202194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471674719828725:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.203548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471674719828740:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.225410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:07.230932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471675336740091:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61403) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471675336740106:2116] held: no
2025-11-27T18:20:07.231133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471673984492650:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:07.199751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:07.204922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471670424861359:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.200941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:07.199757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471676845620307:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:07.228086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471674749710649:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56303
2025-11-27T18:20:07.233818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471669689525279:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.201491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471674334646148:2110] [node 6] ICP22 created new session: [3:7577471674334646174:2110]
2025-11-27T18:20:07.232368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471674749710650:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56303
2025-11-27T18:20:07.204969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:07.238933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471674749710652:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56303
2025-11-27T18:20:07.238976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471674749710649:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:07.239809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471670454743268:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-27T18:20:07.233862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:07.200758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:07.240591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471675336740104:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:07.202868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471674334646174:2110] [node 6] ICS09 handshake done sender: [3:7577471674334646157:2115] self: [3:7577471674334646154:0] peer: [6:7577471673984492655:0] socket: 336 qp: -1
2025-11-27T18:20:07.233904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471669689525279:2049] [node 2] ICP22 created new session: [6:7577471673984492662:2049]
2025-11-27T18:20:07.205021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471670424861359:2048] [node 1] ICP22 created new session: [2:7577471674719828748:2048]
2025-11-27T18:20:07.202487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471676845620289:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.242298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471675336740106:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:07.243243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471674749710639:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:07.204471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471674334646174:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:07.236831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471673984492661:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35144
2025-11-27T18:20:07.205065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471674719828748:2048] [node 1] ICS09 handshake done sender: [2:7577471674719828735:2116] self: [2:7577471674719828737:0] peer: [1:7577471676845620286:0] socket: 339 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.244361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471671041772734:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(27783216ns) {HandleHandshakeStatus:395(27780906ns) {LogHandshakeFail:472(1262327ns) {UpdateErrorStateLog:545(7371ns)} DropOutgoingHandshake:457(756ns) ProcessPendingSessionEvents:495(25774780ns) {unordered DropSessionEvent:512 num# 100 sum# 4574321ns max# 2347764ns}}}
2025-11-27T18:20:07.202514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471676845620289:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.204515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:07.238486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471673984492662:2049] [node 2] ICS09 handshake done sender: [6:7577471673984492648:2114] self: [6:7577471673984492656:0] peer: [2:7577471674719828723:0] socket: 345 qp: -1
2025-11-27T18:20:07.205100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471674719828748:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:07.245663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471670454743269:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.238960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471673984492662:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:07.247062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:07.206760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471676845620302:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.209329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471670039678786:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61049) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471674334646164:2118] held: no
2025-11-27T18:20:07.249592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471671041772733:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.205129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:07.239562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:07.225515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471674334646167:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.227800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471676845620289:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:07.248887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471670454743269:2049] [node 2] ICP22 created new session: [4:7577471674749710656:2049]
2025-11-27T18:20:07.250947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:07.205193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471674719828726:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.240539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471669689525280:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.239044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471676845620284:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.226746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471674334646171:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6229
2025-11-27T18:20:07.249397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471674749710650:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.226247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471674719828720:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.241481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:07.251264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471671041772733:2049] [node 2] ICP22 created new session: [5:7577471675336740118:2049]
2025-11-27T18:20:07.239859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:07.251313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471670039678786:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27957074ns) {HandleHandshakeStatus:395(27955186ns) {LogHandshakeFail:472(10763ns) {UpdateErrorStateLog:545(3983ns)} DropOutgoingHandshake:457(712ns) ProcessPendingSessionEvents:495(27501442ns) {unordered DropSessionEvent:512 num# 100 sum# 27450824ns max# 5536455ns}}}
2025-11-27T18:20:07.240234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471676845620284:2109] [node 6] ICP22 created new session: [1:7577471676845620315:2109]
2025-11-27T18:20:07.253787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471674334646159:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.241575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471676845620315:2109] [node 6] ICS09 handshake done sender: [1:7577471676845620289:2111] self: [1:7577471676845620288:0] peer: [6:7577471673984492658:0] socket: 344 qp: -1
2025-11-27T18:20:07.253828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471674334646159:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.241923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471676845620315:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:07.241974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:07.259787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471674334646149:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(49017121ns) {HandleHandshakeStatus:329(49014574ns) {DropIncomingHandshake:442(12787ns) DropOutgoingHandshake:457(440ns) SwitchToState:179(432ns) ProcessPendingSessionEvents:495(41570904ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 41463448ns max# 8140689ns}}}
2025-11-27T18:20:07.260330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471674334646164:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:07.250698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471674749710656:2049] [node 2] ICS09 handshake done sender: [4:7577471674749710639:2116] self: [4:7577471674749710646:0] peer: [2:7577471674719828724:0] socket: 346 qp: -1
2025-11-27T18:20:07.252204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471675336740118:2049] [node 2] ICS09 handshake done sender: [5:7577471675336740104:2114] self: [5:7577471675336740109:0] peer: [2:7577471674719828721:0] socket: 330 qp: -1
2025-11-27T18:20:07.242045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471676845620313:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31214
2025-11-27T18:20:07.260888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471674334646159:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:07.241817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471669689525280:2050] [node 3] ICP22 created new session: [6:7577471673984492664:2050]
2025-11-27T18:20:07.252208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471675336740105:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:07.250972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471676845620292:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.226685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.251002Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471676845620292:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.241853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471673984492661:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.251182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471670454743270:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.243285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471673984492664:2050] [node 3] ICS09 handshake done sender: [6:7577471673984492650:2116] self: [6:7577471673984492655:0] peer: [3:7577471674334646154:0] socket: 347 qp: -1
2025-11-27T18:20:07.251311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471674749710656:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:07.227447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471674719828720:2109] [node 6] ICP22 created new session: [2:7577471674719828750:2109]
2025-11-27T18:20:07.252355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471675336740118:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:07.244335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471673984492664:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:07.252206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:07.252879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:07.260074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471675336740115:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36362
2025-11-27T18:20:07.244979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:07.261989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471672550652924:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(51526633ns) {HandleHandshakeStatus:329(51524259ns) {DropIncomingHandshake:442(694ns) DropOutgoingHandshake:457(523ns) SwitchToState:179(461ns) ProcessPendingSessionEvents:495(42453226ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 42339645ns max# 7373816ns}}}
2025-11-27T18:20:07.249026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471673984492660:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35144
2025-11-27T18:20:07.262678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471670039678786:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.260751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471671041772734:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.227966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471674719828750:2109] [node 6] ICS09 handshake done sender: [2:7577471674719828725:2111] self: [2:7577471674719828723:0] peer: [6:7577471673984492656:0] socket: 325 qp: -1
2025-11-27T18:20:07.262279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471676845620317:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31214
2025-11-27T18:20:07.260793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:07.228032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471674719828750:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:07.263444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471674749710661:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56303
2025-11-27T18:20:07.262742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.262695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471676845620292:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:07.260960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471669689525278:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61049) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471673984492651:2117] held: no
2025-11-27T18:20:07.263162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471670039678786:2048] [node 1] ICP22 created new session: [3:7577471674334646178:2048]
2025-11-27T18:20:07.266343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471672550652987:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.261457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471671041772734:2050] [node 3] ICP22 created new session: [5:7577471675336740120:2050]
2025-11-27T18:20:07.263489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471674334646178:2048] [node 1] ICS09 handshake done sender: [3:7577471674334646164:2118] self: [3:7577471674334646166:0] peer: [1:7577471676845620290:0] socket: 343 qp: -1
2025-11-27T18:20:07.266617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:07.261662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471675336740120:2050] [node 3] ICS09 handshake done sender: [5:7577471675336740105:2115] self: [5:7577471675336740110:0] peer: [3:7577471674334646155:0] socket: 334 qp: -1
2025-11-27T18:20:07.228071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:07.263844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471674334646178:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:07.260998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471669689525328:2092] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.266841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471672550652987:2107] [node 4] ICP22 created new session: [1:7577471676845620319:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.262004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471675336740120:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:07.267933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471674749710638:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.230697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471674719828718:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.268656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471673984492651:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:07.265887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:07.262456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:07.267361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471672550652925:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(61060685ns) {HandleHandshakeStatus:329(61056815ns) {DropIncomingHandshake:442(14666ns) DropOutgoingHandshake:457(277ns) SwitchToState:179(356ns) ProcessPendingSessionEvents:495(55478011ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 53039778ns max# 7433575ns}}}
2025-11-27T18:20:07.231229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:07.267973Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471674749710638:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.272370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471673984492667:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35144
2025-11-27T18:20:07.267832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471676845620319:2107] [node 4] ICS09 handshake done sender: [1:7577471676845620292:2112] self: [1:7577471676845620291:0] peer: [4:7577471674749710655:0] socket: 335 qp: -1
2025-11-27T18:20:07.263204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471675336740091:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(16896150ns) {HandleHandshakeStatus:395(16894874ns) {LogHandshakeFail:472(272633ns) {UpdateErrorStateLog:545(3606ns)} DropOutgoingHandshake:457(1322ns) ProcessPendingSessionEvents:495(14007469ns) {unordered DropSessionEvent:512 num# 100 sum# 13963216ns max# 3197081ns}}}
2025-11-27T18:20:07.266493Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471674334646148:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(57973531ns) {HandleHandshakeStatus:329(57970277ns) {DropIncomingHandshake:442(522ns) DropOutgoingHandshake:457(253ns) SwitchToState:179(349ns) ProcessPendingSessionEvents:495(41302300ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 38279131ns max# 7983627ns}}}
2025-11-27T18:20:07.268659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471674749710630:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(56623924ns) {HandleHandshakeStatus:329(56621190ns) {DropIncomingHandshake:442(13211ns) DropOutgoingHandshake:457(257ns) SwitchToState:179(342ns) ProcessPendingSessionEvents:495(33148781ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32767281ns max# 3691654ns}}}
2025-11-27T18:20:07.232444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471674719828718:2107] [node 4] ICP22 created new session: [2:7577471674719828752:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.272787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471669689525278:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.273837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471674749710649:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:07.236940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471674719828752:2107] [node 4] ICS09 handshake done sender: [2:7577471674719828726:2112] self: [2:7577471674719828724:0] peer: [4:7577471674749710646:0] socket: 326 qp: -1
2025-11-27T18:20:07.266694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471674334646147:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.268593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471676845620319:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:07.264118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471675336740115:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.272828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:07.266729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:07.237123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471674719828752:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:07.272872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471669689525278:2048] [node 1] ICP22 created new session: [6:7577471673984492670:2048]
2025-11-27T18:20:07.267254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471674334646147:2107] [node 4] ICP22 created new session: [3:7577471674334646181:2107]
2025-11-27T18:20:07.268635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:07.272920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471673984492670:2048] [node 1] ICS09 handshake done sender: [6:7577471673984492651:2117] self: [6:7577471673984492658:0] peer: [1:7577471676845620288:0] socket: 350 qp: -1
2025-11-27T18:20:07.270749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471676845620300:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.270770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471676845620300:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.267303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471674334646181:2107] [node 4] ICS09 handshake done sender: [3:7577471674334646159:2116] self: [3:7577471674334646158:0] peer: [4:7577471674749710657:0] socket: 377 qp: -1
2025-11-27T18:20:07.237166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:07.264166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471675336740091:2108] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.264198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:07.267342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471674334646181:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:07.261601Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471674719828719:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(52119049ns) {HandleHandshakeStatus:329(52116093ns) {DropIncomingHandshake:442(10686ns) DropOutgoingHandshake:457(337ns) SwitchToState:179(228ns) ProcessPendingSessionEvents:495(44709382ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 40664772ns max# 5786457ns}}}
2025-11-27T18:20:07.279636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471676845620317:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.273116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471673984492670:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.267383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471674719828754:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60239
2025-11-27T18:20:07.266035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471671041772732:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.285385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471676845620300:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:07.267834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:07.287033Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471670454743268:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61049) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471674749710649:2119] held: no
2025-11-27T18:20:07.273303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:07.266385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471675336740091:2108] [node 4] ICP22 created new session: [5:7577471675336740122:2108]
2025-11-27T18:20:07.270514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471674719828720:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(35263142ns) {HandleHandshakeStatus:329(35259196ns) {DropIncomingHandshake:442(656ns) DropOutgoingHandshake:457(500ns) SwitchToState:179(2711ns) ProcessPendingSessionEvents:495(24390235ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24273798ns max# 3737394ns}}}
2025-11-27T18:20:07.286069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471674334646182:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6229
2025-11-27T18:20:07.284391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471673984492667:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.267096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471675336740122:2108] [node 4] ICS09 handshake done sender: [5:7577471675336740106:2116] self: [5:7577471675336740112:0] peer: [4:7577471674749710633:0] socket: 357 qp: -1
2025-11-27T18:20:07.287331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471674749710660:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56303
2025-11-27T18:20:07.286152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471674334646148:2110] [node 6] ICP18 handshake (actor [3:7577471674334646167:2119]) is too old
2025-11-27T18:20:07.276582Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471670424861359:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(57386086ns) {HandleHandshakeStatus:329(57383649ns) {DropIncomingHandshake:442(578ns) DropOutgoingHandshake:457(4003ns) SwitchToState:179(283ns) ProcessPendingSessionEvents:495(37552306ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36223877ns max# 3678493ns}}}
2025-11-27T18:20:07.267736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471675336740122:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:07.287589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471670454743269:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(27584430ns) {HandleHandshakeStatus:329(27581936ns) {DropIncomingHandshake:442(680ns) DropOutgoingHandshake:457(3587ns) SwitchToState:179(314ns) ProcessPendingSessionEvents:495(20540382ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20445655ns max# 6927966ns}}}
2025-11-27T18:20:07.267775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:07.287292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471674334646180:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6229
2025-11-27T18:20:07.278020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471675336740124:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36362
2025-11-27T18:20:07.285407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471674719828754:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.289213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471670454743269:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.293526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471670454743268:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.293574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:07.293523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471675336740100:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.293575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471675336740100:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.287125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471669689525326:2096] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.289153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471674719828720:2109] [node 6] ICP18 handshake (actor [2:7577471674719828740:2118]) is too old
2025-11-27T18:20:07.293584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471674749710662:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56303
2025-11-27T18:20:07.287650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471669689525280:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(29949762ns) {HandleHandshakeStatus:329(29947468ns) {DropIncomingHandshake:442(992ns) DropOutgoingHandshake:457(5337ns) SwitchToState:179(279ns) ProcessPendingSessionEvents:495(21469314ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19968577ns max# 3753179ns}}}
2025-11-27T18:20:07.289297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471676845620285:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.293662Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471676845620284:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(39005098ns) {HandleHandshakeStatus:329(39002327ns) {DropIncomingHandshake:442(558982ns) DropOutgoingHandshake:457(972ns) SwitchToState:179(472ns) ProcessPendingSessionEvents:495(33569636ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 31504682ns max# 4267850ns}}}
2025-11-27T18:20:07.293638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471670454743268:2048] [node 1] ICP22 created new session: [4:7577471674749710664:2048]
2025-11-27T18:20:07.295047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471674334646180:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.288414Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471669689525279:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(40577336ns) {HandleHandshakeStatus:329(39259770ns) {DropIncomingHandshake:442(471ns) DropOutgoingHandshake:457(4868ns) SwitchToState:179(341ns) ProcessPendingSessionEvents:495(31041673ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 30920358ns max# 5724317ns}}}
2025-11-27T18:20:07.295522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471674334646182:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.293679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471674749710664:2048] [node 1] ICS09 handshake done sender: [4:7577471674749710649:2119] self: [4:7577471674749710655:0] peer: [1:7577471676845620291:0] socket: 380 qp: -1
2025-11-27T18:20:07.293710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471674749710664:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:07.293743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:07.293889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471674749710638:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:07.296790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471674749710629:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.296805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471674749710650:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:07.294160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:07.296571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471673984492673:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35144
2025-11-27T18:20:07.296933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471674749710662:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.295266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471676845620285:2108] [node 5] ICP22 created new session: [1:7577471676845620323:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.297797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471673984492661:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:07.296653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471670039678787:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:07.298471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471674719828733:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.296969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:07.295725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471676845620323:2108] [node 5] ICS09 handshake done sender: [1:7577471676845620300:2116] self: [1:7577471676845620299:0] peer: [5:7577471675336740123:0] socket: 332 qp: -1
2025-11-27T18:20:07.298510Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471674719828733:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:07.297439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471674749710629:2110] [node 6] ICP22 created new session: [4:7577471674749710666:2110]
2025-11-27T18:20:07.299061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471669689525328:2092] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.295765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471676845620323:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:07.303583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471675336740126:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36362
2025-11-27T18:20:07.301950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:07.298437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471674749710666:2110] [node 6] ICS09 handshake done sender: [4:7577471674749710638:2115] self: [4:7577471674749710636:0] peer: [6:7577471673984492666:0] socket: 385 qp: -1
2025-11-27T18:20:07.296826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:07.304354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471674719828733:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:07.305147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471674334646147:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(26862330ns) {HandleHandshakeStatus:329(26859492ns) {DropIncomingHandshake:442(375ns) DropOutgoingHandshake:457(353ns) SwitchToState:179(356ns) ProcessPendingSessionEvents:495(21096805ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20461388ns max# 4121375ns}}}
2025-11-27T18:20:07.302617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471669689525328:2092] [node 4] ICP22 created new session: [6:7577471673984492674:2092]
2025-11-27T18:20:07.299073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471674749710666:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:07.304677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471675336740124:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:07.304729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471673984492674:2092] [node 4] ICS09 handshake done sender: [6:7577471673984492661:2121] self: [6:7577471673984492666:0] peer: [4:7577471674749710636:0] socket: 386 qp: -1
2025-11-27T18:20:07.299114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:07.305483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471671041772732:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61049) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471675336740115:2121] held: no
2025-11-27T18:20:07.314701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471670424861360:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.304764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471673984492674:2092] [node 4] ICS10 traffic start
2025-11-27T18:20:07.304222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471670454743270:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.305820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.314770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:07.305470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:07.318427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471675336740100:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:07.317604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471674334646185:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6229
2025-11-27T18:20:07.316567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471670424861360:2049] [node 3] ICP22 created new session: [2:7577471674719828758:2049]
2025-11-27T18:20:07.309462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471670454743270:2050] [node 3] ICP22 created new session: [4:7577471674749710668:2050]
2025-11-27T18:20:07.317589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471674719828758:2049] [node 3] ICS09 handshake done sender: [2:7577471674719828733:2114] self: [2:7577471674719828732:0] peer: [3:7577471674334646184:0] socket: 389 qp: -1
2025-11-27T18:20:07.315369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471674749710668:2050] [node 3] ICS09 handshake done sender: [4:7577471674749710650:2120] self: [4:7577471674749710657:0] peer: [3:7577471674334646158:0] socket: 329 qp: -1
2025-11-27T18:20:07.315419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471674749710668:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:07.322136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471675336740092:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:07.318251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471674719828758:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:07.321420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471670039678787:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20459) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471674334646182:2122] held: no
2025-11-27T18:20:07.316366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:07.318299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:07.322397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:07.324159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471674334646147:2107] [node 4] ICP18 handshake (actor [3:7577471674334646180:2121]) is too old
2025-11-27T18:20:07.322607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471675336740092:2109] [node 6] ICP22 created new session: [5:7577471675336740129:2109]
2025-11-27T18:20:07.324787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471672550652987:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(43052441ns) {HandleHandshakeStatus:329(43049944ns) {DropIncomingHandshake:442(10856ns) DropOutgoingHandshake:457(321ns) SwitchToState:179(269ns) ProcessPendingSessionEvents:495(36352536ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36233529ns max# 4604668ns}}}
2025-11-27T18:20:07.323231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471675336740129:2109] [node 6] ICS09 handshake done sender: [5:7577471675336740100:2113] self: [5:7577471675336740097:0] peer: [6:7577471673984492672:0] socket: 360 qp: -1
2025-11-27T18:20:07.323386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471675336740129:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:07.323422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:07.330423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471674334646182:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:07.330810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471669689525326:2096] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2702) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471673984492667:2122] held: no
2025-11-27T18:20:07.330854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471670039678787:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.332489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471671041772732:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17630012ns) {HandleHandshakeStatus:395(17627599ns) {LogHandshakeFail:472(506060ns) {UpdateErrorStateLog:545(4722ns)} DropOutgoingHandshake:457(1531ns) ProcessPendingSessionEvents:495(16654570ns) {unordered DropSessionEvent:512 num# 100 sum# 16607872ns max# 4023724ns}}}
2025-11-27T18:20:07.332437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:07.332766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471670039678787:2049] [node 2] ICP22 created new session: [3:7577471674334646186:2049]
2025-11-27T18:20:07.333074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471674334646186:2049] [node 2] ICS09 handshake done sender: [3:7577471674334646182:2122] self: [3:7577471674334646184:0] peer: [2:7577471674719828732:0] socket: 392 qp: -1
2025-11-27T18:20:07.333122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471674334646186:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:07.333063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471675336740115:2121] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:07.333312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471671041772732:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.333506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471673984492676:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35144
2025-11-27T18:20:07.333631Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471669689525328:2092] [node 4] ICP35 event processing took too much time PendingConnection:279(30391029ns) {HandleHandshakeStatus:329(30388504ns) {DropIncomingHandshake:442(498ns) DropOutgoingHandshake:457(5549ns) SwitchToState:179(307ns) ProcessPendingSessionEvents:495(15957985ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15878951ns max# 4637443ns}}}
2025-11-27T18:20:07.333164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:07.333607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:07.333808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471671041772732:2048] [node 1] ICP22 created new session: [5:7577471675336740131:2048]
2025-11-27T18:20:07.334385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471673984492667:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:07.334422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471676845620285:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(35450740ns) {HandleHandshakeStatus:329(35449550ns) {DropIncomingHandshake:442(9350ns) DropOutgoingHandshake:457(305ns) SwitchToState:179(217ns) ProcessPendingSessionEvents:495(26296472ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25477052ns max# 4009284ns}}}
2025-11-27T18:20:07.334185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471675336740131:2048] [node 1] ICS09 handshake done sender: [5:7577471675336740115:2121] self: [5:7577471675336740123:0] peer: [1:7577471676845620299:0] socket: 384 qp: -1
2025-11-27T18:20:07.334526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471674749710629:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(28932229ns) {HandleHandshakeStatus:329(28930075ns) {DropIncomingHandshake:442(359ns) DropOutgoingHandshake:457(322ns) SwitchToState:179(341ns) ProcessPendingSessionEvents:495(21419596ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21306037ns max# 7409469ns}}}
2025-11-27T18:20:07.334611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471669689525326:2096] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:07.334302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471675336740131:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:07.334659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:07.334343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:07.334784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471669689525326:2096] [node 5] ICP22 created new session: [6:7577471673984492677:2096]
2025-11-27T18:20:07.334892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471673984492677:2096] [node 5] ICS09 handshake done sender: [6:7577471673984492667:2122] self: [6:7577471673984492672:0] peer: [5:7577471675336740097:0] socket: 381 qp: -1
2025-11-27T18:20:07.334931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471673984492677:2096] [node 5] ICS10 traffic start
2025-11-27T18:20:07.334971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:07.335485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471670454743270:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(28799158ns) {HandleHandshakeStatus:329(28797415ns) {DropIncomingHandshake:442(1111ns) DropOutgoingHandshake:457(6696ns) SwitchToState:179(553ns) ProcessPendingSessionEvents:495(13648033ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13563112ns max# 3747955ns}}}
2025-11-27T18:20:07.335653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471674749710629:2110] [node 6] ICP18 handshake (actor [4:7577471674749710662:2124]) is too old
2025-11-27T18:20:07.387026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:07.387255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471674719828748:2048] [node 1] ICS07 socket disconnect 339 reason# EndOfStream
2025-11-27T18:20:07.387281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471674719828748:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.387369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:07.387409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471674719828748:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:07.388659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:07.388861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471674334646178:2048] [node 1] ICS01 socket: 343 reason# EndOfStream
2025-11-27T18:20:07.388901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471670039678786:2048] [node 1] ICP30 unregister session Session# [3:7577471674334646178:2048] VirtualId# [3:7577471674334646166:0]
2025-11-27T18:20:07.388929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471674334646178:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.388997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:07.389048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:07.389926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471670424861359:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61049) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:07.389987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471674719828748:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:07.391110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471670424861359:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61049) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:07.391160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471674719828748:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:07.391786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471670424861359:2048] [node 1] ICP30 unregister session Session# [2:7577471674719828748:2048] VirtualId# [2:7577471674719828737:0]
2025-11-27T18:20:07.391867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:07.391900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471670424861359:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61049) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.391909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61049) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.391996Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:07.392161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471674749710664:2048] [node 1] ICS01 socket: 380 reason# EndOfStream
2025-11-27T18:20:07.392215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471670454743268:2048] [node 1] ICP30 unregister session Session# [4:7577471674749710664:2048] VirtualId# [4:7577471674749710655:0]
2025-11-27T18:20:07.392246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471674749710664:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.392296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:07.392344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:07.393383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:07.393595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471675336740131:2048] [node 1] ICS01 socket: 384 reason# EndOfStream
2025-11-27T18:20:07.393642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471671041772732:2048] [node 1] ICP30 unregister session Session# [5:7577471675336740131:2048] VirtualId# [5:7577471675336740123:0]
2025-11-27T18:20:07.393671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471675336740131:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.393745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:07.393788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:07.397520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:07.397919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471673984492670:2048] [node 1] ICS01 socket: 350 reason# EndOfStream
2025-11-27T18:20:07.398058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471669689525278:2048] [node 1] ICP30 unregister session Session# [6:7577471673984492670:2048] VirtualId# [6:7577471673984492658:0]
2025-11-27T18:20:07.398159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471673984492670:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.398367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:07.398479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:07.401811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471670424861359:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:07.450596Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:07.450839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471674334646186:2049] [node 2] ICS01 socket: 392 reason# EndOfStream
2025-11-27T18:20:07.450883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471670039678787:2049] [node 2] ICP30 unregister session Session# [3:7577471674334646186:2049] VirtualId# [3:7577471674334646184:0]
2025-11-27T18:20:07.450916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471674334646186:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.451003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:07.451058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:07.453409Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:07.453619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471674749710656:2049] [node 2] ICS07 socket disconnect 346 reason# EndOfStream
2025-11-27T18:20:07.453654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471674749710656:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.453725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:07.453778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471674749710656:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:07.454623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471670454743269:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20459) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:07.454698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471674749710656:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:07.454742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471670454743269:2049] [node 2] ICP30 unregister session Session# [4:7577471674749710656:2049] VirtualId# [4:7577471674749710646:0]
2025-11-27T18:20:07.454706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:07.454862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:07.454903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471670454743269:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20459) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.454923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20459) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.454901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471675336740118:2049] [node 2] ICS01 socket: 330 reason# EndOfStream
2025-11-27T18:20:07.454956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471671041772733:2049] [node 2] ICP30 unregister session Session# [5:7577471675336740118:2049] VirtualId# [5:7577471675336740109:0]
2025-11-27T18:20:07.454989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471675336740118:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.455072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:07.455119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:07.461763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:07.462468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471673984492662:2049] [node 2] ICS07 socket disconnect 345 reason# EndOfStream
2025-11-27T18:20:07.462494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471673984492662:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.462564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:07.462630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471673984492662:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:07.463991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471669689525279:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20459) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:07.464043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471673984492662:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:07.464071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471669689525279:2049] [node 2] ICP30 unregister session Session# [6:7577471673984492662:2049] VirtualId# [6:7577471673984492656:0]
2025-11-27T18:20:07.464442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:07.464499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471669689525279:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20459) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.464519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20459) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.465302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471670454743269:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:07.474530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471669689525279:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:07.504080Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:07.504331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471674749710668:2050] [node 3] ICS07 socket disconnect 329 reason# EndOfStream
2025-11-27T18:20:07.504367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471674749710668:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.504445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:07.504489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471674749710668:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:07.505228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471670454743270:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21784) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:07.505362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:07.505356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471674749710668:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:07.505510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471670454743270:2050] [node 3] ICP30 unregister session Session# [4:7577471674749710668:2050] VirtualId# [4:7577471674749710657:0]
2025-11-27T18:20:07.505661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:07.505658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471675336740120:2050] [node 3] ICS01 socket: 334 reason# EndOfStream
2025-11-27T18:20:07.505706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471670454743270:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.505732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.505711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471671041772734:2050] [node 3] ICP30 unregister session Session# [5:7577471675336740120:2050] VirtualId# [5:7577471675336740110:0]
2025-11-27T18:20:07.505741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471675336740120:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.505824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:07.505881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:07.506783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:07.506975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471673984492664:2050] [node 3] ICS07 socket disconnect 347 reason# EndOfStream
2025-11-27T18:20:07.507007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471673984492664:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.507077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:07.507126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471673984492664:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:07.507890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471669689525280:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21784) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:07.507987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471673984492664:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:07.508026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471669689525280:2050] [node 3] ICP30 unregister session Session# [6:7577471673984492664:2050] VirtualId# [6:7577471673984492655:0]
2025-11-27T18:20:07.508141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:07.508187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471669689525280:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.508201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21784) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.515446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471670454743270:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:07.518591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471669689525280:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:07.608067Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:07.608357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471675336740122:2108] [node 4] ICS01 socket: 357 reason# EndOfStream
2025-11-27T18:20:07.608400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471675336740091:2108] [node 4] ICP30 unregister session Session# [5:7577471675336740122:2108] VirtualId# [5:7577471675336740112:0]
2025-11-27T18:20:07.608434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471675336740122:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.608514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:07.608569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:07.609542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:07.609776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471673984492674:2092] [node 4] ICS07 socket disconnect 386 reason# EndOfStream
2025-11-27T18:20:07.609808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471673984492674:2092] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.609881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:07.609920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471673984492674:2092] [node 4] ICS15 start handshake
2025-11-27T18:20:07.610794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471669689525328:2092] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61403) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:07.610906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471673984492674:2092] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:07.610961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471669689525328:2092] [node 4] ICP30 unregister session Session# [6:7577471673984492674:2092] VirtualId# [6:7577471673984492666:0]
2025-11-27T18:20:07.611074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:07.611116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471669689525328:2092] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61403) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.611132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61403) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:07.622002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471669689525328:2092] [node 4] ICP33 wake up from error state
2025-11-27T18:20:07.682679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:07.682916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471673984492677:2096] [node 5] ICS01 socket: 381 reason# EndOfStream
2025-11-27T18:20:07.682959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471669689525326:2096] [node 5] ICP30 unregister session Session# [6:7577471673984492677:2096] VirtualId# [6:7577471673984492672:0]
2025-11-27T18:20:07.682995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471673984492677:2096] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:07.683089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:07.683148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:07.821719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471676987926995:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:07.822329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471676987926996:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:07.845828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471676438496084:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:07.846277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471676438496085:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:07.866088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471675049279982:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:07.866100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471675049279983:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:07.888526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471677338841433:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:07.888566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471677338841434:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:07.888566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471677338841435:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:07.911327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471677229362457:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:07.911371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471677229362458:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:07.912193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471677229362456:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:07.935118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471675408948703:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:07.935178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471675408948704:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:07.935482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471675408948705:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:08.000825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471675408948764:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:08.031858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471681282894354:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:08.078012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471679703916063:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:08.059588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471679344247343:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:08.069825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471680733463444:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:08.035475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471681282894356:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:08.082940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471681633808792:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:08.069790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471679344247344:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:08.078245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471681524329815:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:08.070010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471681282894355:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:08.089913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471679344247345:2111] [node 6] ICP01 ready to work
2025-11-27T18:20:08.089861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471681524329816:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:08.090461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471681633808793:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:08.070510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471680733463445:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:08.086377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471680733463446:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:08.112861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471681633808793:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19257231ns) {Configure:104(19252474ns) {ProcessConfigured:135(52137ns) {StartInitialHandshake:157(49573ns) {DropHandshakes:470(1160ns) {DropIncomingHandshake:442(435ns) DropOutgoingHandshake:457(92ns)} GenerateSessionVirtualId:988(416ns) PrepareNewSessionHandshake:477(1365ns) {SwitchToState:179(576ns)}}}}}
2025-11-27T18:20:08.111434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471676987926995:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(102740656ns) {Configure:104(102731631ns) {ProcessConfigured:135(4718011ns) {StartInitialHandshake:157(2805063ns) {DropHandshakes:470(2419ns) {DropIncomingHandshake:442(684ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(1417ns) PrepareNewSessionHandshake:477(2864ns) {SwitchToState:179(1194ns)}}}}}
2025-11-27T18:20:08.113527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471675408948764:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(99113020ns) {Configure:104(99108353ns) {ProcessConfigured:135(8006965ns) {StartInitialHandshake:157(8004442ns) {DropHandshakes:470(2520ns) {DropIncomingHandshake:442(625ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(1257ns) PrepareNewSessionHandshake:477(5030ns) {SwitchToState:179(2468ns)}}}}}
2025-11-27T18:20:08.113944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471675049279982:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(99355838ns) {Configure:104(99346354ns) {ProcessConfigured:135(7462516ns) {StartInitialHandshake:157(7460009ns) {DropHandshakes:470(1846ns) {DropIncomingHandshake:442(605ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(583ns) PrepareNewSessionHandshake:477(4607ns) {SwitchToState:179(2511ns)}}}}}
2025-11-27T18:20:08.122489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471675049279983:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(38999362ns) {Configure:104(33518107ns) {ProcessConfigured:135(4803266ns) {StartInitialHandshake:157(4800304ns) {DropHandshakes:470(951552ns) {DropIncomingHandshake:442(1918ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(1426ns) PrepareNewSessionHandshake:477(2411ns) {SwitchToState:179(1341ns)}}}}}
2025-11-27T18:20:08.122528Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471681282894356:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21340250ns) {Configure:104(21335775ns) {ProcessConfigured:135(1732918ns) {StartInitialHandshake:157(1730240ns) {DropHandshakes:470(3079ns) {DropIncomingHandshake:442(719ns) DropOutgoingHandshake:457(355ns)} GenerateSessionVirtualId:988(936ns) PrepareNewSessionHandshake:477(11945ns) {SwitchToState:179(1064ns)}}}}}
2025-11-27T18:20:08.126059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471679703916063:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27534685ns) {Configure:104(27526792ns) {ProcessConfigured:135(3263008ns) {StartInitialHandshake:157(3260770ns) {DropHandshakes:470(956ns) {DropIncomingHandshake:442(269ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(899ns) PrepareNewSessionHandshake:477(10791ns) {SwitchToState:179(996ns)}}}}}
2025-11-27T18:20:08.145388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471677338841435:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31697668ns) {Configure:104(31687576ns) {ProcessConfigured:135(4924087ns) {StartInitialHandshake:157(4921805ns) {DropHandshakes:470(687ns) {DropIncomingHandshake:442(241ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(911ns) PrepareNewSessionHandshake:477(1614ns) {SwitchToState:179(672ns)}}}}}
2025-11-27T18:20:08.133932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471679344247345:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30531079ns) {Configure:104(30528076ns) {ProcessConfigured:135(6221407ns) {StartInitialHandshake:157(6219692ns) {DropHandshakes:470(1476506ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(1448ns)} GenerateSessionVirtualId:988(1376ns) PrepareNewSessionHandshake:477(1535ns) {SwitchToState:179(746ns)}}}}}
2025-11-27T18:20:08.145514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471675408948705:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26326889ns) {Configure:104(26321645ns) {ProcessConfigured:135(1731085ns) {StartInitialHandshake:157(1728705ns) {DropHandshakes:470(1384ns) {DropIncomingHandshake:442(293ns) DropOutgoingHandshake:457(166ns)} GenerateSessionVirtualId:988(745ns) PrepareNewSessionHandshake:477(1524ns) {SwitchToState:179(316ns)}}}}}
2025-11-27T18:20:08.134684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471679344247344:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31067915ns) {Configure:104(31062685ns) {ProcessConfigured:135(5273541ns) {StartInitialHandshake:157(5270325ns) {DropHandshakes:470(460173ns) {DropIncomingHandshake:442(263ns) DropOutgoingHandshake:457(646ns)} GenerateSessionVirtualId:988(1221ns) PrepareNewSessionHandshake:477(2474ns) {SwitchToState:179(1194ns)}}}}}
2025-11-27T18:20:08.153747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471675408948704:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(33804610ns) {Configure:104(33800972ns) {ProcessConfigured:135(4752971ns) {StartInitialHandshake:157(4750736ns) {DropHandshakes:470(1966ns) {DropIncomingHandshake:442(449ns) DropOutgoingHandshake:457(410ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:477(1516ns) {SwitchToState:179(90ns)}}}}}
2025-11-27T18:20:08.125989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471681282894354:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29332836ns) {Configure:104(29328337ns) {ProcessConfigured:135(3088035ns) {StartInitialHandshake:157(3085227ns) {DropHandshakes:470(1030ns) {DropIncomingHandshake:442(336ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:477(10110ns) {SwitchToState:179(1287ns)}}}}}
2025-11-27T18:20:08.145711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471679344247343:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(28152814ns) {Configure:104(28147788ns) {ProcessConfigured:135(3967403ns) {StartInitialHandshake:157(3965410ns) {DropHandshakes:470(1247ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(464ns) PrepareNewSessionHandshake:477(2061ns) {SwitchToState:179(932ns)}}}}}
2025-11-27T18:20:08.145741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471677338841434:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(62328761ns) {Configure:104(62320309ns) {ProcessConfigured:135(8333746ns) {StartInitialHandshake:157(8332286ns) {DropHandshakes:470(1149ns) {DropIncomingHandshake:442(359ns) DropOutgoingHandshake:457(231ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:477(1826ns) {SwitchToState:179(638ns)}}}}}
2025-11-27T18:20:08.129481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471677229362458:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25776489ns) {Configure:104(25770761ns) {ProcessConfigured:135(5287406ns) {StartInitialHandshake:157(5284842ns) {DropHandshakes:470(2131ns) {DropIncomingHandshake:442(384ns) DropOutgoingHandshake:457(649ns)} GenerateSessionVirtualId:988(679ns) PrepareNewSessionHandshake:477(2579ns) {SwitchToState:179(1177ns)}}}}}
2025-11-27T18:20:08.134431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471680733463444:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(28232661ns) {Configure:104(28227500ns) {ProcessConfigured:135(519656ns) {StartInitialHandshake:157(517496ns) {DropHandshakes:470(871ns) {DropIncomingHandshake:442(258ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:477(5421ns) {SwitchToState:179(1275ns)}}}}}
2025-11-27T18:20:08.154113Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471677338841433:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34105703ns) {Configure:104(34098902ns) {ProcessConfigured:135(4174950ns) {StartInitialHandshake:157(3725187ns) {DropHandshakes:470(1332ns) {DropIncomingHandshake:442(533ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(1001ns) PrepareNewSessionHandshake:477(2899ns) {SwitchToState:179(1970ns)}}}}}
2025-11-27T18:20:08.152576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471676438496085:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33581661ns) {Configure:104(33576112ns) {ProcessConfigured:135(3828736ns) {StartInitialHandshake:157(3825929ns) {DropHandshakes:470(1557ns) {DropIncomingHandshake:442(532ns) DropOutgoingHandshake:457(116ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(1332610ns) {SwitchToState:179(3119ns)}}}}}
2025-11-27T18:20:08.152647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471676438496084:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34042433ns) {Configure:104(34037501ns) {ProcessConfigured:135(3006543ns) {StartInitialHandshake:157(3003438ns) {DropHandshakes:470(1187ns) {DropIncomingHandshake:442(337ns) DropOutgoingHandshake:457(190ns)} GenerateSessionVirtualId:988(740ns) PrepareNewSessionHandshake:477(2891ns) {SwitchToState:179(1645ns)}}}}}
2025-11-27T18:20:08.153121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471680733463445:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28506270ns) {Configure:104(28497961ns) {ProcessConfigured:135(3730833ns) {StartInitialHandshake:157(3729048ns) {DropHandshakes:470(1091ns) {DropIncomingHandshake:442(288ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(534ns) PrepareNewSessionHandshake:477(1939ns) {SwitchToState:179(743ns)}}}}}
2025-11-27T18:20:08.126095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471681282894355:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25233051ns) {Configure:104(25216207ns) {ProcessConfigured:135(3123485ns) {StartInitialHandshake:157(3120525ns) {DropHandshakes:470(1516556ns) {DropIncomingHandshake:442(339ns) DropOutgoingHandshake:457(1195ns)} GenerateSessionVirtualId:988(1339ns) PrepareNewSessionHandshake:477(1654ns) {SwitchToState:179(643ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.167233Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471680733463446:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(56584982ns) {Configure:104(56578052ns) {ProcessConfigured:135(10893334ns) {StartInitialHandshake:157(10890102ns) {DropHandshakes:470(3490ns) {DropIncomingHandshake:442(721ns) DropOutgoingHandshake:457(353ns)} GenerateSessionVirtualId:988(891ns) PrepareNewSessionHandshake:477(3345ns) {SwitchToState:179(1214ns)}}}}}
2025-11-27T18:20:08.167185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471681633808792:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(68439755ns) {Configure:104(68432647ns) {ProcessConfigured:135(12731017ns) {StartInitialHandshake:157(12727036ns) {DropHandshakes:470(3437456ns) {DropIncomingHandshake:442(855ns) DropOutgoingHandshake:457(1391ns)} GenerateSessionVirtualId:988(1564ns) PrepareNewSessionHandshake:477(4194ns) {SwitchToState:179(2490ns)}}}}}
2025-11-27T18:20:08.141686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471677229362457:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(31437017ns) {Configure:104(31429277ns) {ProcessConfigured:135(4502375ns) {StartInitialHandshake:157(4499695ns) {DropHandshakes:470(1075ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(293ns) PrepareNewSessionHandshake:477(2495ns) {SwitchToState:179(831ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.178809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471681282894361:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15890
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.177519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471681633808807:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32299
2025-11-27T18:20:08.149647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471681524329815:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(34162414ns) {Configure:104(34155168ns) {ProcessConfigured:135(4005553ns) {StartInitialHandshake:157(4002793ns) {DropHandshakes:470(1062ns) {DropIncomingHandshake:442(266ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(868ns) PrepareNewSessionHandshake:477(2692ns) {SwitchToState:179(849ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.196986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471680733463460:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.198178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471679344247356:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47442
2025-11-27T18:20:08.198194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471681282894361:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.152607Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471677229362456:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43854007ns) {Configure:104(43833339ns) {ProcessConfigured:135(3891879ns) {StartInitialHandshake:157(3889623ns) {DropHandshakes:470(1377ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(518ns) PrepareNewSessionHandshake:477(3350ns) {SwitchToState:179(1558ns)}}}}}
2025-11-27T18:20:08.197706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471680733463461:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-11-27T18:20:08.201126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471681633808808:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32299
2025-11-27T18:20:08.152602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471681524329816:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(30324702ns) {Configure:104(30319156ns) {ProcessConfigured:135(8149596ns) {StartInitialHandshake:157(8147320ns) {DropHandshakes:470(2088ns) {DropIncomingHandshake:442(546ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(791ns) PrepareNewSessionHandshake:477(1249ns) {SwitchToState:179(443ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.205134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471681282894373:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15890
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.204321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471681633808807:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.197800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471680733463462:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.207446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471677338841434:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.205574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471680733463462:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.216680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471679703916077:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50547
2025-11-27T18:20:08.215492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471679344247356:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:08.218304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471681282894372:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15890
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.217591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471681524329829:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62207
2025-11-27T18:20:08.215695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471681633808808:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.218558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471681282894360:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.217937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471679703916078:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50547
2025-11-27T18:20:08.214641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471680733463451:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.217862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471681524329831:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62207
2025-11-27T18:20:08.217038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471675049279982:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.218212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471677338841435:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.217498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471679344247360:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47442
2025-11-27T18:20:08.218589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471681282894360:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:08.223773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471679703916077:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.214684Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471680733463451:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.223809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471681524329829:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.228050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471681633808809:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32299
2025-11-27T18:20:08.218611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471681282894373:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.220503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471679344247360:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.216024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471680733463460:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.226121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471679703916078:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.223819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471681524329831:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.230974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471681633808809:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.227665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471679703916063:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.231622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471681633808795:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.225984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471681282894371:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15890
2025-11-27T18:20:08.225356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471679344247353:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.218024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471680733463461:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.227554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471681524329832:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62207
2025-11-27T18:20:08.229294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471675408948705:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.229279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471681282894372:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.223599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471680733463451:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:08.231647Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471681633808795:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:08.225384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471679344247353:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.230891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471681282894359:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.230621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471677229362456:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.225990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471679344247361:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47442
2025-11-27T18:20:08.235476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471681633808812:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32299
2025-11-27T18:20:08.220729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471676438496084:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.231609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471681524329832:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.229402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471675049279983:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.229521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471679344247353:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:08.230918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471681282894359:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.249445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471679703916083:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50547
2025-11-27T18:20:08.241910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471681633808795:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.232697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471680733463453:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.237748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471679344247361:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.232909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471677229362457:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.243623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471681633808807:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.231566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471681282894371:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.232727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471680733463453:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.238524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471679344247354:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.255491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471681524329836:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62207
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.244529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471681633808813:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32299
2025-11-27T18:20:08.233087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471681282894360:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.232803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471680733463444:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.256230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471677229362456:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4670) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471681524329831:2116] held: no
2025-11-27T18:20:08.238554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471679344247354:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:08.281544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471679703916081:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50547
2025-11-27T18:20:08.235643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471681282894367:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.249921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471677338841434:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.232846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:08.269140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471677338841435:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471681633808808:2116] held: no
2025-11-27T18:20:08.269426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:08.269990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471677338841434:2049] [node 2] ICP22 created new session: [4:7577471681633808816:2049]
2025-11-27T18:20:08.269995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471681633808793:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.270048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:08.274752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471681633808793:2109] [node 6] ICP22 created new session: [4:7577471681633808817:2109]
2025-11-27T18:20:08.277350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471681633808816:2049] [node 2] ICS09 handshake done sender: [4:7577471681633808807:2115] self: [4:7577471681633808810:0] peer: [2:7577471680733463449:0] socket: 346 qp: -1
2025-11-27T18:20:08.277412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471681633808816:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:08.277469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:08.279154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471677338841434:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(32661380ns) {HandleHandshakeStatus:329(28298160ns) {DropIncomingHandshake:442(621ns) DropOutgoingHandshake:457(6537ns) SwitchToState:179(572ns) ProcessPendingSessionEvents:495(554526ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 489479ns max# 58361ns}}}
2025-11-27T18:20:08.297448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471681633808817:2109] [node 6] ICS09 handshake done sender: [4:7577471681633808795:2110] self: [4:7577471681633808794:0] peer: [6:7577471679703916079:0] socket: 354 qp: -1
2025-11-27T18:20:08.303389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471681633808817:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:08.305230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:08.280485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471681524329838:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62207
2025-11-27T18:20:08.240003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471679344247343:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.285522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471679703916082:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50547
2025-11-27T18:20:08.235678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471681282894367:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:08.234279Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471680733463453:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.303552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471681524329838:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.301957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471679703916078:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:08.240059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:08.239990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471681282894359:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.234476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471680733463444:2107] [node 4] ICP22 created new session: [2:7577471680733463467:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.306854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471675408948705:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.307059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471677229362457:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61806) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471681524329832:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.240726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471679344247343:2107] [node 4] ICP22 created new session: [3:7577471679344247367:2107]
2025-11-27T18:20:08.241508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471676987926996:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.235531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471680733463467:2107] [node 4] ICS09 handshake done sender: [2:7577471680733463451:2111] self: [2:7577471680733463449:0] peer: [4:7577471681633808810:0] socket: 341 qp: -1
2025-11-27T18:20:08.312805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471681524329816:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.307134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471679703916081:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.235697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471680733463467:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:08.243642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471679344247367:2107] [node 4] ICS09 handshake done sender: [3:7577471679344247353:2114] self: [3:7577471679344247349:0] peer: [4:7577471681633808811:0] socket: 362 qp: -1
2025-11-27T18:20:08.241946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471681282894367:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.326978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471681633808803:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.315469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471681524329837:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62207
2025-11-27T18:20:08.308952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:08.245048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471679344247367:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:08.242062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:08.235748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471676438496085:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.315939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471681524329840:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62207
2025-11-27T18:20:08.327030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471681633808803:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:08.308985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471679703916077:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:08.249983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471679344247354:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:08.244609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471676987926996:2049] [node 3] ICP22 created new session: [1:7577471681282894381:2049]
2025-11-27T18:20:08.236332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:08.249127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471676987926995:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.310575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471675408948705:2050] [node 3] ICP22 created new session: [6:7577471679703916084:2050]
2025-11-27T18:20:08.327105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471681633808820:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32299
2025-11-27T18:20:08.250369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.327896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471681633808820:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.312254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471679703916084:2050] [node 3] ICS09 handshake done sender: [6:7577471679703916078:2116] self: [6:7577471679703916080:0] peer: [3:7577471679344247351:0] socket: 371 qp: -1
2025-11-27T18:20:08.237113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:08.340289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471681524329826:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.238036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471676438496085:2049] [node 3] ICP22 created new session: [2:7577471680733463471:2049]
2025-11-27T18:20:08.312288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471679703916084:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:08.238571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471680733463471:2049] [node 3] ICS09 handshake done sender: [2:7577471680733463453:2113] self: [2:7577471680733463450:0] peer: [3:7577471679344247364:0] socket: 342 qp: -1
2025-11-27T18:20:08.340352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471681524329826:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:08.249173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:08.239309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471680733463456:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.312809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:08.249818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471681282894381:2049] [node 3] ICS09 handshake done sender: [1:7577471681282894360:2111] self: [1:7577471681282894358:0] peer: [3:7577471679344247362:0] socket: 333 qp: -1
2025-11-27T18:20:08.340572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471681524329843:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62207
2025-11-27T18:20:08.317698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471679703916063:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.341160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471681524329840:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.239341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471680733463456:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:08.249821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471676987926995:2048] [node 2] ICP22 created new session: [1:7577471681282894382:2048]
2025-11-27T18:20:08.321704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:08.240009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471680733463471:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:08.342137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471677229362458:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.325794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471679703916063:2109] [node 4] ICP22 created new session: [6:7577471679703916088:2109]
2025-11-27T18:20:08.249861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471681282894381:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:08.240054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:08.266777Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471676438496085:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(30098120ns) {HandleHandshakeStatus:329(30094628ns) {DropIncomingHandshake:442(355ns) DropOutgoingHandshake:457(486ns) SwitchToState:179(490ns) ProcessPendingSessionEvents:495(21014303ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20949419ns max# 11701166ns}}}
2025-11-27T18:20:08.327627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471675408948704:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.269920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471680733463456:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:08.328877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471679703916088:2109] [node 4] ICS09 handshake done sender: [6:7577471679703916077:2115] self: [6:7577471679703916079:0] peer: [4:7577471681633808794:0] socket: 356 qp: -1
2025-11-27T18:20:08.251041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471681282894354:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.281880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471680733463470:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-11-27T18:20:08.329389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471679703916088:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:08.330340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471679703916086:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50547
2025-11-27T18:20:08.270885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471675049279982:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4670) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471679344247356:2117] held: no
2025-11-27T18:20:08.293853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471680733463446:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.251097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.330568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471677338841433:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.349419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471681524329838:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:08.330414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:08.293938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:08.273880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471679344247366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47442
2025-11-27T18:20:08.353587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471681524329826:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:08.251152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471681282894354:2108] [node 5] ICP22 created new session: [1:7577471681282894383:2108]
2025-11-27T18:20:08.332725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471681633808803:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:08.331369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471679703916085:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50547
2025-11-27T18:20:08.293998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471680733463446:2108] [node 5] ICP22 created new session: [2:7577471680733463477:2108]
2025-11-27T18:20:08.301380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471679344247345:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.361267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471677229362456:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(68074183ns) {HandleHandshakeStatus:395(68070446ns) {LogHandshakeFail:472(2838645ns) {UpdateErrorStateLog:545(6801ns)} DropOutgoingHandshake:457(426ns) ProcessPendingSessionEvents:495(65162511ns) {unordered DropSessionEvent:512 num# 100 sum# 65088513ns max# 15079386ns}}}
2025-11-27T18:20:08.334895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471681633808792:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.294061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471680733463477:2108] [node 5] ICS09 handshake done sender: [2:7577471680733463456:2115] self: [2:7577471680733463455:0] peer: [5:7577471681524329835:0] socket: 350 qp: -1
2025-11-27T18:20:08.363967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471681524329816:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.334952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:08.251198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471681282894383:2108] [node 5] ICS09 handshake done sender: [1:7577471681282894367:2115] self: [1:7577471681282894366:0] peer: [5:7577471681524329834:0] socket: 338 qp: -1
2025-11-27T18:20:08.333409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471679703916086:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.364417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.301447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:08.294104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471680733463477:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:08.335515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471681633808792:2108] [node 5] ICP22 created new session: [4:7577471681633808824:2108]
2025-11-27T18:20:08.365122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471681524329816:2108] [node 4] ICP22 created new session: [5:7577471681524329846:2108]
2025-11-27T18:20:08.251236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471681282894383:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:08.294138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:08.334468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471679703916085:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.303386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471679344247345:2111] [node 6] ICP22 created new session: [3:7577471679344247372:2111]
2025-11-27T18:20:08.251274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:08.339449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471681633808824:2108] [node 5] ICS09 handshake done sender: [4:7577471681633808803:2114] self: [4:7577471681633808802:0] peer: [5:7577471681524329841:0] socket: 395 qp: -1
2025-11-27T18:20:08.365143Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471681524329831:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:08.306410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471679344247372:2111] [node 6] ICS09 handshake done sender: [3:7577471679344247354:2115] self: [3:7577471679344247351:0] peer: [6:7577471679703916080:0] socket: 370 qp: -1
2025-11-27T18:20:08.295162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471680733463461:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:08.335396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471675408948764:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:08.256199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:08.342150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471681633808824:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:08.335754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471675408948703:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-27T18:20:08.307235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471679344247372:2111] [node 6] ICS10 traffic start
::ffff:127.1.0.02025-11-27T18:20:08.365461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471681524329846:2108] [node 4] ICS09 handshake done sender: [5:7577471681524329838:2120] self: [5:7577471681524329841:0] peer: [4:7577471681633808802:0] socket: 396 qp: -1
2025-11-27T18:20:08.305236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471676438496084:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.309025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471679344247369:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47442
2025-11-27T18:20:08.345633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:08.365684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471681524329846:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:08.361250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471675408948704:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61806) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471679703916081:2117] held: no
2025-11-27T18:20:08.309190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: 
2025-11-27T18:20:08.309698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:08.361154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471681633808825:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32299
2025-11-27T18:20:08.262080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471681282894382:2048] [node 2] ICS09 handshake done sender: [1:7577471681282894359:2110] self: [1:7577471681282894357:0] peer: [2:7577471680733463466:0] socket: 336 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:08.365838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:08.311494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471679344247360:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:08.270031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471681282894382:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:08.310167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471676438496084:2048] [node 1] ICP22 created new session: [2:7577471680733463479:2048]
2025-11-27T18:20:08.370296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471679703916094:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50547
2025-11-27T18:20:08.366356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471681524329815:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.328860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471675049279983:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.311136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471680733463479:2048] [node 1] ICS09 handshake done sender: [2:7577471680733463461:2117] self: [2:7577471680733463466:0] peer: [1:7577471681282894357:0] socket: 352 qp: -1
2025-11-27T18:20:08.366393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471677229362457:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(28285968ns) {HandleHandshakeStatus:395(28281742ns) {LogHandshakeFail:472(13363ns) {UpdateErrorStateLog:545(5721ns)} DropOutgoingHandshake:457(1385ns) ProcessPendingSessionEvents:495(26488908ns) {unordered DropSessionEvent:512 num# 100 sum# 8688854ns max# 3666045ns}}}
2025-11-27T18:20:08.270084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:08.363224Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471677338841435:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(52683821ns) {HandleHandshakeStatus:395(52680567ns) {LogHandshakeFail:472(18182ns) {UpdateErrorStateLog:545(6809ns)} DropOutgoingHandshake:457(828ns) ProcessPendingSessionEvents:495(52579829ns) {unordered DropSessionEvent:512 num# 100 sum# 52482681ns max# 15070606ns}}}
2025-11-27T18:20:08.366692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:08.312194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471680733463479:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:08.321553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471681282894389:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15890
2025-11-27T18:20:08.329501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:08.366915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471677229362456:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.370483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471679703916095:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50547
2025-11-27T18:20:08.312472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471680733463476:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-11-27T18:20:08.364099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471677338841433:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4670) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471681633808820:2120] held: no
2025-11-27T18:20:08.330941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471681282894363:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.365138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471681633808808:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:08.312546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:08.330976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471681282894363:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:08.368487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471677338841435:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.329483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471680733463476:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.334166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471681282894363:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:08.368530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:08.330526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471676438496085:2049] [node 3] ICP18 handshake (actor [2:7577471680733463476:2120]) is too old
2025-11-27T18:20:08.336111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471681282894356:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.368921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471677338841435:2050] [node 3] ICP22 created new session: [4:7577471681633808827:2050]
2025-11-27T18:20:08.330787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471680733463454:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.336181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:08.369330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471681633808827:2050] [node 3] ICS09 handshake done sender: [4:7577471681633808808:2116] self: [4:7577471681633808811:0] peer: [3:7577471679344247349:0] socket: 363 qp: -1
2025-11-27T18:20:08.340325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471681282894356:2107] [node 4] ICP22 created new session: [1:7577471681282894391:2107]
2025-11-27T18:20:08.330821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471680733463454:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:08.369746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471681633808827:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:08.353491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471681282894365:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.345633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471680733463454:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:08.370021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:08.367065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471681524329815:2109] [node 6] ICP22 created new session: [5:7577471681524329848:2109]
2025-11-27T18:20:08.353534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471681282894365:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:08.371230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471681633808820:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:08.345849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471680733463445:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.353541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471681282894391:2107] [node 4] ICS09 handshake done sender: [1:7577471681282894363:2113] self: [1:7577471681282894362:0] peer: [4:7577471681633808822:0] socket: 391 qp: -1
2025-11-27T18:20:08.367086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:08.345911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:08.329557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471675049279983:2049] [node 2] ICP22 created new session: [3:7577471679344247375:2049]
2025-11-27T18:20:08.368470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471681524329848:2109] [node 6] ICS09 handshake done sender: [5:7577471681524329826:2114] self: [5:7577471681524329824:0] peer: [6:7577471679703916092:0] socket: 347 qp: -1
2025-11-27T18:20:08.357542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471680733463445:2110] [node 6] ICP22 created new session: [2:7577471680733463482:2110]
2025-11-27T18:20:08.353592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471681282894391:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:08.330884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471679344247375:2049] [node 2] ICS09 handshake done sender: [3:7577471679344247360:2118] self: [3:7577471679344247364:0] peer: [2:7577471680733463450:0] socket: 358 qp: -1
2025-11-27T18:20:08.363136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471680733463482:2110] [node 6] ICS09 handshake done sender: [2:7577471680733463454:2114] self: [2:7577471680733463447:0] peer: [6:7577471679703916090:0] socket: 380 qp: -1
2025-11-27T18:20:08.368514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471681524329848:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:08.331910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471679344247375:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:08.364068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471680733463482:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:08.361335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:08.332710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.368548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:08.364438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:08.365777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471681282894365:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:08.368667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471677229362456:2048] [node 1] ICP22 created new session: [5:7577471681524329849:2048]
2025-11-27T18:20:08.345811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471675049279982:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(56177222ns) {HandleHandshakeStatus:395(56174486ns) {LogHandshakeFail:472(1252636ns) {UpdateErrorStateLog:545(7284ns)} DropOutgoingHandshake:457(2969ns) ProcessPendingSessionEvents:495(54107506ns) {unordered DropSessionEvent:512 num# 100 sum# 54033642ns max# 14774809ns}}}
2025-11-27T18:20:08.372152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471675408948705:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(56287123ns) {HandleHandshakeStatus:329(56283685ns) {DropIncomingHandshake:442(971ns) DropOutgoingHandshake:457(5437ns) SwitchToState:179(745ns) ProcessPendingSessionEvents:495(34968387ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 30813693ns max# 4448594ns}}}
2025-11-27T18:20:08.372181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471681633808793:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(96013293ns) {HandleHandshakeStatus:329(96009823ns) {DropIncomingHandshake:442(1080ns) DropOutgoingHandshake:457(267ns) SwitchToState:179(949ns) ProcessPendingSessionEvents:495(55057498ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 49907494ns max# 8919343ns}}}
2025-11-27T18:20:08.369199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471681524329849:2048] [node 1] ICS09 handshake done sender: [5:7577471681524329831:2116] self: [5:7577471681524329834:0] peer: [1:7577471681282894366:0] socket: 355 qp: -1
2025-11-27T18:20:08.368551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471681282894355:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.357481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471679344247355:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.369230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471681524329849:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:08.368604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:08.369778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471680733463444:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(112833639ns) {HandleHandshakeStatus:329(112830082ns) {DropIncomingHandshake:442(339ns) DropOutgoingHandshake:457(929ns) SwitchToState:179(446ns) ProcessPendingSessionEvents:495(104821687ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 95735364ns max# 19967918ns}}}
2025-11-27T18:20:08.357527Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471679344247355:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:08.368872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471681282894355:2109] [node 6] ICP22 created new session: [1:7577471681282894394:2109]
2025-11-27T18:20:08.369622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:08.365088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471679344247356:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:08.370027Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471676438496084:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(40497958ns) {HandleHandshakeStatus:329(40494962ns) {DropIncomingHandshake:442(1191ns) DropOutgoingHandshake:457(7142ns) SwitchToState:179(977ns) ProcessPendingSessionEvents:495(19702307ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18717628ns max# 3417585ns}}}
2025-11-27T18:20:08.366256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471679344247377:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47442
2025-11-27T18:20:08.372575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471677338841433:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.366885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471679344247343:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(107020108ns) {HandleHandshakeStatus:329(107018160ns) {DropIncomingHandshake:442(13114ns) DropOutgoingHandshake:457(418ns) SwitchToState:179(482ns) ProcessPendingSessionEvents:495(63986640ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 59853156ns max# 19798702ns}}}
2025-11-27T18:20:08.369159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471681282894394:2109] [node 6] ICS09 handshake done sender: [1:7577471681282894365:2114] self: [1:7577471681282894364:0] peer: [6:7577471679703916093:0] socket: 392 qp: -1
2025-11-27T18:20:08.373161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:08.372879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471675408948764:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65522) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471679703916086:2121] held: no
2025-11-27T18:20:08.367111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471679344247355:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:08.369358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471676987926995:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(96592565ns) {HandleHandshakeStatus:329(96588984ns) {DropIncomingHandshake:442(570ns) DropOutgoingHandshake:457(642ns) SwitchToState:179(517ns) ProcessPendingSessionEvents:495(43938142ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 43790493ns max# 4796941ns}}}
2025-11-27T18:20:08.369631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471681524329832:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:08.373528Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471679703916063:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(53670107ns) {HandleHandshakeStatus:329(53667427ns) {DropIncomingHandshake:442(2765ns) DropOutgoingHandshake:457(5506ns) SwitchToState:179(688ns) ProcessPendingSessionEvents:495(32990052ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32599353ns max# 4097618ns}}}
2025-11-27T18:20:08.368226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471675049279982:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.373710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471677338841433:2048] [node 1] ICP22 created new session: [4:7577471681633808829:2048]
2025-11-27T18:20:08.373946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471675408948703:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4670) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471679703916085:2120] held: no
2025-11-27T18:20:08.371526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471677229362457:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.368267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:08.374154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471681633808829:2048] [node 1] ICS09 handshake done sender: [4:7577471681633808820:2120] self: [4:7577471681633808822:0] peer: [1:7577471681282894362:0] socket: 394 qp: -1
2025-11-27T18:20:08.371614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:08.368336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471675049279982:2048] [node 1] ICP22 created new session: [3:7577471679344247379:2048]
2025-11-27T18:20:08.374287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471681633808829:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:08.371657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471677229362457:2049] [node 2] ICP22 created new session: [5:7577471681524329853:2049]
2025-11-27T18:20:08.368800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471679344247379:2048] [node 1] ICS09 handshake done sender: [3:7577471679344247356:2117] self: [3:7577471679344247362:0] peer: [1:7577471681282894358:0] socket: 337 qp: -1
2025-11-27T18:20:08.374791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:08.369370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471676987926996:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(104582033ns) {HandleHandshakeStatus:329(104579705ns) {DropIncomingHandshake:442(14250ns) DropOutgoingHandshake:457(260ns) SwitchToState:179(1795ns) ProcessPendingSessionEvents:495(71142308ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 71014119ns max# 23121831ns}}}
2025-11-27T18:20:08.371694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471681524329853:2049] [node 2] ICS09 handshake done sender: [5:7577471681524329832:2117] self: [5:7577471681524329835:0] peer: [2:7577471680733463455:0] socket: 365 qp: -1
2025-11-27T18:20:08.369717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471681282894394:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:08.371721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471681524329853:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:08.369182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471679344247379:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:08.369982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:08.369813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:08.371750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:08.371799Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471676987926995:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:08.371803Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471681524329816:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:08.370360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471679344247377:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:08.375292Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471681282894356:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(35707751ns) {HandleHandshakeStatus:329(35705509ns) {DropIncomingHandshake:442(12186ns) DropOutgoingHandshake:457(354ns) SwitchToState:179(469ns) ProcessPendingSessionEvents:495(6633716ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6291735ns max# 1013778ns}}}
2025-11-27T18:20:08.372059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471681524329850:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62207
2025-11-27T18:20:08.370365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471679344247345:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(44425298ns) {HandleHandshakeStatus:329(44420227ns) {DropIncomingHandshake:442(565ns) DropOutgoingHandshake:457(966ns) SwitchToState:179(752ns) ProcessPendingSessionEvents:495(33032117ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32941268ns max# 9151486ns}}}
2025-11-27T18:20:08.373163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471681524329840:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:08.370588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471679344247344:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:08.374847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471677229362458:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.375266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:08.371570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:08.375310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471677229362458:2050] [node 3] ICP22 created new session: [5:7577471681524329855:2050]
2025-11-27T18:20:08.371619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471679344247344:2109] [node 5] ICP22 created new session: [3:7577471679344247381:2109]
2025-11-27T18:20:08.376320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471681524329855:2050] [node 3] ICS09 handshake done sender: [5:7577471681524329840:2121] self: [5:7577471681524329844:0] peer: [3:7577471679344247352:0] socket: 334 qp: -1
2025-11-27T18:20:08.371664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471679344247381:2109] [node 5] ICS09 handshake done sender: [3:7577471679344247355:2116] self: [3:7577471679344247352:0] peer: [5:7577471681524329844:0] socket: 361 qp: -1
2025-11-27T18:20:08.376357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471681524329855:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:08.376387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:08.371693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471679344247381:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:08.371721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:08.372728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471675049279983:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(40805804ns) {HandleHandshakeStatus:329(40802553ns) {DropIncomingHandshake:442(1467ns) DropOutgoingHandshake:457(5498ns) SwitchToState:179(764ns) ProcessPendingSessionEvents:495(32925862ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32814786ns max# 4364882ns}}}
2025-11-27T18:20:08.376298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471679703916081:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:08.375134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471681633808792:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(36702844ns) {HandleHandshakeStatus:329(36699623ns) {DropIncomingHandshake:442(11954ns) DropOutgoingHandshake:457(533ns) SwitchToState:179(337ns) ProcessPendingSessionEvents:495(25835115ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25735473ns max# 7821680ns}}}
2025-11-27T18:20:08.377052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471679344247344:2109] [node 5] ICP18 handshake (actor [3:7577471679344247377:2122]) is too old
2025-11-27T18:20:08.383322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471675408948704:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:08.384070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471680733463445:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(30457104ns) {HandleHandshakeStatus:329(30455249ns) {DropIncomingHandshake:442(18127ns) DropOutgoingHandshake:457(536ns) SwitchToState:179(491ns) ProcessPendingSessionEvents:495(10405918ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10318461ns max# 1196012ns}}}
2025-11-27T18:20:08.383711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:08.383738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471679703916086:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:08.383820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471675408948704:2049] [node 2] ICP22 created new session: [6:7577471679703916097:2049]
2025-11-27T18:20:08.383888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471679703916097:2049] [node 2] ICS09 handshake done sender: [6:7577471679703916081:2117] self: [6:7577471679703916090:0] peer: [2:7577471680733463447:0] socket: 381 qp: -1
2025-11-27T18:20:08.384029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471679703916097:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:08.384060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:08.384971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471679703916096:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50547
2025-11-27T18:20:08.385334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471675408948764:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.385387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:08.387065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471675408948764:2106] [node 5] ICP22 created new session: [6:7577471679703916099:2106]
2025-11-27T18:20:08.387111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471679703916099:2106] [node 5] ICS09 handshake done sender: [6:7577471679703916086:2121] self: [6:7577471679703916092:0] peer: [5:7577471681524329824:0] socket: 397 qp: -1
2025-11-27T18:20:08.387297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471679703916099:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:08.387459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471679703916085:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:08.387479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:08.389077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471675408948703:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:08.389143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:08.389201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471675408948703:2048] [node 1] ICP22 created new session: [6:7577471679703916101:2048]
2025-11-27T18:20:08.389249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471679703916101:2048] [node 1] ICS09 handshake done sender: [6:7577471679703916085:2120] self: [6:7577471679703916093:0] peer: [1:7577471681282894364:0] socket: 340 qp: -1
2025-11-27T18:20:08.389302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471679703916101:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:08.389351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:08.434399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:08.434677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471680733463479:2048] [node 1] ICS07 socket disconnect 352 reason# EndOfStream
2025-11-27T18:20:08.434712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471680733463479:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.434794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:08.434852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471680733463479:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:08.460192Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:08.460426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471679344247379:2048] [node 1] ICS01 socket: 337 reason# EndOfStream
2025-11-27T18:20:08.460472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471675049279982:2048] [node 1] ICP30 unregister session Session# [3:7577471679344247379:2048] VirtualId# [3:7577471679344247362:0]
2025-11-27T18:20:08.460507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471679344247379:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.460582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:08.460635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:08.461516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471676438496084:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4670) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:08.461573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471680733463479:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:08.462866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471676438496084:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4670) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:08.463115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471680733463479:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:08.463167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471676438496084:2048] [node 1] ICP30 unregister session Session# [2:7577471680733463479:2048] VirtualId# [2:7577471680733463466:0]
2025-11-27T18:20:08.463274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:08.463317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471676438496084:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4670) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.463332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4670) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.463384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:08.463658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471681633808829:2048] [node 1] ICS01 socket: 394 reason# EndOfStream
2025-11-27T18:20:08.463714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471677338841433:2048] [node 1] ICP30 unregister session Session# [4:7577471681633808829:2048] VirtualId# [4:7577471681633808822:0]
2025-11-27T18:20:08.463742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471681633808829:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.463806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:08.463849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:08.464435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:08.464629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471681524329849:2048] [node 1] ICS01 socket: 355 reason# EndOfStream
2025-11-27T18:20:08.464664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471677229362456:2048] [node 1] ICP30 unregister session Session# [5:7577471681524329849:2048] VirtualId# [5:7577471681524329834:0]
2025-11-27T18:20:08.464694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471681524329849:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.464759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:08.464804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:08.465362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:08.465524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471679703916101:2048] [node 1] ICS01 socket: 340 reason# EndOfStream
2025-11-27T18:20:08.465556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471675408948703:2048] [node 1] ICP30 unregister session Session# [6:7577471679703916101:2048] VirtualId# [6:7577471679703916093:0]
2025-11-27T18:20:08.465576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471679703916101:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.465616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:08.465643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:08.523150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:08.523333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471679344247375:2049] [node 2] ICS07 socket disconnect 358 reason# EndOfStream
2025-11-27T18:20:08.523360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471679344247375:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.523419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:08.523457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471679344247375:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:08.525624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:08.525746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471675049279983:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61806) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:08.525782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471679344247375:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:08.525924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471681633808816:2049] [node 2] ICS01 socket: 346 reason# EndOfStream
2025-11-27T18:20:08.526071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471677338841434:2049] [node 2] ICP30 unregister session Session# [4:7577471681633808816:2049] VirtualId# [4:7577471681633808810:0]
2025-11-27T18:20:08.526109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471681633808816:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.526175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:08.526227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:08.526718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471675049279983:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61806) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:08.526761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471679344247375:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:08.526836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471675049279983:2049] [node 2] ICP30 unregister session Session# [3:7577471679344247375:2049] VirtualId# [3:7577471679344247364:0]
2025-11-27T18:20:08.526942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:08.526983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471675049279983:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.526998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.527035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:08.527271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471681524329853:2049] [node 2] ICS01 socket: 365 reason# EndOfStream
2025-11-27T18:20:08.527304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471677229362457:2049] [node 2] ICP30 unregister session Session# [5:7577471681524329853:2049] VirtualId# [5:7577471681524329835:0]
2025-11-27T18:20:08.527324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471681524329853:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.527369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:08.527402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:08.528087Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:08.528234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471679703916097:2049] [node 2] ICS01 socket: 381 reason# EndOfStream
2025-11-27T18:20:08.528263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471675408948704:2049] [node 2] ICP30 unregister session Session# [6:7577471679703916097:2049] VirtualId# [6:7577471679703916090:0]
2025-11-27T18:20:08.528285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471679703916097:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.528345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:08.528379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:08.536359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471675049279983:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:08.567039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:08.567281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471681633808827:2050] [node 3] ICS01 socket: 363 reason# EndOfStream
2025-11-27T18:20:08.567329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471677338841435:2050] [node 3] ICP30 unregister session Session# [4:7577471681633808827:2050] VirtualId# [4:7577471681633808811:0]
2025-11-27T18:20:08.567376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471681633808827:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.567466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:08.567517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:08.568220Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:08.568394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471681524329855:2050] [node 3] ICS07 socket disconnect 334 reason# EndOfStream
2025-11-27T18:20:08.568439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471681524329855:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.568527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:08.568562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471681524329855:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:08.569292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:08.569330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471677229362458:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21177) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:08.569469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471681524329855:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:08.569518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471677229362458:2050] [node 3] ICP30 unregister session Session# [5:7577471681524329855:2050] VirtualId# [5:7577471681524329844:0]
2025-11-27T18:20:08.569519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471679703916084:2050] [node 3] ICS07 socket disconnect 371 reason# EndOfStream
2025-11-27T18:20:08.569552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471679703916084:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.569640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:08.569625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:08.569681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471677229362458:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.569669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471679703916084:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:08.569702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.570290Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471675408948705:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21177) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:08.570390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471679703916084:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:08.570432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471675408948705:2050] [node 3] ICP30 unregister session Session# [6:7577471679703916084:2050] VirtualId# [6:7577471679703916080:0]
2025-11-27T18:20:08.570544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:08.570582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471675408948705:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.570599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.579368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471677229362458:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:08.580371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471675408948705:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:08.629342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:08.629601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471681524329846:2108] [node 4] ICS07 socket disconnect 396 reason# EndOfStream
2025-11-27T18:20:08.629638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471681524329846:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.629719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:08.629756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471681524329846:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:08.630356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471681524329816:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11134) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:08.630468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471681524329846:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:08.630531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471681524329816:2108] [node 4] ICP30 unregister session Session# [5:7577471681524329846:2108] VirtualId# [5:7577471681524329841:0]
2025-11-27T18:20:08.630516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:08.630580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:08.630620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471681524329816:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.630636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.630744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471679703916088:2109] [node 4] ICS07 socket disconnect 356 reason# EndOfStream
2025-11-27T18:20:08.630778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471679703916088:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.630847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:08.630903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471679703916088:2109] [node 4] ICS15 start handshake
2025-11-27T18:20:08.631660Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471679703916063:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11134) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:08.631711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471679703916088:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:08.631792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471679703916063:2109] [node 4] ICP30 unregister session Session# [6:7577471679703916088:2109] VirtualId# [6:7577471679703916079:0]
2025-11-27T18:20:08.631874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:08.631903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471679703916063:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.631917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:08.640843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471681524329816:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:08.641819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471679703916063:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:20:08.704203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:08.704451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471679703916099:2106] [node 5] ICS01 socket: 397 reason# EndOfStream
2025-11-27T18:20:08.704494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471675408948764:2106] [node 5] ICP30 unregister session Session# [6:7577471679703916099:2106] VirtualId# [6:7577471679703916092:0]
2025-11-27T18:20:08.704531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471679703916099:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:08.704602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:08.704650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:08.827249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471678024444672:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:08.827300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471678024444673:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:08.849967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471681130739151:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:08.850020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471681130739152:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:08.872207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471679547261348:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:08.872265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471679547261349:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:08.895437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471679751017379:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:08.895500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471679751017380:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:08.895534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471679751017381:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:08.921733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471681736184422:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:08.921806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471681736184423:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:08.921957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471681736184424:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:08.943079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471678567470314:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:08.943154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471678567470315:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:08.943197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471678567470316:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:09.030641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471682862437672:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.035974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471686031151779:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11791
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.041408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471686031151779:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.066879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471685425706507:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9831
2025-11-27T18:20:09.112689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471684045984740:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:09.112629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471683842228725:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:09.112968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471682862437677:2110] [node 4] ICP01 ready to work
2025-11-27T18:20:09.113021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471682319412034:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:09.112900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471686031151782:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:09.113027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471685425706511:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:09.113127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471683842228726:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:09.113700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471684045984741:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:09.113846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471685425706512:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:09.113964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471682319412035:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:09.114001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471683842228727:2127] [node 6] ICP01 ready to work
2025-11-27T18:20:09.114358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471686031151783:2110] [node 4] ICP01 ready to work
2025-11-27T18:20:09.114987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471682319412036:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:09.115068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471685425706513:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.132234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471682319412045:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38775
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.134004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684045984752:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.135293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682862437688:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-27T18:20:09.135280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471685425706507:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.136126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471683842228738:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27432
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.136397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471683842228739:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27432
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.138455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471682319412045:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.138541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471685425706524:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9831
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.138604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471682319412048:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38775
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.138730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471682319412049:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38775
2025-11-27T18:20:09.138742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471685425706525:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9831
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.138997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471682319412050:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38775
2025-11-27T18:20:09.139287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471685425706526:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9831
2025-11-27T18:20:09.139417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471681130739151:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.138645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471683842228738:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.140232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471685425706524:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.140499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471682319412052:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38775
2025-11-27T18:20:09.140700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471685425706527:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9831
2025-11-27T18:20:09.140709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471682319412029:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.140749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471682319412029:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.141082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471684045984752:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.142117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471682319412048:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.142454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471682862437688:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.142487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471682319412049:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.142544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684045984756:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
2025-11-27T18:20:09.142605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471682862437672:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.142644Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471682319412029:2107] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.142675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471679751017379:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.142771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684045984757:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
2025-11-27T18:20:09.142859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471686031151785:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.142905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471686031151785:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.143089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471678024444672:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.143742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:09.144027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471678024444672:2048] [node 2] ICP22 created new session: [1:7577471682319412055:2048]
2025-11-27T18:20:09.144206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471682319412055:2048] [node 2] ICS09 handshake done sender: [1:7577471682319412029:2107] self: [1:7577471682319412028:0] peer: [2:7577471685425706531:0] socket: 339 qp: -1
2025-11-27T18:20:09.144250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471682319412055:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:09.144294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:09.144522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471685425706525:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.145053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471686031151785:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.145184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471684045984756:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.145314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682862437690:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-27T18:20:09.145330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471679751017381:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.145329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471679547261348:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.145484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471683842228739:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.145589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471686031151782:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.145654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:09.145701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471686031151782:2109] [node 6] ICP22 created new session: [5:7577471686031151799:2109]
2025-11-27T18:20:09.145717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471683842228733:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.145738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471683842228733:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.145941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471686031151799:2109] [node 6] ICS09 handshake done sender: [5:7577471686031151785:2111] self: [5:7577471686031151784:0] peer: [6:7577471682862437689:0] socket: 352 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.145928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471682319412040:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.146495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471686031151799:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:09.145960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471682319412040:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.146705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:09.146761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471683842228743:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27432
2025-11-27T18:20:09.146772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471682862437688:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:09.147429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471682862437672:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.147492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:09.147544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471682862437672:2107] [node 5] ICP22 created new session: [6:7577471682862437691:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.147593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471682862437691:2107] [node 5] ICS09 handshake done sender: [6:7577471682862437688:2115] self: [6:7577471682862437689:0] peer: [5:7577471686031151784:0] socket: 355 qp: -1
2025-11-27T18:20:09.147538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471682319412050:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.147593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471683842228733:2130] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:09.147629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471682862437691:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:09.147659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:09.147740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684045984760:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
2025-11-27T18:20:09.147789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471682319412038:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.147827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471682319412038:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.148982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471683842228725:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.149036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:09.149128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471683842228725:2107] [node 4] ICP22 created new session: [3:7577471683842228747:2107]
2025-11-27T18:20:09.149189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471683842228747:2107] [node 4] ICS09 handshake done sender: [3:7577471683842228733:2130] self: [3:7577471683842228732:0] peer: [4:7577471684045984759:0] socket: 363 qp: -1
2025-11-27T18:20:09.149227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471683842228747:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:09.149277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:09.149409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471684045984756:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:09.149586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471682319412040:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:09.149881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471679751017381:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.150257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:09.150320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471679751017381:2050] [node 3] ICP22 created new session: [4:7577471684045984762:2050]
2025-11-27T18:20:09.150398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471682319412038:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:09.150447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471684045984762:2050] [node 3] ICS09 handshake done sender: [4:7577471684045984756:2118] self: [4:7577471684045984759:0] peer: [3:7577471683842228732:0] socket: 366 qp: -1
2025-11-27T18:20:09.150511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471684045984762:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:09.150554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:09.150660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471685425706526:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.150905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471685425706527:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.150931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471682319412034:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.151025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:09.151073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471682319412034:2108] [node 4] ICP22 created new session: [1:7577471682319412062:2108]
2025-11-27T18:20:09.151114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471682319412062:2108] [node 4] ICS09 handshake done sender: [1:7577471682319412040:2112] self: [1:7577471682319412039:0] peer: [4:7577471684045984758:0] socket: 346 qp: -1
2025-11-27T18:20:09.151160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471682319412062:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:09.151192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.151345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471685425706533:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9831
2025-11-27T18:20:09.152849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471685425706507:2107] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.153107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471686031151801:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11791
2025-11-27T18:20:09.153152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471681130739151:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.153191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471678024444673:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.153234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:09.153293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:09.153298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471681130739151:2048] [node 1] ICP22 created new session: [2:7577471685425706537:2048]
2025-11-27T18:20:09.153336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471678024444673:2049] [node 3] ICP22 created new session: [1:7577471682319412064:2049]
2025-11-27T18:20:09.153356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471685425706537:2048] [node 1] ICS09 handshake done sender: [2:7577471685425706507:2107] self: [2:7577471685425706531:0] peer: [1:7577471682319412028:0] socket: 341 qp: -1
2025-11-27T18:20:09.153393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471685425706537:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:09.153388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471682319412064:2049] [node 3] ICS09 handshake done sender: [1:7577471682319412038:2111] self: [1:7577471682319412037:0] peer: [3:7577471683842228744:0] socket: 345 qp: -1
2025-11-27T18:20:09.153422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:09.153436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471682319412064:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:09.153462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:09.153583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471683842228743:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.157153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684045984761:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.157368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471686031151803:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11791
2025-11-27T18:20:09.157444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471683842228749:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27432
2025-11-27T18:20:09.157566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471682319412052:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.158085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471679547261349:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.159860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471679751017379:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471684045984752:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.160312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471683842228751:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27432
2025-11-27T18:20:09.160482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471683842228752:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27432
2025-11-27T18:20:09.160637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471683842228738:2133] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:09.160815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471685425706517:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.160849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471684045984752:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:09.160913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471679547261348:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471683842228738:2133] held: no
2025-11-27T18:20:09.160868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471685425706517:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.161892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471684045984757:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.161951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471679547261348:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.162006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:09.162060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471679547261348:2048] [node 1] ICP22 created new session: [3:7577471683842228754:2048]
2025-11-27T18:20:09.162105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471683842228754:2048] [node 1] ICS09 handshake done sender: [3:7577471683842228738:2133] self: [3:7577471683842228744:0] peer: [1:7577471682319412037:0] socket: 350 qp: -1
2025-11-27T18:20:09.162154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471683842228754:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:09.162190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:09.162173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471679751017379:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.162203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471685425706517:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:09.162295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:09.162347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471679751017379:2048] [node 1] ICP22 created new session: [4:7577471684045984765:2048]
No RDMA context found for GID: 2025-11-27T18:20:09.162384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471684045984765:2048] [node 1] ICS09 handshake done sender: [4:7577471684045984752:2117] self: [4:7577471684045984758:0] peer: [1:7577471682319412039:0] socket: 347 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:09.162425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471684045984765:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:09.162591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471686031151804:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11791
2025-11-27T18:20:09.162593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:09.163403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471681130739152:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.163486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:09.163527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471681130739152:2049] [node 3] ICP22 created new session: [2:7577471685425706541:2049]
2025-11-27T18:20:09.163582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471685425706541:2049] [node 3] ICS09 handshake done sender: [2:7577471685425706517:2112] self: [2:7577471685425706516:0] peer: [3:7577471683842228753:0] socket: 373 qp: -1
2025-11-27T18:20:09.163627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471685425706541:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.163670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:09.163753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471678024444672:2048] [node 2] ICP18 handshake (actor [1:7577471682319412052:2119]) is too old
2025-11-27T18:20:09.163853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682862437694:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-27T18:20:09.163967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471683842228751:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.164065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471683842228752:2138] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.164051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471686031151801:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.164227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471681736184422:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.164819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471686031151806:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11791
2025-11-27T18:20:09.164827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471682319412042:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.164889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471679751017380:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.164860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471682319412042:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.165703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682862437695:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-27T18:20:09.165910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471685425706519:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.165939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471685425706519:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.168017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471682862437694:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.168019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471682319412042:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:09.168152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471678567470314:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.168165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471686031151807:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11791
2025-11-27T18:20:09.170477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471682319412035:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.170725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:09.170917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471682319412035:2109] [node 5] ICP22 created new session: [1:7577471682319412068:2109]
2025-11-27T18:20:09.170947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471686031151801:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:09.171380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471685425706519:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:09.171195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471682319412068:2109] [node 5] ICS09 handshake done sender: [1:7577471682319412042:2113] self: [1:7577471682319412041:0] peer: [5:7577471686031151805:0] socket: 387 qp: -1
2025-11-27T18:20:09.171255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471682319412068:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:09.171545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:09.171956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471681736184422:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.172035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:09.172089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471681736184422:2048] [node 1] ICP22 created new session: [5:7577471686031151808:2048]
2025-11-27T18:20:09.172080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471685425706511:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.172165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:09.172144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471686031151808:2048] [node 1] ICS09 handshake done sender: [5:7577471686031151801:2116] self: [5:7577471686031151805:0] peer: [1:7577471682319412041:0] socket: 337 qp: -1
2025-11-27T18:20:09.172224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471686031151808:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:09.172208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471682862437695:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.172265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.172347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682862437697:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-27T18:20:09.172212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471679547261349:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26406) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471683842228743:2135] held: no
2025-11-27T18:20:09.172564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471682862437677:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.172219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471685425706511:2108] [node 4] ICP22 created new session: [2:7577471685425706545:2108]
2025-11-27T18:20:09.172267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471685425706545:2108] [node 4] ICS09 handshake done sender: [2:7577471685425706519:2113] self: [2:7577471685425706518:0] peer: [4:7577471684045984768:0] socket: 374 qp: -1
2025-11-27T18:20:09.172296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471685425706545:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:09.172330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.173840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471684045984751:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.173876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471684045984751:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.173870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682862437699:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.174182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471683842228756:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27432
2025-11-27T18:20:09.174255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471682319412044:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.174282Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471682319412044:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.175450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684045984769:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
2025-11-27T18:20:09.175952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471683842228743:2135] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:09.176146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471684045984757:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.176765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471684045984751:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:09.176763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471686031151806:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.176866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471686031151803:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.176844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682862437700:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-27T18:20:09.178720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471682862437695:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:09.178668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471679751017380:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.179083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:09.179140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471679751017380:2049] [node 2] ICP22 created new session: [4:7577471684045984771:2049]
2025-11-27T18:20:09.179108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471679547261349:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.179211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471684045984771:2049] [node 2] ICS09 handshake done sender: [4:7577471684045984757:2119] self: [4:7577471684045984768:0] peer: [2:7577471685425706518:0] socket: 375 qp: -1
2025-11-27T18:20:09.179180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:09.179261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471684045984771:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:09.179278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471679547261349:2049] [node 2] ICP22 created new session: [3:7577471683842228760:2049]
2025-11-27T18:20:09.179303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:09.179350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471683842228760:2049] [node 2] ICS09 handshake done sender: [3:7577471683842228743:2135] self: [3:7577471683842228753:0] peer: [2:7577471685425706516:0] socket: 378 qp: -1
2025-11-27T18:20:09.179396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471683842228760:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:09.179432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:09.179581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471682862437677:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.179626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:09.179683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471682862437677:2110] [node 4] ICP22 created new session: [6:7577471682862437702:2110]
2025-11-27T18:20:09.179745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471682862437702:2110] [node 4] ICS09 handshake done sender: [6:7577471682862437695:2118] self: [6:7577471682862437698:0] peer: [4:7577471684045984750:0] socket: 395 qp: -1
2025-11-27T18:20:09.179742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471682319412044:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:09.179787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471682862437702:2110] [node 4] ICS10 traffic start
2025-11-27T18:20:09.179834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:09.179792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471686031151804:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.180132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471686031151783:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.180506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471681736184424:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.180576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471682319412036:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.180594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471681736184423:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.180648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:09.180694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471682319412036:2110] [node 6] ICP22 created new session: [1:7577471682319412071:2110]
2025-11-27T18:20:09.180734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471682319412071:2110] [node 6] ICS09 handshake done sender: [1:7577471682319412044:2114] self: [1:7577471682319412043:0] peer: [6:7577471682862437696:0] socket: 368 qp: -1
2025-11-27T18:20:09.180763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471682319412071:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:09.180805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:09.181207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471682862437697:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.181482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471683842228735:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.181658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471684045984741:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.181525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471683842228735:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.182825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:09.183122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471684045984741:2109] [node 6] ICP22 created new session: [4:7577471684045984773:2109]
2025-11-27T18:20:09.183374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471684045984773:2109] [node 6] ICS09 handshake done sender: [4:7577471684045984751:2116] self: [4:7577471684045984750:0] peer: [6:7577471682862437698:0] socket: 394 qp: -1
2025-11-27T18:20:09.183411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471682862437699:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.183210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471683842228735:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:09.183430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471684045984773:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:09.183543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.184220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471683842228726:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.184492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:09.184482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471686031151814:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11791
2025-11-27T18:20:09.184555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471683842228726:2108] [node 5] ICP22 created new session: [3:7577471683842228763:2108]
2025-11-27T18:20:09.184443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682862437701:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-27T18:20:09.184609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471683842228763:2108] [node 5] ICS09 handshake done sender: [3:7577471683842228735:2131] self: [3:7577471683842228734:0] peer: [5:7577471686031151811:0] socket: 351 qp: -1
2025-11-27T18:20:09.184644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471683842228763:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:09.184623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471685425706521:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.184657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471685425706521:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.184687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:09.185801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471686031151803:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:09.186180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471681736184424:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.186231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:09.186276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471681736184424:2050] [node 3] ICP22 created new session: [5:7577471686031151815:2050]
2025-11-27T18:20:09.186337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471686031151815:2050] [node 3] ICS09 handshake done sender: [5:7577471686031151803:2117] self: [5:7577471686031151811:0] peer: [3:7577471683842228734:0] socket: 391 qp: -1
2025-11-27T18:20:09.186426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471686031151815:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:09.186469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:09.186454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471685425706521:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:09.187096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471685425706512:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.187275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:09.187410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471678567470314:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30615) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471682862437694:2117] held: no
2025-11-27T18:20:09.187340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471685425706512:2109] [node 5] ICP22 created new session: [2:7577471685425706548:2109]
2025-11-27T18:20:09.187396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471685425706548:2109] [node 5] ICS09 handshake done sender: [2:7577471685425706521:2114] self: [2:7577471685425706520:0] peer: [5:7577471686031151812:0] socket: 361 qp: -1
2025-11-27T18:20:09.187438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471685425706548:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:09.187469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:09.188151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471678567470315:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.189484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471682862437694:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:09.189796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471684045984749:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.189824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471684045984749:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.191480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471679751017380:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:09.191634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471678567470316:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.191718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471685425706523:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.191752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471685425706523:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.192036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471678567470314:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.192083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:09.192533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471678567470314:2048] [node 1] ICP22 created new session: [6:7577471682862437706:2048]
2025-11-27T18:20:09.192577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471682862437706:2048] [node 1] ICS09 handshake done sender: [6:7577471682862437694:2117] self: [6:7577471682862437696:0] peer: [1:7577471682319412043:0] socket: 371 qp: -1
2025-11-27T18:20:09.192618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471682862437706:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:09.192677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.192746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471685425706523:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:09.192796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684045984775:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26196
2025-11-27T18:20:09.193000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471684045984749:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:09.193198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471685425706513:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.193313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:09.193495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471685425706513:2110] [node 6] ICP22 created new session: [2:7577471685425706551:2110]
2025-11-27T18:20:09.193554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471685425706551:2110] [node 6] ICS09 handshake done sender: [2:7577471685425706523:2115] self: [2:7577471685425706522:0] peer: [6:7577471682862437704:0] socket: 362 qp: -1
2025-11-27T18:20:09.193708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471685425706551:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:09.193750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:09.193711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471684045984740:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.193783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:09.193915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471681736184423:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26406) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471686031151804:2118] held: no
2025-11-27T18:20:09.193821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471684045984740:2108] [node 5] ICP22 created new session: [4:7577471684045984777:2108]
2025-11-27T18:20:09.193860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471684045984777:2108] [node 5] ICS09 handshake done sender: [4:7577471684045984749:2115] self: [4:7577471684045984748:0] peer: [5:7577471686031151810:0] socket: 392 qp: -1
2025-11-27T18:20:09.193886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471684045984777:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:09.193913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:09.195235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471684045984775:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.194562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471683842228737:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.194597Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471683842228737:2132] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.197163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471683842228737:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:09.197173Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471678567470315:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26406) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471682862437697:2119] held: no
2025-11-27T18:20:09.197178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471684045984740:2108] [node 5] ICP18 handshake (actor [4:7577471684045984775:2123]) is too old
2025-11-27T18:20:09.197217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471686031151817:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11791
2025-11-27T18:20:09.197538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471683842228727:2127] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.197984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:09.198052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471683842228727:2127] [node 6] ICP22 created new session: [3:7577471683842228766:2127]
2025-11-27T18:20:09.198104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471683842228766:2127] [node 6] ICS09 handshake done sender: [3:7577471683842228737:2132] self: [3:7577471683842228736:0] peer: [6:7577471682862437705:0] socket: 400 qp: -1
2025-11-27T18:20:09.198135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471683842228766:2127] [node 6] ICS10 traffic start
2025-11-27T18:20:09.198167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.198339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682862437708:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.199203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471686031151818:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11791
2025-11-27T18:20:09.199511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471682862437697:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:09.199570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471686031151804:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.199769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682862437709:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-27T18:20:09.200987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471678567470315:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.201027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:09.201080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471678567470315:2049] [node 2] ICP22 created new session: [6:7577471682862437710:2049]
2025-11-27T18:20:09.201116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471682862437710:2049] [node 2] ICS09 handshake done sender: [6:7577471682862437697:2119] self: [6:7577471682862437704:0] peer: [2:7577471685425706522:0] socket: 397 qp: -1
2025-11-27T18:20:09.201138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471682862437710:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:09.201164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:09.201579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471686031151783:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471686031151806:2119] held: no
2025-11-27T18:20:09.202141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471681736184423:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.202190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:09.202237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471681736184423:2049] [node 2] ICP22 created new session: [5:7577471686031151819:2049]
2025-11-27T18:20:09.202278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471686031151819:2049] [node 2] ICS09 handshake done sender: [5:7577471686031151804:2118] self: [5:7577471686031151812:0] peer: [2:7577471685425706520:0] socket: 365 qp: -1
2025-11-27T18:20:09.202303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471686031151819:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:09.202334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:09.203317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471678567470316:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471682862437699:2120] held: no
2025-11-27T18:20:09.203498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471686031151806:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:09.204004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471686031151783:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.204044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:09.204087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471686031151783:2110] [node 4] ICP22 created new session: [5:7577471686031151821:2110]
2025-11-27T18:20:09.204129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471686031151821:2110] [node 4] ICS09 handshake done sender: [5:7577471686031151806:2119] self: [5:7577471686031151810:0] peer: [4:7577471684045984748:0] socket: 372 qp: -1
2025-11-27T18:20:09.204158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471686031151821:2110] [node 4] ICS10 traffic start
2025-11-27T18:20:09.204193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:09.204251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471682862437699:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:09.204899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471678567470316:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.204942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:09.204986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471678567470316:2050] [node 3] ICP22 created new session: [6:7577471682862437712:2050]
2025-11-27T18:20:09.205022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471682862437712:2050] [node 3] ICS09 handshake done sender: [6:7577471682862437699:2120] self: [6:7577471682862437705:0] peer: [3:7577471683842228736:0] socket: 401 qp: -1
2025-11-27T18:20:09.205064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471682862437712:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:09.205092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:09.229550Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:09.229821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471685425706537:2048] [node 1] ICS07 socket disconnect 341 reason# EndOfStream
2025-11-27T18:20:09.229858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471685425706537:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.229938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:09.229992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471685425706537:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:09.230925Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:09.231163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471683842228754:2048] [node 1] ICS01 socket: 350 reason# EndOfStream
2025-11-27T18:20:09.231217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471679547261348:2048] [node 1] ICP30 unregister session Session# [3:7577471683842228754:2048] VirtualId# [3:7577471683842228744:0]
2025-11-27T18:20:09.231251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471683842228754:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.231342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:09.231405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:09.235196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471681130739151:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30615) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:09.235256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471685425706537:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:09.241983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471681130739151:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30615) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:09.242040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471685425706537:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:09.242043Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:09.242078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471681130739151:2048] [node 1] ICP30 unregister session Session# [2:7577471685425706537:2048] VirtualId# [2:7577471685425706531:0]
2025-11-27T18:20:09.242135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:09.242170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471681130739151:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30615) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.242186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30615) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.242277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471684045984765:2048] [node 1] ICS01 socket: 347 reason# EndOfStream
2025-11-27T18:20:09.242317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471679751017379:2048] [node 1] ICP30 unregister session Session# [4:7577471684045984765:2048] VirtualId# [4:7577471684045984758:0]
2025-11-27T18:20:09.242348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471684045984765:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.242426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:09.242472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:09.243347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:09.243560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471686031151808:2048] [node 1] ICS07 socket disconnect 337 reason# EndOfStream
2025-11-27T18:20:09.243596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471686031151808:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.243667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:09.243713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471686031151808:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:09.244490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:09.244535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471681736184422:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30615) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:09.244587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471686031151808:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:09.244621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471681736184422:2048] [node 1] ICP30 unregister session Session# [5:7577471686031151808:2048] VirtualId# [5:7577471686031151805:0]
2025-11-27T18:20:09.244677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471682862437706:2048] [node 1] ICS01 socket: 371 reason# EndOfStream
2025-11-27T18:20:09.244717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471678567470314:2048] [node 1] ICP30 unregister session Session# [6:7577471682862437706:2048] VirtualId# [6:7577471682862437696:0]
2025-11-27T18:20:09.244743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471682862437706:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.244792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:09.244825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:09.244737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:09.244775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471681736184422:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30615) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.244788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30615) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.254195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471681736184422:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:09.268602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:09.268836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471683842228760:2049] [node 2] ICS01 socket: 378 reason# EndOfStream
2025-11-27T18:20:09.268898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471679547261349:2049] [node 2] ICP30 unregister session Session# [3:7577471683842228760:2049] VirtualId# [3:7577471683842228753:0]
2025-11-27T18:20:09.268935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471683842228760:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.269057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:09.269115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:09.271280Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:09.271471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471684045984771:2049] [node 2] ICS07 socket disconnect 375 reason# EndOfStream
2025-11-27T18:20:09.271522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471684045984771:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.271594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:09.271648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471684045984771:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:09.272339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:09.272494Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471679751017380:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26406) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:09.272537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471686031151819:2049] [node 2] ICS01 socket: 365 reason# EndOfStream
2025-11-27T18:20:09.272561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471684045984771:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:09.272602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471679751017380:2049] [node 2] ICP30 unregister session Session# [4:7577471684045984771:2049] VirtualId# [4:7577471684045984768:0]
2025-11-27T18:20:09.272592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471681736184423:2049] [node 2] ICP30 unregister session Session# [5:7577471686031151819:2049] VirtualId# [5:7577471686031151812:0]
2025-11-27T18:20:09.272627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471686031151819:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.272698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:09.272711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:09.272745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:09.272745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471679751017380:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26406) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.272761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26406) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.273368Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:09.273569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471682862437710:2049] [node 2] ICS01 socket: 397 reason# EndOfStream
2025-11-27T18:20:09.273606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471678567470315:2049] [node 2] ICP30 unregister session Session# [6:7577471682862437710:2049] VirtualId# [6:7577471682862437704:0]
2025-11-27T18:20:09.273632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471682862437710:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.273693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:09.273748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:09.282906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471679751017380:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:09.326109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:09.326362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471684045984762:2050] [node 3] ICS07 socket disconnect 366 reason# EndOfStream
2025-11-27T18:20:09.326401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471684045984762:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.326476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:09.326527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471684045984762:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:09.327215Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:09.327420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471686031151815:2050] [node 3] ICS07 socket disconnect 391 reason# EndOfStream
2025-11-27T18:20:09.327463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471686031151815:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.327531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:09.327585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471686031151815:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:09.328239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:09.328402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471681736184424:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10347) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:09.328457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471686031151815:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:09.328496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471681736184424:2050] [node 3] ICP30 unregister session Session# [5:7577471686031151815:2050] VirtualId# [5:7577471686031151811:0]
2025-11-27T18:20:09.328475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471682862437712:2050] [node 3] ICS01 socket: 401 reason# EndOfStream
2025-11-27T18:20:09.328528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471678567470316:2050] [node 3] ICP30 unregister session Session# [6:7577471682862437712:2050] VirtualId# [6:7577471682862437705:0]
2025-11-27T18:20:09.328570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471682862437712:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.328637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:09.328630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:09.328675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:09.328687Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471681736184424:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10347) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.328707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10347) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.329548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471679751017381:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10347) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:09.329607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471684045984762:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:09.329639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471679751017381:2050] [node 3] ICP30 unregister session Session# [4:7577471684045984762:2050] VirtualId# [4:7577471684045984759:0]
2025-11-27T18:20:09.329745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:09.329797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471679751017381:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10347) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.329814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10347) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.338189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471681736184424:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:09.358630Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:09.358871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471682862437702:2110] [node 4] ICS07 socket disconnect 395 reason# EndOfStream
2025-11-27T18:20:09.358906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471682862437702:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.358981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:09.359025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471682862437702:2110] [node 4] ICS15 start handshake
2025-11-27T18:20:09.359848Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471682862437677:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21606) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:09.359908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471682862437702:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:09.359947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471682862437677:2110] [node 4] ICP30 unregister session Session# [6:7577471682862437702:2110] VirtualId# [6:7577471682862437698:0]
2025-11-27T18:20:09.359995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:09.360032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471682862437677:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21606) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.360048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21606) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.361444Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:09.361757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471686031151821:2110] [node 4] ICS01 socket: 372 reason# EndOfStream
2025-11-27T18:20:09.361815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471686031151783:2110] [node 4] ICP30 unregister session Session# [5:7577471686031151821:2110] VirtualId# [5:7577471686031151810:0]
2025-11-27T18:20:09.361849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471686031151821:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.361966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:09.362097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:09.369922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471682862437677:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:20:09.415260Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:09.415508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471682862437691:2107] [node 5] ICS07 socket disconnect 355 reason# EndOfStream
2025-11-27T18:20:09.415548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471682862437691:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.415626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:09.415670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471682862437691:2107] [node 5] ICS15 start handshake
2025-11-27T18:20:09.416642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471682862437672:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3886) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:09.416695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471682862437691:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:09.416736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471682862437672:2107] [node 5] ICP30 unregister session Session# [6:7577471682862437691:2107] VirtualId# [6:7577471682862437689:0]
2025-11-27T18:20:09.416820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:09.416845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471682862437672:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3886) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.416855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3886) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:09.509300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471686002648848:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:09.509411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471686002648849:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:09.529295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471684809659727:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:09.529341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471684809659728:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:09.549524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471684404250780:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:09.549640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471684404250781:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:09.569666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471684687680468:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:09.569708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471684687680469:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:09.569741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471684687680470:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:09.591430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471684812637232:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:09.591506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471684812637234:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:09.591551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471684812637233:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:09.612752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471682600932802:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:09.612765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471682600932803:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:09.612801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471682600932804:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:09.673355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471682600932864:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:09.724071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471686002648913:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:09.724074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471684809659790:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:09.723286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471684404250844:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:09.724804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471684404250845:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:09.724212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471684809659791:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:09.724875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471684404250846:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:09.725103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471684687680532:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:09.725217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471682600932866:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:09.725232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471686002648914:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:09.725243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471684809659792:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:09.725275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471684687680533:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:09.726044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471686002648917:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:09.726069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471686002648848:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(57180365ns) {Configure:104(57173991ns) {ProcessConfigured:135(205037ns) {StartInitialHandshake:157(202103ns) {DropHandshakes:470(2707ns) {DropIncomingHandshake:442(658ns) DropOutgoingHandshake:457(369ns)} GenerateSessionVirtualId:988(1696ns) PrepareNewSessionHandshake:477(1608ns) {SwitchToState:179(558ns)}}}}}
2025-11-27T18:20:09.725231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471684812637296:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:09.725327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471684812637297:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.748849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471686002648929:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48238
2025-11-27T18:20:09.750648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684687680547:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41562
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.757963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471684812637306:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43610
2025-11-27T18:20:09.754210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471684404250845:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18805993ns) {Configure:104(18694604ns) {ProcessConfigured:135(589136ns) {StartInitialHandshake:157(585882ns) {DropHandshakes:470(2572ns) {DropIncomingHandshake:442(665ns) DropOutgoingHandshake:457(365ns)} GenerateSessionVirtualId:988(615ns) PrepareNewSessionHandshake:477(508794ns) {SwitchToState:179(4782ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.773638Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471684809659792:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31344753ns) {Configure:104(31338135ns) {ProcessConfigured:135(7601935ns) {StartInitialHandshake:157(7598884ns) {DropHandshakes:470(3118ns) {DropIncomingHandshake:442(853ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(1206ns) PrepareNewSessionHandshake:477(2787ns) {SwitchToState:179(1299ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.776043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471686002648929:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.774041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471684809659801:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42260
2025-11-27T18:20:09.775363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471684687680547:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.797074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471684812637306:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.779994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471684687680468:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.775043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471684809659801:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.797780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471684812637296:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.797888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471684812637312:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43610
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.800451Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471682600932866:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(48877907ns) {Configure:104(48867980ns) {ProcessConfigured:135(3231007ns) {StartInitialHandshake:157(3212779ns) {DropHandshakes:470(3146ns) {DropIncomingHandshake:442(591ns) DropOutgoingHandshake:457(425ns)} GenerateSessionVirtualId:988(1276ns) PrepareNewSessionHandshake:477(3148ns) {SwitchToState:179(1716ns)}}}}}
2025-11-27T18:20:09.821576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471684687680540:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.809602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471686002648921:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.789568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471684809659804:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42260
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.798346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471684809659727:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-27T18:20:09.803549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471684809659804:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:09.827236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471684809659809:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42260
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.821632Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471684687680540:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.809657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471686002648921:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.830116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471684404250855:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28482
2025-11-27T18:20:09.830054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471686002648930:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48238
2025-11-27T18:20:09.830306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471684404250858:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28482
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.830227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471686002648931:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48238
2025-11-27T18:20:09.831734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471686002648916:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.832405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471684687680540:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:09.831768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471686002648916:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.834184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471684404250855:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.849528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471684809659812:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42260
2025-11-27T18:20:09.841499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471684812637314:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43610
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.837688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471686002648931:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.837911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684687680549:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41562
2025-11-27T18:20:09.837670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471684404250861:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28482
2025-11-27T18:20:09.845394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471684404250858:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.854083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471684404250780:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.855929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471684404250862:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28482
2025-11-27T18:20:09.841658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471686002648930:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.856032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471686002648932:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48238
2025-11-27T18:20:09.857223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471684812637306:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.857595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471684812637315:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43610
2025-11-27T18:20:09.859613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471684404250861:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.859718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471684404250862:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.855712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471684687680533:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.855777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:09.856386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471686002648921:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:09.863734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471684809659809:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.860243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471684812637296:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.863928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471684404250781:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.860185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471686002648916:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:09.855829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471684687680533:2108] [node 5] ICP22 created new session: [4:7577471684687680551:2108]
2025-11-27T18:20:09.860353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:09.866754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471686002648936:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48238
2025-11-27T18:20:09.860406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471684812637296:2108] [node 4] ICP22 created new session: [5:7577471684812637316:2108]
2025-11-27T18:20:09.865279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471684809659811:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42260
2025-11-27T18:20:09.856351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471684687680549:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.863891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471684687680551:2108] [node 5] ICS09 handshake done sender: [4:7577471684687680540:2113] self: [4:7577471684687680539:0] peer: [5:7577471684812637313:0] socket: 345 qp: -1
2025-11-27T18:20:09.864323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471684687680551:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:09.868218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471686002648848:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.864367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:09.868275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:09.868325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471686002648848:2048] [node 2] ICP22 created new session: [1:7577471686002648938:2048]
2025-11-27T18:20:09.868343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471686002648919:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.868367Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471686002648919:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.868377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471686002648938:2048] [node 2] ICS09 handshake done sender: [1:7577471686002648916:2112] self: [1:7577471686002648915:0] peer: [2:7577471684809659808:0] socket: 342 qp: -1
2025-11-27T18:20:09.860462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471684812637316:2108] [node 4] ICS09 handshake done sender: [5:7577471684812637306:2115] self: [5:7577471684812637313:0] peer: [4:7577471684687680539:0] socket: 353 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.863690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471684812637316:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:09.868443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682600932880:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52797
2025-11-27T18:20:09.868509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471684809659811:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.868408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471686002648938:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:09.868441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:09.864245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:09.870179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471684809659796:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.870251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471682600932880:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.866411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471684812637312:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.873382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471682600932864:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.879527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684687680552:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41562
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.870276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471684812637315:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.870208Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471684809659796:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.873009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471684809659801:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:09.890018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471684404250852:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.890667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682600932881:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52797
2025-11-27T18:20:09.889969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471684809659727:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.890652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471686002648932:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.881156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471684812637234:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.891198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471684687680469:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.890544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471684809659796:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:09.890064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471684404250852:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.890717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471686002648913:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.891008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:09.889668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471684812637305:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.891070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471684809659727:2048] [node 1] ICP22 created new session: [2:7577471684809659816:2048]
2025-11-27T18:20:09.890987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:09.889706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471684812637305:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.891131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471684809659816:2048] [node 1] ICS09 handshake done sender: [2:7577471684809659801:2114] self: [2:7577471684809659808:0] peer: [1:7577471686002648915:0] socket: 349 qp: -1
2025-11-27T18:20:09.891272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471686002648913:2107] [node 4] ICP22 created new session: [1:7577471686002648941:2107]
2025-11-27T18:20:09.891268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471684809659816:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:09.891338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471686002648941:2107] [node 4] ICS09 handshake done sender: [1:7577471686002648921:2114] self: [1:7577471686002648920:0] peer: [4:7577471684687680548:0] socket: 348 qp: -1
2025-11-27T18:20:09.891308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:09.891378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471686002648941:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:09.891450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:09.891498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471684687680547:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:09.892005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471684812637305:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:09.892086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471684404250852:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.892347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471684404250868:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28482
2025-11-27T18:20:09.892073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471684687680468:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.892167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.893816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682600932883:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52797
2025-11-27T18:20:09.893779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471684812637297:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.892551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471684687680468:2048] [node 1] ICP22 created new session: [4:7577471684687680556:2048]
2025-11-27T18:20:09.893784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471684404250846:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.892606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471684687680556:2048] [node 1] ICS09 handshake done sender: [4:7577471684687680547:2116] self: [4:7577471684687680548:0] peer: [1:7577471686002648920:0] socket: 351 qp: -1
2025-11-27T18:20:09.894221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:09.893855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471684404250862:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:09.893027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471684687680556:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:09.894881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471682600932880:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:09.893300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:09.894367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471684812637297:2109] [node 6] ICP22 created new session: [5:7577471684812637323:2109]
2025-11-27T18:20:09.894795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:09.894757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471684812637323:2109] [node 6] ICS09 handshake done sender: [5:7577471684812637305:2114] self: [5:7577471684812637304:0] peer: [6:7577471682600932882:0] socket: 390 qp: -1
2025-11-27T18:20:09.894931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471684404250846:2109] [node 5] ICP22 created new session: [3:7577471684404250870:2109]
2025-11-27T18:20:09.895247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471684812637323:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:09.895262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471684404250870:2109] [node 5] ICS09 handshake done sender: [3:7577471684404250852:2113] self: [3:7577471684404250851:0] peer: [5:7577471684812637320:0] socket: 382 qp: -1
2025-11-27T18:20:09.895574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:09.895444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471684404250781:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.897765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471682600932864:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.895822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471684404250870:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:09.899972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:09.897628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471684809659798:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.896415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:09.896483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:09.897674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471684809659798:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.896849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471684404250781:2049] [node 2] ICP22 created new session: [3:7577471684404250872:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.898607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471684809659728:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.897595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471684404250872:2049] [node 2] ICS09 handshake done sender: [3:7577471684404250862:2119] self: [3:7577471684404250867:0] peer: [2:7577471684809659795:0] socket: 371 qp: -1
2025-11-27T18:20:09.898660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:09.897632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471684404250872:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:09.900380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471682600932864:2107] [node 5] ICP22 created new session: [6:7577471682600932884:2107]
2025-11-27T18:20:09.898690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471684809659728:2049] [node 3] ICP22 created new session: [2:7577471684809659818:2049]
2025-11-27T18:20:09.901241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471686002648848:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(25378316ns) {HandleHandshakeStatus:329(25372515ns) {DropIncomingHandshake:442(10666ns) DropOutgoingHandshake:457(374ns) SwitchToState:179(129ns) ProcessPendingSessionEvents:495(23100396ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23016458ns max# 6615097ns}}}
2025-11-27T18:20:09.900558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471682600932884:2107] [node 5] ICS09 handshake done sender: [6:7577471682600932880:2115] self: [6:7577471682600932882:0] peer: [5:7577471684812637304:0] socket: 391 qp: -1
2025-11-27T18:20:09.901471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471684812637322:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43610
2025-11-27T18:20:09.898716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471684809659818:2049] [node 3] ICS09 handshake done sender: [2:7577471684809659796:2111] self: [2:7577471684809659795:0] peer: [3:7577471684404250867:0] socket: 363 qp: -1
2025-11-27T18:20:09.901361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471686002648936:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.900903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471682600932884:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:09.898736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471684809659818:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:09.899838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:09.900952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:09.898754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:09.902009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471684687680533:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(39959216ns) {HandleHandshakeStatus:329(39950278ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(386ns) SwitchToState:179(251ns) ProcessPendingSessionEvents:495(28914394ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28798651ns max# 8009879ns}}}
2025-11-27T18:20:09.902433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471684809659798:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:09.903023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471684812637234:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17007) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471684812637315:2119] held: no
2025-11-27T18:20:09.904101Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471682600932864:2107] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.904907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682600932887:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52797
2025-11-27T18:20:09.906028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471684809659791:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.906070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:09.906207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471684809659791:2107] [node 4] ICP22 created new session: [2:7577471684809659823:2107]
2025-11-27T18:20:09.906249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471684809659823:2107] [node 4] ICS09 handshake done sender: [2:7577471684809659798:2112] self: [2:7577471684809659797:0] peer: [4:7577471684687680555:0] socket: 365 qp: -1
2025-11-27T18:20:09.907049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471684809659823:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:09.907828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471684812637315:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:09.907121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:09.911166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471686002648919:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.911550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471684812637234:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.911682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:09.911797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471684812637234:2050] [node 3] ICP22 created new session: [5:7577471684812637326:2050]
2025-11-27T18:20:09.911744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682600932888:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52797
2025-11-27T18:20:09.911894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471684812637326:2050] [node 3] ICS09 handshake done sender: [5:7577471684812637315:2119] self: [5:7577471684812637320:0] peer: [3:7577471684404250851:0] socket: 383 qp: -1
2025-11-27T18:20:09.911929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471684812637326:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:09.911979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.912384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471684812637325:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43610
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.912497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471684404250875:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28482
2025-11-27T18:20:09.912588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471684404250876:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28482
2025-11-27T18:20:09.912895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471682600932881:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.912601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471686002648849:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.913057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471682600932887:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.912657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.912711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471686002648849:2049] [node 3] ICP22 created new session: [1:7577471686002648948:2049]
2025-11-27T18:20:09.912750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471686002648948:2049] [node 3] ICS09 handshake done sender: [1:7577471686002648919:2113] self: [1:7577471686002648918:0] peer: [3:7577471684404250865:0] socket: 346 qp: -1
2025-11-27T18:20:09.912780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471686002648948:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:09.912807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.913365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684687680558:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41562
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.913322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471682600932866:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.913169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471684404250875:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.913689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471684687680542:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.913703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471684809659803:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.913835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471684404250858:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:09.913399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471682600932803:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.913713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471684687680542:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.913734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471684809659803:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.914521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684687680559:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41562
2025-11-27T18:20:09.913601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682600932889:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52797
2025-11-27T18:20:09.915521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471684812637325:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.914757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471684687680549:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:09.915661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471684404250780:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.915995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471684812637328:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43610
2025-11-27T18:20:09.916163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:09.915877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471684687680469:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.916073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471684809659825:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42260
2025-11-27T18:20:09.916210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471684404250780:2048] [node 1] ICP22 created new session: [3:7577471684404250878:2048]
2025-11-27T18:20:09.916122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:09.916235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471684812637233:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.916258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471684404250878:2048] [node 1] ICS09 handshake done sender: [3:7577471684404250858:2117] self: [3:7577471684404250865:0] peer: [1:7577471686002648918:0] socket: 355 qp: -1
2025-11-27T18:20:09.916102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471684809659803:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:09.916165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471684687680469:2049] [node 2] ICP22 created new session: [4:7577471684687680562:2049]
2025-11-27T18:20:09.916400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471684809659792:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.916440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471684404250878:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:09.916212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471684687680562:2049] [node 2] ICS09 handshake done sender: [4:7577471684687680549:2117] self: [4:7577471684687680555:0] peer: [2:7577471684809659797:0] socket: 366 qp: -1
2025-11-27T18:20:09.916526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:09.916957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:09.916815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471684809659792:2109] [node 6] ICP22 created new session: [2:7577471684809659827:2109]
2025-11-27T18:20:09.916246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471684687680562:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:20:09.916333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
::ffff:127.1.0.0
2025-11-27T18:20:09.916975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471684809659827:2109] [node 6] ICS09 handshake done sender: [2:7577471684809659803:2115] self: [2:7577471684809659802:0] peer: [6:7577471682600932891:0] socket: 368 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.917086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471684809659827:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:09.917336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471684687680542:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:09.917123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:09.917343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471684809659800:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.917395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682600932892:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52797
2025-11-27T18:20:09.917401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682600932893:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52797
2025-11-27T18:20:09.917367Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471684809659800:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.917787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471684687680559:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.918230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471682600932881:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:09.918233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471684687680532:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.918268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:09.918341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471684687680532:2109] [node 6] ICP22 created new session: [4:7577471684687680564:2109]
2025-11-27T18:20:09.918993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471684687680564:2109] [node 6] ICS09 handshake done sender: [4:7577471684687680542:2114] self: [4:7577471684687680541:0] peer: [6:7577471682600932890:0] socket: 392 qp: -1
2025-11-27T18:20:09.918768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471682600932887:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:09.918770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471684812637328:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.919029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471684687680564:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:09.919140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471684812637232:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.919118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:09.919559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471682600932866:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.920186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471686002648925:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.920222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471686002648925:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.920179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:09.920232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471682600932866:2109] [node 4] ICP22 created new session: [6:7577471682600932894:2109]
2025-11-27T18:20:09.920339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471682600932894:2109] [node 4] ICS09 handshake done sender: [6:7577471682600932881:2116] self: [6:7577471682600932890:0] peer: [4:7577471684687680541:0] socket: 393 qp: -1
2025-11-27T18:20:09.920380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471682600932894:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:09.920436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:09.921427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471686002648925:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:09.921330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471682600932803:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.921341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471684404250780:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:09.921496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:09.921539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471682600932803:2049] [node 2] ICP22 created new session: [6:7577471682600932896:2049]
2025-11-27T18:20:09.921642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471682600932896:2049] [node 2] ICS09 handshake done sender: [6:7577471682600932887:2118] self: [6:7577471682600932891:0] peer: [2:7577471684809659802:0] socket: 369 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.921675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471682600932896:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:09.921707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:09.921825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471686002648914:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.921844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471684812637331:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43610
2025-11-27T18:20:09.921874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:09.922045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471686002648914:2109] [node 5] ICP22 created new session: [1:7577471686002648951:2109]
2025-11-27T18:20:09.922182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471686002648951:2109] [node 5] ICS09 handshake done sender: [1:7577471686002648925:2116] self: [1:7577471686002648923:0] peer: [5:7577471684812637330:0] socket: 400 qp: -1
2025-11-27T18:20:09.922168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471684809659800:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:09.922215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471686002648951:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:09.922301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:09.922530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471684812637328:2122] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.923500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471684812637332:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43610
2025-11-27T18:20:09.923568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471684809659790:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.923748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:09.923790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471684809659790:2108] [node 5] ICP22 created new session: [2:7577471684809659830:2108]
2025-11-27T18:20:09.923907Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471684812637232:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28348) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471684812637328:2122] held: no
2025-11-27T18:20:09.923918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471684809659830:2108] [node 5] ICS09 handshake done sender: [2:7577471684809659800:2113] self: [2:7577471684809659799:0] peer: [5:7577471684812637329:0] socket: 367 qp: -1
2025-11-27T18:20:09.923959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471684809659830:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:09.923989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:09.924353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471684812637325:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:09.925095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471684812637232:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.925232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:09.925294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471684812637232:2048] [node 1] ICP22 created new session: [5:7577471684812637333:2048]
2025-11-27T18:20:09.925317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471684812637233:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.925341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:09.925340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471684812637333:2048] [node 1] ICS09 handshake done sender: [5:7577471684812637328:2122] self: [5:7577471684812637330:0] peer: [1:7577471686002648923:0] socket: 404 qp: -1
2025-11-27T18:20:09.925401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471684812637233:2049] [node 2] ICP22 created new session: [5:7577471684812637334:2049]
2025-11-27T18:20:09.925417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471684812637333:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:09.925427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471684812637334:2049] [node 2] ICS09 handshake done sender: [5:7577471684812637325:2121] self: [5:7577471684812637329:0] peer: [2:7577471684809659799:0] socket: 343 qp: -1
2025-11-27T18:20:09.925465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471684687680470:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.925448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:09.925464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471684812637334:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:09.925522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471682600932889:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.925485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:09.925724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471684404250854:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.925762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471684404250854:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.925950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471682600932803:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:09.926468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471682600932888:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.926566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471682600932804:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.926725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471682600932802:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:09.926848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471684404250857:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.927221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471686002648924:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:09.926877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471684404250857:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.927250Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471686002648924:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:09.927547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471684404250854:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.928327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471684404250844:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.928339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471684687680567:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41562
2025-11-27T18:20:09.928476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:09.928527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471684404250844:2107] [node 4] ICP22 created new session: [3:7577471684404250883:2107]
2025-11-27T18:20:09.928514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471686002648924:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.928575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471684404250883:2107] [node 4] ICS09 handshake done sender: [3:7577471684404250854:2114] self: [3:7577471684404250853:0] peer: [4:7577471684687680566:0] socket: 403 qp: -1
2025-11-27T18:20:09.928696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471686002648917:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.928616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471684404250883:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:09.928759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:09.928776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471684687680559:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:09.928747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682600932900:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52797
2025-11-27T18:20:09.928930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471686002648917:2110] [node 6] ICP22 created new session: [1:7577471686002648954:2110]
2025-11-27T18:20:09.928671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:09.929564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471684404250857:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:09.929221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471682600932888:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:09.930078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471682600932901:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52797
2025-11-27T18:20:09.929922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471686002648954:2110] [node 6] ICS09 handshake done sender: [1:7577471686002648924:2115] self: [1:7577471686002648922:0] peer: [6:7577471682600932899:0] socket: 401 qp: -1
2025-11-27T18:20:09.930222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471682600932802:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.930291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:09.930260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471684687680470:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17007) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471684687680559:2120] held: no
2025-11-27T18:20:09.930006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471686002648954:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:09.930346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471682600932802:2048] [node 1] ICP22 created new session: [6:7577471682600932902:2048]
2025-11-27T18:20:09.930554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471684404250845:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:09.930163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:09.930394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471682600932902:2048] [node 1] ICS09 handshake done sender: [6:7577471682600932888:2119] self: [6:7577471682600932899:0] peer: [1:7577471686002648922:0] socket: 402 qp: -1
2025-11-27T18:20:09.930519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471682600932902:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:09.930583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:09.930981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:09.931051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471684404250845:2110] [node 6] ICP22 created new session: [3:7577471684404250885:2110]
2025-11-27T18:20:09.931090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471684404250885:2110] [node 6] ICS09 handshake done sender: [3:7577471684404250857:2116] self: [3:7577471684404250856:0] peer: [6:7577471682600932898:0] socket: 406 qp: -1
2025-11-27T18:20:09.931126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471684404250885:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:09.931180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:09.931235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471684687680470:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.931578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:09.931644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471684687680470:2050] [node 3] ICP22 created new session: [4:7577471684687680568:2050]
2025-11-27T18:20:09.931700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471684687680568:2050] [node 3] ICS09 handshake done sender: [4:7577471684687680559:2120] self: [4:7577471684687680566:0] peer: [3:7577471684404250853:0] socket: 405 qp: -1
2025-11-27T18:20:09.931739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471684687680568:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:09.931829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:09.933389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471682600932889:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:09.933707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471682600932804:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17007) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471682600932889:2120] held: no
2025-11-27T18:20:09.934202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471682600932804:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:09.934285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:09.934334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471682600932804:2050] [node 3] ICP22 created new session: [6:7577471682600932904:2050]
2025-11-27T18:20:09.934374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471682600932904:2050] [node 3] ICS09 handshake done sender: [6:7577471682600932889:2120] self: [6:7577471682600932898:0] peer: [3:7577471684404250856:0] socket: 407 qp: -1
2025-11-27T18:20:09.934404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471682600932904:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:09.934438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:09.954986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:09.955157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471684404250878:2048] [node 1] ICS07 socket disconnect 355 reason# EndOfStream
2025-11-27T18:20:09.955198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471684404250878:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.954745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:09.955020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471684809659816:2048] [node 1] ICS01 socket: 349 reason# EndOfStream
2025-11-27T18:20:09.955062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471684809659727:2048] [node 1] ICP30 unregister session Session# [2:7577471684809659816:2048] VirtualId# [2:7577471684809659808:0]
2025-11-27T18:20:09.955250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:09.955095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471684809659816:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.955292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471684404250878:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:09.955179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:09.955230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:09.956893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471684404250780:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28348) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:09.956952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471684404250878:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:09.956990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471684404250780:2048] [node 1] ICP30 unregister session Session# [3:7577471684404250878:2048] VirtualId# [3:7577471684404250865:0]
2025-11-27T18:20:09.957098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:09.957138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471684404250780:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28348) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.957153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28348) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.959009Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:09.959252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471684687680556:2048] [node 1] ICS07 socket disconnect 351 reason# EndOfStream
2025-11-27T18:20:09.959311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471684687680556:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.959407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:09.959452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471684687680556:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:09.960178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:09.960334Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471684687680468:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28348) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:09.960409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471684812637333:2048] [node 1] ICS01 socket: 404 reason# EndOfStream
2025-11-27T18:20:09.960650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471684812637232:2048] [node 1] ICP30 unregister session Session# [5:7577471684812637333:2048] VirtualId# [5:7577471684812637330:0]
2025-11-27T18:20:09.960392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471684687680556:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:09.960687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471684812637333:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.960505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471684687680468:2048] [node 1] ICP30 unregister session Session# [4:7577471684687680556:2048] VirtualId# [4:7577471684687680548:0]
2025-11-27T18:20:09.960750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:09.960837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:09.960792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:09.960880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471684687680468:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28348) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.960896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28348) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.961995Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:09.962212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471682600932902:2048] [node 1] ICS07 socket disconnect 402 reason# EndOfStream
2025-11-27T18:20:09.962247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471682600932902:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:09.962315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:09.962358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471682600932902:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:09.963190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471682600932802:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28348) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:09.963247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471682600932902:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:09.963307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471682600932802:2048] [node 1] ICP30 unregister session Session# [6:7577471682600932902:2048] VirtualId# [6:7577471682600932899:0]
2025-11-27T18:20:09.963371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:09.963402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471682600932802:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28348) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.963415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28348) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:09.966918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471684404250780:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:09.970202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471684687680468:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:09.973055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471682600932802:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:10.013297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:10.013572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471684404250872:2049] [node 2] ICS07 socket disconnect 371 reason# EndOfStream
2025-11-27T18:20:10.013615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471684404250872:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.013710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:10.013780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471684404250872:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:10.015935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471684404250781:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:10.016004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471684404250872:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:10.016568Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:10.016781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471684687680562:2049] [node 2] ICS07 socket disconnect 366 reason# EndOfStream
2025-11-27T18:20:10.017076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471684404250781:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:10.016808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471684687680562:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.016905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:10.017159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471684404250872:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:10.017198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471684404250781:2049] [node 2] ICP30 unregister session Session# [3:7577471684404250872:2049] VirtualId# [3:7577471684404250867:0]
2025-11-27T18:20:10.016971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471684687680562:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:10.017339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:10.017383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471684404250781:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.017398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.017695Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471684687680469:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:10.017792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471684687680562:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:10.017865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471684687680469:2049] [node 2] ICP30 unregister session Session# [4:7577471684687680562:2049] VirtualId# [4:7577471684687680555:0]
2025-11-27T18:20:10.017855Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:10.018000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471684812637334:2049] [node 2] ICS07 socket disconnect 343 reason# EndOfStream
2025-11-27T18:20:10.018020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:10.018060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471684687680469:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.018037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471684812637334:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.018073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.018090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:10.018118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471684812637334:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:10.018799Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471684812637233:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:10.018919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471684812637334:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:10.018965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471684812637233:2049] [node 2] ICP30 unregister session Session# [5:7577471684812637334:2049] VirtualId# [5:7577471684812637329:0]
2025-11-27T18:20:10.018960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:10.019062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:10.019107Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471684812637233:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.019120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.019151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471682600932896:2049] [node 2] ICS07 socket disconnect 369 reason# EndOfStream
2025-11-27T18:20:10.019181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471682600932896:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.019245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:10.019294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471682600932896:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:10.020009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471682600932803:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:10.020061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471682600932896:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:10.020105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471682600932803:2049] [node 2] ICP30 unregister session Session# [6:7577471682600932896:2049] VirtualId# [6:7577471682600932891:0]
2025-11-27T18:20:10.020232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:10.020264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471682600932803:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.020287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5285) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.028369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471684687680469:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:10.029378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471684812637233:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:10.030406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471682600932803:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:10.062607Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:10.062838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471684687680568:2050] [node 3] ICS01 socket: 405 reason# EndOfStream
2025-11-27T18:20:10.062874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471684687680470:2050] [node 3] ICP30 unregister session Session# [4:7577471684687680568:2050] VirtualId# [4:7577471684687680566:0]
2025-11-27T18:20:10.062895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471684687680568:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.062970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:10.063034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:10.063759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:10.063909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471684812637326:2050] [node 3] ICS01 socket: 383 reason# EndOfStream
2025-11-27T18:20:10.063941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471684812637234:2050] [node 3] ICP30 unregister session Session# [5:7577471684812637326:2050] VirtualId# [5:7577471684812637320:0]
2025-11-27T18:20:10.063962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471684812637326:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.064012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:10.064043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:10.064710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:10.064933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471682600932904:2050] [node 3] ICS01 socket: 407 reason# EndOfStream
2025-11-27T18:20:10.064963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471682600932804:2050] [node 3] ICP30 unregister session Session# [6:7577471682600932904:2050] VirtualId# [6:7577471682600932898:0]
2025-11-27T18:20:10.064980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471682600932904:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.065030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:10.065068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:10.116314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:10.116518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471684812637316:2108] [node 4] ICS01 socket: 353 reason# EndOfStream
2025-11-27T18:20:10.116570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471684812637296:2108] [node 4] ICP30 unregister session Session# [5:7577471684812637316:2108] VirtualId# [5:7577471684812637313:0]
2025-11-27T18:20:10.116598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471684812637316:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.116680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:10.116716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:10.117477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:10.117630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471682600932894:2109] [node 4] ICS07 socket disconnect 393 reason# EndOfStream
2025-11-27T18:20:10.117653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471682600932894:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.117701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:10.117733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471682600932894:2109] [node 4] ICS15 start handshake
2025-11-27T18:20:10.118450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471682600932866:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23202) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:10.118505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471682600932894:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:10.118563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471682600932866:2109] [node 4] ICP30 unregister session Session# [6:7577471682600932894:2109] VirtualId# [6:7577471682600932890:0]
2025-11-27T18:20:10.118603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:10.118628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471682600932866:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.118637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.128723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471682600932866:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:20:10.156881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:10.157074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471682600932884:2107] [node 5] ICS07 socket disconnect 391 reason# EndOfStream
2025-11-27T18:20:10.157114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471682600932884:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.157187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:10.157229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471682600932884:2107] [node 5] ICS15 start handshake
2025-11-27T18:20:10.158115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471682600932864:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23210) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:10.158273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471682600932884:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:10.158331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471682600932864:2107] [node 5] ICP30 unregister session Session# [6:7577471682600932884:2107] VirtualId# [6:7577471682600932882:0]
2025-11-27T18:20:10.158378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:10.158418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471682600932864:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23210) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.158438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23210) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.168625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471682600932864:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:10.250570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471687118548380:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:10.250606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471687118548381:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:10.272259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471690353177693:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:10.272270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471690353177692:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:10.294959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471686559677455:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:10.295054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471686559677454:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:10.318528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471688262814556:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:10.318623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471688262814557:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:10.318793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471688262814558:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:10.339090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471688494072867:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:10.339138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471688494072869:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:10.339302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471688494072868:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:10.363265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471690456542971:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:10.363264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471690456542970:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:10.363322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471690456542972:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:10.499077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471688494072932:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:10.503765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471686559677519:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:10.499013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471690353177756:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:10.508068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471686559677520:2111] [node 6] ICP01 ready to work
2025-11-27T18:20:10.508774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471686559677521:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:10.506184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471690353177757:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:10.509542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471690456543032:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:10.502617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471687118548444:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:10.506109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471687118548443:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:10.509858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471687118548445:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:10.513813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471690353177758:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:10.513680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471690456543033:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:10.519478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471688494072933:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:10.527683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471688262814629:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:10.529358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471688262814628:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:10.546544Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471686559677519:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21143113ns) {Configure:104(21139810ns) {ProcessConfigured:135(53049ns) {StartInitialHandshake:157(50763ns) {DropHandshakes:470(2700ns) {DropIncomingHandshake:442(503ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(713ns) PrepareNewSessionHandshake:477(1610ns) {SwitchToState:179(423ns)}}}}}
2025-11-27T18:20:10.547598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471687118548380:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(113058881ns) {Configure:104(113051898ns) {ProcessConfigured:135(3283183ns) {StartInitialHandshake:157(3280658ns) {DropHandshakes:470(493031ns) {DropIncomingHandshake:442(2287ns) DropOutgoingHandshake:457(337ns)} GenerateSessionVirtualId:988(4689ns) PrepareNewSessionHandshake:477(2306ns) {SwitchToState:179(1226ns)}}}}}
2025-11-27T18:20:10.564552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471690353177692:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53367434ns) {Configure:104(53361726ns) {ProcessConfigured:135(4957055ns) {StartInitialHandshake:157(4955145ns) {DropHandshakes:470(1637ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(910ns) PrepareNewSessionHandshake:477(11578ns) {SwitchToState:179(1024ns)}}}}}
2025-11-27T18:20:10.556134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471686559677454:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(41771496ns) {Configure:104(41766606ns) {ProcessConfigured:135(745836ns) {StartInitialHandshake:157(743588ns) {DropHandshakes:470(4928ns) {DropIncomingHandshake:442(521ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(1830ns) {SwitchToState:179(586ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.569421Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471690456542971:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46921748ns) {Configure:104(46915020ns) {ProcessConfigured:135(2787658ns) {StartInitialHandshake:157(2784363ns) {DropHandshakes:470(2523ns) {DropIncomingHandshake:442(613ns) DropOutgoingHandshake:457(467ns)} GenerateSessionVirtualId:988(2251ns) PrepareNewSessionHandshake:477(3059ns) {SwitchToState:179(1263ns)}}}}}
2025-11-27T18:20:10.568142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471690353177693:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(30330555ns) {Configure:104(30326310ns) {ProcessConfigured:135(43789ns) {StartInitialHandshake:157(41888ns) {DropHandshakes:470(2521ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(351ns)} GenerateSessionVirtualId:988(689ns) PrepareNewSessionHandshake:477(767ns) {SwitchToState:179(318ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.598592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471690456543037:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48879
2025-11-27T18:20:10.598763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471690456543038:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48879
2025-11-27T18:20:10.599263Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471688262814557:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(61572703ns) {Configure:104(61567962ns) {ProcessConfigured:135(13120990ns) {StartInitialHandshake:157(13119534ns) {DropHandshakes:470(2891ns) {DropIncomingHandshake:442(809ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(703ns) PrepareNewSessionHandshake:477(2787ns) {SwitchToState:179(1682ns)}}}}}
2025-11-27T18:20:10.568603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471690353177768:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16190
2025-11-27T18:20:10.573103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471687118548444:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24360123ns) {Configure:104(24355684ns) {ProcessConfigured:135(44595ns) {StartInitialHandshake:157(42653ns) {DropHandshakes:470(1873ns) {DropIncomingHandshake:442(797ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(356ns) PrepareNewSessionHandshake:477(1298ns) {SwitchToState:179(328ns)}}}}}
2025-11-27T18:20:10.589892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471688494072868:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51900120ns) {Configure:104(51453265ns) {ProcessConfigured:135(73267ns) {StartInitialHandshake:157(69719ns) {DropHandshakes:470(1250ns) {DropIncomingHandshake:442(254ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(476ns) PrepareNewSessionHandshake:477(1742ns) {SwitchToState:179(522ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.615909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471690456542970:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(63595846ns) {Configure:104(63589690ns) {ProcessConfigured:135(15760108ns) {StartInitialHandshake:157(15759033ns) {DropHandshakes:470(624ns) {DropIncomingHandshake:442(127ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:477(12538ns) {SwitchToState:179(11723ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.614308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471690353177768:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.616354Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471687118548381:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(67991130ns) {Configure:104(67985740ns) {ProcessConfigured:135(34964939ns) {StartInitialHandshake:157(34962397ns) {DropHandshakes:470(10995110ns) {DropIncomingHandshake:442(485ns) DropOutgoingHandshake:457(2119ns)} GenerateSessionVirtualId:988(1897ns) PrepareNewSessionHandshake:477(7621ns) {SwitchToState:179(2377ns)}}}}}
2025-11-27T18:20:10.616353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471690456543037:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.614479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471690353177774:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16190
2025-11-27T18:20:10.616411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471688494072867:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(71570418ns) {Configure:104(71560982ns) {ProcessConfigured:135(7685627ns) {StartInitialHandshake:157(7683510ns) {DropHandshakes:470(984ns) {DropIncomingHandshake:442(276ns) DropOutgoingHandshake:457(172ns)} GenerateSessionVirtualId:988(473ns) PrepareNewSessionHandshake:477(2497ns) {SwitchToState:179(1338ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.619702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471686559677520:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(68939225ns) {Configure:104(68934886ns) {ProcessConfigured:135(6789110ns) {StartInitialHandshake:157(1699085ns) {DropHandshakes:470(2780ns) {DropIncomingHandshake:442(1142ns) DropOutgoingHandshake:457(443ns)} GenerateSessionVirtualId:988(1115ns) PrepareNewSessionHandshake:477(2341ns) {SwitchToState:179(1439ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.618992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471690353177773:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16190
2025-11-27T18:20:10.620420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471690353177773:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.620607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471690353177692:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.619824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471688262814628:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(67236606ns) {Configure:104(67229195ns) {ProcessConfigured:135(6484921ns) {StartInitialHandshake:157(6481975ns) {DropHandshakes:470(2046ns) {DropIncomingHandshake:442(583ns) DropOutgoingHandshake:457(277ns)} GenerateSessionVirtualId:988(690ns) PrepareNewSessionHandshake:477(257638ns) {SwitchToState:179(1969ns)}}}}}
2025-11-27T18:20:10.620667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471690456543038:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.621656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471690456542972:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51884467ns) {Configure:104(51879405ns) {ProcessConfigured:135(44572ns) {StartInitialHandshake:157(42955ns) {DropHandshakes:470(1639ns) {DropIncomingHandshake:442(377ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(384ns) PrepareNewSessionHandshake:477(943ns) {SwitchToState:179(312ns)}}}}}
2025-11-27T18:20:10.622201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471690456542971:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.622368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471690456542970:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.620419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471688494072869:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(71148831ns) {Configure:104(71144744ns) {ProcessConfigured:135(6825260ns) {StartInitialHandshake:157(6823679ns) {DropHandshakes:470(1532ns) {DropIncomingHandshake:442(395ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(1001ns) {SwitchToState:179(748ns)}}}}}
2025-11-27T18:20:10.620033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471688262814556:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(74658292ns) {Configure:104(74654086ns) {ProcessConfigured:135(13862950ns) {StartInitialHandshake:157(13861062ns) {DropHandshakes:470(1129ns) {DropIncomingHandshake:442(209ns) DropOutgoingHandshake:457(147ns)} GenerateSessionVirtualId:988(427ns) PrepareNewSessionHandshake:477(1886ns) {SwitchToState:179(971ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.622541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471687118548447:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.622586Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471687118548447:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.623265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471687118548451:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.623297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471687118548451:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.620556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471688262814634:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-27T18:20:10.624089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471690353177762:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.630701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471690456543052:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48879
2025-11-27T18:20:10.622575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471688262814635:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-27T18:20:10.624128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471690353177762:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.626264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471688262814636:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-27T18:20:10.631983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471690353177762:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:10.631572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471687118548447:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:10.631797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471688262814635:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.631927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471687118548451:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.633593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471687118548380:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.632002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471688262814636:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.634275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471690353177758:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.635619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471687118548445:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.632245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471688262814634:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.634539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471686559677527:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.634336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:10.638425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471688494072945:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.640814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471690456543038:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:10.634576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471686559677527:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.633183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471688262814556:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.635664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:10.638516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471690353177758:2110] [node 6] ICP22 created new session: [2:7577471690353177779:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.653613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471686559677536:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36391
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.642379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471690456543053:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48879
2025-11-27T18:20:10.638787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471688494072946:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.639119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471690353177779:2110] [node 6] ICS09 handshake done sender: [2:7577471690353177762:2112] self: [2:7577471690353177761:0] peer: [6:7577471690456543051:0] socket: 349 qp: -1
2025-11-27T18:20:10.633201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471688262814557:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.642656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471688494072948:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:20:10.648848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471688494072943:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:20:10.657574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471688494072947:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:20:10.635711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471687118548445:2109] [node 6] ICP22 created new session: [1:7577471687118548462:2109]
2025-11-27T18:20:10.657313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471690456542970:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.633272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471688262814558:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.660703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471688494072946:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.661495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471686559677527:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:10.635763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471687118548462:2109] [node 6] ICS09 handshake done sender: [1:7577471687118548451:2112] self: [1:7577471687118548450:0] peer: [6:7577471690456543050:0] socket: 358 qp: -1
2025-11-27T18:20:10.663462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471688494072945:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.639157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471690353177779:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:10.657367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:10.634551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471688262814645:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-27T18:20:10.635797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471687118548462:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:10.665586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471688494072869:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.639194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:10.663438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471686559677535:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36391
2025-11-27T18:20:10.658661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471690456542970:2048] [node 1] ICP22 created new session: [6:7577471690456543055:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.640813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471690353177760:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.649350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471688262814645:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.666104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471688494072947:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.635838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:10.640843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471690353177760:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.673940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471686559677523:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.657050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471688262814650:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.666845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471688494072932:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.659158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471690456543055:2048] [node 1] ICS09 handshake done sender: [6:7577471690456543038:2112] self: [6:7577471690456543050:0] peer: [1:7577471687118548450:0] socket: 363 qp: -1
2025-11-27T18:20:10.636987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471687118548449:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.647874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471690353177774:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.666087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471688262814636:2123] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:10.673975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471686559677523:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.670681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471688494072943:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.674786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471688262814651:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-27T18:20:10.677366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471688262814653:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-27T18:20:10.637009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471687118548449:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.660954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471690456543055:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:10.676854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471686559677519:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.648147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471690353177780:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16190
2025-11-27T18:20:10.674004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471688494072868:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.638383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:10.677387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471688262814556:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.676893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:10.660994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:10.683641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471688494072948:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.638935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471687118548380:2048] [node 2] ICP22 created new session: [1:7577471687118548461:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.676935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471686559677519:2107] [node 4] ICP22 created new session: [3:7577471686559677538:2107]
2025-11-27T18:20:10.664152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471690456543037:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.638959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471687118548449:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:10.648577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471690353177782:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16190
2025-11-27T18:20:10.683774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471688494072867:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.684756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:10.676976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471686559677538:2107] [node 4] ICS09 handshake done sender: [3:7577471686559677527:2114] self: [3:7577471686559677526:0] peer: [4:7577471688262814649:0] socket: 354 qp: -1
2025-11-27T18:20:10.673668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471690456542971:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.650223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471690353177760:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:10.640915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471687118548461:2048] [node 2] ICS09 handshake done sender: [1:7577471687118548447:2110] self: [1:7577471687118548446:0] peer: [2:7577471690353177776:0] socket: 355 qp: -1
2025-11-27T18:20:10.677008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471686559677538:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:10.684526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:10.663596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471690353177778:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16190
2025-11-27T18:20:10.642213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471687118548461:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:10.684572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471690456542971:2049] [node 2] ICP22 created new session: [6:7577471690456543057:2049]
2025-11-27T18:20:10.670217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471690353177782:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.686121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471690456543057:2049] [node 2] ICS09 handshake done sender: [6:7577471690456543037:2111] self: [6:7577471690456543051:0] peer: [2:7577471690353177761:0] socket: 362 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.685304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471688262814556:2048] [node 1] ICP22 created new session: [4:7577471688262814655:2048]
2025-11-27T18:20:10.648909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:10.686332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471690456543057:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:10.677037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:10.686497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:10.659079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471687118548443:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.686414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471686559677536:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.660801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:10.670264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471690353177757:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.661295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471687118548443:2107] [node 4] ICP22 created new session: [1:7577471687118548469:2107]
2025-11-27T18:20:10.686911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471688262814655:2048] [node 1] ICS09 handshake done sender: [4:7577471688262814636:2123] self: [4:7577471688262814647:0] peer: [1:7577471687118548448:0] socket: 367 qp: -1
2025-11-27T18:20:10.686771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471686559677535:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.687654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471688262814655:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:10.690911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471690456543054:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48879
2025-11-27T18:20:10.665514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471687118548469:2107] [node 4] ICS09 handshake done sender: [1:7577471687118548449:2111] self: [1:7577471687118548448:0] peer: [4:7577471688262814647:0] socket: 356 qp: -1
2025-11-27T18:20:10.670309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:10.692556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471688494072957:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:20:10.687884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471688262814638:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.666271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471687118548469:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:10.687906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471688262814638:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.670348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471690353177757:2107] [node 4] ICP22 created new session: [2:7577471690353177785:2107]
2025-11-27T18:20:10.666311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:10.687724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471686559677523:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:10.691858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471690456543054:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.688651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471688262814651:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.683287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471687118548443:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(23559152ns) {HandleHandshakeStatus:329(23405670ns) {DropIncomingHandshake:442(2201ns) DropOutgoingHandshake:457(785ns) SwitchToState:179(352ns) ProcessPendingSessionEvents:495(7616290ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7561941ns max# 7168637ns}}}
2025-11-27T18:20:10.670397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471690353177785:2107] [node 4] ICS09 handshake done sender: [2:7577471690353177760:2111] self: [2:7577471690353177759:0] peer: [4:7577471688262814648:0] socket: 348 qp: -1
2025-11-27T18:20:10.690960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471686559677454:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.688751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471688262814652:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31532
2025-11-27T18:20:10.693069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471690456543033:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.689058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:10.691410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471686559677521:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.683545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471687118548468:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47112
2025-11-27T18:20:10.692449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:10.691256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471688262814635:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:10.670431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471690353177785:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.687060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471687118548456:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.692714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471688262814634:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:10.692873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471686559677521:2109] [node 5] ICP22 created new session: [3:7577471686559677543:2109]
2025-11-27T18:20:10.693371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471688262814558:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.692922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471686559677543:2109] [node 5] ICS09 handshake done sender: [3:7577471686559677523:2112] self: [3:7577471686559677522:0] peer: [5:7577471688494072953:0] socket: 351 qp: -1
2025-11-27T18:20:10.698399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:10.698662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471688262814558:2050] [node 3] ICP22 created new session: [4:7577471688262814660:2050]
2025-11-27T18:20:10.692959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471686559677543:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:10.693004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:10.698953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471688262814557:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:10.694923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471686559677519:2107] [node 4] ICP18 handshake (actor [3:7577471686559677535:2117]) is too old
::ffff:127.1.0.0
2025-11-27T18:20:10.699136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471688494072945:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:10.699364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471688494072959:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:20:10.699447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471688262814660:2050] [node 3] ICS09 handshake done sender: [4:7577471688262814635:2122] self: [4:7577471688262814649:0] peer: [3:7577471686559677526:0] socket: 366 qp: -1
2025-11-27T18:20:10.699504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471688262814660:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:10.699566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.670467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:10.701419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471688494072958:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
2025-11-27T18:20:10.700613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471686559677544:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36391
2025-11-27T18:20:10.687087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471687118548456:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.701527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:10.677425Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471690353177773:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.690980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471687118548468:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.721720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471686559677544:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.733357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471688494072940:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.733404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471688494072940:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.701589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471688262814557:2049] [node 2] ICP22 created new session: [4:7577471688262814662:2049]
2025-11-27T18:20:10.734035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471688494072869:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.686291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471690353177764:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.734109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:10.734165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471688494072869:2050] [node 3] ICP22 created new session: [5:7577471688494072961:2050]
2025-11-27T18:20:10.734219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471688494072961:2050] [node 3] ICS09 handshake done sender: [5:7577471688494072945:2118] self: [5:7577471688494072953:0] peer: [3:7577471686559677522:0] socket: 370 qp: -1
2025-11-27T18:20:10.734256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471688494072961:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:10.734299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:10.736767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471688494072948:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:10.737192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471688494072960:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22066
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.693032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471687118548458:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.725417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471688262814662:2049] [node 2] ICS09 handshake done sender: [4:7577471688262814634:2121] self: [4:7577471688262814648:0] peer: [2:7577471690353177759:0] socket: 365 qp: -1
2025-11-27T18:20:10.722020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471686559677547:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36391
2025-11-27T18:20:10.742523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471690456542971:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(72706635ns) {HandleHandshakeStatus:329(72703934ns) {DropIncomingHandshake:442(1779ns) DropOutgoingHandshake:457(5865ns) SwitchToState:179(1644ns) ProcessPendingSessionEvents:495(53477601ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 52448288ns max# 11056478ns}}}
2025-11-27T18:20:10.693057Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471687118548458:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.686315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471690353177764:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.725467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471688262814662:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:10.743885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471688494072940:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:10.691041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471690353177692:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.744869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471686559677549:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36391
2025-11-27T18:20:10.737567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471688262814556:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23263352ns) {HandleHandshakeStatus:329(21967368ns) {DropIncomingHandshake:442(697ns) DropOutgoingHandshake:457(5375ns) SwitchToState:179(212ns) ProcessPendingSessionEvents:495(7785465ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7690122ns max# 1964937ns}}}
2025-11-27T18:20:10.698176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471687118548456:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:10.744048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471688494072867:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.691487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:10.701482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471687118548471:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47112
2025-11-27T18:20:10.737722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:10.691803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471690353177692:2048] [node 1] ICP22 created new session: [2:7577471690353177789:2048]
2025-11-27T18:20:10.747468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:10.692600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471690353177789:2048] [node 1] ICS09 handshake done sender: [2:7577471690353177773:2117] self: [2:7577471690353177776:0] peer: [1:7577471687118548446:0] socket: 361 qp: -1
2025-11-27T18:20:10.748115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471686559677536:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:10.747716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471688494072867:2048] [node 1] ICP22 created new session: [5:7577471688494072965:2048]
2025-11-27T18:20:10.705629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471687118548380:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(64492993ns) {HandleHandshakeStatus:329(64490535ns) {DropIncomingHandshake:442(282ns) DropOutgoingHandshake:457(583ns) SwitchToState:179(413ns) ProcessPendingSessionEvents:495(38966207ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 35266917ns max# 6594376ns}}}
2025-11-27T18:20:10.692744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471690353177789:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:10.754136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471686559677454:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.748533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471688494072965:2048] [node 1] ICS09 handshake done sender: [5:7577471688494072948:2121] self: [5:7577471688494072956:0] peer: [1:7577471687118548455:0] socket: 394 qp: -1
2025-11-27T18:20:10.750753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471688494072965:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:10.713665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471687118548444:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.754577Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471690456542970:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(41382534ns) {HandleHandshakeStatus:329(41378536ns) {DropIncomingHandshake:442(1118ns) DropOutgoingHandshake:457(6078ns) SwitchToState:179(666ns) ProcessPendingSessionEvents:495(37526046ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 37408136ns max# 6606064ns}}}
2025-11-27T18:20:10.693239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.754210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:10.725435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471687118548458:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:10.755065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471688262814638:2124] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.698418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471690353177758:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(50505887ns) {HandleHandshakeStatus:329(50503926ns) {DropIncomingHandshake:442(487ns) DropOutgoingHandshake:457(420ns) SwitchToState:179(330ns) ProcessPendingSessionEvents:495(45303340ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 44775231ns max# 7293011ns}}}
2025-11-27T18:20:10.754868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:10.754396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471688494072933:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.755639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471690456543061:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48879
2025-11-27T18:20:10.729899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:10.754825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:10.755109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471688494072933:2110] [node 6] ICP22 created new session: [5:7577471688494072967:2110]
2025-11-27T18:20:10.755652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471688494072967:2110] [node 6] ICS09 handshake done sender: [5:7577471688494072940:2114] self: [5:7577471688494072938:0] peer: [6:7577471690456543060:0] socket: 397 qp: -1
2025-11-27T18:20:10.756587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471688494072967:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:10.756623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:10.755426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471686559677454:2048] [node 1] ICP22 created new session: [3:7577471686559677550:2048]
2025-11-27T18:20:10.700710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471690353177780:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.745387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471690353177757:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(25084718ns) {HandleHandshakeStatus:329(25079638ns) {DropIncomingHandshake:442(245ns) DropOutgoingHandshake:457(415ns) SwitchToState:179(157ns) ProcessPendingSessionEvents:495(20810064ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20708727ns max# 9697779ns}}}
2025-11-27T18:20:10.755686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471686559677550:2048] [node 1] ICS09 handshake done sender: [3:7577471686559677536:2118] self: [3:7577471686559677542:0] peer: [1:7577471687118548457:0] socket: 393 qp: -1
2025-11-27T18:20:10.757321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471690456543062:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48879
2025-11-27T18:20:10.750262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471690353177758:2110] [node 6] ICP18 handshake (actor [2:7577471690353177774:2118]) is too old
2025-11-27T18:20:10.756557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471686559677550:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:10.729960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471687118548444:2108] [node 5] ICP22 created new session: [1:7577471687118548474:2108]
2025-11-27T18:20:10.750591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471690353177764:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:10.757232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:10.742880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471687118548474:2108] [node 5] ICS09 handshake done sender: [1:7577471687118548456:2113] self: [1:7577471687118548455:0] peer: [5:7577471688494072956:0] socket: 389 qp: -1
2025-11-27T18:20:10.743747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471687118548474:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:10.744194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471687118548471:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.761456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471690353177756:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.744872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:10.761520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:10.761934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471690353177756:2109] [node 5] ICP22 created new session: [2:7577471690353177792:2109]
2025-11-27T18:20:10.766534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471690353177792:2109] [node 5] ICS09 handshake done sender: [2:7577471690353177764:2113] self: [2:7577471690353177763:0] peer: [5:7577471688494072955:0] socket: 350 qp: -1
2025-11-27T18:20:10.767140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471690353177792:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:10.767443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:10.748071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471687118548381:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.775785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471690456543054:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:10.748474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:10.753431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471687118548381:2049] [node 3] ICP22 created new session: [1:7577471687118548477:2049]
2025-11-27T18:20:10.754440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471687118548477:2049] [node 3] ICS09 handshake done sender: [1:7577471687118548458:2114] self: [1:7577471687118548457:0] peer: [3:7577471686559677542:0] socket: 391 qp: -1
2025-11-27T18:20:10.754629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471687118548477:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:10.754665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:10.776474Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471690353177692:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(64758677ns) {HandleHandshakeStatus:329(64756182ns) {DropIncomingHandshake:442(859ns) DropOutgoingHandshake:457(5872ns) SwitchToState:179(708ns) ProcessPendingSessionEvents:495(62135696ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 60643385ns max# 8414306ns}}}
2025-11-27T18:20:10.776740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471688494072943:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:10.758977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471687118548381:2049] [node 3] ICP18 handshake (actor [1:7577471687118548471:2116]) is too old
2025-11-27T18:20:10.759065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471687118548476:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47112
2025-11-27T18:20:10.767417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471687118548380:2048] [node 2] ICP18 handshake (actor [1:7577471687118548468:2115]) is too old
2025-11-27T18:20:10.778280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471690456543061:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.778626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471690456543033:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12886) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471690456543054:2119] held: no
2025-11-27T18:20:10.778707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471688494072947:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:10.778943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471688262814558:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(65967110ns) {HandleHandshakeStatus:329(65965378ns) {DropIncomingHandshake:442(3106ns) DropOutgoingHandshake:457(624162ns) SwitchToState:179(805ns) ProcessPendingSessionEvents:495(59582665ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 59461923ns max# 12619221ns}}}
2025-11-27T18:20:10.778393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471690353177757:2107] [node 4] ICP18 handshake (actor [2:7577471690353177780:2120]) is too old
2025-11-27T18:20:10.779449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471688262814629:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.779534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471688494072867:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32611665ns) {HandleHandshakeStatus:329(32608788ns) {DropIncomingHandshake:442(924ns) DropOutgoingHandshake:457(13805ns) SwitchToState:179(892ns) ProcessPendingSessionEvents:495(21331916ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20718024ns max# 6795822ns}}}
2025-11-27T18:20:10.779652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471688494072868:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.780202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471688494072932:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.780259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:10.780275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:10.780502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:10.780535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471688494072932:2108] [node 4] ICP22 created new session: [5:7577471688494072969:2108]
2025-11-27T18:20:10.780525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471688262814629:2108] [node 5] ICP22 created new session: [4:7577471688262814664:2108]
2025-11-27T18:20:10.780546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471688494072868:2049] [node 2] ICP22 created new session: [5:7577471688494072970:2049]
2025-11-27T18:20:10.781242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471688494072969:2108] [node 4] ICS09 handshake done sender: [5:7577471688494072947:2120] self: [5:7577471688494072954:0] peer: [4:7577471688262814637:0] socket: 392 qp: -1
2025-11-27T18:20:10.785325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471688262814664:2108] [node 5] ICS09 handshake done sender: [4:7577471688262814638:2124] self: [4:7577471688262814637:0] peer: [5:7577471688494072954:0] socket: 385 qp: -1
2025-11-27T18:20:10.785411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471688494072970:2049] [node 2] ICS09 handshake done sender: [5:7577471688494072943:2116] self: [5:7577471688494072955:0] peer: [2:7577471690353177763:0] socket: 368 qp: -1
2025-11-27T18:20:10.785368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471688262814664:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:10.785454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471688494072970:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:10.789492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471688494072969:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:10.791092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471688262814557:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(78346611ns) {HandleHandshakeStatus:329(78344696ns) {DropIncomingHandshake:442(458ns) DropOutgoingHandshake:457(3736ns) SwitchToState:179(302ns) ProcessPendingSessionEvents:495(32327804ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32202342ns max# 6666650ns}}}
2025-11-27T18:20:10.789532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:10.793535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471687118548444:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(71880698ns) {HandleHandshakeStatus:329(67840254ns) {DropIncomingHandshake:442(1726ns) DropOutgoingHandshake:457(939ns) SwitchToState:179(556ns) ProcessPendingSessionEvents:495(29841674ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29432025ns max# 6322480ns}}}
2025-11-27T18:20:10.793457Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471690353177756:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(16195013ns) {HandleHandshakeStatus:329(16193164ns) {DropIncomingHandshake:442(11967ns) DropOutgoingHandshake:457(713ns) SwitchToState:179(635ns) ProcessPendingSessionEvents:495(3804603ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3736746ns max# 957871ns}}}
2025-11-27T18:20:10.799610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471686559677454:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(26191388ns) {HandleHandshakeStatus:329(26189230ns) {DropIncomingHandshake:442(1516ns) DropOutgoingHandshake:457(5747ns) SwitchToState:179(685ns) ProcessPendingSessionEvents:495(20329708ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19514229ns max# 6331816ns}}}
2025-11-27T18:20:10.797114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471690456543033:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.791765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:10.789558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:10.809571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471688262814629:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(29396211ns) {HandleHandshakeStatus:329(29394123ns) {DropIncomingHandshake:442(13843ns) DropOutgoingHandshake:457(308ns) SwitchToState:179(753ns) ProcessPendingSessionEvents:495(11320931ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11255320ns max# 4819181ns}}}
2025-11-27T18:20:10.811395Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471688494072933:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(24948218ns) {HandleHandshakeStatus:329(24944859ns) {DropIncomingHandshake:442(14372ns) DropOutgoingHandshake:457(212ns) SwitchToState:179(456ns) ProcessPendingSessionEvents:495(21814750ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21399788ns max# 6313743ns}}}
2025-11-27T18:20:10.807093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.806936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471686559677547:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.814860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471690353177772:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.814903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471690353177772:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.816249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471690353177772:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:10.807755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471690456543033:2105] [node 5] ICP22 created new session: [6:7577471690456543064:2105]
2025-11-27T18:20:10.810033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471686559677455:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.812023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471690456543064:2105] [node 5] ICS09 handshake done sender: [6:7577471690456543054:2119] self: [6:7577471690456543060:0] peer: [5:7577471688494072938:0] socket: 399 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.818490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471686559677553:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36391
2025-11-27T18:20:10.812067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471690456543064:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:10.819251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471688494072868:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:10.820858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471690353177693:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.813682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:10.822137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:10.813977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471690456542972:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.816160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471690456543063:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48879
2025-11-27T18:20:10.822421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471690353177693:2049] [node 3] ICP22 created new session: [2:7577471690353177795:2049]
2025-11-27T18:20:10.825049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471686559677534:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.817446Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471690456543033:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(19614759ns) {HandleHandshakeStatus:329(19611974ns) {DropIncomingHandshake:442(716ns) DropOutgoingHandshake:457(379ns) SwitchToState:179(491ns) ProcessPendingSessionEvents:495(1417ns)}}
2025-11-27T18:20:10.824726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471690353177795:2049] [node 3] ICS09 handshake done sender: [2:7577471690353177772:2116] self: [2:7577471690353177771:0] peer: [3:7577471686559677552:0] socket: 418 qp: -1
2025-11-27T18:20:10.825082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471686559677534:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.834021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471686559677547:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:10.834230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471686559677455:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15935) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471686559677547:2120] held: no
2025-11-27T18:20:10.834011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471690456543063:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.834189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471690456543032:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:10.834695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471688262814641:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:10.834742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471688262814641:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:10.824851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471690353177795:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:10.825030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:10.844041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471688494072932:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(55920687ns) {HandleHandshakeStatus:329(55918609ns) {DropIncomingHandshake:442(1226ns) DropOutgoingHandshake:457(5211ns) SwitchToState:179(557ns) ProcessPendingSessionEvents:495(39715928ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 34327510ns max# 5793132ns}}}
2025-11-27T18:20:10.853193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471686559677534:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:10.846263Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471688262814641:2125] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:10.855649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471686559677455:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20678947ns) {HandleHandshakeStatus:395(20676461ns) {LogHandshakeFail:472(12782ns) {UpdateErrorStateLog:545(4493ns)} DropOutgoingHandshake:457(489ns) ProcessPendingSessionEvents:495(20602499ns) {unordered DropSessionEvent:512 num# 100 sum# 5274434ns max# 3180744ns}}}
2025-11-27T18:20:10.854188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471690456543068:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48879
2025-11-27T18:20:10.855737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471686559677455:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.857089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471688262814628:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.855825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:10.854607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471690456543069:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48879
2025-11-27T18:20:10.855876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471686559677455:2049] [node 2] ICP22 created new session: [3:7577471686559677555:2049]
2025-11-27T18:20:10.855918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471686559677555:2049] [node 2] ICS09 handshake done sender: [3:7577471686559677547:2120] self: [3:7577471686559677552:0] peer: [2:7577471690353177771:0] socket: 420 qp: -1
2025-11-27T18:20:10.855954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471686559677555:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:10.855987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:10.856612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471686559677520:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:10.857391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:10.858031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471686559677520:2111] [node 6] ICP22 created new session: [3:7577471686559677557:2111]
2025-11-27T18:20:10.865082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471686559677557:2111] [node 6] ICS09 handshake done sender: [3:7577471686559677534:2116] self: [3:7577471686559677533:0] peer: [6:7577471690456543066:0] socket: 417 qp: -1
2025-11-27T18:20:10.866023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471690353177693:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(35846645ns) {HandleHandshakeStatus:329(35843916ns) {DropIncomingHandshake:442(18204ns) DropOutgoingHandshake:457(1443ns) SwitchToState:179(794ns) ProcessPendingSessionEvents:495(21956011ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20833251ns max# 9125037ns}}}
2025-11-27T18:20:10.857620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471690456543061:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:10.858233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:10.866949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471686559677557:2111] [node 6] ICS10 traffic start
2025-11-27T18:20:10.858286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471688262814628:2109] [node 6] ICP22 created new session: [4:7577471688262814667:2109]
2025-11-27T18:20:10.867262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:10.863603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471688262814667:2109] [node 6] ICS09 handshake done sender: [4:7577471688262814641:2125] self: [4:7577471688262814640:0] peer: [6:7577471690456543067:0] socket: 382 qp: -1
2025-11-27T18:20:10.874675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471686559677520:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(19031441ns) {HandleHandshakeStatus:329(19027885ns) {DropIncomingHandshake:442(24536ns) DropOutgoingHandshake:457(313ns) SwitchToState:179(371ns) ProcessPendingSessionEvents:495(6416294ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6362335ns max# 5416360ns}}}
2025-11-27T18:20:10.865314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471688262814667:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:10.865362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:10.867110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471690456543063:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:10.868634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471690456542972:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.868700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:10.868744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471690456542972:2050] [node 3] ICP22 created new session: [6:7577471690456543070:2050]
2025-11-27T18:20:10.868804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471690456543070:2050] [node 3] ICS09 handshake done sender: [6:7577471690456543061:2120] self: [6:7577471690456543066:0] peer: [3:7577471686559677533:0] socket: 383 qp: -1
2025-11-27T18:20:10.868836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471690456543070:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:10.868863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:10.885703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471688262814628:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(28000428ns) {HandleHandshakeStatus:329(27997959ns) {DropIncomingHandshake:442(189129ns) DropOutgoingHandshake:457(412ns) SwitchToState:179(464ns) ProcessPendingSessionEvents:495(6473563ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6421238ns max# 2560165ns}}}
2025-11-27T18:20:10.886329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471690456543032:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:10.887561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:10.887643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471690456543032:2107] [node 4] ICP22 created new session: [6:7577471690456543072:2107]
2025-11-27T18:20:10.887709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471690456543072:2107] [node 4] ICS09 handshake done sender: [6:7577471690456543063:2122] self: [6:7577471690456543067:0] peer: [4:7577471688262814640:0] socket: 369 qp: -1
2025-11-27T18:20:10.887750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471690456543072:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:10.887785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:10.888521Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471690456543032:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:10.889344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471690456542972:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:10.941446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:10.941880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471686559677550:2048] [node 1] ICS07 socket disconnect 393 reason# EndOfStream
2025-11-27T18:20:10.941917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471686559677550:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.941992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:10.942029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471686559677550:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:10.941446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:10.941670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471690353177789:2048] [node 1] ICS07 socket disconnect 361 reason# EndOfStream
2025-11-27T18:20:10.941707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471690353177789:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.941815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:10.941858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471690353177789:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:10.947454Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471686559677454:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:10.947496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471686559677550:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:10.950418Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:10.950653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471688494072965:2048] [node 1] ICS01 socket: 394 reason# EndOfStream
2025-11-27T18:20:10.950690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471688494072867:2048] [node 1] ICP30 unregister session Session# [5:7577471688494072965:2048] VirtualId# [5:7577471688494072956:0]
2025-11-27T18:20:10.951597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:10.951650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471688494072965:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.951781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471688262814655:2048] [node 1] ICS07 socket disconnect 367 reason# EndOfStream
2025-11-27T18:20:10.956885Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:10.957165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471690456543055:2048] [node 1] ICS07 socket disconnect 363 reason# EndOfStream
2025-11-27T18:20:10.957193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471690456543055:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.953197Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471686559677454:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:10.951713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:10.953132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471690353177692:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:10.954250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471690353177789:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:10.957286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:10.954805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471686559677550:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:10.951745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:10.951822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471688262814655:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:10.958504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471690456543055:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:10.959215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471690456542970:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:10.965487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471690456543055:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:10.966152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471690456542970:2048] [node 1] ICP30 unregister session Session# [6:7577471690456543055:2048] VirtualId# [6:7577471690456543050:0]
2025-11-27T18:20:10.966569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:10.966598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471690456542970:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.966607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.965911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471686559677454:2048] [node 1] ICP30 unregister session Session# [3:7577471686559677550:2048] VirtualId# [3:7577471686559677542:0]
2025-11-27T18:20:10.951907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:10.951955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471688262814655:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:10.953292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471688262814556:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:10.954336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471688262814655:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:10.954377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471688262814556:2048] [node 1] ICP30 unregister session Session# [4:7577471688262814655:2048] VirtualId# [4:7577471688262814647:0]
2025-11-27T18:20:10.954494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:10.954549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471688262814556:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.954568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.973485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471688262814556:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:10.966766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:10.981928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471690353177692:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:10.981989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471690353177789:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:10.982024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471690353177692:2048] [node 1] ICP30 unregister session Session# [2:7577471690353177789:2048] VirtualId# [2:7577471690353177776:0]
2025-11-27T18:20:10.982150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:10.982193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471690353177692:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.982206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.994726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471690456542970:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:10.966796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471686559677454:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.966809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2232) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:10.981788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [3:7577471686559677454:2048] [node 1] ICP35 event processing took too much time StateWork:294(20492822ns) {HandleHandshakeStatus:395(20482544ns) {LogHandshakeFail:472(5777ns) {UpdateErrorStateLog:545(1568ns)} DropOutgoingHandshake:457(573ns) ProcessPendingSessionEvents:495(201ns) UnregisterSession:538(11121467ns) {ProcessPendingSessionEvents:495(1132ns) SwitchToInitialState:202(2466ns) {SwitchToState:179(1387ns)}} TransitToErrorState:752(6602925ns) {SwitchToState:179(3811ns) ProcessPendingSessionEvents:495(232ns) DropHandshakes:470(6558973ns) {DropIncomingHandshake:442(2187ns) DropOutgoingHandshake:457(291ns)}}}}
2025-11-27T18:20:10.993604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471686559677454:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:11.033593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:11.033847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471686559677555:2049] [node 2] ICS01 socket: 420 reason# EndOfStream
2025-11-27T18:20:11.033889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471686559677455:2049] [node 2] ICP30 unregister session Session# [3:7577471686559677555:2049] VirtualId# [3:7577471686559677552:0]
2025-11-27T18:20:11.033925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471686559677555:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.034015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:11.034066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:11.035247Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:11.035454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471688494072970:2049] [node 2] ICS07 socket disconnect 368 reason# EndOfStream
2025-11-27T18:20:11.035488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471688494072970:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.035553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:11.035598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471688494072970:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:11.035727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:11.035967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471688262814662:2049] [node 2] ICS07 socket disconnect 365 reason# EndOfStream
2025-11-27T18:20:11.035994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471688262814662:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.036069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:11.036106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471688262814662:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:11.036822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:11.037054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471690456543057:2049] [node 2] ICS07 socket disconnect 362 reason# EndOfStream
2025-11-27T18:20:11.039239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471688262814557:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15935) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:11.037083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471690456543057:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.037752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471688494072868:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15935) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:11.037150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:11.037189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471690456543057:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:11.039189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471688494072970:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:11.041666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471688262814662:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:11.042300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471688262814557:2049] [node 2] ICP30 unregister session Session# [4:7577471688262814662:2049] VirtualId# [4:7577471688262814648:0]
2025-11-27T18:20:11.042674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:11.042717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471688262814557:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15935) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.039498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471688494072868:2049] [node 2] ICP30 unregister session Session# [5:7577471688494072970:2049] VirtualId# [5:7577471688494072955:0]
2025-11-27T18:20:11.039636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:11.042739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15935) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.042294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471690456542971:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15935) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:11.039685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471688494072868:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15935) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.043706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471690456543057:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:11.043811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471690456542971:2049] [node 2] ICP30 unregister session Session# [6:7577471690456543057:2049] VirtualId# [6:7577471690456543051:0]
2025-11-27T18:20:11.043949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:11.044061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471690456542971:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15935) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.044086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15935) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.039701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15935) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.050325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471688494072868:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:11.052351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471688262814557:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:11.053378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471690456542971:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:11.131015Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:11.131251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471688262814660:2050] [node 3] ICS07 socket disconnect 366 reason# EndOfStream
2025-11-27T18:20:11.131289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471688262814660:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.131378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:11.131429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471688262814660:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:11.132315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:11.132517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471688494072961:2050] [node 3] ICS07 socket disconnect 370 reason# EndOfStream
2025-11-27T18:20:11.132550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471688494072961:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.132638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:11.132682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471688494072961:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:11.133001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471688262814558:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10126) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:11.133069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471688262814660:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:11.133120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471688262814558:2050] [node 3] ICP30 unregister session Session# [4:7577471688262814660:2050] VirtualId# [4:7577471688262814649:0]
2025-11-27T18:20:11.133228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:11.133284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471688262814558:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10126) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.133305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10126) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.133422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471688494072869:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10126) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:11.133479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471688494072961:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:11.133546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471688494072869:2050] [node 3] ICP30 unregister session Session# [5:7577471688494072961:2050] VirtualId# [5:7577471688494072953:0]
2025-11-27T18:20:11.133581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:11.133637Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471688494072869:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10126) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.133651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10126) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.133472Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:11.133686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471690456543070:2050] [node 3] ICS07 socket disconnect 383 reason# EndOfStream
2025-11-27T18:20:11.133723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471690456543070:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.133798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:11.133833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471690456543070:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:11.134501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471690456542972:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10126) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:11.134632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471690456543070:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:11.134668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471690456542972:2050] [node 3] ICP30 unregister session Session# [6:7577471690456543070:2050] VirtualId# [6:7577471690456543066:0]
2025-11-27T18:20:11.134781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:11.134845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471690456542972:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10126) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.134864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10126) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.143495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471688494072869:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:11.144506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471690456542972:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:11.203902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:11.204122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471688494072969:2108] [node 4] ICS07 socket disconnect 392 reason# EndOfStream
2025-11-27T18:20:11.204149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471688494072969:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.204243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:11.204299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471688494072969:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:11.205379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:11.205576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471690456543072:2107] [node 4] ICS07 socket disconnect 369 reason# EndOfStream
2025-11-27T18:20:11.205612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471690456543072:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.205695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:11.205180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471688494072932:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11387) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:11.207045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471688494072969:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:11.207107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471688494072932:2108] [node 4] ICP30 unregister session Session# [5:7577471688494072969:2108] VirtualId# [5:7577471688494072954:0]
2025-11-27T18:20:11.205737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471690456543072:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:11.207591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:11.207637Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471688494072932:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11387) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.207656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11387) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.208847Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471690456543032:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11387) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:11.208911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471690456543072:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:11.208943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471690456543032:2107] [node 4] ICP30 unregister session Session# [6:7577471690456543072:2107] VirtualId# [6:7577471690456543067:0]
2025-11-27T18:20:11.209028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:11.209060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471690456543032:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11387) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.209079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11387) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.217222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471688494072932:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:11.218283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471690456543032:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:11.283265Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:11.283560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471690456543064:2105] [node 5] ICS01 socket: 399 reason# EndOfStream
2025-11-27T18:20:11.283612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471690456543033:2105] [node 5] ICP30 unregister session Session# [6:7577471690456543064:2105] VirtualId# [6:7577471690456543060:0]
2025-11-27T18:20:11.283651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471690456543064:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.283724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:11.283817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:11.374029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471694364743131:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:11.374030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471694364743130:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:11.396070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471692352188283:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:11.396191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471692352188284:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:11.416707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471694814763012:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:11.416731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471694814763013:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:11.439035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471692155053066:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:11.439090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471692155053068:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:11.439257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471692155053067:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:11.463162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471691743678524:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:11.463546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471691743678525:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:11.463626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471691743678526:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:11.485278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471693353989022:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:11.485318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471693353989023:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:11.485749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471693353989021:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:11.630380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471692352188347:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:11.671912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471691743678587:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:11.674482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471692155053131:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:11.672064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471692352188349:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.686366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471693353989083:2105] [node 4] ICP01 ready to work
2025-11-27T18:20:11.674447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471694364743195:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:11.671758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471692352188348:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:11.689810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471694814763102:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:11.689601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471694364743196:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:11.690345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471693353989084:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:11.690860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471694814763103:2116] [node 5] ICP01 ready to work
2025-11-27T18:20:11.695367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471694814763106:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34808
2025-11-27T18:20:11.705780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471694814763111:2125] [node 6] ICP01 ready to work
2025-11-27T18:20:11.706059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471691743678588:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:11.706242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471692155053138:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:11.708263Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471691743678524:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(105198797ns) {Configure:104(105190966ns) {ProcessConfigured:135(68907ns) {StartInitialHandshake:157(67489ns) {DropHandshakes:470(2896ns) {DropIncomingHandshake:442(714ns) DropOutgoingHandshake:457(439ns)} GenerateSessionVirtualId:988(829ns) PrepareNewSessionHandshake:477(1456ns) {SwitchToState:179(408ns)}}}}}
2025-11-27T18:20:11.689603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471694364743197:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.764713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471694364743195:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(43289337ns) {Configure:104(43285924ns) {ProcessConfigured:135(70338ns) {StartInitialHandshake:157(67371ns) {DropHandshakes:470(6230ns) {DropIncomingHandshake:442(894ns) DropOutgoingHandshake:457(395ns)} GenerateSessionVirtualId:988(1078ns) PrepareNewSessionHandshake:477(2033ns) {SwitchToState:179(763ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.767282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471692352188283:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49425778ns) {Configure:104(49420307ns) {ProcessConfigured:135(47229ns) {StartInitialHandshake:157(45656ns) {DropHandshakes:470(3019ns) {DropIncomingHandshake:442(820ns) DropOutgoingHandshake:457(636ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:477(1553ns) {SwitchToState:179(657ns)}}}}}
2025-11-27T18:20:11.762592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471692155053139:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43570
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.777868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471692352188356:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25691
2025-11-27T18:20:11.765090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471694364743198:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47157
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.790432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471692155053140:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43570
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.765169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471694364743199:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47157
2025-11-27T18:20:11.783556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471692352188356:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.765358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471694364743200:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47157
2025-11-27T18:20:11.801167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471692155053140:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.783446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471694364743199:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.808543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471692155053066:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.792602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471694364743209:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47157
2025-11-27T18:20:11.825761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471694814763115:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34808
2025-11-27T18:20:11.797889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471694364743130:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(162718826ns) {Configure:104(162712329ns) {ProcessConfigured:135(6871513ns) {StartInitialHandshake:157(6869022ns) {DropHandshakes:470(2739ns) {DropIncomingHandshake:442(804ns) DropOutgoingHandshake:457(461ns)} GenerateSessionVirtualId:988(677ns) PrepareNewSessionHandshake:477(3366ns) {SwitchToState:179(1644ns)}}}}}
2025-11-27T18:20:11.833202Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471691743678525:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(128278628ns) {Configure:104(128269408ns) {ProcessConfigured:135(32739077ns) {StartInitialHandshake:157(32734764ns) {DropHandshakes:470(3036ns) {DropIncomingHandshake:442(702ns) DropOutgoingHandshake:457(363ns)} GenerateSessionVirtualId:988(588ns) PrepareNewSessionHandshake:477(3749ns) {SwitchToState:179(1499ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.831401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471694814763106:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.809549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471694364743209:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.836774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471691743678526:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(110871664ns) {Configure:104(110868088ns) {ProcessConfigured:135(41071ns) {StartInitialHandshake:157(39938ns) {DropHandshakes:470(1102ns) {DropIncomingHandshake:442(317ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(378ns) PrepareNewSessionHandshake:477(691ns) {SwitchToState:179(218ns)}}}}}
2025-11-27T18:20:11.832631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471694814763115:2141] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.836756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471692155053147:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43570
2025-11-27T18:20:11.836247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471694814763013:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.821516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471694364743202:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.821559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471694364743202:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.839296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471692352188353:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.838914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471692155053067:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(98498607ns) {Configure:104(98489080ns) {ProcessConfigured:135(2530172ns) {StartInitialHandshake:157(2527965ns) {DropHandshakes:470(299538ns) {DropIncomingHandshake:442(411ns) DropOutgoingHandshake:457(1217ns)} GenerateSessionVirtualId:988(1595ns) PrepareNewSessionHandshake:477(1530ns) {SwitchToState:179(548ns)}}}}}
2025-11-27T18:20:11.832304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471694364743202:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.839335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471692352188353:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.839367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471692352188348:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(56415790ns) {Configure:104(56409722ns) {ProcessConfigured:135(3131269ns) {StartInitialHandshake:157(3128807ns) {DropHandshakes:470(1205ns) {DropIncomingHandshake:442(347ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(354ns) PrepareNewSessionHandshake:477(1961ns) {SwitchToState:179(982ns)}}}}}
2025-11-27T18:20:11.834229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471694364743198:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.841534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471692155053066:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471692155053140:2115] held: no
2025-11-27T18:20:11.839672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471692352188361:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25691
2025-11-27T18:20:11.836113Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471694364743197:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(86029362ns) {Configure:104(86026314ns) {ProcessConfigured:135(2054675ns) {StartInitialHandshake:157(1303893ns) {DropHandshakes:470(3457ns) {DropIncomingHandshake:442(621ns) DropOutgoingHandshake:457(746ns)} GenerateSessionVirtualId:988(1556ns) PrepareNewSessionHandshake:477(2010ns) {SwitchToState:179(915ns)}}}}}
2025-11-27T18:20:11.843075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471691743678595:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23044
2025-11-27T18:20:11.848357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471693353989021:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(76249471ns) {Configure:104(76243983ns) {ProcessConfigured:135(950684ns) {StartInitialHandshake:157(948556ns) {DropHandshakes:470(1447ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(373ns) PrepareNewSessionHandshake:477(1301ns) {SwitchToState:179(108ns)}}}}}
2025-11-27T18:20:11.844555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471692155053139:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.843107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471692352188361:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.848559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471693353989023:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(60128492ns) {Configure:104(60121936ns) {ProcessConfigured:135(584341ns) {StartInitialHandshake:157(582089ns) {DropHandshakes:470(2421ns) {DropIncomingHandshake:442(504ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(747ns) PrepareNewSessionHandshake:477(1582ns) {SwitchToState:179(554ns)}}}}}
2025-11-27T18:20:11.836630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471694364743196:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.849108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471692155053140:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:11.837158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:11.843371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471692352188283:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.843597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471692352188353:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:11.850667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471693353989022:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(76869602ns) {Configure:104(76862037ns) {ProcessConfigured:135(1578358ns) {StartInitialHandshake:157(1575981ns) {DropHandshakes:470(1309ns) {DropIncomingHandshake:442(375ns) DropOutgoingHandshake:457(175ns)} GenerateSessionVirtualId:988(540ns) PrepareNewSessionHandshake:477(1796ns) {SwitchToState:179(935ns)}}}}}
2025-11-27T18:20:11.849711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471691743678595:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.849285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471692352188284:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.837797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471694364743131:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(111561922ns) {Configure:104(111554041ns) {ProcessConfigured:135(32645ns) {StartInitialHandshake:157(31595ns) {DropHandshakes:470(3444ns) {DropIncomingHandshake:442(277ns) DropOutgoingHandshake:457(423ns)} GenerateSessionVirtualId:988(344ns) PrepareNewSessionHandshake:477(494ns) {SwitchToState:179(164ns)}}}}}
2025-11-27T18:20:11.850364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471691743678596:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23044
2025-11-27T18:20:11.849936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471692155053066:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.837945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471694364743196:2107] [node 4] ICP22 created new session: [1:7577471694364743218:2107]
2025-11-27T18:20:11.852149Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471694814763013:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471694814763115:2141] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.854534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471693353989086:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60262
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.854806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471693353989087:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60262
2025-11-27T18:20:11.855173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471693353989087:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.852420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471691743678526:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.849970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:11.849340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:11.853731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471694814763121:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34808
2025-11-27T18:20:11.856653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471691743678587:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16804449ns) {Configure:104(16799608ns) {ProcessConfigured:135(1989967ns) {StartInitialHandshake:157(1987756ns) {DropHandshakes:470(1173ns) {DropIncomingHandshake:442(315ns) DropOutgoingHandshake:457(232ns)} GenerateSessionVirtualId:988(338ns) PrepareNewSessionHandshake:477(1067ns) {SwitchToState:179(251ns)}}}}}
2025-11-27T18:20:11.856901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471694814763108:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.839328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471694364743218:2107] [node 4] ICS09 handshake done sender: [1:7577471694364743202:2115] self: [1:7577471694364743201:0] peer: [4:7577471692155053146:0] socket: 375 qp: -1
2025-11-27T18:20:11.850528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471692155053066:2048] [node 1] ICP22 created new session: [4:7577471692155053150:2048]
2025-11-27T18:20:11.850301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471692352188284:2049] [node 3] ICP22 created new session: [2:7577471692352188367:2049]
2025-11-27T18:20:11.856944Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471694814763108:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.839727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471694364743218:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:11.852248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471692155053150:2048] [node 1] ICS09 handshake done sender: [4:7577471692155053140:2115] self: [4:7577471692155053146:0] peer: [1:7577471694364743201:0] socket: 381 qp: -1
2025-11-27T18:20:11.861430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471693353989086:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.852880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471692352188349:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18268560ns) {Configure:104(18261658ns) {ProcessConfigured:135(2306800ns) {StartInitialHandshake:157(2303613ns) {DropHandshakes:470(2642ns) {DropIncomingHandshake:442(644ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(1020ns) PrepareNewSessionHandshake:477(2559ns) {SwitchToState:179(994ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.840581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:11.853655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471692155053150:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:11.853171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471692352188367:2049] [node 3] ICS09 handshake done sender: [2:7577471692352188353:2112] self: [2:7577471692352188352:0] peer: [3:7577471694814763119:0] socket: 364 qp: -1
2025-11-27T18:20:11.862427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471691743678596:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.843852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471694364743208:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.856605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:11.862489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471693353989021:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.863078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471694814763108:2139] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.853597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471692352188367:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:11.843875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471694364743208:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.863397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471691743678601:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23044
2025-11-27T18:20:11.853753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:11.843914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471694364743200:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.864030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471694814763115:2141] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:11.863689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471691743678525:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.848629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471694364743208:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:11.866426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471691743678607:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23044
2025-11-27T18:20:11.863742Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471692352188283:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471692352188361:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.865428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471694814763013:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.866632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471692352188368:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25691
2025-11-27T18:20:11.850516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471694364743130:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.865626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471694814763103:2116] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.866481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471691743678606:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23044
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.867357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471692352188351:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.866447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:11.866829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:11.867008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471694814763013:2049] [node 2] ICP22 created new session: [3:7577471694814763127:2049]
2025-11-27T18:20:11.850565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:11.867059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471694814763103:2116] [node 5] ICP22 created new session: [3:7577471694814763128:2116]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.867381Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471692352188351:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.867577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471691743678601:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.852910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471694364743130:2048] [node 2] ICP22 created new session: [1:7577471694364743224:2048]
2025-11-27T18:20:11.869292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471693353989083:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.868823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471692155053154:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43570
2025-11-27T18:20:11.867191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471694814763123:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34808
2025-11-27T18:20:11.867609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471691743678607:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.856580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471694364743224:2048] [node 2] ICS09 handshake done sender: [1:7577471694364743208:2117] self: [1:7577471694364743207:0] peer: [2:7577471692352188365:0] socket: 384 qp: -1
2025-11-27T18:20:11.870671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471692155053144:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.867376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471694814763127:2049] [node 2] ICS09 handshake done sender: [3:7577471694814763115:2141] self: [3:7577471694814763119:0] peer: [2:7577471692352188352:0] socket: 370 qp: -1
2025-11-27T18:20:11.856758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471694364743224:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:11.874311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471692352188361:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:11.867626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471691743678609:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23044
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.870705Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471692155053144:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.875690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471692352188372:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25691
2025-11-27T18:20:11.868252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471691743678606:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.867955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471694814763127:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:11.874210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471692155053155:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43570
2025-11-27T18:20:11.858667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.867965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471694814763128:2116] [node 5] ICS09 handshake done sender: [3:7577471694814763108:2139] self: [3:7577471694814763107:0] peer: [5:7577471691743678603:0] socket: 358 qp: -1
2025-11-27T18:20:11.876450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471692352188283:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.874355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471692155053154:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.864316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471694364743204:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.868637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471691743678524:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.877839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471693353989101:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60262
2025-11-27T18:20:11.868001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:11.879566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471693353989104:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60262
2025-11-27T18:20:11.868622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471694814763128:2116] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.868653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471691743678588:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.876487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:11.875231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471691743678595:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:11.868656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471694814763124:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34808
2025-11-27T18:20:11.877639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471691743678526:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.864348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471694364743204:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.874885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471692155053142:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.876785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471692352188283:2048] [node 1] ICP22 created new session: [2:7577471692352188375:2048]
2025-11-27T18:20:11.879740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471693353989101:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.867954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471694364743227:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47157
2025-11-27T18:20:11.868835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471694364743214:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.874918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471692155053142:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.881786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471693353989023:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.879188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:11.868663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:11.877075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471692352188375:2048] [node 1] ICS09 handshake done sender: [2:7577471692352188361:2115] self: [2:7577471692352188365:0] peer: [1:7577471694364743207:0] socket: 385 qp: -1
No RDMA context found for GID: 2025-11-27T18:20:11.879573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471691743678526:2050] [node 3] ICP22 created new session: [5:7577471691743678615:2050]
2025-11-27T18:20:11.868856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471694364743214:2119] [node 5] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-27T18:20:11.875411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471692155053144:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:11.877657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471692352188351:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:11.882238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471693353989086:2110] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:11.868976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471694814763123:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.869186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471694364743227:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.883114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471693353989021:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.877931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471692352188372:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.879643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471691743678615:2050] [node 3] ICS09 handshake done sender: [5:7577471691743678595:2112] self: [5:7577471691743678603:0] peer: [3:7577471694814763107:0] socket: 369 qp: -1
2025-11-27T18:20:11.876719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471692155053068:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.874233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471694814763124:2146] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.876825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471692155053131:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.879808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471691743678615:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:11.871091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471694364743204:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:11.885834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471693353989105:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60262
2025-11-27T18:20:11.875036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471694814763012:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.878075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471692352188375:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:11.885968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: 2025-11-27T18:20:11.878634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
::ffff:127.1.0.0
2025-11-27T18:20:11.876987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471694814763117:2142] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.877899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:11.875668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471694364743195:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.880073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:11.876154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:11.877934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471692155053155:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.877013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471694814763117:2142] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.886775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471693353989107:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60262
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:11.881308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471691743678613:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23044
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.882166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471692352188347:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.876755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471694364743195:2110] [node 6] ICP22 created new session: [1:7577471694364743230:2110]
2025-11-27T18:20:11.881698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471694814763117:2142] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:11.878115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471692155053131:2108] [node 5] ICP22 created new session: [4:7577471692155053158:2108]
2025-11-27T18:20:11.882211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:11.886785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471693353989021:2048] [node 1] ICP22 created new session: [6:7577471693353989108:2048]
2025-11-27T18:20:11.882171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471694814763102:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.877336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471694364743230:2110] [node 6] ICS09 handshake done sender: [1:7577471694364743204:2116] self: [1:7577471694364743203:0] peer: [6:7577471693353989098:0] socket: 376 qp: -1
2025-11-27T18:20:11.882921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471692352188347:2108] [node 5] ICP22 created new session: [2:7577471692352188378:2108]
2025-11-27T18:20:11.887788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471693353989108:2048] [node 1] ICS09 handshake done sender: [6:7577471693353989086:2110] self: [6:7577471693353989098:0] peer: [1:7577471694364743203:0] socket: 378 qp: -1
2025-11-27T18:20:11.879806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471692155053158:2108] [node 5] ICS09 handshake done sender: [4:7577471692155053144:2117] self: [4:7577471692155053143:0] peer: [5:7577471691743678612:0] socket: 374 qp: -1
2025-11-27T18:20:11.882539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471694814763122:2144] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.885565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471692352188378:2108] [node 5] ICS09 handshake done sender: [2:7577471692352188351:2111] self: [2:7577471692352188350:0] peer: [5:7577471691743678608:0] socket: 363 qp: -1
2025-11-27T18:20:11.877448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471694364743217:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.882564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471694814763122:2144] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.885855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471692352188378:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:11.883689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:11.877491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471694364743217:2120] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.879822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471692155053067:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.887818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471693353989108:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:11.885841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471694814763102:2107] [node 4] ICP22 created new session: [3:7577471694814763135:2107]
2025-11-27T18:20:11.877601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471694364743230:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:11.885892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:11.880099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471692155053158:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:11.883726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471691743678588:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12970) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471691743678606:2118] held: no
2025-11-27T18:20:11.886766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471694814763135:2107] [node 4] ICS09 handshake done sender: [3:7577471694814763117:2142] self: [3:7577471694814763116:0] peer: [4:7577471692155053157:0] socket: 401 qp: -1
2025-11-27T18:20:11.886540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471692352188363:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.880147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:11.886566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471692352188363:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.886783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471694814763122:2144] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:11.880178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471692155053142:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:11.889161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:11.891107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471692352188284:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(30372087ns) {HandleHandshakeStatus:329(30357445ns) {DropIncomingHandshake:442(12479ns) DropOutgoingHandshake:457(412ns) SwitchToState:179(197ns) ProcessPendingSessionEvents:495(21718472ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20574991ns max# 3540109ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.887741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471694814763135:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:11.877897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:11.891044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471693353989087:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:11.885313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471692155053138:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.885590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:11.881501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471694364743214:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:11.887789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:11.886490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471692155053138:2109] [node 6] ICP22 created new session: [4:7577471692155053164:2109]
2025-11-27T18:20:11.882743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471694364743130:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(23874100ns) {HandleHandshakeStatus:329(23871944ns) {DropIncomingHandshake:442(11316ns) DropOutgoingHandshake:457(159ns) SwitchToState:179(145ns) ProcessPendingSessionEvents:495(11688181ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11208771ns max# 1953303ns}}}
2025-11-27T18:20:11.887283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471692155053164:2109] [node 6] ICS09 handshake done sender: [4:7577471692155053142:2116] self: [4:7577471692155053141:0] peer: [6:7577471693353989103:0] socket: 383 qp: -1
2025-11-27T18:20:11.891090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471694814763111:2125] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:11.887331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471692155053164:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:11.883775Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471694364743196:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(32255995ns) {HandleHandshakeStatus:329(32253992ns) {DropIncomingHandshake:442(10767ns) DropOutgoingHandshake:457(283ns) SwitchToState:179(327ns) ProcessPendingSessionEvents:495(25464459ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25342008ns max# 3705683ns}}}
2025-11-27T18:20:11.885578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471694364743197:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.887734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:11.893167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:11.891402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471693353989109:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60262
2025-11-27T18:20:11.886320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:11.893793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471694814763111:2125] [node 6] ICP22 created new session: [3:7577471694814763139:2125]
2025-11-27T18:20:11.892778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471693353989105:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.889602Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471692155053068:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63623) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471692155053154:2120] held: no
2025-11-27T18:20:11.891533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471692155053163:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43570
2025-11-27T18:20:11.886362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471694364743197:2108] [node 5] ICP22 created new session: [1:7577471694364743233:2108]
2025-11-27T18:20:11.889172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471691743678606:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:11.893991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471693353989083:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.888578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471694364743233:2108] [node 5] ICS09 handshake done sender: [1:7577471694364743214:2119] self: [1:7577471694364743213:0] peer: [5:7577471691743678610:0] socket: 362 qp: -1
2025-11-27T18:20:11.894049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:11.889162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471694364743233:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:11.889540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:11.894152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471694364743217:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:11.890427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471691743678614:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23044
2025-11-27T18:20:11.893983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471694814763137:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34808
2025-11-27T18:20:11.893174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471692352188363:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:11.894105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471693353989083:2105] [node 4] ICP22 created new session: [6:7577471693353989113:2105]
2025-11-27T18:20:11.892520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471691743678588:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.894298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471694814763139:2125] [node 6] ICS09 handshake done sender: [3:7577471694814763122:2144] self: [3:7577471694814763120:0] peer: [6:7577471693353989106:0] socket: 408 qp: -1
2025-11-27T18:20:11.894412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471692352188379:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25691
2025-11-27T18:20:11.894166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471693353989113:2105] [node 4] ICS09 handshake done sender: [6:7577471693353989087:2111] self: [6:7577471693353989103:0] peer: [4:7577471692155053141:0] socket: 388 qp: -1
2025-11-27T18:20:11.894344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471694814763139:2125] [node 6] ICS10 traffic start
2025-11-27T18:20:11.892733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:11.894215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471693353989113:2105] [node 4] ICS10 traffic start
2025-11-27T18:20:11.895842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471694364743131:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.896504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471692352188348:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.894778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:11.893926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471691743678588:2108] [node 4] ICP22 created new session: [5:7577471691743678619:2108]
2025-11-27T18:20:11.894252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:11.896701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:11.893994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471691743678619:2108] [node 4] ICS09 handshake done sender: [5:7577471691743678606:2118] self: [5:7577471691743678612:0] peer: [4:7577471692155053143:0] socket: 394 qp: -1
2025-11-27T18:20:11.894023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471691743678619:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:11.897002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471694364743131:2049] [node 3] ICP22 created new session: [1:7577471694364743235:2049]
2025-11-27T18:20:11.894091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:11.895026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471691743678616:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23044
2025-11-27T18:20:11.896383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471691743678525:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471691743678596:2113] held: no
2025-11-27T18:20:11.897027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471691743678601:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:11.896550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:11.894932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471693353989101:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:11.897503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471693353989023:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.897058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471692352188348:2107] [node 4] ICP22 created new session: [2:7577471692352188383:2107]
2025-11-27T18:20:11.898811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471694364743235:2049] [node 3] ICS09 handshake done sender: [1:7577471694364743217:2120] self: [1:7577471694364743216:0] peer: [3:7577471694814763132:0] socket: 371 qp: -1
2025-11-27T18:20:11.898346Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471692155053154:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:11.897584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:11.897105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471692352188383:2107] [node 4] ICS09 handshake done sender: [2:7577471692352188363:2116] self: [2:7577471692352188362:0] peer: [4:7577471692155053161:0] socket: 402 qp: -1
2025-11-27T18:20:11.898852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471694364743235:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:11.898301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471693353989023:2050] [node 3] ICP22 created new session: [6:7577471693353989115:2050]
2025-11-27T18:20:11.897410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471692352188383:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:11.900526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471694814763103:2116] [node 5] ICP35 event processing took too much time PendingConnection:279(26048944ns) {HandleHandshakeStatus:329(25952816ns) {DropIncomingHandshake:442(480ns) DropOutgoingHandshake:457(414ns) SwitchToState:179(305ns) ProcessPendingSessionEvents:495(22654086ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21309935ns max# 2029561ns}}}
2025-11-27T18:20:11.899051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:11.897440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:11.898933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471693353989115:2050] [node 3] ICS09 handshake done sender: [6:7577471693353989101:2116] self: [6:7577471693353989106:0] peer: [3:7577471694814763120:0] socket: 409 qp: -1
2025-11-27T18:20:11.901356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471694814763123:2145] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:11.899064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471693353989115:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:11.901420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471691743678524:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471691743678601:2116] held: no
2025-11-27T18:20:11.899355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:11.901911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471691743678526:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16506442ns) {HandleHandshakeStatus:329(16500978ns) {DropIncomingHandshake:442(356ns) DropOutgoingHandshake:457(5397ns) SwitchToState:179(126ns) ProcessPendingSessionEvents:495(12021188ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11923274ns max# 1638107ns}}}
2025-11-27T18:20:11.901945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471691743678596:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:11.904047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471691743678525:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.904304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:11.904530Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471692155053131:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(21883824ns) {HandleHandshakeStatus:329(21881700ns) {DropIncomingHandshake:442(13127ns) DropOutgoingHandshake:457(181ns) SwitchToState:179(189ns) ProcessPendingSessionEvents:495(16538119ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15940981ns max# 2489691ns}}}
2025-11-27T18:20:11.905346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471692155053068:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.904853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471692352188379:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.904702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471691743678525:2049] [node 2] ICP22 created new session: [5:7577471691743678621:2049]
2025-11-27T18:20:11.905394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:11.905327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471691743678621:2049] [node 2] ICS09 handshake done sender: [5:7577471691743678596:2113] self: [5:7577471691743678608:0] peer: [2:7577471692352188350:0] socket: 372 qp: -1
2025-11-27T18:20:11.905528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471691743678621:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:11.905574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471692155053068:2050] [node 3] ICP22 created new session: [4:7577471692155053167:2050]
2025-11-27T18:20:11.905828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471692155053167:2050] [node 3] ICS09 handshake done sender: [4:7577471692155053154:2120] self: [4:7577471692155053157:0] peer: [3:7577471694814763116:0] socket: 403 qp: -1
No RDMA context found for GID: 2025-11-27T18:20:11.905858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471692155053167:2050] [node 3] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:20:11.905899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:11.905982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471692155053166:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43570
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.906195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471693353989111:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60262
2025-11-27T18:20:11.906234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471692155053138:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16199535ns) {HandleHandshakeStatus:329(16197011ns) {DropIncomingHandshake:442(253ns) DropOutgoingHandshake:457(241ns) SwitchToState:179(347ns) ProcessPendingSessionEvents:495(11051845ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10940332ns max# 1206070ns}}}
2025-11-27T18:20:11.905809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471692352188347:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18672750ns) {HandleHandshakeStatus:329(18671161ns) {DropIncomingHandshake:442(11590ns) DropOutgoingHandshake:457(237ns) SwitchToState:179(291ns) ProcessPendingSessionEvents:495(11165224ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10458378ns max# 1195640ns}}}
2025-11-27T18:20:11.905846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471692352188382:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25691
2025-11-27T18:20:11.906332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:11.907313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471694364743195:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(22195923ns) {HandleHandshakeStatus:329(22194492ns) {DropIncomingHandshake:442(1175ns) DropOutgoingHandshake:457(897ns) SwitchToState:179(290ns) ProcessPendingSessionEvents:495(17943004ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17843192ns max# 1755295ns}}}
2025-11-27T18:20:11.907391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471694364743195:2110] [node 6] ICP18 handshake (actor [1:7577471694364743227:2121]) is too old
2025-11-27T18:20:11.907093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471693353989022:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.907380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471692352188348:2107] [node 4] ICP18 handshake (actor [2:7577471692352188379:2120]) is too old
2025-11-27T18:20:11.908174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471693353989111:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.908278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471694814763012:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.908014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471691743678524:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.908399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471692352188369:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.908560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:11.908611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:11.908373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471692155053155:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:11.909278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471691743678524:2048] [node 1] ICP22 created new session: [5:7577471691743678623:2048]
2025-11-27T18:20:11.909326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471694814763012:2048] [node 1] ICP22 created new session: [3:7577471694814763141:2048]
2025-11-27T18:20:11.908436Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471692352188369:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.909763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471691743678623:2048] [node 1] ICS09 handshake done sender: [5:7577471691743678601:2116] self: [5:7577471691743678610:0] peer: [1:7577471694364743213:0] socket: 391 qp: -1
2025-11-27T18:20:11.909961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471691743678623:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:11.910632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:11.910533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471694814763141:2048] [node 1] ICS09 handshake done sender: [3:7577471694814763123:2145] self: [3:7577471694814763132:0] peer: [1:7577471694364743216:0] socket: 395 qp: -1
2025-11-27T18:20:11.910436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471693353989084:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:11.910455Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471693353989021:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22824208ns) {HandleHandshakeStatus:329(22822326ns) {DropIncomingHandshake:442(429ns) DropOutgoingHandshake:457(3952ns) SwitchToState:179(132ns) ProcessPendingSessionEvents:495(14590888ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14508667ns max# 1852239ns}}}
2025-11-27T18:20:11.911414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471692352188382:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.911787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471691743678605:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:11.910794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471694814763141:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:11.911791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471692352188369:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:11.911938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471694364743197:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(22633881ns) {HandleHandshakeStatus:329(22509131ns) {DropIncomingHandshake:442(13199ns) DropOutgoingHandshake:457(375ns) SwitchToState:179(187ns) ProcessPendingSessionEvents:495(13276049ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12620014ns max# 1989416ns}}}
2025-11-27T18:20:11.910834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:11.912026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471692155053067:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.911837Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471691743678605:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:11.912934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471692352188349:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.913003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:11.913868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471692352188349:2109] [node 6] ICP22 created new session: [2:7577471692352188387:2109]
2025-11-27T18:20:11.914569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471692352188387:2109] [node 6] ICS09 handshake done sender: [2:7577471692352188369:2118] self: [2:7577471692352188366:0] peer: [6:7577471693353989117:0] socket: 415 qp: -1
2025-11-27T18:20:11.914289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:11.914603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471692352188387:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:11.914938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471692155053067:2049] [node 2] ICP22 created new session: [4:7577471692155053169:2049]
2025-11-27T18:20:11.914892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:11.915333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471691743678605:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:11.915109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471692155053169:2049] [node 2] ICS09 handshake done sender: [4:7577471692155053155:2121] self: [4:7577471692155053161:0] peer: [2:7577471692352188362:0] socket: 407 qp: -1
2025-11-27T18:20:11.915158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471692155053169:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:11.916142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471691743678587:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:11.915716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:11.917468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471693353989022:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471693353989105:2119] held: no
2025-11-27T18:20:11.919767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:11.919943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471691743678587:2109] [node 6] ICP22 created new session: [5:7577471691743678626:2109]
2025-11-27T18:20:11.919997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471691743678626:2109] [node 6] ICS09 handshake done sender: [5:7577471691743678605:2117] self: [5:7577471691743678604:0] peer: [6:7577471693353989118:0] socket: 379 qp: -1
2025-11-27T18:20:11.920028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471691743678626:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:11.920064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:11.919910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471694814763111:2125] [node 6] ICP35 event processing took too much time PendingConnection:279(20035388ns) {HandleHandshakeStatus:329(20033302ns) {DropIncomingHandshake:442(13420ns) DropOutgoingHandshake:457(334ns) SwitchToState:179(506ns) ProcessPendingSessionEvents:495(14297778ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14201303ns max# 1235738ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.921478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471693353989119:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60262
2025-11-27T18:20:11.921504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471693353989023:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17048566ns) {HandleHandshakeStatus:329(17046626ns) {DropIncomingHandshake:442(954ns) DropOutgoingHandshake:457(4334ns) SwitchToState:179(169ns) ProcessPendingSessionEvents:495(13079950ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11977990ns max# 1218077ns}}}
2025-11-27T18:20:11.921861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471694814763012:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:11.922520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471693353989023:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:11.922999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471692155053067:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:11.923266Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471693353989084:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1114) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471693353989111:2122] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:11.923967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471693353989105:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:11.924066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471693353989120:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60262
2025-11-27T18:20:11.926220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471693353989022:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.926310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:11.926327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471693353989111:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:11.926353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471693353989022:2049] [node 2] ICP22 created new session: [6:7577471693353989121:2049]
2025-11-27T18:20:11.926399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471693353989121:2049] [node 2] ICS09 handshake done sender: [6:7577471693353989105:2119] self: [6:7577471693353989117:0] peer: [2:7577471692352188366:0] socket: 416 qp: -1
2025-11-27T18:20:11.926438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471693353989121:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:11.926484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:11.926695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471693353989084:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:11.926737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:11.926781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471693353989084:2106] [node 5] ICP22 created new session: [6:7577471693353989123:2106]
2025-11-27T18:20:11.926836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471693353989123:2106] [node 5] ICS09 handshake done sender: [6:7577471693353989111:2122] self: [6:7577471693353989118:0] peer: [5:7577471691743678604:0] socket: 354 qp: -1
2025-11-27T18:20:11.926870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471693353989123:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:11.926918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:11.953167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:11.953435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471692352188375:2048] [node 1] ICS01 socket: 385 reason# EndOfStream
2025-11-27T18:20:11.953481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471692352188283:2048] [node 1] ICP30 unregister session Session# [2:7577471692352188375:2048] VirtualId# [2:7577471692352188365:0]
2025-11-27T18:20:11.953533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471692352188375:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.953721Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:11.953920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471694814763141:2048] [node 1] ICS07 socket disconnect 395 reason# EndOfStream
2025-11-27T18:20:11.953963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471694814763141:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.954028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:11.954067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471694814763141:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:11.954666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:11.954747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:11.956716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471694814763012:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13752) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:11.956759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471694814763141:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:11.956782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:11.961419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471692155053150:2048] [node 1] ICS01 socket: 381 reason# EndOfStream
2025-11-27T18:20:11.961487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471692155053066:2048] [node 1] ICP30 unregister session Session# [4:7577471692155053150:2048] VirtualId# [4:7577471692155053146:0]
2025-11-27T18:20:11.961533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471692155053150:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.961623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:11.961669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:11.962617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471694814763012:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13752) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:11.962703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471694814763141:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:11.962757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471694814763012:2048] [node 1] ICP30 unregister session Session# [3:7577471694814763141:2048] VirtualId# [3:7577471694814763132:0]
2025-11-27T18:20:11.962868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:11.962908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471694814763012:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13752) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.962924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13752) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.964478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:11.964695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471693353989108:2048] [node 1] ICS07 socket disconnect 378 reason# EndOfStream
2025-11-27T18:20:11.964734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471693353989108:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.964813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:11.964856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471693353989108:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:11.965379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:11.965606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471691743678623:2048] [node 1] ICS01 socket: 391 reason# EndOfStream
2025-11-27T18:20:11.965658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471691743678524:2048] [node 1] ICP30 unregister session Session# [5:7577471691743678623:2048] VirtualId# [5:7577471691743678610:0]
2025-11-27T18:20:11.965693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471691743678623:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:11.965732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471693353989021:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13752) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:11.965892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:11.965968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:11.965788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471693353989108:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:11.965935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471693353989021:2048] [node 1] ICP30 unregister session Session# [6:7577471693353989108:2048] VirtualId# [6:7577471693353989098:0]
2025-11-27T18:20:11.966043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:11.966078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471693353989021:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13752) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.966099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13752) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:11.971856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471694814763012:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:11.976014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471693353989021:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:12.029817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:12.030102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471694814763127:2049] [node 2] ICS01 socket: 370 reason# EndOfStream
2025-11-27T18:20:12.030154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471694814763013:2049] [node 2] ICP30 unregister session Session# [3:7577471694814763127:2049] VirtualId# [3:7577471694814763119:0]
2025-11-27T18:20:12.030191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471694814763127:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.030274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:12.030325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:12.032164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:12.032393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471692155053169:2049] [node 2] ICS07 socket disconnect 407 reason# EndOfStream
2025-11-27T18:20:12.032430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471692155053169:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.032497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:12.032549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471692155053169:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:12.033240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471692155053067:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23396) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:12.033229Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:12.033315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471692155053169:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:12.033356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471692155053067:2049] [node 2] ICP30 unregister session Session# [4:7577471692155053169:2049] VirtualId# [4:7577471692155053161:0]
2025-11-27T18:20:12.033459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471691743678621:2049] [node 2] ICS01 socket: 372 reason# EndOfStream
2025-11-27T18:20:12.033494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:12.033504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471691743678525:2049] [node 2] ICP30 unregister session Session# [5:7577471691743678621:2049] VirtualId# [5:7577471691743678608:0]
2025-11-27T18:20:12.033536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471692155053067:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.033535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471691743678621:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.033559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.033610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:12.033666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:12.034437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:12.034631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471693353989121:2049] [node 2] ICS01 socket: 416 reason# EndOfStream
2025-11-27T18:20:12.034673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471693353989022:2049] [node 2] ICP30 unregister session Session# [6:7577471693353989121:2049] VirtualId# [6:7577471693353989117:0]
2025-11-27T18:20:12.034701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471693353989121:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.034776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:12.034821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:12.043585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471692155053067:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:12.096603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:12.096894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471692155053167:2050] [node 3] ICS01 socket: 403 reason# EndOfStream
2025-11-27T18:20:12.096945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471692155053068:2050] [node 3] ICP30 unregister session Session# [4:7577471692155053167:2050] VirtualId# [4:7577471692155053157:0]
2025-11-27T18:20:12.096981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471692155053167:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.097096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:12.097152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:12.097536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:12.097681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471691743678615:2050] [node 3] ICS01 socket: 369 reason# EndOfStream
2025-11-27T18:20:12.097711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471691743678526:2050] [node 3] ICP30 unregister session Session# [5:7577471691743678615:2050] VirtualId# [5:7577471691743678603:0]
2025-11-27T18:20:12.097729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471691743678615:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.097810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:12.097858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:12.098541Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:12.098754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471693353989115:2050] [node 3] ICS07 socket disconnect 409 reason# EndOfStream
2025-11-27T18:20:12.098804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471693353989115:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.098883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:12.098919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471693353989115:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:12.099647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471693353989023:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63623) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:12.099728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471693353989115:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:12.099771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471693353989023:2050] [node 3] ICP30 unregister session Session# [6:7577471693353989115:2050] VirtualId# [6:7577471693353989106:0]
2025-11-27T18:20:12.099875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:12.099930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471693353989023:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63623) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.099952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63623) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.110175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471693353989023:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:12.163570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:12.163854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471691743678619:2108] [node 4] ICS01 socket: 394 reason# EndOfStream
2025-11-27T18:20:12.163904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471691743678588:2108] [node 4] ICP30 unregister session Session# [5:7577471691743678619:2108] VirtualId# [5:7577471691743678612:0]
2025-11-27T18:20:12.163942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471691743678619:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.164084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:12.164170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:12.164603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:12.164794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471693353989113:2105] [node 4] ICS07 socket disconnect 388 reason# EndOfStream
2025-11-27T18:20:12.164838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471693353989113:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.164942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:12.164986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471693353989113:2105] [node 4] ICS15 start handshake
2025-11-27T18:20:12.165607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471693353989083:2105] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12970) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:12.165696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471693353989113:2105] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:12.165742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471693353989083:2105] [node 4] ICP30 unregister session Session# [6:7577471693353989113:2105] VirtualId# [6:7577471693353989103:0]
2025-11-27T18:20:12.165790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:12.165829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471693353989083:2105] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12970) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.165848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12970) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.175683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471693353989083:2105] [node 4] ICP33 wake up from error state
2025-11-27T18:20:12.213534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:12.213793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471693353989123:2106] [node 5] ICS01 socket: 354 reason# EndOfStream
2025-11-27T18:20:12.213848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471693353989084:2106] [node 5] ICP30 unregister session Session# [6:7577471693353989123:2106] VirtualId# [6:7577471693353989118:0]
2025-11-27T18:20:12.213898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471693353989123:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.213967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:12.214007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:12.316342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471698076303590:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:12.316596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471698076303591:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:12.339236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471694963918730:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:12.339271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471694963918731:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:12.363326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471696170660328:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:12.363546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471696170660327:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:12.388041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471697901369471:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:12.388094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471697901369472:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:12.388196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471697901369473:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:12.421566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471697491974074:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:12.421626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471697491974075:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:12.421665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471697491974076:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:12.446430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471696048169725:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:12.446496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471696048169727:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:12.446807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471696048169726:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:12.516958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471698076303655:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:12.522952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471698076303656:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:12.542607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471694963918794:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:12.542620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471696170660391:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:12.536142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471698076303657:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:12.552225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471694963918795:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:12.552978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471697901369534:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:12.553577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471696048169789:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:12.553620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471694963918796:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:12.553687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471697901369535:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:12.553582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471696048169788:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:12.553587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471697491974138:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:12.552682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471696170660392:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:12.553234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471696170660393:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:12.554488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471697491974139:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:12.562346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471698076303591:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(55780719ns) {Configure:104(55773268ns) {ProcessConfigured:135(532683ns) {StartInitialHandshake:157(531030ns) {DropHandshakes:470(2095ns) {DropIncomingHandshake:442(523ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(1016ns) PrepareNewSessionHandshake:477(1282ns) {SwitchToState:179(655ns)}}}}}
2025-11-27T18:20:12.562627Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471696170660327:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42757042ns) {Configure:104(42751353ns) {ProcessConfigured:135(558495ns) {StartInitialHandshake:157(556575ns) {DropHandshakes:470(824ns) {DropIncomingHandshake:442(317ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(323ns) PrepareNewSessionHandshake:477(5409ns) {SwitchToState:179(1406ns)}}}}}
2025-11-27T18:20:12.563531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471697901369472:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20758170ns) {Configure:104(20749230ns) {ProcessConfigured:135(487906ns) {StartInitialHandshake:157(486029ns) {DropHandshakes:470(1196ns) {DropIncomingHandshake:442(320ns) DropOutgoingHandshake:457(227ns)} GenerateSessionVirtualId:988(502ns) PrepareNewSessionHandshake:477(1940ns) {SwitchToState:179(997ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.570666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471697901369471:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(52582190ns) {Configure:104(52575038ns) {ProcessConfigured:135(733363ns) {StartInitialHandshake:157(730527ns) {DropHandshakes:470(1593ns) {DropIncomingHandshake:442(506ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(587ns) PrepareNewSessionHandshake:477(206203ns) {SwitchToState:179(2192ns)}}}}}
2025-11-27T18:20:12.572373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471694963918731:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16403947ns) {Configure:104(16399686ns) {ProcessConfigured:135(660422ns) {StartInitialHandshake:157(658313ns) {DropHandshakes:470(1120ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:477(1985ns) {SwitchToState:179(926ns)}}}}}
2025-11-27T18:20:12.572305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471696048169725:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16946462ns) {Configure:104(16941747ns) {ProcessConfigured:135(524092ns) {StartInitialHandshake:157(522022ns) {DropHandshakes:470(704ns) {DropIncomingHandshake:442(271ns) DropOutgoingHandshake:457(101ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:477(1574ns) {SwitchToState:179(839ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.572855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471696170660408:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43585
2025-11-27T18:20:12.573028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471696048169789:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16454795ns) {Configure:104(16450048ns) {ProcessConfigured:135(1272131ns) {StartInitialHandshake:157(1270476ns) {DropHandshakes:470(1053ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(961ns) PrepareNewSessionHandshake:477(1416ns) {SwitchToState:179(419ns)}}}}}
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:12.574923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471697491974153:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57904
2025-11-27T18:20:12.573756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471696048169726:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17023370ns) {Configure:104(17019135ns) {ProcessConfigured:135(805230ns) {StartInitialHandshake:157(802958ns) {DropHandshakes:470(1254ns) {DropIncomingHandshake:442(361ns) DropOutgoingHandshake:457(252ns)} GenerateSessionVirtualId:988(738ns) PrepareNewSessionHandshake:477(2099ns) {SwitchToState:179(1029ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.577475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471696170660407:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43585
2025-11-27T18:20:12.578211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471697901369545:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31332
2025-11-27T18:20:12.577979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471697491974153:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.577448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471696048169727:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18609596ns) {Configure:104(18605131ns) {ProcessConfigured:135(2455969ns) {StartInitialHandshake:157(2453558ns) {DropHandshakes:470(1319ns) {DropIncomingHandshake:442(504ns) DropOutgoingHandshake:457(169ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:477(1764ns) {SwitchToState:179(202ns)}}}}}
2025-11-27T18:20:12.579847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471697901369545:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.579076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471696170660407:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.579809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471697491974076:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.579027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471698076303671:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65083
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.581004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471698076303664:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.579906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471696170660327:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.581042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471697901369471:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.581046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471698076303664:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.580487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471696170660399:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.583919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471696048169800:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13177
2025-11-27T18:20:12.582579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471698076303671:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.580515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471696170660399:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.585467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471694963918807:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:784
2025-11-27T18:20:12.590132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471696048169800:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.590311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471696048169727:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.581034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471696170660409:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43585
2025-11-27T18:20:12.585470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471694963918810:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:784
2025-11-27T18:20:12.583937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471698076303663:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.584810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471696170660409:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.583982Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471698076303663:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.589533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471694963918808:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:784
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.591356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471697491974156:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57904
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.589703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471694963918809:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:784
2025-11-27T18:20:12.585430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471696170660408:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.583998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471698076303672:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65083
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.592746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471697901369551:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31332
2025-11-27T18:20:12.589800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471696170660399:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:12.590608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471694963918808:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.585385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471698076303664:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.590890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471694963918810:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.596333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471697491974155:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57904
2025-11-27T18:20:12.591538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471698076303672:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.597420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471697901369551:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.592254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471696170660414:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43585
2025-11-27T18:20:12.598194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471697491974076:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16810) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471697491974153:2116] held: no
2025-11-27T18:20:12.591766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471698076303663:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:12.592232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471694963918809:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.594136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471696170660395:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.600198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471697491974155:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.600428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471697901369471:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15358) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471697901369545:2114] held: no
2025-11-27T18:20:12.592744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471694963918814:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:784
2025-11-27T18:20:12.592766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471698076303591:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.594153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471696170660392:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.600524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471697901369552:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31332
2025-11-27T18:20:12.594238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471698076303676:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65083
2025-11-27T18:20:12.592755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471694963918807:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.600985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471697491974075:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.594166Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471696170660395:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.594502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471698076303673:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65083
2025-11-27T18:20:12.594196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471694963918730:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.594212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:12.595419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471698076303656:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.594935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471696170660392:2109] [node 5] ICP22 created new session: [3:7577471696170660416:2109]
2025-11-27T18:20:12.605415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471696048169805:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13177
2025-11-27T18:20:12.595423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:12.595383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471696170660412:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43585
2025-11-27T18:20:12.595493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471698076303591:2049] [node 3] ICP22 created new session: [1:7577471698076303679:2049]
2025-11-27T18:20:12.595402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471696170660416:2109] [node 5] ICS09 handshake done sender: [3:7577471696170660399:2112] self: [3:7577471696170660397:0] peer: [5:7577471697491974154:0] socket: 366 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.596333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471698076303662:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.598184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471694963918814:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.606488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471697491974153:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:12.596346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:12.595445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471696170660416:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:12.602070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471694963918802:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.606814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471697491974160:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57904
2025-11-27T18:20:12.606811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471697901369555:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31332
2025-11-27T18:20:12.597393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:12.608469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471696048169807:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13177
2025-11-27T18:20:12.596366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471698076303662:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.607685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471697901369545:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:12.602091Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471694963918802:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.607018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471697491974076:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.597894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471696170660407:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:12.604494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471694963918802:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:12.596400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471698076303656:2107] [node 4] ICP22 created new session: [1:7577471698076303680:2107]
2025-11-27T18:20:12.610344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471696048169800:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:12.607176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471697491974155:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:12.608784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471697901369555:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.599051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471696170660327:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.606484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471694963918795:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.611762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471696048169807:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.597425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471698076303679:2049] [node 3] ICS09 handshake done sender: [1:7577471698076303664:2114] self: [1:7577471698076303658:0] peer: [3:7577471696170660410:0] socket: 362 qp: -1
2025-11-27T18:20:12.607612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471697491974159:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57904
2025-11-27T18:20:12.609684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471697901369473:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.606787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471694963918804:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.599152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471696170660395:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:12.598622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471698076303679:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:12.611946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471696048169806:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13177
2025-11-27T18:20:12.608557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471697491974075:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.600155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:12.610298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471697901369471:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.612730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471696048169727:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.606813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471694963918804:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.598672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:12.608756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:12.610689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:12.606901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471694963918821:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:784
2025-11-27T18:20:12.600491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471696170660327:2048] [node 1] ICP22 created new session: [3:7577471696170660420:2048]
2025-11-27T18:20:12.598968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471698076303680:2107] [node 4] ICS09 handshake done sender: [1:7577471698076303663:2113] self: [1:7577471698076303660:0] peer: [4:7577471697901369550:0] socket: 364 qp: -1
2025-11-27T18:20:12.613786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471696048169789:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.610739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471697901369471:2048] [node 1] ICP22 created new session: [4:7577471697901369557:2048]
No RDMA context found for GID: 2025-11-27T18:20:12.601023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471696170660420:2048] [node 1] ICS09 handshake done sender: [3:7577471696170660407:2116] self: [3:7577471696170660410:0] peer: [1:7577471698076303658:0] socket: 369 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:12.609662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471697491974159:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.607625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:12.615620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471696048169806:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.600175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471698076303680:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:12.610051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471697491974076:2050] [node 3] ICP22 created new session: [5:7577471697491974161:2050]
2025-11-27T18:20:12.611758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471697901369557:2048] [node 1] ICS09 handshake done sender: [4:7577471697901369545:2114] self: [4:7577471697901369550:0] peer: [1:7577471698076303660:0] socket: 372 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.607971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471694963918795:2109] [node 5] ICP22 created new session: [2:7577471694963918823:2109]
2025-11-27T18:20:12.600569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:12.601328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471696170660420:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:12.615782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471696048169808:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13177
2025-11-27T18:20:12.610425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471697491974074:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.612149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471697901369557:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:12.609700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471694963918730:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15358) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471694963918809:2118] held: no
2025-11-27T18:20:12.601238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471698076303676:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.616001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:12.601659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471696170660412:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.611439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:12.612196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.610270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471694963918823:2109] [node 5] ICS09 handshake done sender: [2:7577471694963918802:2111] self: [2:7577471694963918797:0] peer: [5:7577471697491974157:0] socket: 392 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.601669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471698076303673:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.616772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471696048169727:2050] [node 3] ICP22 created new session: [6:7577471696048169811:2050]
2025-11-27T18:20:12.625512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471697901369543:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.611490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471697491974161:2050] [node 3] ICS09 handshake done sender: [5:7577471697491974153:2116] self: [5:7577471697491974154:0] peer: [3:7577471696170660397:0] socket: 370 qp: -1
2025-11-27T18:20:12.602008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471696170660391:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.603132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471698076303662:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:12.616792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471696048169725:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.602168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:12.625547Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471697901369543:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.605050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471698076303590:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.617347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471696048169808:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.628986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471697901369559:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31332
2025-11-27T18:20:12.605143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:12.602203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:12.610310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471694963918823:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.606085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471698076303590:2048] [node 2] ICP22 created new session: [1:7577471698076303688:2048]
2025-11-27T18:20:12.629717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471697901369543:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:12.618052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471696048169811:2050] [node 3] ICS09 handshake done sender: [6:7577471696048169800:2115] self: [6:7577471696048169804:0] peer: [3:7577471696170660394:0] socket: 377 qp: -1
2025-11-27T18:20:12.611530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471697491974161:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:20:12.602241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471696170660391:2110] [node 6] ICP22 created new session: [3:7577471696170660422:2110]
::ffff:127.1.0.0
2025-11-27T18:20:12.629886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471697901369561:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31332
2025-11-27T18:20:12.606684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471698076303688:2048] [node 2] ICS09 handshake done sender: [1:7577471698076303662:2112] self: [1:7577471698076303659:0] peer: [2:7577471694963918817:0] socket: 368 qp: -1
2025-11-27T18:20:12.610347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:12.630009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471697901369473:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16810) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471697901369555:2118] held: no
2025-11-27T18:20:12.611575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:12.606721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471698076303688:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:12.618309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471696048169811:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:12.604198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471696170660422:2110] [node 6] ICS09 handshake done sender: [3:7577471696170660395:2111] self: [3:7577471696170660394:0] peer: [6:7577471696048169804:0] socket: 361 qp: -1
2025-11-27T18:20:12.611975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471697491974075:2049] [node 2] ICP22 created new session: [5:7577471697491974163:2049]
2025-11-27T18:20:12.611924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471694963918804:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:12.618536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471696048169788:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.604212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471696170660328:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.606897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:12.616803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471694963918731:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.612185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471697491974163:2049] [node 2] ICS09 handshake done sender: [5:7577471697491974155:2117] self: [5:7577471697491974157:0] peer: [2:7577471694963918797:0] socket: 394 qp: -1
2025-11-27T18:20:12.631845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471697901369561:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.618638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:12.604600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471696170660422:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:12.614614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471698076303665:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.617013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:12.631893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471697901369534:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.618907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471696048169809:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13177
2025-11-27T18:20:12.614648Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471698076303665:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.612728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471697491974163:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:12.618064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471694963918731:2049] [node 3] ICP22 created new session: [2:7577471694963918827:2049]
2025-11-27T18:20:12.604643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:12.633598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471697901369560:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31332
2025-11-27T18:20:12.618745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471698076303665:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:12.618822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471694963918827:2049] [node 3] ICS09 handshake done sender: [2:7577471694963918804:2113] self: [2:7577471694963918799:0] peer: [3:7577471696170660423:0] socket: 396 qp: -1
2025-11-27T18:20:12.611682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471696170660403:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.616006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:12.625972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471696048169809:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.620046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471698076303667:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.633786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:12.617313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471697491974149:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.620078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471698076303667:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.634990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471697901369534:2109] [node 6] ICP22 created new session: [4:7577471697901369563:2109]
2025-11-27T18:20:12.617341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471697491974149:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.621690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471694963918827:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:12.628359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471696048169726:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.611712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471696170660403:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.629008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471697491974149:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:12.634913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471696048169817:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13177
2025-11-27T18:20:12.638406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471697901369563:2109] [node 6] ICS09 handshake done sender: [4:7577471697901369543:2113] self: [4:7577471697901369541:0] peer: [6:7577471696048169814:0] socket: 418 qp: -1
2025-11-27T18:20:12.625635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471698076303655:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.629204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:12.623294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:12.629755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471698076303655:2108] [node 5] ICP22 created new session: [1:7577471698076303692:2108]
2025-11-27T18:20:12.631491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471697491974139:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.629937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471698076303692:2108] [node 5] ICS09 handshake done sender: [1:7577471698076303665:2115] self: [1:7577471698076303661:0] peer: [5:7577471697491974162:0] socket: 360 qp: -1
2025-11-27T18:20:12.631541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:12.630509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471698076303692:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:12.638526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471696048169818:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13177
2025-11-27T18:20:12.616716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471696170660403:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:12.627622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471694963918809:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:12.638462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471697901369559:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.630553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:12.621678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471696170660425:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43585
2025-11-27T18:20:12.633321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471697491974139:2110] [node 6] ICP22 created new session: [5:7577471697491974170:2110]
2025-11-27T18:20:12.639792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471696048169819:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13177
2025-11-27T18:20:12.629496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471694963918806:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.640044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471696048169806:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:12.633566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471697491974168:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57904
2025-11-27T18:20:12.638697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471697901369563:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:12.629518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471694963918806:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.633570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471698076303667:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:12.638505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471698076303657:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.625720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471696170660393:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.638565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:12.639082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471698076303657:2110] [node 6] ICP22 created new session: [1:7577471698076303694:2110]
2025-11-27T18:20:12.625775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:12.640600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471698076303694:2110] [node 6] ICS09 handshake done sender: [1:7577471698076303667:2116] self: [1:7577471698076303666:0] peer: [6:7577471696048169812:0] socket: 410 qp: -1
2025-11-27T18:20:12.625980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471696170660393:2107] [node 4] ICP22 created new session: [3:7577471696170660427:2107]
No RDMA context found for GID: 2025-11-27T18:20:12.640931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471698076303694:2110] [node 6] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:20:12.627696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471696170660427:2107] [node 4] ICS09 handshake done sender: [3:7577471696170660403:2115] self: [3:7577471696170660402:0] peer: [4:7577471697901369556:0] socket: 408 qp: -1
2025-11-27T18:20:12.640968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:12.628259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471696170660427:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:12.641317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471698076303656:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(34201428ns) {HandleHandshakeStatus:329(34055354ns) {DropIncomingHandshake:442(8896ns) DropOutgoingHandshake:457(255ns) SwitchToState:179(235ns) ProcessPendingSessionEvents:495(27454004ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27311391ns max# 2702681ns}}}
2025-11-27T18:20:12.628986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:12.644381Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471698076303591:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(37598805ns) {HandleHandshakeStatus:329(37596531ns) {DropIncomingHandshake:442(966ns) DropOutgoingHandshake:457(593ns) SwitchToState:179(385ns) ProcessPendingSessionEvents:495(28480103ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25897818ns max# 2780955ns}}}
2025-11-27T18:20:12.639811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471696170660392:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(30598439ns) {HandleHandshakeStatus:329(30595887ns) {DropIncomingHandshake:442(12553ns) DropOutgoingHandshake:457(233ns) SwitchToState:179(977ns) ProcessPendingSessionEvents:495(23194273ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22393659ns max# 1712082ns}}}
2025-11-27T18:20:12.649196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471698076303590:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(35578775ns) {HandleHandshakeStatus:329(35576612ns) {DropIncomingHandshake:442(10732ns) DropOutgoingHandshake:457(270ns) SwitchToState:179(443ns) ProcessPendingSessionEvents:495(30550618ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29589144ns max# 3169623ns}}}
2025-11-27T18:20:12.640643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471696170660328:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4099) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471696170660412:2119] held: no
2025-11-27T18:20:12.633770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471697491974167:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57904
2025-11-27T18:20:12.645660Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471696170660327:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32425023ns) {HandleHandshakeStatus:329(32422940ns) {DropIncomingHandshake:442(429ns) DropOutgoingHandshake:457(4043ns) SwitchToState:179(482ns) ProcessPendingSessionEvents:495(27185075ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27066106ns max# 2742824ns}}}
2025-11-27T18:20:12.646011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471696170660412:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:12.636817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471697491974170:2110] [node 6] ICS09 handshake done sender: [5:7577471697491974149:2115] self: [5:7577471697491974145:0] peer: [6:7577471696048169810:0] socket: 411 qp: -1
2025-11-27T18:20:12.648488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471696170660328:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.636918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471697491974074:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15358) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471697491974159:2119] held: no
2025-11-27T18:20:12.638750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:12.649406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:12.630447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471694963918730:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.640795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471696048169807:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:12.640329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471697901369555:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:12.637988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471697491974170:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:12.649597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471696170660328:2049] [node 2] ICP22 created new session: [3:7577471696170660429:2049]
2025-11-27T18:20:12.644332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471696048169725:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.630718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:12.640728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471697901369472:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.639144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:12.650331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471696170660429:2049] [node 2] ICS09 handshake done sender: [3:7577471696170660412:2119] self: [3:7577471696170660423:0] peer: [2:7577471694963918799:0] socket: 397 qp: -1
2025-11-27T18:20:12.644694Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471696048169788:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25722) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471696048169808:2119] held: no
2025-11-27T18:20:12.641641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471697901369473:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.645550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:12.641874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:12.645771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471696048169725:2048] [node 1] ICP22 created new session: [6:7577471696048169821:2048]
2025-11-27T18:20:12.646972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471696048169789:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.643867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471697901369473:2050] [node 3] ICP22 created new session: [4:7577471697901369568:2050]
2025-11-27T18:20:12.647047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:12.647049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471696048169821:2048] [node 1] ICS09 handshake done sender: [6:7577471696048169806:2117] self: [6:7577471696048169812:0] peer: [1:7577471698076303666:0] socket: 412 qp: -1
2025-11-27T18:20:12.647267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471696048169821:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:12.644468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471697901369568:2050] [node 3] ICS09 handshake done sender: [4:7577471697901369555:2118] self: [4:7577471697901369556:0] peer: [3:7577471696170660402:0] socket: 384 qp: -1
2025-11-27T18:20:12.647300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:12.644503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471697901369568:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:12.648009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471696048169789:2107] [node 5] ICP22 created new session: [6:7577471696048169822:2107]
2025-11-27T18:20:12.645804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:12.648560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471696048169822:2107] [node 5] ICS09 handshake done sender: [6:7577471696048169807:2118] self: [6:7577471696048169810:0] peer: [5:7577471697491974145:0] socket: 415 qp: -1
2025-11-27T18:20:12.651604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471697901369570:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31332
2025-11-27T18:20:12.648710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471696048169820:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13177
2025-11-27T18:20:12.652051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471697901369546:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.648789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471696048169822:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:12.652082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471697901369546:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.648824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:12.654916Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471697901369472:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4099) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471697901369561:2121] held: no
2025-11-27T18:20:12.650399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471696048169808:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:12.655535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471697901369561:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:12.654809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471696048169726:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4099) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471696048169809:2120] held: no
2025-11-27T18:20:12.630768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471694963918730:2048] [node 1] ICP22 created new session: [2:7577471694963918829:2048]
2025-11-27T18:20:12.640058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471697491974167:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.650979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471696170660429:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:12.632126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471694963918829:2048] [node 1] ICS09 handshake done sender: [2:7577471694963918809:2118] self: [2:7577471694963918817:0] peer: [1:7577471698076303659:0] socket: 380 qp: -1
2025-11-27T18:20:12.651148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471696170660391:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(38275538ns) {HandleHandshakeStatus:329(38134245ns) {DropIncomingHandshake:442(510ns) DropOutgoingHandshake:457(258ns) SwitchToState:179(350ns) ProcessPendingSessionEvents:495(33951016ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 31398227ns max# 2273273ns}}}
2025-11-27T18:20:12.647001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471697491974159:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:12.632851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471694963918829:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:12.647543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471697491974074:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.658075Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471696048169727:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(36259578ns) {HandleHandshakeStatus:329(36257185ns) {DropIncomingHandshake:442(1480ns) DropOutgoingHandshake:457(5379ns) SwitchToState:179(300ns) ProcessPendingSessionEvents:495(22668451ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22540777ns max# 2966404ns}}}
2025-11-27T18:20:12.633322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:12.648274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471697491974172:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57904
2025-11-27T18:20:12.636877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471694963918806:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:12.651779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:12.648665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:12.638550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471694963918794:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.648723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471697491974074:2048] [node 1] ICP22 created new session: [5:7577471697491974173:2048]
2025-11-27T18:20:12.639634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:12.640582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471694963918794:2110] [node 6] ICP22 created new session: [2:7577471694963918832:2110]
2025-11-27T18:20:12.648783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471697491974173:2048] [node 1] ICS09 handshake done sender: [5:7577471697491974159:2119] self: [5:7577471697491974162:0] peer: [1:7577471698076303661:0] socket: 409 qp: -1
2025-11-27T18:20:12.658875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471696048169788:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.649415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471697491974173:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:12.641782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471694963918832:2110] [node 6] ICS09 handshake done sender: [2:7577471694963918806:2115] self: [2:7577471694963918800:0] peer: [6:7577471696048169816:0] socket: 378 qp: -1
2025-11-27T18:20:12.641816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471694963918832:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:12.649572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471697491974172:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.649600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:12.644014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:12.650268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471697491974138:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:12.644320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471694963918805:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:12.644351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471694963918805:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:12.658607Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471697491974139:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(20243468ns) {HandleHandshakeStatus:329(20241443ns) {DropIncomingHandshake:442(236ns) DropOutgoingHandshake:457(190ns) SwitchToState:179(431ns) ProcessPendingSessionEvents:495(9928426ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9313056ns max# 2133383ns}}}
2025-11-27T18:20:12.646711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471694963918805:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:12.659790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471697491974075:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(40219783ns) {HandleHandshakeStatus:329(40213417ns) {DropIncomingHandshake:442(790ns) DropOutgoingHandshake:457(5688ns) SwitchToState:179(458ns) ProcessPendingSessionEvents:495(30694764ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29388222ns max# 3675562ns}}}
2025-11-27T18:20:12.648470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471694963918796:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.649189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:12.649454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471694963918796:2107] [node 4] ICP22 created new session: [2:7577471694963918835:2107]
2025-11-27T18:20:12.649974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471694963918835:2107] [node 4] ICS09 handshake done sender: [2:7577471694963918805:2114] self: [2:7577471694963918801:0] peer: [4:7577471697901369566:0] socket: 427 qp: -1
2025-11-27T18:20:12.659983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471696048169809:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:12.660003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471698076303657:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(17236364ns) {HandleHandshakeStatus:329(17233800ns) {DropIncomingHandshake:442(17078ns) DropOutgoingHandshake:457(378ns) SwitchToState:179(552ns) ProcessPendingSessionEvents:495(11696181ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11607908ns max# 1404158ns}}}
2025-11-27T18:20:12.660019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:12.650009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471694963918835:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:12.659970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471697901369472:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.660239Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471696170660393:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(29786458ns) {HandleHandshakeStatus:329(29773585ns) {DropIncomingHandshake:442(13701ns) DropOutgoingHandshake:457(390ns) SwitchToState:179(463ns) ProcessPendingSessionEvents:495(25025142ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21905246ns max# 1885348ns}}}
2025-11-27T18:20:12.660409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:12.660455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471696048169788:2108] [node 4] ICP22 created new session: [6:7577471696048169825:2108]
2025-11-27T18:20:12.650283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:12.660452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471697901369472:2049] [node 2] ICP22 created new session: [4:7577471697901369572:2049]
2025-11-27T18:20:12.660827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471697901369546:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:12.661024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471697901369572:2049] [node 2] ICS09 handshake done sender: [4:7577471697901369561:2121] self: [4:7577471697901369566:0] peer: [2:7577471694963918801:0] socket: 428 qp: -1
2025-11-27T18:20:12.651603Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471694963918795:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(38315676ns) {HandleHandshakeStatus:329(38312994ns) {DropIncomingHandshake:442(160950ns) DropOutgoingHandshake:457(447ns) SwitchToState:179(366ns) ProcessPendingSessionEvents:495(30691632ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 30540611ns max# 2275299ns}}}
2025-11-27T18:20:12.661423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471696048169825:2108] [node 4] ICS09 handshake done sender: [6:7577471696048169808:2119] self: [6:7577471696048169814:0] peer: [4:7577471697901369541:0] socket: 419 qp: -1
2025-11-27T18:20:12.661230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471697901369572:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:12.661636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471696048169825:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:12.656135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471694963918731:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(33935164ns) {HandleHandshakeStatus:329(33932685ns) {DropIncomingHandshake:442(13081ns) DropOutgoingHandshake:457(672ns) SwitchToState:179(240ns) ProcessPendingSessionEvents:495(20245414ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19637601ns max# 1938736ns}}}
2025-11-27T18:20:12.661696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471697901369535:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:12.661673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:12.661762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471697901369534:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(19785597ns) {HandleHandshakeStatus:329(19783182ns) {DropIncomingHandshake:442(10945ns) DropOutgoingHandshake:457(177ns) SwitchToState:179(317ns) ProcessPendingSessionEvents:495(10494881ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10405408ns max# 2178687ns}}}
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:12.661952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:12.662601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471697491974176:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57904
2025-11-27T18:20:12.663048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471698076303655:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(29333246ns) {HandleHandshakeStatus:329(29330159ns) {DropIncomingHandshake:442(478724ns) DropOutgoingHandshake:457(712ns) SwitchToState:179(210ns) ProcessPendingSessionEvents:495(21685444ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21572015ns max# 1988031ns}}}
2025-11-27T18:20:12.662847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:12.662459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471696048169725:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:12.664198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471697491974138:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25722) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471697491974172:2123] held: no
2025-11-27T18:20:12.662906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471697901369535:2108] [node 5] ICP22 created new session: [4:7577471697901369574:2108]
2025-11-27T18:20:12.662538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471696048169726:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.664568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471697491974172:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:12.662627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:12.662951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471697901369574:2108] [node 5] ICS09 handshake done sender: [4:7577471697901369546:2115] self: [4:7577471697901369544:0] peer: [5:7577471697491974175:0] socket: 363 qp: -1
2025-11-27T18:20:12.662678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471696048169726:2049] [node 2] ICP22 created new session: [6:7577471696048169827:2049]
2025-11-27T18:20:12.662720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471696048169827:2049] [node 2] ICS09 handshake done sender: [6:7577471696048169809:2120] self: [6:7577471696048169816:0] peer: [2:7577471694963918800:0] socket: 390 qp: -1
2025-11-27T18:20:12.664739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577471697491974139:2110] [node 6] ICP18 handshake (actor [5:7577471697491974167:2121]) is too old
2025-11-27T18:20:12.662973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471697901369574:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:12.662748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471696048169827:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:12.663007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:12.662841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:12.666124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471697491974138:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:12.667967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:12.668026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471697491974138:2108] [node 4] ICP22 created new session: [5:7577471697491974177:2108]
2025-11-27T18:20:12.668229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471697491974177:2108] [node 4] ICS09 handshake done sender: [5:7577471697491974172:2123] self: [5:7577471697491974175:0] peer: [4:7577471697901369544:0] socket: 365 qp: -1
2025-11-27T18:20:12.668311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471697491974177:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:12.668350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:12.706008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:12.706258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471694963918829:2048] [node 1] ICS01 socket: 380 reason# EndOfStream
2025-11-27T18:20:12.706310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471694963918730:2048] [node 1] ICP30 unregister session Session# [2:7577471694963918829:2048] VirtualId# [2:7577471694963918817:0]
2025-11-27T18:20:12.706345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471694963918829:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.706429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:12.706482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:12.707411Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:12.707627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471696170660420:2048] [node 1] ICS07 socket disconnect 369 reason# EndOfStream
2025-11-27T18:20:12.707669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471696170660420:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.707763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:12.707827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471696170660420:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:12.709792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:12.709932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471696170660327:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15358) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:12.710017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471697901369557:2048] [node 1] ICS01 socket: 372 reason# EndOfStream
2025-11-27T18:20:12.709974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471696170660420:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:12.710057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471697901369471:2048] [node 1] ICP30 unregister session Session# [4:7577471697901369557:2048] VirtualId# [4:7577471697901369550:0]
2025-11-27T18:20:12.710092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471697901369557:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.710242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:12.710296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:12.710789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471696170660327:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15358) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:12.710844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471696170660420:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:12.710898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471696170660327:2048] [node 1] ICP30 unregister session Session# [3:7577471696170660420:2048] VirtualId# [3:7577471696170660410:0]
2025-11-27T18:20:12.711001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:12.711039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471696170660327:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15358) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.711059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15358) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.711170Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:12.711415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471697491974173:2048] [node 1] ICS01 socket: 409 reason# EndOfStream
2025-11-27T18:20:12.711455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471697491974074:2048] [node 1] ICP30 unregister session Session# [5:7577471697491974173:2048] VirtualId# [5:7577471697491974162:0]
2025-11-27T18:20:12.711494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471697491974173:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.711557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:12.711604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:12.712097Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:12.712357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471696048169821:2048] [node 1] ICS01 socket: 412 reason# EndOfStream
2025-11-27T18:20:12.712397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471696048169725:2048] [node 1] ICP30 unregister session Session# [6:7577471696048169821:2048] VirtualId# [6:7577471696048169812:0]
2025-11-27T18:20:12.712423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471696048169821:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.712485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:12.712529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:12.720457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471696170660327:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:12.794665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:12.794891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471696170660429:2049] [node 2] ICS01 socket: 397 reason# EndOfStream
2025-11-27T18:20:12.794932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471696170660328:2049] [node 2] ICP30 unregister session Session# [3:7577471696170660429:2049] VirtualId# [3:7577471696170660423:0]
2025-11-27T18:20:12.794962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471696170660429:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.795048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:12.795098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:12.798436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:12.798654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471697901369572:2049] [node 2] ICS01 socket: 428 reason# EndOfStream
2025-11-27T18:20:12.798696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471697901369472:2049] [node 2] ICP30 unregister session Session# [4:7577471697901369572:2049] VirtualId# [4:7577471697901369566:0]
2025-11-27T18:20:12.798728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471697901369572:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.798795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:12.798845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:12.799554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:12.799742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471697491974163:2049] [node 2] ICS07 socket disconnect 394 reason# EndOfStream
2025-11-27T18:20:12.799771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471697491974163:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.799836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:12.799884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471697491974163:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:12.800675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:12.800769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471697491974075:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4099) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:12.800838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471697491974163:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:12.800877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471697491974075:2049] [node 2] ICP30 unregister session Session# [5:7577471697491974163:2049] VirtualId# [5:7577471697491974157:0]
2025-11-27T18:20:12.800947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471696048169827:2049] [node 2] ICS01 socket: 390 reason# EndOfStream
2025-11-27T18:20:12.800979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:12.800987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471696048169726:2049] [node 2] ICP30 unregister session Session# [6:7577471696048169827:2049] VirtualId# [6:7577471696048169816:0]
2025-11-27T18:20:12.801017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471696048169827:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.801013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471697491974075:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4099) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.801077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:12.801035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4099) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.801120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:12.811425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471697491974075:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:12.864512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:12.864779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471697901369568:2050] [node 3] ICS01 socket: 384 reason# EndOfStream
2025-11-27T18:20:12.864837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471697901369473:2050] [node 3] ICP30 unregister session Session# [4:7577471697901369568:2050] VirtualId# [4:7577471697901369556:0]
2025-11-27T18:20:12.864874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471697901369568:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.864968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:12.865059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:12.865705Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:12.865934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471697491974161:2050] [node 3] ICS01 socket: 370 reason# EndOfStream
2025-11-27T18:20:12.865971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471697491974076:2050] [node 3] ICP30 unregister session Session# [5:7577471697491974161:2050] VirtualId# [5:7577471697491974154:0]
2025-11-27T18:20:12.865998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471697491974161:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.866084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:12.866155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:12.866611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:12.866833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471696048169811:2050] [node 3] ICS07 socket disconnect 377 reason# EndOfStream
2025-11-27T18:20:12.866867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471696048169811:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.866933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:12.866982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471696048169811:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:12.867780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471696048169727:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16810) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:12.867826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471696048169811:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:12.867918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471696048169727:2050] [node 3] ICP30 unregister session Session# [6:7577471696048169811:2050] VirtualId# [6:7577471696048169804:0]
2025-11-27T18:20:12.868028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:12.868070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471696048169727:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16810) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.868085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16810) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:12.878153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471696048169727:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:12.923655Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:12.923873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471697491974177:2108] [node 4] ICS01 socket: 365 reason# EndOfStream
2025-11-27T18:20:12.923906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471697491974138:2108] [node 4] ICP30 unregister session Session# [5:7577471697491974177:2108] VirtualId# [5:7577471697491974175:0]
2025-11-27T18:20:12.923938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471697491974177:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.923997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:12.924031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:12.924701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:12.924881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471696048169825:2108] [node 4] ICS01 socket: 419 reason# EndOfStream
2025-11-27T18:20:12.924919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471696048169788:2108] [node 4] ICP30 unregister session Session# [6:7577471696048169825:2108] VirtualId# [6:7577471696048169814:0]
2025-11-27T18:20:12.924937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471696048169825:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:12.925001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:12.925029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:13.015341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:13.015558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471696048169822:2107] [node 5] ICS01 socket: 415 reason# EndOfStream
2025-11-27T18:20:13.015594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471696048169789:2107] [node 5] ICP30 unregister session Session# [6:7577471696048169822:2107] VirtualId# [6:7577471696048169810:0]
2025-11-27T18:20:13.015634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471696048169822:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.015705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:13.015755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:13.127566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471701572314219:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:13.127659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471701572314220:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:13.149150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471701143229885:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:13.149212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471701143229886:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:13.169993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471702383660288:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:13.170068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471702383660289:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:13.191274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471703242282734:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:13.191332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471703242282735:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:13.191372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471703242282736:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:13.212781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471699691640866:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:13.212866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471699691640867:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:13.212916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471699691640868:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:13.234496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471702640894876:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:13.234542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471702640894877:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:13.234573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471702640894878:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:13.338382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471701572314285:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:13.348446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471702383660352:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:13.348673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471702640894939:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:13.349233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471701572314286:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:13.349281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471701143229949:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:13.349377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471699691640930:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:13.349752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471702383660353:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:13.349749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471701143229950:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:13.350131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471701572314287:2111] [node 6] ICP01 ready to work
2025-11-27T18:20:13.350298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471699691640931:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:13.350378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471701143229951:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:13.350485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471702383660354:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:13.350551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471702640894941:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:13.349407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471703242282798:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:13.350276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471703242282799:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.361681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471701143229962:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25693
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.361830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471701143229963:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25693
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.362989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471701572314298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11286
2025-11-27T18:20:13.363596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471701572314299:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11286
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.364590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471701143229967:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25693
2025-11-27T18:20:13.364591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471701572314300:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11286
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.365145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471701572314304:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11286
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.366796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471702383660365:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27398
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.366877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471702383660366:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27398
2025-11-27T18:20:13.366937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471702383660367:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27398
2025-11-27T18:20:13.367332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471702383660368:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27398
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.370369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471699691640945:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58432
No RDMA context found for GID: 2025-11-27T18:20:13.370959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471699691640945:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.371057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703242282813:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41258
2025-11-27T18:20:13.371165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703242282814:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41258
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.371361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703242282815:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41258
2025-11-27T18:20:13.371822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471701143229962:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.372024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471701143229968:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25693
2025-11-27T18:20:13.372132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471701572314298:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.372176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471701143229967:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.372235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471701572314299:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.372327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471701143229963:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.372336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471701572314300:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.372136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471703242282813:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.372399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471701143229969:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25693
2025-11-27T18:20:13.372243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471703242282735:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:13.372578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471701143229957:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.372605Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471701143229957:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.372628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471703242282814:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.372718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471701572314304:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.372733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471703242282734:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:13.372800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471701572314305:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11286
2025-11-27T18:20:13.373400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471701572314293:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.373426Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471701572314293:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.373504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471701143229968:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.373922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471701143229885:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.374114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471701143229957:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:13.374262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703242282818:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41258
2025-11-27T18:20:13.374480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471701572314289:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.374529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471701572314289:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.374946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471702383660365:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.375103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471702383660367:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.375085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471701143229949:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.375175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:13.375214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471702383660366:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.375227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471701143229949:2107] [node 4] ICP22 created new session: [2:7577471701143229975:2107]
2025-11-27T18:20:13.375292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471701143229975:2107] [node 4] ICS09 handshake done sender: [2:7577471701143229957:2113] self: [2:7577471701143229956:0] peer: [4:7577471703242282816:0] socket: 380 qp: -1
2025-11-27T18:20:13.375304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471701572314293:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:13.375334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471701143229975:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:13.375397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:13.375458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471703242282813:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:13.375706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471702383660368:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.375784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471703242282735:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.375844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471701572314305:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.375875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:13.375898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471701572314285:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.375922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471703242282735:2049] [node 2] ICP22 created new session: [4:7577471703242282820:2049]
2025-11-27T18:20:13.375965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:13.375998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471703242282820:2049] [node 2] ICS09 handshake done sender: [4:7577471703242282813:2116] self: [4:7577471703242282816:0] peer: [2:7577471701143229956:0] socket: 397 qp: -1
2025-11-27T18:20:13.376036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471703242282820:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:13.376028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471701572314285:2107] [node 4] ICP22 created new session: [1:7577471701572314312:2107]
2025-11-27T18:20:13.376086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471701572314312:2107] [node 4] ICS09 handshake done sender: [1:7577471701572314293:2115] self: [1:7577471701572314292:0] peer: [4:7577471703242282817:0] socket: 395 qp: -1
2025-11-27T18:20:13.376084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:13.376083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471702383660289:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:13.376118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471701572314312:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:13.376179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:13.376335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471702383660288:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:13.377050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471701143229969:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.377159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471701143229955:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.377176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471701143229955:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.377725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471701572314291:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.377746Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471701572314291:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.378002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703242282819:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41258
2025-11-27T18:20:13.378507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471701572314289:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:13.379242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471703242282815:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.379301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471701572314219:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.379369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471701143229977:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25693
2025-11-27T18:20:13.379426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:13.379481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471701572314219:2048] [node 2] ICP22 created new session: [1:7577471701572314316:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.379423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471702383660376:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27398
2025-11-27T18:20:13.379543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471701572314316:2048] [node 2] ICS09 handshake done sender: [1:7577471701572314289:2113] self: [1:7577471701572314288:0] peer: [2:7577471701143229974:0] socket: 387 qp: -1
2025-11-27T18:20:13.379559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471699691640947:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58432
2025-11-27T18:20:13.379598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471701572314316:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:13.379633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:13.379905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471701143229955:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:13.380621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471701143229886:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.380705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:13.380776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471701143229886:2049] [node 3] ICP22 created new session: [2:7577471701143229981:2049]
2025-11-27T18:20:13.380821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471701143229981:2049] [node 3] ICS09 handshake done sender: [2:7577471701143229955:2112] self: [2:7577471701143229954:0] peer: [3:7577471702383660373:0] socket: 379 qp: -1
2025-11-27T18:20:13.380781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471702383660367:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:13.380783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471703242282734:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471703242282814:2117] held: no
2025-11-27T18:20:13.380868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471701143229981:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:13.380931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:13.381107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471702383660289:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.381157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:13.381192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471702383660289:2049] [node 2] ICP22 created new session: [3:7577471702383660377:2049]
2025-11-27T18:20:13.381231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471702383660377:2049] [node 2] ICS09 handshake done sender: [3:7577471702383660367:2118] self: [3:7577471702383660373:0] peer: [2:7577471701143229954:0] socket: 392 qp: -1
2025-11-27T18:20:13.381275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471702383660377:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:13.381307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:13.381507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471703242282814:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.382144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471699691640948:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58432
2025-11-27T18:20:13.382122Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471701143229968:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:13.382248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471703242282734:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.382336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:13.382386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471703242282734:2048] [node 1] ICP22 created new session: [4:7577471703242282822:2048]
2025-11-27T18:20:13.382438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471703242282822:2048] [node 1] ICS09 handshake done sender: [4:7577471703242282814:2117] self: [4:7577471703242282817:0] peer: [1:7577471701572314292:0] socket: 398 qp: -1
2025-11-27T18:20:13.382475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471703242282822:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:13.382502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:13.382639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471701572314291:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:13.383132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471701572314220:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.383215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:13.383263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471701572314220:2049] [node 3] ICP22 created new session: [1:7577471701572314320:2049]
2025-11-27T18:20:13.383334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471701572314320:2049] [node 3] ICS09 handshake done sender: [1:7577471701572314291:2114] self: [1:7577471701572314290:0] peer: [3:7577471702383660375:0] socket: 391 qp: -1
2025-11-27T18:20:13.383393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471701572314320:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:13.383458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:13.383576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471703242282736:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.384771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471699691640949:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58432
2025-11-27T18:20:13.384829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471702640894955:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:20:13.385137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471702383660360:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.385174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471702383660360:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.386237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471701143229885:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.386310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:13.386289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471702640894956:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:20:13.386357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471701143229885:2048] [node 1] ICP22 created new session: [2:7577471701143229983:2048]
2025-11-27T18:20:13.386405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471701143229983:2048] [node 1] ICS09 handshake done sender: [2:7577471701143229968:2119] self: [2:7577471701143229974:0] peer: [1:7577471701572314288:0] socket: 399 qp: -1
2025-11-27T18:20:13.386439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471701143229983:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:13.386501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:13.387389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471701143229885:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.387527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471702383660379:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27398
2025-11-27T18:20:13.388284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471702383660288:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471702383660368:2119] held: no
2025-11-27T18:20:13.388375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471699691640949:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.388507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471699691640930:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:13.388499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471702640894955:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.388629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471702640894941:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:13.388742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471703242282807:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.388774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471703242282807:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.389339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471703242282809:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.389362Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471703242282809:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.389480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471699691640948:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.389536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471702383660368:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:13.389589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471699691640951:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58432
2025-11-27T18:20:13.389697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471699691640866:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:13.390195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471702383660288:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.390241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:13.390280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471702383660288:2048] [node 1] ICP22 created new session: [3:7577471702383660382:2048]
2025-11-27T18:20:13.390318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471702383660382:2048] [node 1] ICS09 handshake done sender: [3:7577471702383660368:2119] self: [3:7577471702383660375:0] peer: [1:7577471701572314290:0] socket: 394 qp: -1
2025-11-27T18:20:13.390347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471702383660382:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.390383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.390478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703242282827:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41258
2025-11-27T18:20:13.390985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471699691640953:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58432
2025-11-27T18:20:13.390603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471703242282807:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.391268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471702640894958:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:20:13.391261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471701572314295:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.391309Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471701572314295:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.391473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471703242282809:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:13.391888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471703242282798:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.392237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:13.392329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471703242282798:2108] [node 5] ICP22 created new session: [4:7577471703242282830:2108]
2025-11-27T18:20:13.392383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471703242282830:2108] [node 5] ICS09 handshake done sender: [4:7577471703242282807:2114] self: [4:7577471703242282806:0] peer: [5:7577471699691640950:0] socket: 378 qp: -1
2025-11-27T18:20:13.392474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471703242282830:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:13.392558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471699691640947:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.392595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:13.392727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471699691640949:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:13.393045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471702383660360:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:13.393058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471701572314295:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:13.393062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471702640894956:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.393139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471702640894955:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:13.393140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471699691640930:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.393219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:13.393494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471699691640930:2108] [node 4] ICP22 created new session: [5:7577471699691640955:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.393892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471701572314286:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.393896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471699691640955:2108] [node 4] ICS09 handshake done sender: [5:7577471699691640949:2119] self: [5:7577471699691640950:0] peer: [4:7577471703242282806:0] socket: 406 qp: -1
2025-11-27T18:20:13.394147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471699691640955:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:13.393947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:13.394182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:13.394200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471701572314286:2109] [node 5] ICP22 created new session: [1:7577471701572314323:2109]
2025-11-27T18:20:13.394269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471702640894959:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:20:13.394255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471701572314323:2109] [node 5] ICS09 handshake done sender: [1:7577471701572314295:2116] self: [1:7577471701572314294:0] peer: [5:7577471699691640952:0] socket: 415 qp: -1
2025-11-27T18:20:13.394294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471701572314323:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:13.394328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:13.394388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471702640894941:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.394473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:13.394515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471702640894941:2107] [node 4] ICP22 created new session: [6:7577471702640894960:2107]
2025-11-27T18:20:13.394556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471702640894960:2107] [node 4] ICS09 handshake done sender: [6:7577471702640894955:2116] self: [6:7577471702640894957:0] peer: [4:7577471703242282808:0] socket: 417 qp: -1
2025-11-27T18:20:13.394586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471702640894960:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:13.394634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:13.394820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471702383660352:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.394843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471703242282799:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.394876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:13.394924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471702383660352:2107] [node 4] ICP22 created new session: [3:7577471702383660385:2107]
2025-11-27T18:20:13.394937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:13.394966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471702383660385:2107] [node 4] ICS09 handshake done sender: [3:7577471702383660360:2113] self: [3:7577471702383660359:0] peer: [4:7577471703242282826:0] socket: 403 qp: -1
2025-11-27T18:20:13.394998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471702383660385:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:13.394985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471703242282799:2110] [node 6] ICP22 created new session: [4:7577471703242282832:2110]
2025-11-27T18:20:13.395023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:13.395029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471703242282832:2110] [node 6] ICS09 handshake done sender: [4:7577471703242282809:2115] self: [4:7577471703242282808:0] peer: [6:7577471702640894957:0] socket: 382 qp: -1
2025-11-27T18:20:13.395052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471703242282832:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:13.395079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:13.395967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471702640894959:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.396016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471699691640867:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:13.396112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471702640894876:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:13.397220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471701572314297:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.397253Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471701572314297:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.398767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471702640894877:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.399028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471701572314297:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:13.399141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471702640894962:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.399228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471701143229959:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.400072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471699691640954:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58432
2025-11-27T18:20:13.399262Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471701143229959:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.400318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471701572314287:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.400597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:13.400580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471701143229961:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.400645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471701572314287:2111] [node 6] ICP22 created new session: [1:7577471701572314326:2111]
2025-11-27T18:20:13.400697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471699691640866:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471699691640948:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:13.400617Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471701143229961:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.400693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471701572314326:2111] [node 6] ICS09 handshake done sender: [1:7577471701572314297:2117] self: [1:7577471701572314296:0] peer: [6:7577471702640894963:0] socket: 418 qp: -1
2025-11-27T18:20:13.401559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471701572314326:2111] [node 6] ICS10 traffic start
2025-11-27T18:20:13.401603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.401632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471702640894966:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:20:13.401833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471702383660387:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27398
2025-11-27T18:20:13.402444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471701143229959:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:13.402626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471699691640948:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:13.402805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471703242282736:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471703242282815:2118] held: no
2025-11-27T18:20:13.403214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471701143229961:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:13.403711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471702640894962:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.403933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471699691640951:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.404007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471699691640866:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.404028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471701143229950:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.404045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:13.404103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471699691640866:2048] [node 1] ICP22 created new session: [5:7577471699691640960:2048]
2025-11-27T18:20:13.404120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:13.404173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471701143229950:2108] [node 5] ICP22 created new session: [2:7577471701143229988:2108]
2025-11-27T18:20:13.404149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471699691640960:2048] [node 1] ICS09 handshake done sender: [5:7577471699691640948:2118] self: [5:7577471699691640952:0] peer: [1:7577471701572314294:0] socket: 416 qp: -1
2025-11-27T18:20:13.404204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471699691640960:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:13.404226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471701143229988:2108] [node 5] ICS09 handshake done sender: [2:7577471701143229959:2114] self: [2:7577471701143229958:0] peer: [5:7577471699691640957:0] socket: 371 qp: -1
2025-11-27T18:20:13.404272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471701143229988:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:13.404288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:13.404360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:13.404404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471703242282815:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:13.405303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471702640894956:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:13.405498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471703242282736:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.405573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:13.405617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471703242282736:2050] [node 3] ICP22 created new session: [4:7577471703242282834:2050]
2025-11-27T18:20:13.405675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471703242282834:2050] [node 3] ICS09 handshake done sender: [4:7577471703242282815:2118] self: [4:7577471703242282826:0] peer: [3:7577471702383660359:0] socket: 407 qp: -1
2025-11-27T18:20:13.405708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471703242282834:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:13.405731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:13.405734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471702640894876:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5676) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471702640894956:2117] held: no
2025-11-27T18:20:13.405809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471702383660387:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.405959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471701143229951:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.406064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:13.406109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471701143229951:2110] [node 6] ICP22 created new session: [2:7577471701143229990:2110]
2025-11-27T18:20:13.406159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471701143229990:2110] [node 6] ICS09 handshake done sender: [2:7577471701143229961:2115] self: [2:7577471701143229960:0] peer: [6:7577471702640894965:0] socket: 426 qp: -1
2025-11-27T18:20:13.406193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471701143229990:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:13.406220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:13.406243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471702640894876:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.406278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:13.406316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471702640894876:2048] [node 1] ICP22 created new session: [6:7577471702640894968:2048]
2025-11-27T18:20:13.406351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471702640894968:2048] [node 1] ICS09 handshake done sender: [6:7577471702640894956:2117] self: [6:7577471702640894963:0] peer: [1:7577471701572314296:0] socket: 419 qp: -1
2025-11-27T18:20:13.406376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471702640894968:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:13.406394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.406607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471699691640958:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58432
2025-11-27T18:20:13.407074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471699691640868:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:13.407243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471699691640947:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:13.407368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471702383660362:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.407415Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471702383660362:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.408533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471702383660362:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:13.408816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471699691640867:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.408862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:13.408906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471699691640867:2049] [node 2] ICP22 created new session: [5:7577471699691640963:2049]
2025-11-27T18:20:13.408985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471699691640963:2049] [node 2] ICS09 handshake done sender: [5:7577471699691640947:2117] self: [5:7577471699691640957:0] peer: [2:7577471701143229958:0] socket: 414 qp: -1
2025-11-27T18:20:13.409027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471699691640963:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:13.409069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:13.409057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471702640894878:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.409097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471702383660353:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.409180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471702640894967:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:20:13.409164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:13.409207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471702383660353:2109] [node 5] ICP22 created new session: [3:7577471702383660390:2109]
2025-11-27T18:20:13.409276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471702383660390:2109] [node 5] ICS09 handshake done sender: [3:7577471702383660362:2114] self: [3:7577471702383660361:0] peer: [5:7577471699691640962:0] socket: 404 qp: -1
2025-11-27T18:20:13.409316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471702383660390:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:13.409344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:13.411050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471702640894877:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471702640894959:2119] held: no
2025-11-27T18:20:13.411107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471702383660364:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.411138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471702383660364:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.412101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471702383660364:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:13.412304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471702640894959:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:13.412414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471702383660354:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.412477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:13.412536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471702383660354:2110] [node 6] ICP22 created new session: [3:7577471702383660393:2110]
2025-11-27T18:20:13.412587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471702383660393:2110] [node 6] ICS09 handshake done sender: [3:7577471702383660364:2115] self: [3:7577471702383660363:0] peer: [6:7577471702640894971:0] socket: 430 qp: -1
2025-11-27T18:20:13.412622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471702383660393:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:13.412656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.412978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471702640894970:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:20:13.413110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471702640894877:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.413152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:13.413193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471702640894877:2049] [node 2] ICP22 created new session: [6:7577471702640894973:2049]
2025-11-27T18:20:13.413243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471702640894973:2049] [node 2] ICS09 handshake done sender: [6:7577471702640894959:2119] self: [6:7577471702640894965:0] peer: [2:7577471701143229960:0] socket: 427 qp: -1
2025-11-27T18:20:13.413298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471702640894973:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:13.413329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:13.414083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471699691640867:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:13.414329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471699691640868:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471699691640951:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.415372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471699691640965:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58432
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.415683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471702640894972:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:20:13.415783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471699691640951:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:13.415993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471699691640868:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.416035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:13.416024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471702640894878:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471702640894962:2120] held: no
2025-11-27T18:20:13.416082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471699691640868:2050] [node 3] ICP22 created new session: [5:7577471699691640966:2050]
2025-11-27T18:20:13.416130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471699691640966:2050] [node 3] ICS09 handshake done sender: [5:7577471699691640951:2120] self: [5:7577471699691640962:0] peer: [3:7577471702383660361:0] socket: 420 qp: -1
2025-11-27T18:20:13.416159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471699691640966:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:13.416211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:13.416566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471702640894970:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.416837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471702640894962:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:13.417068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471702640894878:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.417118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:13.417154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471702640894878:2050] [node 3] ICP22 created new session: [6:7577471702640894975:2050]
2025-11-27T18:20:13.417197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471702640894975:2050] [node 3] ICS09 handshake done sender: [6:7577471702640894962:2120] self: [6:7577471702640894971:0] peer: [3:7577471702383660363:0] socket: 375 qp: -1
2025-11-27T18:20:13.417230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471702640894975:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:13.417277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:13.418209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471702640894939:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:13.460517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471699691640941:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:13.460555Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471699691640941:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:13.461644Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471699691640941:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.462015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471702640894978:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40453
2025-11-27T18:20:13.462064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471699691640931:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:13.462144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:13.462194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471699691640931:2109] [node 6] ICP22 created new session: [5:7577471699691640969:2109]
2025-11-27T18:20:13.462240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471699691640969:2109] [node 6] ICS09 handshake done sender: [5:7577471699691640941:2115] self: [5:7577471699691640940:0] peer: [6:7577471702640894977:0] socket: 383 qp: -1
2025-11-27T18:20:13.462276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471699691640969:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:13.462330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:13.462533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471702640894970:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:13.462759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471702640894939:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:13.462806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:13.462850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471702640894939:2106] [node 5] ICP22 created new session: [6:7577471702640894979:2106]
2025-11-27T18:20:13.462899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471702640894979:2106] [node 5] ICS09 handshake done sender: [6:7577471702640894970:2123] self: [6:7577471702640894977:0] peer: [5:7577471699691640940:0] socket: 369 qp: -1
2025-11-27T18:20:13.462927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471702640894979:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:13.462957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:13.487839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:13.488009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471701143229983:2048] [node 1] ICS07 socket disconnect 399 reason# EndOfStream
2025-11-27T18:20:13.488037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471701143229983:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.488110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:13.488153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471701143229983:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:13.489193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:13.489425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471702383660382:2048] [node 1] ICS01 socket: 394 reason# EndOfStream
2025-11-27T18:20:13.489468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471702383660288:2048] [node 1] ICP30 unregister session Session# [3:7577471702383660382:2048] VirtualId# [3:7577471702383660375:0]
2025-11-27T18:20:13.489501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471702383660382:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.489585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:13.489658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:13.491347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471701143229885:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5676) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:13.491388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471701143229983:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:13.492103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:13.492363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471701143229885:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5676) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:13.492429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471701143229983:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:13.492268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471703242282822:2048] [node 1] ICS01 socket: 398 reason# EndOfStream
2025-11-27T18:20:13.492459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471701143229885:2048] [node 1] ICP30 unregister session Session# [2:7577471701143229983:2048] VirtualId# [2:7577471701143229974:0]
2025-11-27T18:20:13.492331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471703242282734:2048] [node 1] ICP30 unregister session Session# [4:7577471703242282822:2048] VirtualId# [4:7577471703242282817:0]
2025-11-27T18:20:13.492498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:13.492353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471703242282822:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.492530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471701143229885:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5676) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.492414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:13.492545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5676) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.492468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:13.493244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:13.493464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471699691640960:2048] [node 1] ICS01 socket: 416 reason# EndOfStream
2025-11-27T18:20:13.493508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471699691640866:2048] [node 1] ICP30 unregister session Session# [5:7577471699691640960:2048] VirtualId# [5:7577471699691640952:0]
2025-11-27T18:20:13.493534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471699691640960:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.493596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:13.493640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:13.494178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:13.494358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471702640894968:2048] [node 1] ICS01 socket: 419 reason# EndOfStream
2025-11-27T18:20:13.494393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471702640894876:2048] [node 1] ICP30 unregister session Session# [6:7577471702640894968:2048] VirtualId# [6:7577471702640894963:0]
2025-11-27T18:20:13.494426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471702640894968:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.494479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:13.494522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:13.517251Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:13.517519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471702383660377:2049] [node 2] ICS07 socket disconnect 392 reason# EndOfStream
2025-11-27T18:20:13.517555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471702383660377:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.517639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:13.517688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471702383660377:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:13.520364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:13.520573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471703242282820:2049] [node 2] ICS07 socket disconnect 397 reason# EndOfStream
2025-11-27T18:20:13.520606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471703242282820:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.520671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:13.520715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471703242282820:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:13.520883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471702383660289:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23908) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:13.520929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471702383660377:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:13.521430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471703242282735:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23908) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:13.521483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471703242282820:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:13.521519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471703242282735:2049] [node 2] ICP30 unregister session Session# [4:7577471703242282820:2049] VirtualId# [4:7577471703242282816:0]
2025-11-27T18:20:13.521651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:13.521638Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:13.521691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471703242282735:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.521706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.521801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471702383660289:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23908) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:13.521848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471702383660377:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:13.521850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471699691640963:2049] [node 2] ICS07 socket disconnect 414 reason# EndOfStream
2025-11-27T18:20:13.521889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471699691640963:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.521884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471702383660289:2049] [node 2] ICP30 unregister session Session# [3:7577471702383660377:2049] VirtualId# [3:7577471702383660373:0]
2025-11-27T18:20:13.521953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:13.521995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471699691640963:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:13.522025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:13.522071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471702383660289:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.522084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.522700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471699691640867:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23908) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:13.522734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471699691640963:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:13.522752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471699691640867:2049] [node 2] ICP30 unregister session Session# [5:7577471699691640963:2049] VirtualId# [5:7577471699691640957:0]
2025-11-27T18:20:13.522823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:13.522845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471699691640867:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.522856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.522857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:13.523079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471702640894973:2049] [node 2] ICS01 socket: 427 reason# EndOfStream
2025-11-27T18:20:13.523135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471702640894877:2049] [node 2] ICP30 unregister session Session# [6:7577471702640894973:2049] VirtualId# [6:7577471702640894965:0]
2025-11-27T18:20:13.523172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471702640894973:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.523244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:13.523289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:13.531425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471703242282735:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:13.531455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471702383660289:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:13.532439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471699691640867:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:13.562786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:13.563070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471703242282834:2050] [node 3] ICS01 socket: 407 reason# EndOfStream
2025-11-27T18:20:13.563117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471703242282736:2050] [node 3] ICP30 unregister session Session# [4:7577471703242282834:2050] VirtualId# [4:7577471703242282826:0]
2025-11-27T18:20:13.563155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471703242282834:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.563231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:13.563285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:13.563888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:13.564099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471699691640966:2050] [node 3] ICS01 socket: 420 reason# EndOfStream
2025-11-27T18:20:13.564141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471699691640868:2050] [node 3] ICP30 unregister session Session# [5:7577471699691640966:2050] VirtualId# [5:7577471699691640962:0]
2025-11-27T18:20:13.564168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471699691640966:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.564228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:13.564263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:13.564810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:13.564987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471702640894975:2050] [node 3] ICS01 socket: 375 reason# EndOfStream
2025-11-27T18:20:13.565028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471702640894878:2050] [node 3] ICP30 unregister session Session# [6:7577471702640894975:2050] VirtualId# [6:7577471702640894971:0]
2025-11-27T18:20:13.565053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471702640894975:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.565117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:13.565151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:13.590267Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:13.590435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471699691640955:2108] [node 4] ICS07 socket disconnect 406 reason# EndOfStream
2025-11-27T18:20:13.590475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471699691640955:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.590560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:13.590596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471699691640955:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:13.591298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471699691640930:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10913) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:13.591354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471699691640955:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:13.591390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471699691640930:2108] [node 4] ICP30 unregister session Session# [5:7577471699691640955:2108] VirtualId# [5:7577471699691640950:0]
2025-11-27T18:20:13.591482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:13.591521Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471699691640930:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.591473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:13.591541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.591675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471702640894960:2107] [node 4] ICS07 socket disconnect 417 reason# EndOfStream
2025-11-27T18:20:13.591715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471702640894960:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.591797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:13.591859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471702640894960:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:13.592565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471702640894941:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10913) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:13.592621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471702640894960:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:13.592655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471702640894941:2107] [node 4] ICP30 unregister session Session# [6:7577471702640894960:2107] VirtualId# [6:7577471702640894957:0]
2025-11-27T18:20:13.592691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:13.592760Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471702640894941:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.592784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.601069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471699691640930:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:13.603113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471702640894941:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:13.651019Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:13.651262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471702640894979:2106] [node 5] ICS07 socket disconnect 369 reason# EndOfStream
2025-11-27T18:20:13.651299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471702640894979:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:13.651398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:13.651452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471702640894979:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:13.652314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471702640894939:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:13.652373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471702640894979:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:13.652415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471702640894939:2106] [node 5] ICP30 unregister session Session# [6:7577471702640894979:2106] VirtualId# [6:7577471702640894977:0]
2025-11-27T18:20:13.652519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:13.652569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471702640894939:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:13.652590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:13.753410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471702797005960:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:13.753494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471702797005961:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:13.777394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471701205275075:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:13.777440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471701205275076:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:13.796205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471702631262610:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:13.796266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471702631262611:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:13.816176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471700407439551:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:13.816231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471700407439552:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:13.816329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471700407439553:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:13.836961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471701726836831:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:13.837009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471701726836832:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:13.837040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471701726836833:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:13.856982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471700986691166:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:13.857027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471700986691167:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:13.857057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471700986691168:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:13.927856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471700986691228:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:13.938715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471701205275135:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39944
2025-11-27T18:20:13.942079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471702631262670:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34100
2025-11-27T18:20:13.942299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471702797006030:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:13.981616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471702631262674:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:14.001109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471702797006032:2110] [node 5] ICP01 ready to work
2025-11-27T18:20:13.999837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471701205275141:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:14.003867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471707091973329:2111] [node 6] ICP01 ready to work
2025-11-27T18:20:14.005545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471704702406910:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:13.981964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471702631262675:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:14.003469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471701205275142:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:14.005178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471706926229974:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:14.006182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471705500242439:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:14.006126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471704702406911:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:14.006355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471706021804190:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:14.006611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471706021804191:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:14.008579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471702631262610:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23603332ns) {Configure:104(23598745ns) {ProcessConfigured:135(873647ns) {StartInitialHandshake:157(870847ns) {DropHandshakes:470(123477ns) {DropIncomingHandshake:442(873ns) DropOutgoingHandshake:457(771ns)} GenerateSessionVirtualId:988(1693ns) PrepareNewSessionHandshake:477(1627ns) {SwitchToState:179(766ns)}}}}}
2025-11-27T18:20:14.006922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471705281658538:2110] [node 4] ICP01 ready to work
2025-11-27T18:20:14.011036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471700986691167:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45255728ns) {Configure:104(45248064ns) {ProcessConfigured:135(498338ns) {StartInitialHandshake:157(436265ns) {DropHandshakes:470(2234ns) {DropIncomingHandshake:442(1040ns) DropOutgoingHandshake:457(422ns)} GenerateSessionVirtualId:988(624ns) PrepareNewSessionHandshake:477(1758ns) {SwitchToState:179(865ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.040889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471707091973336:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14405
2025-11-27T18:20:14.042037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471702631262670:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.046435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471706021804190:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19175578ns) {Configure:104(19172071ns) {ProcessConfigured:135(52506ns) {StartInitialHandshake:157(50138ns) {DropHandshakes:470(3324ns) {DropIncomingHandshake:442(641ns) DropOutgoingHandshake:457(1025ns)} GenerateSessionVirtualId:988(571ns) PrepareNewSessionHandshake:477(1432ns) {SwitchToState:179(891ns)}}}}}
2025-11-27T18:20:14.042184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471702631262610:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.051173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471702797006023:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.046956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471706021804192:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2308
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.056537Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471700407439552:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28702079ns) {Configure:104(28698416ns) {ProcessConfigured:135(1303547ns) {StartInitialHandshake:157(1300615ns) {DropHandshakes:470(4207ns) {DropIncomingHandshake:442(852ns) DropOutgoingHandshake:457(418ns)} GenerateSessionVirtualId:988(966ns) PrepareNewSessionHandshake:477(2025ns) {SwitchToState:179(907ns)}}}}}
2025-11-27T18:20:14.047253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471706021804201:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2308
2025-11-27T18:20:14.051218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471702797006023:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.059588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471701726836832:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44396385ns) {Configure:104(44391204ns) {ProcessConfigured:135(3158611ns) {StartInitialHandshake:157(2385622ns) {DropHandshakes:470(1531ns) {DropIncomingHandshake:442(656ns) DropOutgoingHandshake:457(264ns)} GenerateSessionVirtualId:988(698ns) PrepareNewSessionHandshake:477(2428ns) {SwitchToState:179(1093ns)}}}}}
2025-11-27T18:20:14.060007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471706021804201:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.059460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471706926229987:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34100
2025-11-27T18:20:14.058969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707091973336:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.060365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471701726836831:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.061812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471701205275142:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(38311530ns) {Configure:104(38306975ns) {ProcessConfigured:135(1623300ns) {StartInitialHandshake:157(1620587ns) {DropHandshakes:470(1436ns) {DropIncomingHandshake:442(381ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(542ns) PrepareNewSessionHandshake:477(2133ns) {SwitchToState:179(1123ns)}}}}}
2025-11-27T18:20:14.060083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471704702406910:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(37020797ns) {Configure:104(37009046ns) {ProcessConfigured:135(1965194ns) {StartInitialHandshake:157(1963311ns) {DropHandshakes:470(3800ns) {DropIncomingHandshake:442(756ns) DropOutgoingHandshake:457(685ns)} GenerateSessionVirtualId:988(756ns) PrepareNewSessionHandshake:477(3301ns) {SwitchToState:179(1824ns)}}}}}
2025-11-27T18:20:14.064282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471706926229987:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.060435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471705281658538:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32809158ns) {Configure:104(32806638ns) {ProcessConfigured:135(268277ns) {StartInitialHandshake:157(266639ns) {DropHandshakes:470(1047ns) {DropIncomingHandshake:442(235ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(435ns) PrepareNewSessionHandshake:477(1154ns) {SwitchToState:179(515ns)}}}}}
2025-11-27T18:20:14.061846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471701205275141:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(38985163ns) {Configure:104(38978620ns) {ProcessConfigured:135(651610ns) {StartInitialHandshake:157(649113ns) {DropHandshakes:470(7600ns) {DropIncomingHandshake:442(473ns) DropOutgoingHandshake:457(283ns)} GenerateSessionVirtualId:988(396ns) PrepareNewSessionHandshake:477(2047ns) {SwitchToState:179(734ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.061954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471701205275135:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.062710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471701205275075:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.071581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707091973334:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.062847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471705500242448:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39944
2025-11-27T18:20:14.071627Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471707091973334:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.072425Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471702797006023:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.070681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705500242448:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.072664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471702797006021:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.072709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471702797006021:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.070946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471705500242453:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39944
2025-11-27T18:20:14.076113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471706926229988:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34100
2025-11-27T18:20:14.083225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471704702406911:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21532935ns) {Configure:104(21526502ns) {ProcessConfigured:135(3410706ns) {StartInitialHandshake:157(1822214ns) {DropHandshakes:470(4210ns) {DropIncomingHandshake:442(479ns) DropOutgoingHandshake:457(345ns)} GenerateSessionVirtualId:988(2557ns) PrepareNewSessionHandshake:477(3517ns) {SwitchToState:179(1511ns)}}}}}
2025-11-27T18:20:14.083017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471706926229989:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34100
2025-11-27T18:20:14.083630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471702631262670:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:14.084412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705500242453:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.084418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471702631262610:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.084115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471702797005961:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.084439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471706021804192:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.084439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:14.084486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:14.085015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471707091973338:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14405
2025-11-27T18:20:14.084807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471702631262610:2048] [node 1] ICP22 created new session: [3:7577471706926229991:2048]
2025-11-27T18:20:14.085527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471702797005961:2049] [node 3] ICP22 created new session: [1:7577471707091973345:2049]
2025-11-27T18:20:14.087247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471706926229989:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.087368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471707091973345:2049] [node 3] ICS09 handshake done sender: [1:7577471702797006023:2108] self: [1:7577471702797006022:0] peer: [3:7577471706926229986:0] socket: 373 qp: -1
2025-11-27T18:20:14.087366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471706926229991:2048] [node 1] ICS09 handshake done sender: [3:7577471702631262670:2107] self: [3:7577471706926229986:0] peer: [1:7577471702797006022:0] socket: 374 qp: -1
2025-11-27T18:20:14.087416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471706926229991:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:14.087413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471707091973345:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:14.087459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:14.087501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:14.089465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471704702406920:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46940
2025-11-27T18:20:14.089511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471704702406923:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46940
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.089517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471706021804209:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2308
2025-11-27T18:20:14.089633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471704702406924:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46940
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.091623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707091973338:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.091821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471707091973341:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14405
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.092565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471707091973334:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:14.091756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471705500242458:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39944
2025-11-27T18:20:14.092639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471707091973342:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14405
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.095326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471706021804201:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:14.093470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471704702406924:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.092981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471702797006021:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:14.095554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471705500242459:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39944
2025-11-27T18:20:14.095603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471704702406923:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.096153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471701205275135:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:14.096117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471704702406920:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.096261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471701726836831:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.096489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:14.097022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707091973341:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.096979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471701205275075:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.096778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471701726836831:2048] [node 1] ICP22 created new session: [5:7577471706021804210:2048]
2025-11-27T18:20:14.096845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471700407439552:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.097188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471702797006032:2110] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.097308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:14.097536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471700407439553:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.099384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471702631262611:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.097716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471706021804210:2048] [node 1] ICS09 handshake done sender: [5:7577471706021804201:2115] self: [5:7577471706021804207:0] peer: [1:7577471707091973333:0] socket: 385 qp: -1
2025-11-27T18:20:14.097225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:14.100956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471706926229993:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34100
2025-11-27T18:20:14.107435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471706926229978:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.107473Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471706926229978:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.107665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471706926229993:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.098998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471701205275075:2048] [node 1] ICP22 created new session: [2:7577471705500242460:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.097766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471706021804210:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:14.097787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471702797006032:2110] [node 5] ICP22 created new session: [1:7577471707091973349:2110]
2025-11-27T18:20:14.099570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471705500242460:2048] [node 1] ICS09 handshake done sender: [2:7577471701205275135:2107] self: [2:7577471705500242449:0] peer: [1:7577471702797006020:0] socket: 375 qp: -1
2025-11-27T18:20:14.097813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707091973342:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.115503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471706926229997:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34100
2025-11-27T18:20:14.098850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471702797005960:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.098779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.099130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:14.099621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471702797005960:2048] [node 2] ICP22 created new session: [1:7577471707091973350:2048]
2025-11-27T18:20:14.115664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471706926229978:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:14.101907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471705500242460:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.101929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471707091973349:2110] [node 5] ICS09 handshake done sender: [1:7577471707091973334:2115] self: [1:7577471707091973333:0] peer: [5:7577471706021804207:0] socket: 384 qp: -1
2025-11-27T18:20:14.119312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471702631262674:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.101987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471707091973349:2110] [node 5] ICS10 traffic start
2025-11-27T18:20:14.101949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:14.126440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471705281658549:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-27T18:20:14.119396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471704702406931:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46940
2025-11-27T18:20:14.126440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471704702406932:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46940
2025-11-27T18:20:14.119392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:14.110520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705500242445:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.110545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471705500242445:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.102043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:14.104880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471707091973350:2048] [node 2] ICS09 handshake done sender: [1:7577471702797006021:2107] self: [1:7577471702797006020:0] peer: [2:7577471705500242449:0] socket: 372 qp: -1
2025-11-27T18:20:14.111103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705500242459:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.119449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471702631262674:2108] [node 4] ICP22 created new session: [3:7577471706926229999:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.120200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471706926229999:2108] [node 4] ICS09 handshake done sender: [3:7577471706926229978:2113] self: [3:7577471706926229977:0] peer: [4:7577471704702406930:0] socket: 381 qp: -1
2025-11-27T18:20:14.111203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705500242443:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.107583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471707091973350:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:14.111221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471705500242443:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.120245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471706926229999:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:14.131199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471706021804212:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2308
2025-11-27T18:20:14.107633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:14.121679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:14.131642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471704702406924:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:14.116885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471705500242445:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:14.118591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471701205275075:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20782997ns) {HandleHandshakeStatus:329(20780698ns) {DropIncomingHandshake:442(854ns) DropOutgoingHandshake:457(4797ns) SwitchToState:179(164ns) ProcessPendingSessionEvents:495(4732639ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4679578ns max# 4228920ns}}}
2025-11-27T18:20:14.131718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471705281658550:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-27T18:20:14.139125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471704702406920:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:14.118749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471705500242443:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:14.138917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471705281658550:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.123959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471701205275076:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.139158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471700986691228:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.139536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471706021804212:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.124012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.124057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471701205275076:2049] [node 3] ICP22 created new session: [2:7577471705500242466:2049]
2025-11-27T18:20:14.124098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471705500242466:2049] [node 3] ICS09 handshake done sender: [2:7577471705500242443:2114] self: [2:7577471705500242442:0] peer: [3:7577471706926229994:0] socket: 397 qp: -1
2025-11-27T18:20:14.139790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471706021804200:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.124131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471705500242466:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:14.139921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471706926229989:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:14.139864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471705281658551:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-27T18:20:14.139830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471706021804200:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.124158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:14.139981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471705281658549:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.139982Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471700407439552:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471704702406924:2117] held: no
2025-11-27T18:20:14.125031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471701205275142:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.140378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471700986691168:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.140653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471702797006032:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(31856147ns) {HandleHandshakeStatus:329(31853594ns) {DropIncomingHandshake:442(278ns) DropOutgoingHandshake:457(283ns) SwitchToState:179(443ns) ProcessPendingSessionEvents:495(21552743ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21459171ns max# 4917079ns}}}
2025-11-27T18:20:14.140393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471705281658551:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.141920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471702631262611:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471706926229989:2118] held: no
2025-11-27T18:20:14.141554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471700986691167:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.142686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471702797005960:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(33866951ns) {HandleHandshakeStatus:329(33864892ns) {DropIncomingHandshake:442(885ns) DropOutgoingHandshake:457(323ns) SwitchToState:179(751ns) ProcessPendingSessionEvents:495(15805683ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15723940ns max# 4865681ns}}}
2025-11-27T18:20:14.142868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471701726836831:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(43305151ns) {HandleHandshakeStatus:329(43302752ns) {DropIncomingHandshake:442(744ns) DropOutgoingHandshake:457(5229ns) SwitchToState:179(531ns) ProcessPendingSessionEvents:495(37374308ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 37254193ns max# 5306774ns}}}
2025-11-27T18:20:14.142890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471702631262674:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(20531920ns) {HandleHandshakeStatus:329(20529109ns) {DropIncomingHandshake:442(17299ns) DropOutgoingHandshake:457(569ns) SwitchToState:179(342ns) ProcessPendingSessionEvents:495(11691480ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11607630ns max# 3373022ns}}}
2025-11-27T18:20:14.142975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471701726836832:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.143030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471702631262611:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.143066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471706926229980:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.143095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471706926229980:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.143105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:14.125074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:14.142165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471700407439553:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.143888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471705281658556:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.143167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471702631262611:2049] [node 2] ICP22 created new session: [3:7577471706926230002:2049]
2025-11-27T18:20:14.143332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471706021804200:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:14.125109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471701205275142:2108] [node 4] ICP22 created new session: [2:7577471705500242468:2108]
2025-11-27T18:20:14.125151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471705500242468:2108] [node 4] ICS09 handshake done sender: [2:7577471705500242445:2115] self: [2:7577471705500242444:0] peer: [4:7577471704702406928:0] socket: 398 qp: -1
2025-11-27T18:20:14.125177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471705500242468:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:14.125202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:14.143493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705500242456:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.143521Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471705500242456:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.144144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705500242447:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.144170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471705500242447:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.144071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471702797006032:2110] [node 5] ICP18 handshake (actor [1:7577471707091973338:2118]) is too old
2025-11-27T18:20:14.142232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:14.143532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471706926230002:2049] [node 2] ICS09 handshake done sender: [3:7577471706926229989:2118] self: [3:7577471706926229994:0] peer: [2:7577471705500242442:0] socket: 401 qp: -1
2025-11-27T18:20:14.143284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471700407439553:2050] [node 3] ICP22 created new session: [4:7577471704702406933:2050]
No RDMA context found for GID: 2025-11-27T18:20:14.143336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471704702406933:2050] [node 3] ICS09 handshake done sender: [4:7577471704702406920:2114] self: [4:7577471704702406930:0] peer: [3:7577471706926229977:0] socket: 406 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:14.147061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471706021804190:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.143572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471706926230002:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:14.145516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471705281658555:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-27T18:20:14.143561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471704702406933:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:14.162800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471707091973348:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14405
2025-11-27T18:20:14.143604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:14.147137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:14.143598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:14.163633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471705500242456:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:14.144157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471700407439552:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.153679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471706021804190:2109] [node 6] ICP22 created new session: [5:7577471706021804216:2109]
2025-11-27T18:20:14.153071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:14.164349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471705500242447:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:14.155854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471700407439552:2049] [node 2] ICP22 created new session: [4:7577471704702406935:2049]
2025-11-27T18:20:14.153750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471706021804216:2109] [node 6] ICS09 handshake done sender: [5:7577471706021804200:2114] self: [5:7577471706021804199:0] peer: [6:7577471705281658552:0] socket: 395 qp: -1
2025-11-27T18:20:14.164389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471705281658550:2125] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:14.161135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471706021804216:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:14.155908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471704702406935:2049] [node 2] ICS09 handshake done sender: [4:7577471704702406924:2117] self: [4:7577471704702406928:0] peer: [2:7577471705500242444:0] socket: 408 qp: -1
2025-11-27T18:20:14.161193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:14.155947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471704702406935:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:14.166292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471705500242439:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.155989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.166641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:14.166974Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471700986691228:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1033) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471705281658550:2125] held: no
2025-11-27T18:20:14.166937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471704702406939:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46940
2025-11-27T18:20:14.166845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471705500242439:2110] [node 6] ICP22 created new session: [2:7577471705500242473:2110]
2025-11-27T18:20:14.167003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471705500242473:2110] [node 6] ICS09 handshake done sender: [2:7577471705500242456:2119] self: [2:7577471705500242455:0] peer: [6:7577471705281658554:0] socket: 380 qp: -1
No RDMA context found for GID: 2025-11-27T18:20:14.167477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471705500242473:2110] [node 6] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:20:14.167566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:14.168048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471705281658557:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-27T18:20:14.168169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707091973348:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.168548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471701205275141:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.168632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:14.168281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471700986691228:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.168685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471701205275141:2109] [node 5] ICP22 created new session: [2:7577471705500242475:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.168329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:14.168839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471705500242475:2109] [node 5] ICS09 handshake done sender: [2:7577471705500242447:2116] self: [2:7577471705500242446:0] peer: [5:7577471706021804215:0] socket: 421 qp: -1
2025-11-27T18:20:14.168367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471700986691228:2106] [node 5] ICP22 created new session: [6:7577471705281658559:2106]
2025-11-27T18:20:14.168876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471705500242475:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:14.168403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471705281658559:2106] [node 5] ICS09 handshake done sender: [6:7577471705281658550:2125] self: [6:7577471705281658552:0] peer: [5:7577471706021804199:0] socket: 416 qp: -1
2025-11-27T18:20:14.168948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:14.168431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471705281658559:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:14.168460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:14.169029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471700407439553:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(25352888ns) {HandleHandshakeStatus:329(25350263ns) {DropIncomingHandshake:442(418ns) DropOutgoingHandshake:457(5941ns) SwitchToState:179(698ns) ProcessPendingSessionEvents:495(20356973ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20253708ns max# 7341793ns}}}
2025-11-27T18:20:14.168975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471705281658558:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-27T18:20:14.169100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471700407439553:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:14.169060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471702797005960:2048] [node 2] ICP18 handshake (actor [1:7577471707091973341:2119]) is too old
2025-11-27T18:20:14.169769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471706926229980:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:14.169739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471705281658557:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.171313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471702631262675:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.171517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471705281658551:2126] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:14.171361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:14.171468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471702631262675:2110] [node 6] ICP22 created new session: [3:7577471706926230006:2110]
2025-11-27T18:20:14.172300Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471706021804190:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(23028877ns) {HandleHandshakeStatus:329(23025701ns) {DropIncomingHandshake:442(14886ns) DropOutgoingHandshake:457(197ns) SwitchToState:179(633ns) ProcessPendingSessionEvents:495(6053898ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5822977ns max# 760089ns}}}
2025-11-27T18:20:14.171796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471706926230006:2110] [node 6] ICS09 handshake done sender: [3:7577471706926229980:2114] self: [3:7577471706926229979:0] peer: [6:7577471705281658553:0] socket: 418 qp: -1
2025-11-27T18:20:14.172207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471706926230006:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:14.172361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.177710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471700986691167:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471705281658551:2126] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.177870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471706021804218:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2308
2025-11-27T18:20:14.177738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471705281658538:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.178462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471705281658562:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.178470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471706021804220:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2308
2025-11-27T18:20:14.178663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471706021804219:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2308
2025-11-27T18:20:14.180469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471704702406939:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.179108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471700986691167:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.178910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471706021804218:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.180331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:14.181011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471706021804212:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:14.181415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471700407439551:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.180937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471700986691167:2049] [node 2] ICP22 created new session: [6:7577471705281658564:2049]
2025-11-27T18:20:14.181356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471705281658564:2049] [node 2] ICS09 handshake done sender: [6:7577471705281658551:2126] self: [6:7577471705281658554:0] peer: [2:7577471705500242455:0] socket: 426 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.181394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471705281658564:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:14.182004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471704702406922:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.182040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471704702406922:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.181958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471701726836832:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.181674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:14.182269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471705500242477:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39944
2025-11-27T18:20:14.182366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.182309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471705281658555:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.182441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471701726836832:2049] [node 2] ICP22 created new session: [5:7577471706021804222:2049]
2025-11-27T18:20:14.182965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471706021804222:2049] [node 2] ICS09 handshake done sender: [5:7577471706021804212:2118] self: [5:7577471706021804215:0] peer: [2:7577471705500242446:0] socket: 425 qp: -1
2025-11-27T18:20:14.183154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471706021804222:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:14.183226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471706926230005:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34100
2025-11-27T18:20:14.183505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707091973331:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.183198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:14.183543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471707091973331:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.184884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471704702406922:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:14.185095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471706021804191:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.185154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471704702406911:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.185194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:14.185447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471706021804220:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.185475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471704702406911:2109] [node 6] ICP22 created new session: [4:7577471704702406942:2109]
2025-11-27T18:20:14.185527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471704702406942:2109] [node 6] ICS09 handshake done sender: [4:7577471704702406922:2115] self: [4:7577471704702406921:0] peer: [6:7577471705281658563:0] socket: 429 qp: -1
2025-11-27T18:20:14.185447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471701205275141:2109] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:14.185583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471704702406942:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:14.185616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:14.185776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471701726836833:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.186052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471707091973331:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:14.186154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471704702406919:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.186188Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471704702406919:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.187134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471706926229985:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.187207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471702797006030:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.187702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:14.188703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471702797006030:2109] [node 4] ICP22 created new session: [1:7577471707091973357:2109]
2025-11-27T18:20:14.191161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471705281658549:2124] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:14.187168Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471706926229985:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.189513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471706926229985:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:14.192837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471707091973357:2109] [node 4] ICS09 handshake done sender: [1:7577471707091973331:2114] self: [1:7577471707091973330:0] peer: [4:7577471704702406940:0] socket: 391 qp: -1
2025-11-27T18:20:14.189715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471706926230005:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.193147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471707091973357:2109] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.192599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471702631262675:2110] [node 6] ICP18 handshake (actor [3:7577471706926230005:2121]) is too old
2025-11-27T18:20:14.194919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471700986691168:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.193200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:14.193021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471706926229974:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.193099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:14.195088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471706021804226:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2308
2025-11-27T18:20:14.195358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:14.195755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471700986691168:2050] [node 3] ICP22 created new session: [6:7577471705281658568:2050]
2025-11-27T18:20:14.193627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471706926229974:2109] [node 5] ICP22 created new session: [3:7577471706926230009:2109]
2025-11-27T18:20:14.196462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471705281658568:2050] [node 3] ICS09 handshake done sender: [6:7577471705281658549:2124] self: [6:7577471705281658553:0] peer: [3:7577471706926229979:0] socket: 419 qp: -1
2025-11-27T18:20:14.194979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471706926230009:2109] [node 5] ICS09 handshake done sender: [3:7577471706926229985:2115] self: [3:7577471706926229984:0] peer: [5:7577471706021804225:0] socket: 432 qp: -1
2025-11-27T18:20:14.196633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471705281658568:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:14.195165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471706926230009:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:14.196671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:14.195235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:14.197361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471700986691166:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.198678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471700407439551:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17720) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471704702406939:2120] held: no
2025-11-27T18:20:14.198757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471706021804220:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:14.199635Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471701726836833:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471706021804220:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.199866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471704702406944:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46940
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.200719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471705281658567:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-27T18:20:14.200819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707091973335:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.200862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471707091973335:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.200770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471701726836833:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.200863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:14.200921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471701726836833:2050] [node 3] ICP22 created new session: [5:7577471706021804228:2050]
2025-11-27T18:20:14.201140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471704702406919:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:14.201345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471704702406939:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:14.200985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471706021804228:2050] [node 3] ICS09 handshake done sender: [5:7577471706021804220:2121] self: [5:7577471706021804225:0] peer: [3:7577471706926229984:0] socket: 438 qp: -1
2025-11-27T18:20:14.202567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471707091973335:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.201021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471706021804228:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.201736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:14.203669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471704702406946:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46940
2025-11-27T18:20:14.203442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471707091973329:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.203737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471706021804227:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2308
2025-11-27T18:20:14.205717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471704702406910:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.203602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:14.205478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471707091973329:2111] [node 6] ICP22 created new session: [1:7577471707091973360:2111]
2025-11-27T18:20:14.205724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471700407439551:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.206194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471707091973360:2111] [node 6] ICS09 handshake done sender: [1:7577471707091973335:2116] self: [1:7577471707091973332:0] peer: [6:7577471705281658570:0] socket: 379 qp: -1
2025-11-27T18:20:14.206334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471707091973360:2111] [node 6] ICS10 traffic start
2025-11-27T18:20:14.206158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:14.206492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:14.206975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471706021804218:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:14.206326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471704702406910:2108] [node 5] ICP22 created new session: [4:7577471704702406947:2108]
2025-11-27T18:20:14.206331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:14.207046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471705281658557:2129] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:14.206484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471700407439551:2048] [node 1] ICP22 created new session: [4:7577471704702406948:2048]
2025-11-27T18:20:14.206532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471704702406948:2048] [node 1] ICS09 handshake done sender: [4:7577471704702406939:2120] self: [4:7577471704702406940:0] peer: [1:7577471707091973330:0] socket: 433 qp: -1
2025-11-27T18:20:14.206622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471704702406948:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:14.207842Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471700986691168:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:14.206659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:14.208017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471706021804191:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.206668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471704702406947:2108] [node 5] ICS09 handshake done sender: [4:7577471704702406919:2113] self: [4:7577471704702406918:0] peer: [5:7577471706021804224:0] socket: 428 qp: -1
2025-11-27T18:20:14.206881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471704702406947:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:14.208151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:14.206919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:14.208215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471706021804191:2108] [node 4] ICP22 created new session: [5:7577471706021804230:2108]
2025-11-27T18:20:14.208297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471706021804230:2108] [node 4] ICS09 handshake done sender: [5:7577471706021804218:2119] self: [5:7577471706021804224:0] peer: [4:7577471704702406918:0] socket: 435 qp: -1
2025-11-27T18:20:14.208342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471706021804230:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:14.208306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471705281658538:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.208399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:14.208413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:14.208444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471705281658538:2110] [node 4] ICP22 created new session: [6:7577471705281658572:2110]
2025-11-27T18:20:14.208501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471705281658572:2110] [node 4] ICS09 handshake done sender: [6:7577471705281658557:2129] self: [6:7577471705281658563:0] peer: [4:7577471704702406921:0] socket: 430 qp: -1
2025-11-27T18:20:14.208521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471705281658572:2110] [node 4] ICS10 traffic start
2025-11-27T18:20:14.208542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:14.211586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471700986691166:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17720) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471705281658555:2127] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.213007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471705281658571:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11803
2025-11-27T18:20:14.213716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471705281658555:2127] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:14.214112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471704702406911:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:14.213858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471700986691166:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.213931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:14.213969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471700986691166:2048] [node 1] ICP22 created new session: [6:7577471705281658574:2048]
2025-11-27T18:20:14.214019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471705281658574:2048] [node 1] ICS09 handshake done sender: [6:7577471705281658555:2127] self: [6:7577471705281658570:0] peer: [1:7577471707091973332:0] socket: 415 qp: -1
2025-11-27T18:20:14.214056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471705281658574:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:14.214113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:14.259442Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:14.259701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471705500242460:2048] [node 1] ICS01 socket: 375 reason# EndOfStream
2025-11-27T18:20:14.259751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471701205275075:2048] [node 1] ICP30 unregister session Session# [2:7577471705500242460:2048] VirtualId# [2:7577471705500242449:0]
2025-11-27T18:20:14.259786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471705500242460:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.259873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:14.259947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:14.260385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:14.260553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471706926229991:2048] [node 1] ICS07 socket disconnect 374 reason# EndOfStream
2025-11-27T18:20:14.260576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471706926229991:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.260646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:14.260715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471706926229991:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:14.262989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471702631262610:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17720) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:14.263030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471706926229991:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:14.264186Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:14.264460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471704702406948:2048] [node 1] ICS01 socket: 433 reason# EndOfStream
2025-11-27T18:20:14.264529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471700407439551:2048] [node 1] ICP30 unregister session Session# [4:7577471704702406948:2048] VirtualId# [4:7577471704702406940:0]
2025-11-27T18:20:14.264589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471704702406948:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.264761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:14.264825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:14.264825Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471702631262610:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17720) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:14.264958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471706926229991:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:14.265012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471702631262610:2048] [node 1] ICP30 unregister session Session# [3:7577471706926229991:2048] VirtualId# [3:7577471706926229986:0]
2025-11-27T18:20:14.265117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:14.265153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471702631262610:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17720) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.265167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17720) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.266154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:14.266366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471706021804210:2048] [node 1] ICS07 socket disconnect 385 reason# EndOfStream
2025-11-27T18:20:14.266397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471706021804210:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.266477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:14.266523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471706021804210:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:14.267224Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471701726836831:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17720) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:14.267242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:14.267324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471706021804210:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:14.267374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471701726836831:2048] [node 1] ICP30 unregister session Session# [5:7577471706021804210:2048] VirtualId# [5:7577471706021804207:0]
2025-11-27T18:20:14.267481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:14.267521Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471701726836831:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17720) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.267475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471705281658574:2048] [node 1] ICS01 socket: 415 reason# EndOfStream
2025-11-27T18:20:14.267535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17720) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.267520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471700986691166:2048] [node 1] ICP30 unregister session Session# [6:7577471705281658574:2048] VirtualId# [6:7577471705281658570:0]
2025-11-27T18:20:14.267550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471705281658574:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.267616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:14.267658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:14.274869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471702631262610:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:14.276915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471701726836831:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:14.313050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:14.313342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471706926230002:2049] [node 2] ICS01 socket: 401 reason# EndOfStream
2025-11-27T18:20:14.313386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471702631262611:2049] [node 2] ICP30 unregister session Session# [3:7577471706926230002:2049] VirtualId# [3:7577471706926229994:0]
2025-11-27T18:20:14.313420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471706926230002:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.313512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:14.313562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:14.316130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:14.316380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471704702406935:2049] [node 2] ICS01 socket: 408 reason# EndOfStream
2025-11-27T18:20:14.316422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471700407439552:2049] [node 2] ICP30 unregister session Session# [4:7577471704702406935:2049] VirtualId# [4:7577471704702406928:0]
2025-11-27T18:20:14.316454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471704702406935:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.316528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:14.316579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:14.317200Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:14.317408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471706021804222:2049] [node 2] ICS07 socket disconnect 425 reason# EndOfStream
2025-11-27T18:20:14.317446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471706021804222:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.317526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:14.317595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471706021804222:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:14.318234Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:14.318382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471705281658564:2049] [node 2] ICS01 socket: 426 reason# EndOfStream
2025-11-27T18:20:14.318426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471700986691167:2049] [node 2] ICP30 unregister session Session# [6:7577471705281658564:2049] VirtualId# [6:7577471705281658554:0]
2025-11-27T18:20:14.318456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471705281658564:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.318523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:14.318565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:14.318737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471701726836832:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2204) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:14.318813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471706021804222:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:14.318860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471701726836832:2049] [node 2] ICP30 unregister session Session# [5:7577471706021804222:2049] VirtualId# [5:7577471706021804215:0]
2025-11-27T18:20:14.318967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:14.319004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471701726836832:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2204) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.319039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2204) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.329185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471701726836832:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:14.381895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:14.382128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471704702406933:2050] [node 3] ICS07 socket disconnect 406 reason# EndOfStream
2025-11-27T18:20:14.382168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471704702406933:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.382242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:14.382288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471704702406933:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:14.383105Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471700407439553:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13445) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:14.383169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471704702406933:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:14.383217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471700407439553:2050] [node 3] ICP30 unregister session Session# [4:7577471704702406933:2050] VirtualId# [4:7577471704702406930:0]
2025-11-27T18:20:14.383317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:14.383453Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471700407439553:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13445) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.383478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13445) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.383111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:14.383328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471706021804228:2050] [node 3] ICS01 socket: 438 reason# EndOfStream
2025-11-27T18:20:14.383396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471701726836833:2050] [node 3] ICP30 unregister session Session# [5:7577471706021804228:2050] VirtualId# [5:7577471706021804225:0]
2025-11-27T18:20:14.383426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471706021804228:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.383495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:14.383531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:14.389508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:14.389825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471705281658568:2050] [node 3] ICS07 socket disconnect 419 reason# EndOfStream
2025-11-27T18:20:14.389860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471705281658568:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.389989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:14.390045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471705281658568:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:14.390866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471700986691168:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13445) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:14.390920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471705281658568:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:14.390971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471700986691168:2050] [node 3] ICP30 unregister session Session# [6:7577471705281658568:2050] VirtualId# [6:7577471705281658553:0]
2025-11-27T18:20:14.391085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:14.391119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471700986691168:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13445) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.391146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13445) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.400813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471700986691168:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:14.427864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:14.428067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471706021804230:2108] [node 4] ICS07 socket disconnect 435 reason# EndOfStream
2025-11-27T18:20:14.428101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471706021804230:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.428182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:14.428233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471706021804230:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:14.428923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:14.429135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471705281658572:2110] [node 4] ICS07 socket disconnect 430 reason# EndOfStream
2025-11-27T18:20:14.429092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471706021804191:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23735) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:14.429183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471705281658572:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.429161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471706021804230:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:14.429225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471706021804191:2108] [node 4] ICP30 unregister session Session# [5:7577471706021804230:2108] VirtualId# [5:7577471706021804224:0]
2025-11-27T18:20:14.429294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:14.429340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471705281658572:2110] [node 4] ICS15 start handshake
2025-11-27T18:20:14.429361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:14.429400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471706021804191:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.429475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.430198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471705281658538:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23735) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:14.430272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471705281658572:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:14.430342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471705281658538:2110] [node 4] ICP30 unregister session Session# [6:7577471705281658572:2110] VirtualId# [6:7577471705281658563:0]
2025-11-27T18:20:14.430383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:14.430447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471705281658538:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.430466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23735) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:14.439740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471706021804191:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:14.440796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471705281658538:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:20:14.488150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:14.488356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471705281658559:2106] [node 5] ICS01 socket: 416 reason# EndOfStream
2025-11-27T18:20:14.488392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471700986691228:2106] [node 5] ICP30 unregister session Session# [6:7577471705281658559:2106] VirtualId# [6:7577471705281658552:0]
2025-11-27T18:20:14.488422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471705281658559:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:14.488489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:14.488528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:14.572566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471707726620547:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:14.572697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471707726620546:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:14.592408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471705969689399:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:14.592639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471705969689400:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:14.613611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471707280084254:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:14.613641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471707280084255:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:14.634434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471703703649945:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:14.634465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471703703649946:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:14.634483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471703703649947:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:14.655801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471705419474938:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:14.655839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471705419474939:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:14.655849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471705419474940:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:14.678144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471707732177278:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:14.678251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471707732177279:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:14.678309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471707732177280:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:14.751745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471707732177340:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:14.841761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471705419475002:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:14.841712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471707726620610:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:14.836739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471707280084332:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:14.847199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471705969689480:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:14.851894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471703703650017:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:14.851861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471707732177342:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:14.852794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471707726620611:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:14.853199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471705969689481:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:14.853059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471705419475003:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:14.853017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471703703650018:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:14.853376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471707726620612:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:14.852827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471707280084336:2125] [node 6] ICP01 ready to work
2025-11-27T18:20:14.853882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471707280084337:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:14.854686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471705969689482:2111] [node 6] ICP01 ready to work
2025-11-27T18:20:14.855188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471707726620546:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(103839705ns) {Configure:104(103835797ns) {ProcessConfigured:135(513775ns) {StartInitialHandshake:157(510626ns) {DropHandshakes:470(2767ns) {DropIncomingHandshake:442(595ns) DropOutgoingHandshake:457(510ns)} GenerateSessionVirtualId:988(909ns) PrepareNewSessionHandshake:477(2262ns) {SwitchToState:179(753ns)}}}}}
2025-11-27T18:20:14.855669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471707732177340:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(103543000ns) {Configure:104(103527996ns) {ProcessConfigured:135(149676ns) {StartInitialHandshake:157(147453ns) {DropHandshakes:470(6135ns) {DropIncomingHandshake:442(524ns) DropOutgoingHandshake:457(305ns)} GenerateSessionVirtualId:988(1017ns) PrepareNewSessionHandshake:477(3198ns) {SwitchToState:179(1717ns)}}}}}
2025-11-27T18:20:14.863853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471707280084255:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(46752044ns) {Configure:104(46742470ns) {ProcessConfigured:135(2351771ns) {StartInitialHandshake:157(2350014ns) {DropHandshakes:470(1549ns) {DropIncomingHandshake:442(360ns) DropOutgoingHandshake:457(382ns)} GenerateSessionVirtualId:988(664ns) PrepareNewSessionHandshake:477(8963ns) {SwitchToState:179(1163ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.873692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703703650025:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.873748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471705419475010:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52541
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.874412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471707280084344:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11102
2025-11-27T18:20:14.876752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471705419475010:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.876978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471703703650025:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.877110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471707280084345:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11102
2025-11-27T18:20:14.877246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471705969689490:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5234
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.877757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471703703649945:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.877331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471705969689489:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5234
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.878262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471707726620626:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-27T18:20:14.878380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705969689490:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.878735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471707726620627:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-27T18:20:14.881028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705969689489:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.881882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471705419475018:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52541
2025-11-27T18:20:14.881748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707726620618:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.881399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471705969689498:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5234
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:14.885162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705969689498:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:14.884962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471707280084345:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.885057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471707280084344:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.881778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471707726620618:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.881838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471707726620628:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-27T18:20:14.885765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471705969689499:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5234
2025-11-27T18:20:14.885748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471707280084353:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11102
2025-11-27T18:20:14.885135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471707726620629:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-27T18:20:14.885754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471707280084352:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11102
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.886520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471705969689399:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.886479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471707280084354:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11102
2025-11-27T18:20:14.889570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471705969689500:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5234
2025-11-27T18:20:14.886948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471707280084254:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.885143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707726620626:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.888903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471707280084352:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.885363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471707726620618:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:14.889982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471707280084353:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.889002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471707726620610:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.890250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:14.890262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707726620628:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.891008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703703650034:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-27T18:20:14.890290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707726620616:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.891420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471705419475018:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.891967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705969689499:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.891030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703703650035:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-27T18:20:14.892354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471705419474939:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.890316Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471707726620616:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.893163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705969689488:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.892752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471707280084354:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.890337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471707726620610:2107] [node 4] ICP22 created new session: [1:7577471707726620632:2107]
2025-11-27T18:20:14.893406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471703703650025:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:14.893290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471707280084255:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.890485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471707726620632:2107] [node 4] ICS09 handshake done sender: [1:7577471707726620618:2113] self: [1:7577471707726620617:0] peer: [4:7577471703703650033:0] socket: 383 qp: -1
2025-11-27T18:20:14.893198Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471705969689488:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.893859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705969689500:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.890541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707726620614:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.894429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471703703650035:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.894559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705969689486:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.890572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471707726620614:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.890659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471707726620632:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.894484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471703703649945:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.895867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471707732177356:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34930
2025-11-27T18:20:14.894611Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471705969689486:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.890702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:14.895950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471705969689488:2130] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:14.892923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707726620627:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.895616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:14.896903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471705969689505:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5234
2025-11-27T18:20:14.893467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471707726620616:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:14.894194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471707726620614:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:14.895911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471703703649945:2048] [node 1] ICP22 created new session: [4:7577471703703650036:2048]
2025-11-27T18:20:14.896858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471707280084361:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11102
2025-11-27T18:20:14.895917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471707726620546:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.896423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471703703650036:2048] [node 1] ICS09 handshake done sender: [4:7577471703703650025:2122] self: [4:7577471703703650033:0] peer: [1:7577471707726620617:0] socket: 390 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.896703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:14.897484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471705969689486:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:14.896665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471703703650036:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:14.896855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471707726620546:2048] [node 2] ICP22 created new session: [1:7577471707726620638:2048]
2025-11-27T18:20:14.896738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471703703649947:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.897824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471707280084343:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.896916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471707726620638:2048] [node 2] ICS09 handshake done sender: [1:7577471707726620614:2111] self: [1:7577471707726620613:0] peer: [2:7577471705969689502:0] socket: 381 qp: -1
2025-11-27T18:20:14.897802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471705969689480:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.897780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471705419475021:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52541
2025-11-27T18:20:14.899536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471705969689489:2131] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:14.896823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:14.897864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471707280084343:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.897079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471707726620638:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:14.900259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:14.901072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471707280084344:2131] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:14.897505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:14.900284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471705969689400:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.901057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471705969689480:2108] [node 5] ICP22 created new session: [2:7577471705969689509:2108]
2025-11-27T18:20:14.899801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707726620629:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.899973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471707726620547:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.902956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471705419475018:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:14.902465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471705969689509:2108] [node 5] ICS09 handshake done sender: [2:7577471705969689488:2130] self: [2:7577471705969689487:0] peer: [5:7577471705419475020:0] socket: 405 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.903007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471707280084254:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.900276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:14.902522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471705969689509:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:14.903038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471705419474939:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471705419475018:2117] held: no
2025-11-27T18:20:14.903292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471707280084343:2130] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:14.901489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471707726620547:2049] [node 3] ICP22 created new session: [1:7577471707726620640:2049]
2025-11-27T18:20:14.911453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471707732177357:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34930
2025-11-27T18:20:14.909381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:14.903098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:14.901542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471707726620640:2049] [node 3] ICS09 handshake done sender: [1:7577471707726620616:2112] self: [1:7577471707726620615:0] peer: [3:7577471707280084357:0] socket: 379 qp: -1
2025-11-27T18:20:14.903169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:14.909453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471707280084254:2048] [node 1] ICP22 created new session: [3:7577471707280084364:2048]
2025-11-27T18:20:14.911328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471707280084364:2048] [node 1] ICS09 handshake done sender: [3:7577471707280084344:2131] self: [3:7577471707280084357:0] peer: [1:7577471707726620615:0] socket: 388 qp: -1
2025-11-27T18:20:14.901573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471707726620640:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:14.909513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471705969689400:2049] [node 3] ICP22 created new session: [2:7577471705969689510:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.911371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471707280084364:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:14.912314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471707732177356:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.902881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:14.911326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471705969689399:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.911619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471707280084362:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11102
2025-11-27T18:20:14.914645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703703650040:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-27T18:20:14.912292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471707726620610:2107] [node 4] ICP18 handshake (actor [1:7577471707726620628:2118]) is too old
2025-11-27T18:20:14.914040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471707732177278:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.911512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471705969689510:2049] [node 3] ICS09 handshake done sender: [2:7577471705969689486:2129] self: [2:7577471705969689485:0] peer: [3:7577471707280084360:0] socket: 404 qp: -1
2025-11-27T18:20:14.911752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:14.911574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471705969689510:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:14.915116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471705419474939:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.914084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471707732177357:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.914911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471707280084332:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.911657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:14.914791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471707732177342:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.915277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:14.915337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471703703650030:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.915358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:14.911700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471705969689399:2048] [node 1] ICP22 created new session: [2:7577471705969689512:2048]
2025-11-27T18:20:14.915321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471705419474939:2049] [node 2] ICP22 created new session: [5:7577471705419475023:2049]
2025-11-27T18:20:14.915510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471707280084332:2107] [node 4] ICP22 created new session: [3:7577471707280084366:2107]
2025-11-27T18:20:14.915629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471705419475023:2049] [node 2] ICS09 handshake done sender: [5:7577471705419475018:2117] self: [5:7577471705419475020:0] peer: [2:7577471705969689487:0] socket: 408 qp: -1
2025-11-27T18:20:14.915866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471707280084366:2107] [node 4] ICS09 handshake done sender: [3:7577471707280084343:2130] self: [3:7577471707280084342:0] peer: [4:7577471703703650037:0] socket: 418 qp: -1
2025-11-27T18:20:14.911731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:14.915962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471705419475023:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:14.915971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471707280084366:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:14.911957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471705969689512:2048] [node 1] ICS09 handshake done sender: [2:7577471705969689489:2131] self: [2:7577471705969689502:0] peer: [1:7577471707726620613:0] socket: 386 qp: -1
2025-11-27T18:20:14.915381Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471703703650030:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.916149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:14.912226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471705969689512:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:14.915690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471703703650035:2126] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:14.916018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:14.913402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:14.917690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471703703649947:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.917783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:14.917848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471703703649947:2050] [node 3] ICP22 created new session: [4:7577471703703650041:2050]
2025-11-27T18:20:14.917891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471703703650041:2050] [node 3] ICS09 handshake done sender: [4:7577471703703650035:2126] self: [4:7577471703703650037:0] peer: [3:7577471707280084342:0] socket: 420 qp: -1
2025-11-27T18:20:14.918014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471703703650041:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:14.918093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:14.918136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471707280084354:2137] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:14.919370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707726620620:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.919420Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471707726620620:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.920297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471703703649945:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21481490ns) {HandleHandshakeStatus:329(21479628ns) {DropIncomingHandshake:442(788ns) DropOutgoingHandshake:457(5423ns) SwitchToState:179(371ns) ProcessPendingSessionEvents:495(18878145ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18767378ns max# 6299780ns}}}
2025-11-27T18:20:14.920182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471707726620546:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(20238945ns) {HandleHandshakeStatus:329(20237858ns) {DropIncomingHandshake:442(8875ns) DropOutgoingHandshake:457(256ns) SwitchToState:179(233ns) ProcessPendingSessionEvents:495(18356376ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18233450ns max# 6310873ns}}}
2025-11-27T18:20:14.920387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471707726620547:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17985808ns) {HandleHandshakeStatus:329(17983551ns) {DropIncomingHandshake:442(1073ns) DropOutgoingHandshake:457(742ns) SwitchToState:179(303ns) ProcessPendingSessionEvents:495(6955830ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6871294ns max# 1431393ns}}}
2025-11-27T18:20:14.920159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471707280084255:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.920470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471703703650030:2123] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:14.920444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:14.920483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703703650042:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-27T18:20:14.920494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471707280084255:2049] [node 2] ICP22 created new session: [3:7577471707280084370:2049]
2025-11-27T18:20:14.921387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471707280084370:2049] [node 2] ICS09 handshake done sender: [3:7577471707280084354:2137] self: [3:7577471707280084360:0] peer: [2:7577471705969689485:0] socket: 410 qp: -1
2025-11-27T18:20:14.921416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471707280084370:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:14.922274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.922358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471705969689480:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(20330460ns) {HandleHandshakeStatus:329(20327723ns) {DropIncomingHandshake:442(15346ns) DropOutgoingHandshake:457(232ns) SwitchToState:179(186ns) ProcessPendingSessionEvents:495(6580597ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6386942ns max# 1538235ns}}}
2025-11-27T18:20:14.922366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471703703650017:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.922423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:14.922464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471703703650017:2109] [node 6] ICP22 created new session: [4:7577471703703650045:2109]
2025-11-27T18:20:14.922465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471707732177360:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34930
2025-11-27T18:20:14.922505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471703703650045:2109] [node 6] ICS09 handshake done sender: [4:7577471703703650030:2123] self: [4:7577471703703650029:0] peer: [6:7577471707732177359:0] socket: 398 qp: -1
2025-11-27T18:20:14.922532Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471703703649947:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:14.922552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471703703650045:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:14.922639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.923094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471705969689400:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(19204178ns) {HandleHandshakeStatus:329(19202553ns) {DropIncomingHandshake:442(14117ns) DropOutgoingHandshake:457(175ns) SwitchToState:179(320ns) ProcessPendingSessionEvents:495(5441890ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4605758ns max# 756484ns}}}
2025-11-27T18:20:14.923067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471707726620620:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:14.924405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471705419475025:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52541
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.927048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471707732177357:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:14.928865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471703703650042:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.929004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471705419475026:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52541
2025-11-27T18:20:14.929159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471707732177342:2109] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.930374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471707732177361:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34930
2025-11-27T18:20:14.930407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:14.931060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471705419475026:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.930410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471707732177362:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34930
2025-11-27T18:20:14.930693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471707732177342:2109] [node 4] ICP22 created new session: [6:7577471707732177364:2109]
2025-11-27T18:20:14.931059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471707732177364:2109] [node 4] ICS09 handshake done sender: [6:7577471707732177357:2116] self: [6:7577471707732177359:0] peer: [4:7577471703703650029:0] socket: 432 qp: -1
2025-11-27T18:20:14.931716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471707726620611:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.931776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:14.931366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471705419474940:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.931192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471707732177364:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:14.931829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471707726620611:2110] [node 6] ICP22 created new session: [1:7577471707726620644:2110]
2025-11-27T18:20:14.931682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:14.931693Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471705969689399:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:14.931839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471707726620547:2049] [node 3] ICP18 handshake (actor [1:7577471707726620629:2119]) is too old
2025-11-27T18:20:14.931816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471707732177361:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.931974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471707280084349:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.931877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471707726620644:2110] [node 6] ICS09 handshake done sender: [1:7577471707726620620:2114] self: [1:7577471707726620619:0] peer: [6:7577471707732177358:0] socket: 422 qp: -1
2025-11-27T18:20:14.932101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471707732177356:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:14.932070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471707726620644:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:14.932312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471705419475025:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.932228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471707732177363:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34930
2025-11-27T18:20:14.932260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:14.932028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471707280084349:2134] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.933401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471707732177278:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.932809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471705419474938:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.933587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.934742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471705419475028:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52541
2025-11-27T18:20:14.933717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471707732177278:2048] [node 1] ICP22 created new session: [6:7577471707732177366:2048]
2025-11-27T18:20:14.934279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471707732177366:2048] [node 1] ICS09 handshake done sender: [6:7577471707732177356:2115] self: [6:7577471707732177358:0] peer: [1:7577471707726620619:0] socket: 423 qp: -1
2025-11-27T18:20:14.934317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471707732177366:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:14.934345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471707732177280:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.934889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:14.936988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703703650048:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-27T18:20:14.937916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471707732177363:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.937943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471703703650048:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.938058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471707732177279:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.938100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471703703649946:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.937981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471707280084347:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.938033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471707280084347:2133] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.938202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471707726620625:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.942382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705969689496:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.938241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471707726620625:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.943453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471707280084349:2134] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:14.942443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471705969689496:2133] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.942747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471705969689497:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.942776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471705969689497:2134] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.944848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471707726620625:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:14.944665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471705969689496:2133] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:14.945377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471707280084347:2133] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:14.945330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471705419475030:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52541
2025-11-27T18:20:14.945215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471705969689497:2134] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:14.945989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471707726620612:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.946002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471705419475028:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.945405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471707280084337:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.945621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471705969689482:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.946960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:14.945791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:14.946325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.947211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471705419475003:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.947045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471707726620612:2109] [node 5] ICP22 created new session: [1:7577471707726620647:2109]
2025-11-27T18:20:14.946014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471705969689482:2111] [node 6] ICP22 created new session: [2:7577471705969689519:2111]
2025-11-27T18:20:14.949162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471703703650032:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.946963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471707280084337:2108] [node 5] ICP22 created new session: [3:7577471707280084374:2108]
2025-11-27T18:20:14.949757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471705419475026:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:14.947938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471707726620647:2109] [node 5] ICS09 handshake done sender: [1:7577471707726620625:2115] self: [1:7577471707726620624:0] peer: [5:7577471705419475029:0] socket: 409 qp: -1
2025-11-27T18:20:14.949212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471703703650032:2124] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.946544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471705969689519:2111] [node 6] ICS09 handshake done sender: [2:7577471705969689496:2133] self: [2:7577471705969689494:0] peer: [6:7577471707732177370:0] socket: 438 qp: -1
2025-11-27T18:20:14.948333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471707726620647:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:14.947267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471707280084374:2108] [node 5] ICS09 handshake done sender: [3:7577471707280084349:2134] self: [3:7577471707280084348:0] peer: [5:7577471705419475027:0] socket: 434 qp: -1
2025-11-27T18:20:14.949859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471703703650051:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54799
2025-11-27T18:20:14.947312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471707280084336:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.946581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471705969689519:2111] [node 6] ICS10 traffic start
2025-11-27T18:20:14.949790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:14.947538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:14.954603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471703703650048:2129] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:14.947209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.961480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471705419474940:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24107) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471705419475026:2120] held: no
2025-11-27T18:20:14.947542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471707280084374:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:14.957355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471703703649946:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471703703650048:2129] held: no
2025-11-27T18:20:14.963210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471705419474938:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24038) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471705419475025:2119] held: no
2025-11-27T18:20:14.949183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471705969689481:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.963835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471703703650032:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:14.964144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471705419475031:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52541
2025-11-27T18:20:14.947741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471707280084336:2125] [node 6] ICP22 created new session: [3:7577471707280084375:2125]
2025-11-27T18:20:14.949238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:14.949319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471705969689481:2107] [node 4] ICP22 created new session: [2:7577471705969689521:2107]
2025-11-27T18:20:14.948009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:14.949678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471707280084375:2125] [node 6] ICS09 handshake done sender: [3:7577471707280084347:2133] self: [3:7577471707280084346:0] peer: [6:7577471707732177367:0] socket: 415 qp: -1
2025-11-27T18:20:14.965331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471703703650018:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.950093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471707280084375:2125] [node 6] ICS10 traffic start
2025-11-27T18:20:14.953617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471705969689521:2107] [node 4] ICS09 handshake done sender: [2:7577471705969689497:2134] self: [2:7577471705969689495:0] peer: [4:7577471703703650050:0] socket: 440 qp: -1
2025-11-27T18:20:14.967645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471705419474940:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.954267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471705969689521:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:14.967740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471707732177279:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471707732177363:2120] held: no
2025-11-27T18:20:14.965553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471703703649946:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.954320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:14.950129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:14.966026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:14.966077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471703703649946:2049] [node 2] ICP22 created new session: [4:7577471703703650053:2049]
2025-11-27T18:20:14.967350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:14.968198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:14.968288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471705419474940:2050] [node 3] ICP22 created new session: [5:7577471705419475034:2050]
2025-11-27T18:20:14.967687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471703703650018:2108] [node 5] ICP22 created new session: [4:7577471703703650054:2108]
2025-11-27T18:20:14.967753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471703703650053:2049] [node 2] ICS09 handshake done sender: [4:7577471703703650048:2129] self: [4:7577471703703650050:0] peer: [2:7577471705969689495:0] socket: 397 qp: -1
2025-11-27T18:20:14.967881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471703703650054:2108] [node 5] ICS09 handshake done sender: [4:7577471703703650032:2124] self: [4:7577471703703650031:0] peer: [5:7577471705419475032:0] socket: 412 qp: -1
2025-11-27T18:20:14.968329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471705419475034:2050] [node 3] ICS09 handshake done sender: [5:7577471705419475026:2120] self: [5:7577471705419475027:0] peer: [3:7577471707280084348:0] socket: 435 qp: -1
2025-11-27T18:20:14.967953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471703703650053:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:14.968390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471705419475034:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:14.968214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471703703650054:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:14.968476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:14.968243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:14.968262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:14.968642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471705419475025:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.970297Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471707732177280:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24107) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471707732177361:2118] held: no
2025-11-27T18:20:14.970631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471707732177371:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34930
2025-11-27T18:20:14.971153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471705969689482:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(22925454ns) {HandleHandshakeStatus:329(22921416ns) {DropIncomingHandshake:442(14878ns) DropOutgoingHandshake:457(2366ns) SwitchToState:179(315ns) ProcessPendingSessionEvents:495(19139831ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18118590ns max# 3873801ns}}}
2025-11-27T18:20:14.971235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471705419475003:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471705419475028:2121] held: no
2025-11-27T18:20:14.971613Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471707726620612:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(22571452ns) {HandleHandshakeStatus:329(22462722ns) {DropIncomingHandshake:442(250079ns) DropOutgoingHandshake:457(503ns) SwitchToState:179(230ns) ProcessPendingSessionEvents:495(17459611ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17229666ns max# 4761707ns}}}
2025-11-27T18:20:14.971523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471705969689481:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(19979917ns) {HandleHandshakeStatus:329(19977844ns) {DropIncomingHandshake:442(12791ns) DropOutgoingHandshake:457(266ns) SwitchToState:179(358ns) ProcessPendingSessionEvents:495(7387121ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6698412ns max# 1287625ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:14.971819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471705419474938:2048] [node 1] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:14.972033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471705419475033:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52541
2025-11-27T18:20:14.972216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:14.972216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471707280084337:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(22949988ns) {HandleHandshakeStatus:329(22947737ns) {DropIncomingHandshake:442(24839ns) DropOutgoingHandshake:457(448ns) SwitchToState:179(348ns) ProcessPendingSessionEvents:495(18069314ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17790131ns max# 3993502ns}}}
2025-11-27T18:20:14.972278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471705419474938:2048] [node 1] ICP22 created new session: [5:7577471705419475037:2048]
2025-11-27T18:20:14.972391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471705419475037:2048] [node 1] ICS09 handshake done sender: [5:7577471705419475025:2119] self: [5:7577471705419475029:0] peer: [1:7577471707726620624:0] socket: 411 qp: -1
2025-11-27T18:20:14.972342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471707732177372:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34930
2025-11-27T18:20:14.972442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471705419475037:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:14.972475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:14.973058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471707732177361:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:14.973115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471707732177363:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:14.973370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471705419475028:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:14.973452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471707732177279:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.973619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:14.973756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471707732177279:2049] [node 2] ICP22 created new session: [6:7577471707732177374:2049]
2025-11-27T18:20:14.973848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471707732177373:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34930
2025-11-27T18:20:14.973917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471707732177374:2049] [node 2] ICS09 handshake done sender: [6:7577471707732177363:2120] self: [6:7577471707732177370:0] peer: [2:7577471705969689494:0] socket: 441 qp: -1
2025-11-27T18:20:14.973974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471707732177374:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:14.974007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:14.974985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471707732177280:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.975067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:14.975111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471707732177280:2050] [node 3] ICP22 created new session: [6:7577471707732177376:2050]
2025-11-27T18:20:14.975150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471707732177376:2050] [node 3] ICS09 handshake done sender: [6:7577471707732177361:2118] self: [6:7577471707732177367:0] peer: [3:7577471707280084346:0] socket: 436 qp: -1
2025-11-27T18:20:14.975182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471707732177376:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:14.975235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:14.976218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471705419475003:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.976311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:14.976354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471705419475003:2108] [node 4] ICP22 created new session: [5:7577471705419475039:2108]
2025-11-27T18:20:14.976414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471705419475039:2108] [node 4] ICS09 handshake done sender: [5:7577471705419475028:2121] self: [5:7577471705419475032:0] peer: [4:7577471703703650031:0] socket: 442 qp: -1
2025-11-27T18:20:14.976460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471705419475039:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:14.976494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:14.977950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471707732177373:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.978108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471707732177340:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:14.978329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471705419475014:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:14.978363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471705419475014:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:14.979423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471705419475014:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:14.979609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471707732177379:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34930
2025-11-27T18:20:14.979708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471705419475002:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:14.980022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:14.980080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471705419475002:2110] [node 6] ICP22 created new session: [5:7577471705419475042:2110]
2025-11-27T18:20:14.980138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471705419475042:2110] [node 6] ICS09 handshake done sender: [5:7577471705419475014:2116] self: [5:7577471705419475013:0] peer: [6:7577471707732177378:0] socket: 382 qp: -1
2025-11-27T18:20:14.980170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471705419475042:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:14.980153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471707732177373:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:14.980373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:14.980720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471707732177340:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15821) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471707732177373:2123] held: no
2025-11-27T18:20:14.982702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471707732177340:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:14.982775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:14.982831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471707732177340:2107] [node 5] ICP22 created new session: [6:7577471707732177380:2107]
2025-11-27T18:20:14.982880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471707732177380:2107] [node 5] ICS09 handshake done sender: [6:7577471707732177373:2123] self: [6:7577471707732177378:0] peer: [5:7577471705419475013:0] socket: 389 qp: -1
2025-11-27T18:20:14.982915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471707732177380:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:14.982971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:15.027611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:15.027880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471705969689512:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-11-27T18:20:15.027926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471705969689399:2048] [node 1] ICP30 unregister session Session# [2:7577471705969689512:2048] VirtualId# [2:7577471705969689502:0]
2025-11-27T18:20:15.027961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471705969689512:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.028050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:15.028136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:15.028850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:15.029073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471707280084364:2048] [node 1] ICS01 socket: 388 reason# EndOfStream
2025-11-27T18:20:15.029132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471707280084254:2048] [node 1] ICP30 unregister session Session# [3:7577471707280084364:2048] VirtualId# [3:7577471707280084357:0]
2025-11-27T18:20:15.029162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471707280084364:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.029246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:15.029353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:15.031554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:15.031778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471703703650036:2048] [node 1] ICS07 socket disconnect 390 reason# EndOfStream
2025-11-27T18:20:15.031813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471703703650036:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.031908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:15.031982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471703703650036:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:15.032512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:15.032739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471705419475037:2048] [node 1] ICS01 socket: 411 reason# EndOfStream
2025-11-27T18:20:15.032792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471705419474938:2048] [node 1] ICP30 unregister session Session# [5:7577471705419475037:2048] VirtualId# [5:7577471705419475029:0]
2025-11-27T18:20:15.032816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471705419475037:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.032835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471703703649945:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24038) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:15.032950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:15.032895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471703703650036:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:15.032989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:15.033033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471703703649945:2048] [node 1] ICP30 unregister session Session# [4:7577471703703650036:2048] VirtualId# [4:7577471703703650033:0]
2025-11-27T18:20:15.033364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:15.033411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471703703649945:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24038) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:15.033435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24038) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:15.033701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:15.033900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471707732177366:2048] [node 1] ICS07 socket disconnect 423 reason# EndOfStream
2025-11-27T18:20:15.033934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471707732177366:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.033998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:15.034045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471707732177366:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:15.034745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471707732177278:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24038) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:15.034849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471707732177366:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:15.034899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471707732177278:2048] [node 1] ICP30 unregister session Session# [6:7577471707732177366:2048] VirtualId# [6:7577471707732177358:0]
2025-11-27T18:20:15.034997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:15.035061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471707732177278:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24038) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:15.035081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24038) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:15.042903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471703703649945:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:15.044895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471707732177278:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:15.096437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:15.096672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471707280084370:2049] [node 2] ICS07 socket disconnect 410 reason# EndOfStream
2025-11-27T18:20:15.096700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471707280084370:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.096775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:15.096824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471707280084370:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:15.098940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471707280084255:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29204) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:15.098979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471707280084370:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:15.100204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:15.100426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471703703650053:2049] [node 2] ICS01 socket: 397 reason# EndOfStream
2025-11-27T18:20:15.100477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471703703649946:2049] [node 2] ICP30 unregister session Session# [4:7577471703703650053:2049] VirtualId# [4:7577471703703650050:0]
2025-11-27T18:20:15.100510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471703703650053:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.100619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:15.100678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:15.100730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471707280084255:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29204) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:15.100832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471707280084370:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:15.100878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471707280084255:2049] [node 2] ICP30 unregister session Session# [3:7577471707280084370:2049] VirtualId# [3:7577471707280084360:0]
2025-11-27T18:20:15.100939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:15.100972Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471707280084255:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29204) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:15.100986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29204) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:15.101777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:15.102014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471705419475023:2049] [node 2] ICS01 socket: 408 reason# EndOfStream
2025-11-27T18:20:15.102055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471705419474939:2049] [node 2] ICP30 unregister session Session# [5:7577471705419475023:2049] VirtualId# [5:7577471705419475020:0]
2025-11-27T18:20:15.102098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471705419475023:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.102165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:15.102227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:15.102927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:15.103120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471707732177374:2049] [node 2] ICS01 socket: 441 reason# EndOfStream
2025-11-27T18:20:15.103176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471707732177279:2049] [node 2] ICP30 unregister session Session# [6:7577471707732177374:2049] VirtualId# [6:7577471707732177370:0]
2025-11-27T18:20:15.103210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471707732177374:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.103273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:15.103301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:15.179748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:15.179971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471703703650041:2050] [node 3] ICS07 socket disconnect 420 reason# EndOfStream
2025-11-27T18:20:15.180009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471703703650041:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.180091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:15.180134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471703703650041:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:15.180986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471703703649947:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24107) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:15.181053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471703703650041:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:15.181098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471703703649947:2050] [node 3] ICP30 unregister session Session# [4:7577471703703650041:2050] VirtualId# [4:7577471703703650037:0]
2025-11-27T18:20:15.181131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:15.181300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:15.181371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471705419475034:2050] [node 3] ICS01 socket: 435 reason# EndOfStream
2025-11-27T18:20:15.181372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471703703649947:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24107) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:15.181421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471705419474940:2050] [node 3] ICP30 unregister session Session# [5:7577471705419475034:2050] VirtualId# [5:7577471705419475027:0]
2025-11-27T18:20:15.181395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24107) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:15.181454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471705419475034:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.181541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:15.181592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:15.182355Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:15.182533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471707732177376:2050] [node 3] ICS01 socket: 436 reason# EndOfStream
2025-11-27T18:20:15.182568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471707732177280:2050] [node 3] ICP30 unregister session Session# [6:7577471707732177376:2050] VirtualId# [6:7577471707732177367:0]
2025-11-27T18:20:15.182600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471707732177376:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.182663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:15.182706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:15.245924Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:15.246150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471705419475039:2108] [node 4] ICS01 socket: 442 reason# EndOfStream
2025-11-27T18:20:15.246213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471705419475003:2108] [node 4] ICP30 unregister session Session# [5:7577471705419475039:2108] VirtualId# [5:7577471705419475032:0]
2025-11-27T18:20:15.246252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471705419475039:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.246347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:15.246430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:15.246841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:15.247068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471707732177364:2109] [node 4] ICS07 socket disconnect 432 reason# EndOfStream
2025-11-27T18:20:15.247105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471707732177364:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.247208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:15.247281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471707732177364:2109] [node 4] ICS15 start handshake
2025-11-27T18:20:15.248001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471707732177342:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:15.248042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471707732177364:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:15.248136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471707732177342:2109] [node 4] ICP30 unregister session Session# [6:7577471707732177364:2109] VirtualId# [6:7577471707732177359:0]
2025-11-27T18:20:15.248278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:15.248330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471707732177342:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:15.248345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4054) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:15.257944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471707732177342:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:20:15.330669Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:15.330892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471707732177380:2107] [node 5] ICS01 socket: 389 reason# EndOfStream
2025-11-27T18:20:15.330950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471707732177340:2107] [node 5] ICP30 unregister session Session# [6:7577471707732177380:2107] VirtualId# [6:7577471707732177378:0]
2025-11-27T18:20:15.331004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471707732177380:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:15.331107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:15.331159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:15.431578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471711977844489:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:15.431593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471711977844488:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:15.455704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471710743685247:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:15.455772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471710743685248:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:15.478918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471708806378771:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:15.478943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471708806378770:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:15.500981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471709757529134:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:15.501298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471709757529136:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:15.501525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471709757529135:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:15.523871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471711069462236:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:15.523920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471711069462237:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:15.524167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471711069462238:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:15.547720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471711971986896:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:15.547976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471711971986897:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:15.548117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471711971986898:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:15.636143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471711971986957:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:15.642282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471711977844553:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:15.675556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471708806378860:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:15.670606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471711971986961:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:15.683213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471710743685311:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:15.670910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471711977844554:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:15.685962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471709757529215:2116] [node 6] ICP01 ready to work
2025-11-27T18:20:15.686385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471708806378862:2125] [node 6] ICP01 ready to work
2025-11-27T18:20:15.686085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471710743685313:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:15.685831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471711069462300:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:15.683225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471711977844555:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:15.704863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471709757529216:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:15.702230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471708806378861:2115] [node 5] ICP01 ready to work
2025-11-27T18:20:15.709780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471711069462301:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:15.693365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471710743685312:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:15.769854Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471711971986898:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26600010ns) {Configure:104(26594439ns) {ProcessConfigured:135(16779137ns) {StartInitialHandshake:157(16775405ns) {DropHandshakes:470(6814ns) {DropIncomingHandshake:442(1088ns) DropOutgoingHandshake:457(3558ns)} GenerateSessionVirtualId:988(1138ns) PrepareNewSessionHandshake:477(5051ns) {SwitchToState:179(2654ns)}}}}}
2025-11-27T18:20:15.791047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471710743685247:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(53416430ns) {Configure:104(53402376ns) {ProcessConfigured:135(92192ns) {StartInitialHandshake:157(85396ns) {DropHandshakes:470(3543ns) {DropIncomingHandshake:442(885ns) DropOutgoingHandshake:457(508ns)} GenerateSessionVirtualId:988(893ns) PrepareNewSessionHandshake:477(1889ns) {SwitchToState:179(800ns)}}}}}
2025-11-27T18:20:15.815407Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471710743685248:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(95723126ns) {Configure:104(95712462ns) {ProcessConfigured:135(5108012ns) {StartInitialHandshake:157(5105730ns) {DropHandshakes:470(1511ns) {DropIncomingHandshake:442(529ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(547ns) PrepareNewSessionHandshake:477(3234ns) {SwitchToState:179(1892ns)}}}}}
2025-11-27T18:20:15.815773Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471711977844554:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(106794561ns) {Configure:104(106789599ns) {ProcessConfigured:135(11263599ns) {StartInitialHandshake:157(11261878ns) {DropHandshakes:470(817ns) {DropIncomingHandshake:442(233ns) DropOutgoingHandshake:457(119ns)} GenerateSessionVirtualId:988(138ns) PrepareNewSessionHandshake:477(1619ns) {SwitchToState:179(42ns)}}}}}
2025-11-27T18:20:15.815749Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471711971986897:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(109862748ns) {Configure:104(109854953ns) {ProcessConfigured:135(11065741ns) {StartInitialHandshake:157(11064187ns) {DropHandshakes:470(671ns) {DropIncomingHandshake:442(215ns) DropOutgoingHandshake:457(104ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(1912ns) {SwitchToState:179(776ns)}}}}}
2025-11-27T18:20:15.818108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471709757529135:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(119101900ns) {Configure:104(119093181ns) {ProcessConfigured:135(5371082ns) {StartInitialHandshake:157(5368436ns) {DropHandshakes:470(2393ns) {DropIncomingHandshake:442(827ns) DropOutgoingHandshake:457(398ns)} GenerateSessionVirtualId:988(784ns) PrepareNewSessionHandshake:477(9307ns) {SwitchToState:179(5712ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.822724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471711069462238:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(89601685ns) {Configure:104(89593956ns) {ProcessConfigured:135(4757542ns) {StartInitialHandshake:157(4755035ns) {DropHandshakes:470(3363ns) {DropIncomingHandshake:442(786ns) DropOutgoingHandshake:457(493ns)} GenerateSessionVirtualId:988(666ns) PrepareNewSessionHandshake:477(1415ns) {SwitchToState:179(1010ns)}}}}}
2025-11-27T18:20:15.821805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471709757529134:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(117999959ns) {Configure:104(117992636ns) {ProcessConfigured:135(10953349ns) {StartInitialHandshake:157(6484750ns) {DropHandshakes:470(3508ns) {DropIncomingHandshake:442(736ns) DropOutgoingHandshake:457(496ns)} GenerateSessionVirtualId:988(1249ns) PrepareNewSessionHandshake:477(2591ns) {SwitchToState:179(1537ns)}}}}}
2025-11-27T18:20:15.817718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471711977844489:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(127469907ns) {Configure:104(127461708ns) {ProcessConfigured:135(12064283ns) {StartInitialHandshake:157(12060533ns) {DropHandshakes:470(2976ns) {DropIncomingHandshake:442(732ns) DropOutgoingHandshake:457(502ns)} GenerateSessionVirtualId:988(1463ns) PrepareNewSessionHandshake:477(4852ns) {SwitchToState:179(2617ns)}}}}}
2025-11-27T18:20:15.825843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471709757529217:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-11-27T18:20:15.825471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471711977844488:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(105348066ns) {Configure:104(105340250ns) {ProcessConfigured:135(3714174ns) {StartInitialHandshake:157(3712608ns) {DropHandshakes:470(1353ns) {DropIncomingHandshake:442(510ns) DropOutgoingHandshake:457(329ns)} GenerateSessionVirtualId:988(447ns) PrepareNewSessionHandshake:477(3215ns) {SwitchToState:179(1634ns)}}}}}
2025-11-27T18:20:15.826330Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471708806378771:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(133875830ns) {Configure:104(133867324ns) {ProcessConfigured:135(3942562ns) {StartInitialHandshake:157(3940544ns) {DropHandshakes:470(2284ns) {DropIncomingHandshake:442(763ns) DropOutgoingHandshake:457(370ns)} GenerateSessionVirtualId:988(745ns) PrepareNewSessionHandshake:477(2212ns) {SwitchToState:179(896ns)}}}}}
2025-11-27T18:20:15.825747Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471711069462236:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(113265689ns) {Configure:104(113253101ns) {ProcessConfigured:135(4171514ns) {StartInitialHandshake:157(4169375ns) {DropHandshakes:470(770485ns) {DropIncomingHandshake:442(640ns) DropOutgoingHandshake:457(1397ns)} GenerateSessionVirtualId:988(1775ns) PrepareNewSessionHandshake:477(2192ns) {SwitchToState:179(918ns)}}}}}
2025-11-27T18:20:15.825868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471709757529136:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(86312364ns) {Configure:104(86306159ns) {ProcessConfigured:135(2639896ns) {StartInitialHandshake:157(2637633ns) {DropHandshakes:470(4422ns) {DropIncomingHandshake:442(605ns) DropOutgoingHandshake:457(261ns)} GenerateSessionVirtualId:988(531ns) PrepareNewSessionHandshake:477(3113ns) {SwitchToState:179(1468ns)}}}}}
2025-11-27T18:20:15.830001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471708806378860:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(68307558ns) {Configure:104(68301476ns) {ProcessConfigured:135(3181603ns) {StartInitialHandshake:157(3178436ns) {DropHandshakes:470(2590ns) {DropIncomingHandshake:442(724ns) DropOutgoingHandshake:457(369ns)} GenerateSessionVirtualId:988(1684ns) PrepareNewSessionHandshake:477(3349ns) {SwitchToState:179(1824ns)}}}}}
2025-11-27T18:20:15.831417Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471711069462237:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(68657094ns) {Configure:104(68649289ns) {ProcessConfigured:135(711462ns) {StartInitialHandshake:157(708669ns) {DropHandshakes:470(307197ns) {DropIncomingHandshake:442(1179ns) DropOutgoingHandshake:457(442ns)} GenerateSessionVirtualId:988(990ns) PrepareNewSessionHandshake:477(1894ns) {SwitchToState:179(912ns)}}}}}
2025-11-27T18:20:15.831581Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471708806378770:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(81220305ns) {Configure:104(81211253ns) {ProcessConfigured:135(3774134ns) {StartInitialHandshake:157(3771975ns) {DropHandshakes:470(1305ns) {DropIncomingHandshake:442(402ns) DropOutgoingHandshake:457(164ns)} GenerateSessionVirtualId:988(420ns) PrepareNewSessionHandshake:477(1906ns) {SwitchToState:179(909ns)}}}}}
2025-11-27T18:20:15.833634Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471711971986957:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(71854219ns) {Configure:104(71849032ns) {ProcessConfigured:135(3302659ns) {StartInitialHandshake:157(3299901ns) {DropHandshakes:470(1116ns) {DropIncomingHandshake:442(349ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(424ns) PrepareNewSessionHandshake:477(2341ns) {SwitchToState:179(1022ns)}}}}}
2025-11-27T18:20:15.834346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471710743685312:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(33914584ns) {Configure:104(33906696ns) {ProcessConfigured:135(2298613ns) {StartInitialHandshake:157(2296626ns) {DropHandshakes:470(1634ns) {DropIncomingHandshake:442(374ns) DropOutgoingHandshake:457(262ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:477(2393ns) {SwitchToState:179(1455ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.836774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471711977844553:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(48029948ns) {Configure:104(40022798ns) {ProcessConfigured:135(2957787ns) {StartInitialHandshake:157(2955805ns) {DropHandshakes:470(2545ns) {DropIncomingHandshake:442(779ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:477(4042ns) {SwitchToState:179(2117ns)}}}}}
2025-11-27T18:20:15.839334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471708806378861:2115] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(54007227ns) {Configure:104(50024885ns) {ProcessConfigured:135(1485881ns) {StartInitialHandshake:157(1484426ns) {DropHandshakes:470(1372ns) {DropIncomingHandshake:442(347ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(2384ns) {SwitchToState:179(1143ns)}}}}}
2025-11-27T18:20:15.838936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471711069462302:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21597
2025-11-27T18:20:15.841437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471709757529217:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.841918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471711069462302:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.844431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471711069462237:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.841972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471711977844562:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
2025-11-27T18:20:15.844946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471709757529135:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.842727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471711977844566:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.844986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471711977844563:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
2025-11-27T18:20:15.851395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471710743685317:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.849711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471709757529224:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-11-27T18:20:15.849342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471711977844562:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.851442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471710743685317:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:15.852771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471709757529224:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.850808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471711977844563:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.852494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471710743685315:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.855680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471709757529216:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18933829ns) {Configure:104(18922201ns) {ProcessConfigured:135(1389869ns) {StartInitialHandshake:157(1387122ns) {DropHandshakes:470(3023ns) {DropIncomingHandshake:442(414ns) DropOutgoingHandshake:457(392ns)} GenerateSessionVirtualId:988(511ns) PrepareNewSessionHandshake:477(3231ns) {SwitchToState:179(1677ns)}}}}}
2025-11-27T18:20:15.856423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471711069462300:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19711531ns) {Configure:104(19703108ns) {ProcessConfigured:135(2150836ns) {StartInitialHandshake:157(2149009ns) {DropHandshakes:470(344783ns) {DropIncomingHandshake:442(1288ns) DropOutgoingHandshake:457(190ns)} GenerateSessionVirtualId:988(1269ns) PrepareNewSessionHandshake:477(1369ns) {SwitchToState:179(601ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.852528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471710743685315:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.856695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471708806378869:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42045
2025-11-27T18:20:15.852741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471711977844566:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:15.857716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471708806378873:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42045
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.856819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471710743685327:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21550
2025-11-27T18:20:15.857049Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471709757529215:2116] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21629792ns) {Configure:104(21620721ns) {ProcessConfigured:135(2600413ns) {StartInitialHandshake:157(2598442ns) {DropHandshakes:470(1122ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(159ns)} GenerateSessionVirtualId:988(532ns) PrepareNewSessionHandshake:477(1932ns) {SwitchToState:179(879ns)}}}}}
2025-11-27T18:20:15.853906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471711977844570:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
2025-11-27T18:20:15.857085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471710743685328:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21550
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.857117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471711069462315:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21597
2025-11-27T18:20:15.858235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471708806378872:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42045
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.858847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471708806378873:2143] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.856296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471711977844555:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19435289ns) {Configure:104(19430586ns) {ProcessConfigured:135(1155061ns) {StartInitialHandshake:157(1152794ns) {DropHandshakes:470(1423ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(384ns)} GenerateSessionVirtualId:988(712ns) PrepareNewSessionHandshake:477(1888ns) {SwitchToState:179(792ns)}}}}}
2025-11-27T18:20:15.859268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471711069462315:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.858898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471708806378869:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.859415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471711977844575:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
2025-11-27T18:20:15.859559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471708806378862:2125] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16786714ns) {Configure:104(16779556ns) {ProcessConfigured:135(1120045ns) {StartInitialHandshake:157(1117916ns) {DropHandshakes:470(2076ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(685ns)} GenerateSessionVirtualId:988(537ns) PrepareNewSessionHandshake:477(1856ns) {SwitchToState:179(841ns)}}}}}
2025-11-27T18:20:15.859561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471708806378877:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42045
2025-11-27T18:20:15.859605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471711069462236:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.859883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471711977844570:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.857617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471710743685317:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.857619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471710743685315:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:15.860017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471708806378770:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.860540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471711069462318:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21597
2025-11-27T18:20:15.860819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471711977844559:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.860811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471708806378872:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.861119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471709757529233:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-11-27T18:20:15.860864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471711977844559:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:15.861061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471708806378877:2144] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.858252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471710743685329:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21550
2025-11-27T18:20:15.861450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471711977844558:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.859666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471710743685330:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21550
2025-11-27T18:20:15.861934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471711069462302:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:15.861491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471711977844558:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:15.859979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471710743685328:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.862237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471709757529235:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-11-27T18:20:15.863293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471711977844559:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:15.860032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471710743685311:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:15.862591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471711069462237:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.860070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:15.864485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471711977844561:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.860345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471710743685311:2109] [node 5] ICP22 created new session: [2:7577471710743685333:2109]
2025-11-27T18:20:15.864519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471711977844561:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:15.860843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471710743685327:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.860901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471710743685313:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:15.867545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:15.869416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471711971986975:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-27T18:20:15.868689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471711069462237:2049] [node 2] ICP22 created new session: [5:7577471711069462320:2049]
2025-11-27T18:20:15.860939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:15.869576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471709757529217:2128] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:15.869505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471711977844558:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:15.869917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471708806378884:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42045
2025-11-27T18:20:15.868948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471711069462320:2049] [node 2] ICS09 handshake done sender: [5:7577471711069462302:2111] self: [5:7577471711069462314:0] peer: [2:7577471710743685316:0] socket: 388 qp: -1
2025-11-27T18:20:15.861370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471710743685329:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.869572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471711977844561:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:15.870969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471709757529135:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:15.861371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471710743685313:2107] [node 4] ICP22 created new session: [2:7577471710743685334:2107]
2025-11-27T18:20:15.877803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471711971986975:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.869433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471711069462320:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:15.869817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471711977844554:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:15.884668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:15.861461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471710743685333:2109] [node 5] ICS09 handshake done sender: [2:7577471710743685317:2112] self: [2:7577471710743685316:0] peer: [5:7577471711069462314:0] socket: 386 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.869950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.883556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471711971986897:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.885339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471708806378884:2146] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.869934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:15.886120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471709757529135:2049] [node 2] ICP22 created new session: [4:7577471709757529237:2049]
2025-11-27T18:20:15.861488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471710743685334:2107] [node 4] ICS09 handshake done sender: [2:7577471710743685315:2111] self: [2:7577471710743685314:0] peer: [4:7577471709757529226:0] socket: 384 qp: -1
2025-11-27T18:20:15.886087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471708806378771:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.886176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471709757529233:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.888357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471711069462321:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21597
2025-11-27T18:20:15.870290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471711977844554:2109] [node 5] ICP22 created new session: [1:7577471711977844581:2109]
2025-11-27T18:20:15.861510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471710743685333:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:15.890642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471708806378866:2138] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.890689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471711069462236:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28863) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471711069462315:2116] held: no
2025-11-27T18:20:15.881509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471711977844581:2109] [node 5] ICS09 handshake done sender: [1:7577471711977844559:2113] self: [1:7577471711977844557:0] peer: [5:7577471711069462319:0] socket: 407 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.887807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471709757529237:2049] [node 2] ICS09 handshake done sender: [4:7577471709757529217:2128] self: [4:7577471709757529226:0] peer: [2:7577471710743685314:0] socket: 385 qp: -1
2025-11-27T18:20:15.861629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471710743685334:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:15.890668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471708806378885:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42045
2025-11-27T18:20:15.881564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471711977844581:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:15.887830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471709757529136:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.861665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471710743685247:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.890686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471708806378866:2138] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:15.887850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471709757529237:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:15.883912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471711977844575:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.861669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:15.888499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471709757529236:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-11-27T18:20:15.896596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471708806378873:2143] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.897971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471711971986978:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-27T18:20:15.883954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471711977844489:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:15.861678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:15.888549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:15.883999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:15.884045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471711977844489:2049] [node 3] ICP22 created new session: [1:7577471711977844582:2049]
2025-11-27T18:20:15.883573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471710743685330:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.897091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471711069462315:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:15.894968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471709757529236:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.884098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471711977844582:2049] [node 3] ICS09 handshake done sender: [1:7577471711977844558:2112] self: [1:7577471711977844556:0] peer: [3:7577471708806378881:0] socket: 406 qp: -1
2025-11-27T18:20:15.889387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471710743685326:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.899066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471711069462236:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:15.889424Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471710743685326:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:15.899236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471708806378866:2138] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:15.884131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471711977844582:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:15.899185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471711971986977:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-27T18:20:15.899218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:15.892615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471710743685324:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.901285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471708806378770:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:15.902998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471708806378860:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:15.884173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.903288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.892650Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471710743685324:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:15.901289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471711069462236:2048] [node 1] ICP22 created new session: [5:7577471711069462325:2048]
2025-11-27T18:20:15.884663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:15.905192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471711971986975:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.895848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471710743685340:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21550
2025-11-27T18:20:15.905992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471709757529242:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-11-27T18:20:15.902354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471711069462325:2048] [node 1] ICS09 handshake done sender: [5:7577471711069462315:2116] self: [5:7577471711069462319:0] peer: [1:7577471711977844557:0] socket: 411 qp: -1
2025-11-27T18:20:15.885394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471711977844488:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:15.904256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471708806378860:2107] [node 4] ICP22 created new session: [3:7577471708806378889:2107]
2025-11-27T18:20:15.906360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471709757529243:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-11-27T18:20:15.908370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471711971986897:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:15.904326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.897372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471710743685326:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:15.886177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:15.908432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:15.904379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471708806378770:2048] [node 1] ICP22 created new session: [3:7577471708806378890:2048]
2025-11-27T18:20:15.903272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471711069462325:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:15.887296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471711977844488:2048] [node 2] ICP22 created new session: [1:7577471711977844585:2048]
2025-11-27T18:20:15.908535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471709757529136:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15780) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471709757529233:2135] held: no
2025-11-27T18:20:15.898842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471710743685324:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:15.904895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471708806378889:2107] [node 4] ICS09 handshake done sender: [3:7577471708806378866:2138] self: [3:7577471708806378865:0] peer: [4:7577471709757529238:0] socket: 422 qp: -1
2025-11-27T18:20:15.904135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:15.904012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471710743685312:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:15.888061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471711977844585:2048] [node 2] ICS09 handshake done sender: [1:7577471711977844561:2114] self: [1:7577471711977844560:0] peer: [2:7577471710743685336:0] socket: 405 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.913995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471711971986979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-27T18:20:15.932535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471709757529242:2138] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.905128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471708806378890:2048] [node 1] ICS09 handshake done sender: [3:7577471708806378873:2143] self: [3:7577471708806378881:0] peer: [1:7577471711977844556:0] socket: 410 qp: -1
2025-11-27T18:20:15.905014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471710743685247:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28863) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471710743685328:2117] held: no
2025-11-27T18:20:15.908373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471711069462324:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21597
2025-11-27T18:20:15.914877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471711971986897:2049] [node 2] ICP22 created new session: [6:7577471711971986980:2049]
2025-11-27T18:20:15.888096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471711977844585:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:15.905954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471708806378889:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:15.929918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471711069462317:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.915586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471711971986979:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.905176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:15.888155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:15.953761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471709757529136:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(41223818ns) {HandleHandshakeStatus:395(41221247ns) {LogHandshakeFail:472(1696529ns) {UpdateErrorStateLog:545(7740ns)} DropOutgoingHandshake:457(3217ns) ProcessPendingSessionEvents:495(33237858ns) {unordered DropSessionEvent:512 num# 100 sum# 33181930ns max# 5597664ns}}}
2025-11-27T18:20:15.953948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471709757529233:2135] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:15.929959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471711069462317:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:15.921781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471711971986957:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.908060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471710743685312:2110] [node 6] ICP22 created new session: [2:7577471710743685344:2110]
2025-11-27T18:20:15.957441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471711977844574:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.906611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471708806378890:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:15.932643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471711069462324:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.954945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471709757529136:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:15.957503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471711977844574:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:15.906658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:15.921604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471710743685344:2110] [node 6] ICS09 handshake done sender: [2:7577471710743685326:2115] self: [2:7577471710743685325:0] peer: [6:7577471711971986976:0] socket: 431 qp: -1
2025-11-27T18:20:15.922711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471711971986980:2049] [node 2] ICS09 handshake done sender: [6:7577471711971986975:2116] self: [6:7577471711971986976:0] peer: [2:7577471710743685325:0] socket: 434 qp: -1
2025-11-27T18:20:15.950684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471711069462301:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.921671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471710743685344:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:15.955008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.914636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:15.923446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471711971986980:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:15.955189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471711069462317:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:15.925512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471708806378860:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(21473099ns) {HandleHandshakeStatus:329(21469501ns) {DropIncomingHandshake:442(14488ns) DropOutgoingHandshake:457(377ns) SwitchToState:179(594ns) ProcessPendingSessionEvents:495(606275ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 549117ns max# 66376ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.929402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:15.959683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471709757529136:2050] [node 3] ICP22 created new session: [4:7577471709757529244:2050]
2025-11-27T18:20:15.929130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.961745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471711069462237:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(67374462ns) {HandleHandshakeStatus:329(67372113ns) {DropIncomingHandshake:442(557ns) DropOutgoingHandshake:457(13550ns) SwitchToState:179(373ns) ProcessPendingSessionEvents:495(43886159ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 43752828ns max# 8375056ns}}}
2025-11-27T18:20:15.930066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471708806378888:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42045
2025-11-27T18:20:15.998101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471711977844565:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.931259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471710743685311:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(36700203ns) {HandleHandshakeStatus:329(36696989ns) {DropIncomingHandshake:442(224ns) DropOutgoingHandshake:457(465ns) SwitchToState:179(375ns) ProcessPendingSessionEvents:495(33391264ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 33277142ns max# 8327645ns}}}
2025-11-27T18:20:15.931158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471711971986978:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:15.961471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471709757529134:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.980626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471711069462300:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:15.932990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471708806378884:2146] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:15.931343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471710743685248:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:15.998163Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471711977844565:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:15.988804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471711069462329:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21597
2025-11-27T18:20:15.937528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471711971986896:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.931412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:15.945392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471708806378771:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:15.942498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471710743685248:2049] [node 3] ICP22 created new session: [2:7577471710743685346:2049]
2025-11-27T18:20:15.959625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:15.957842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471710743685346:2049] [node 3] ICS09 handshake done sender: [2:7577471710743685324:2114] self: [2:7577471710743685323:0] peer: [3:7577471708806378886:0] socket: 430 qp: -1
2025-11-27T18:20:15.965711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471708806378771:2049] [node 2] ICP22 created new session: [3:7577471708806378895:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.957903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471710743685346:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:15.965762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471709757529244:2050] [node 3] ICS09 handshake done sender: [4:7577471709757529233:2135] self: [4:7577471709757529238:0] peer: [3:7577471708806378865:0] socket: 424 qp: -1
2025-11-27T18:20:16.005682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471711977844565:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.989571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:15.971917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471708806378895:2049] [node 2] ICS09 handshake done sender: [3:7577471708806378884:2146] self: [3:7577471708806378886:0] peer: [2:7577471710743685323:0] socket: 432 qp: -1
2025-11-27T18:20:16.022036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471711977844489:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(37561646ns) {HandleHandshakeStatus:329(37558156ns) {DropIncomingHandshake:442(144ns) DropOutgoingHandshake:457(611ns) SwitchToState:179(373ns) ProcessPendingSessionEvents:495(37211400ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 37050057ns max# 7288346ns}}}
2025-11-27T18:20:15.980473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471709757529244:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:15.957950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:15.942867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471711971986984:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-27T18:20:16.022871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471711977844488:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(44155289ns) {HandleHandshakeStatus:329(43478201ns) {DropIncomingHandshake:442(14494ns) DropOutgoingHandshake:457(362ns) SwitchToState:179(1206ns) ProcessPendingSessionEvents:495(40608638ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 40469362ns max# 6952107ns}}}
2025-11-27T18:20:15.971966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471708806378895:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:16.025658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471711977844574:2120] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:15.997829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471711069462300:2109] [node 6] ICP22 created new session: [5:7577471711069462332:2109]
2025-11-27T18:20:15.980541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:15.971957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471710743685313:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(43372499ns) {HandleHandshakeStatus:329(43369293ns) {DropIncomingHandshake:442(464ns) DropOutgoingHandshake:457(359ns) SwitchToState:179(374ns) ProcessPendingSessionEvents:495(40950097ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 40779423ns max# 6838011ns}}}
2025-11-27T18:20:15.965761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471711971986984:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.046068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471711977844554:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(60919803ns) {HandleHandshakeStatus:329(60916993ns) {DropIncomingHandshake:442(38494ns) DropOutgoingHandshake:457(521ns) SwitchToState:179(600ns) ProcessPendingSessionEvents:495(40785352ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 40065615ns max# 6257685ns}}}
2025-11-27T18:20:15.992367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:16.011197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471711069462332:2109] [node 6] ICS09 handshake done sender: [5:7577471711069462317:2117] self: [5:7577471711069462316:0] peer: [6:7577471711971986981:0] socket: 389 qp: -1
2025-11-27T18:20:15.989365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471709757529230:2133] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.050034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471711977844555:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:15.968607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471711971986898:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:15.978734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471710743685312:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(73778282ns) {HandleHandshakeStatus:329(73774619ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(296ns) SwitchToState:179(555ns) ProcessPendingSessionEvents:495(36378383ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36286331ns max# 9033556ns}}}
2025-11-27T18:20:16.025305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471708806378879:2145] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.017853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471711069462332:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:15.989412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471709757529230:2133] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.025352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471708806378879:2145] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.018193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471711971986986:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-27T18:20:16.050105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:16.029528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:16.033897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471709757529135:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(60278033ns) {HandleHandshakeStatus:329(60275870ns) {DropIncomingHandshake:442(548ns) DropOutgoingHandshake:457(5984ns) SwitchToState:179(408ns) ProcessPendingSessionEvents:495(39162590ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39014978ns max# 7152099ns}}}
2025-11-27T18:20:15.979306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471710743685311:2109] [node 5] ICP18 handshake (actor [2:7577471710743685327:2116]) is too old
2025-11-27T18:20:16.050197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471711977844553:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.051801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471708806378871:2141] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.050667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471711971986979:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:16.035284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471711069462330:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21597
2025-11-27T18:20:16.049562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471711069462329:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.050979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471711971986988:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.049705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471711069462238:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:16.051320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471711971986957:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:16.051061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471711069462330:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.051955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471716266954285:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-27T18:20:16.050247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:15.980640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471710743685328:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:16.051484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471715364429630:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21597
2025-11-27T18:20:16.041928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471709757529230:2133] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:16.052131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:16.051843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471708806378871:2141] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.049950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471709757529136:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(86465336ns) {HandleHandshakeStatus:329(86462785ns) {DropIncomingHandshake:442(1147ns) DropOutgoingHandshake:457(373ns) SwitchToState:179(653ns) ProcessPendingSessionEvents:495(1858ns)}}
2025-11-27T18:20:16.009709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471710743685313:2107] [node 4] ICP18 handshake (actor [2:7577471710743685329:2118]) is too old
2025-11-27T18:20:16.064855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471711069462301:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28829) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471711069462324:2120] held: no
2025-11-27T18:20:16.050295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471711977844553:2107] [node 4] ICP22 created new session: [1:7577471716272811888:2107]
2025-11-27T18:20:16.050515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471710743685247:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(33580157ns) {HandleHandshakeStatus:395(33114291ns) {LogHandshakeFail:472(16641ns) {UpdateErrorStateLog:545(6133ns)} DropOutgoingHandshake:457(967ns) ProcessPendingSessionEvents:495(32494029ns) {unordered DropSessionEvent:512 num# 100 sum# 15064170ns max# 7807505ns}}}
2025-11-27T18:20:16.053859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471711971986957:2107] [node 5] ICP22 created new session: [6:7577471716266954286:2107]
2025-11-27T18:20:16.050713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471716272811888:2107] [node 4] ICS09 handshake done sender: [1:7577471711977844565:2117] self: [1:7577471711977844564:0] peer: [4:7577471709757529245:0] socket: 443 qp: -1
2025-11-27T18:20:16.051846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471708806378879:2145] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:16.074316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471711069462324:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:16.050022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471709757529216:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.062811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471716266954286:2107] [node 5] ICS09 handshake done sender: [6:7577471711971986979:2119] self: [6:7577471711971986981:0] peer: [5:7577471711069462316:0] socket: 449 qp: -1
2025-11-27T18:20:16.050985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471716272811888:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:16.050801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471710743685247:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:16.050865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:16.061921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471708806378862:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.051026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:16.063129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471716266954286:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:16.050969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:16.051071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471709757529216:2108] [node 5] ICP22 created new session: [4:7577471714052496546:2108]
2025-11-27T18:20:16.051017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471710743685247:2048] [node 1] ICP22 created new session: [2:7577471715038652644:2048]
2025-11-27T18:20:16.051075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471711977844555:2110] [node 6] ICP22 created new session: [1:7577471716272811887:2110]
2025-11-27T18:20:16.051517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471714052496546:2108] [node 5] ICS09 handshake done sender: [4:7577471709757529230:2133] self: [4:7577471709757529229:0] peer: [5:7577471711069462331:0] socket: 400 qp: -1
2025-11-27T18:20:16.052092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471715038652644:2048] [node 1] ICS09 handshake done sender: [2:7577471710743685328:2117] self: [2:7577471710743685336:0] peer: [1:7577471711977844560:0] socket: 409 qp: -1
2025-11-27T18:20:16.051954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471716272811887:2110] [node 6] ICS09 handshake done sender: [1:7577471711977844574:2120] self: [1:7577471711977844572:0] peer: [6:7577471711971986985:0] socket: 446 qp: -1
2025-11-27T18:20:16.051999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471714052496546:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:16.053983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471715038652644:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:16.052137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471716272811887:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:16.061526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:16.053285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:16.052171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:16.069413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471714052496545:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-11-27T18:20:16.102612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471711977844489:2049] [node 3] ICP18 handshake (actor [1:7577471711977844575:2121]) is too old
2025-11-27T18:20:16.064815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:16.066202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471708806378862:2125] [node 6] ICP22 created new session: [3:7577471713101346194:2125]
2025-11-27T18:20:16.066680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471713101346194:2125] [node 6] ICS09 handshake done sender: [3:7577471708806378879:2145] self: [3:7577471708806378878:0] peer: [6:7577471711971986987:0] socket: 417 qp: -1
2025-11-27T18:20:16.067019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471713101346194:2125] [node 6] ICS10 traffic start
2025-11-27T18:20:16.073282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:16.073508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471708806378871:2141] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:16.095922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471708806378861:2115] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.105183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:16.064132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:16.081925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471711971986978:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:16.110462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471709757529242:2138] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:16.104160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471711971986897:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(146469826ns) {HandleHandshakeStatus:329(146467308ns) {DropIncomingHandshake:442(2122ns) DropOutgoingHandshake:457(7375ns) SwitchToState:179(432ns) ProcessPendingSessionEvents:495(121992973ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 111183706ns max# 12774008ns}}}
2025-11-27T18:20:16.110230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471711971986988:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.110606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471708806378861:2115] [node 5] ICP22 created new session: [3:7577471713101346197:2115]
2025-11-27T18:20:16.110304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471711971986896:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:16.110679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471713101346197:2115] [node 5] ICS09 handshake done sender: [3:7577471708806378871:2141] self: [3:7577471708806378870:0] peer: [5:7577471715364429631:0] socket: 450 qp: -1
2025-11-27T18:20:16.110454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:16.111043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471713101346197:2115] [node 5] ICS10 traffic start
2025-11-27T18:20:16.111390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471710743685248:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(141336842ns) {HandleHandshakeStatus:329(141333189ns) {DropIncomingHandshake:442(14489ns) DropOutgoingHandshake:457(510ns) SwitchToState:179(752ns) ProcessPendingSessionEvents:495(91945490ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 91609626ns max# 17077906ns}}}
2025-11-27T18:20:16.110517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471711971986896:2048] [node 1] ICP22 created new session: [6:7577471716266954289:2048]
2025-11-27T18:20:16.114483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471709757529134:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28863) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471709757529242:2138] held: no
2025-11-27T18:20:16.110586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471716266954289:2048] [node 1] ICS09 handshake done sender: [6:7577471711971986978:2118] self: [6:7577471711971986985:0] peer: [1:7577471711977844572:0] socket: 447 qp: -1
2025-11-27T18:20:16.110634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471716266954289:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:16.110919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471711971986898:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15780) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471711971986984:2120] held: no
2025-11-27T18:20:16.110961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:16.118766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471711971986896:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:16.111083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:16.119780Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471711069462301:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(39207602ns) {HandleHandshakeStatus:395(39205777ns) {LogHandshakeFail:472(14106ns) {UpdateErrorStateLog:545(5236ns)} DropOutgoingHandshake:457(951ns) ProcessPendingSessionEvents:495(38727014ns) {unordered DropSessionEvent:512 num# 100 sum# 18654689ns max# 11128110ns}}}
2025-11-27T18:20:16.112119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471708806378771:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(127741682ns) {HandleHandshakeStatus:329(127738135ns) {DropIncomingHandshake:442(827ns) DropOutgoingHandshake:457(6601ns) SwitchToState:179(570ns) ProcessPendingSessionEvents:495(47301349ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 46909496ns max# 15987079ns}}}
2025-11-27T18:20:16.115416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471708806378770:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(164500183ns) {HandleHandshakeStatus:329(164497218ns) {DropIncomingHandshake:442(1238ns) DropOutgoingHandshake:457(7117ns) SwitchToState:179(1531ns) ProcessPendingSessionEvents:495(136332129ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 136141704ns max# 13746665ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.121349Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471708806378861:2115] [node 5] ICP35 event processing took too much time PendingConnection:279(23742477ns) {HandleHandshakeStatus:329(23738113ns) {DropIncomingHandshake:442(21554ns) DropOutgoingHandshake:457(5504ns) SwitchToState:179(925ns) ProcessPendingSessionEvents:495(6031799ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5957816ns max# 3928147ns}}}
2025-11-27T18:20:16.120759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471711069462301:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:16.121886Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471708806378862:2125] [node 6] ICP35 event processing took too much time PendingConnection:279(57488844ns) {HandleHandshakeStatus:329(57485143ns) {DropIncomingHandshake:442(18737ns) DropOutgoingHandshake:457(545ns) SwitchToState:179(617ns) ProcessPendingSessionEvents:495(37283743ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 37173698ns max# 11533687ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.120852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:16.134001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471711977844553:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(60094689ns) {HandleHandshakeStatus:329(60092351ns) {DropIncomingHandshake:442(13814ns) DropOutgoingHandshake:457(125ns) SwitchToState:179(145ns) ProcessPendingSessionEvents:495(58378649ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 58223504ns max# 7782793ns}}}
2025-11-27T18:20:16.120904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471711069462301:2108] [node 4] ICP22 created new session: [5:7577471715364429633:2108]
2025-11-27T18:20:16.141807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471716266954288:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-27T18:20:16.120954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471715364429633:2108] [node 4] ICS09 handshake done sender: [5:7577471711069462324:2120] self: [5:7577471711069462331:0] peer: [4:7577471709757529229:0] socket: 399 qp: -1
2025-11-27T18:20:16.157073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471711977844555:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(69205985ns) {HandleHandshakeStatus:329(69202890ns) {DropIncomingHandshake:442(12713ns) DropOutgoingHandshake:457(556ns) SwitchToState:179(375ns) ProcessPendingSessionEvents:495(56346655ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 56218784ns max# 6855331ns}}}
2025-11-27T18:20:16.120967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471715364429632:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21597
2025-11-27T18:20:16.120994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471715364429633:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:16.121041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:16.125824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471711069462329:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:16.151357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471711069462238:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15780) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471711069462329:2121] held: no
2025-11-27T18:20:16.155745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471711069462300:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(130404949ns) {HandleHandshakeStatus:329(130401534ns) {DropIncomingHandshake:442(953ns) DropOutgoingHandshake:457(622ns) SwitchToState:179(756ns) ProcessPendingSessionEvents:495(61239601ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 55955831ns max# 7726512ns}}}
2025-11-27T18:20:16.156356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7577471711069462300:2109] [node 6] ICP18 handshake (actor [5:7577471711069462330:2122]) is too old
2025-11-27T18:20:16.158058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471711971986898:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(45568325ns) {HandleHandshakeStatus:395(45565511ns) {LogHandshakeFail:472(25946ns) {UpdateErrorStateLog:545(5989ns)} DropOutgoingHandshake:457(1464ns) ProcessPendingSessionEvents:495(38497687ns) {unordered DropSessionEvent:512 num# 100 sum# 38429661ns max# 7405860ns}}}
2025-11-27T18:20:16.158297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471711971986961:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:16.158404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471711971986984:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:16.158512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471711069462238:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:16.157471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471709757529216:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(60343332ns) {HandleHandshakeStatus:329(60336549ns) {DropIncomingHandshake:442(274154ns) DropOutgoingHandshake:457(342ns) SwitchToState:179(327ns) ProcessPendingSessionEvents:495(47768371ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 47631961ns max# 7133123ns}}}
2025-11-27T18:20:16.158603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:16.162146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471711971986898:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:16.162480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471711971986957:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(59412440ns) {HandleHandshakeStatus:329(59410443ns) {DropIncomingHandshake:442(686ns) DropOutgoingHandshake:457(4625ns) SwitchToState:179(233ns) ProcessPendingSessionEvents:495(40933364ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 35956678ns max# 7927586ns}}}
2025-11-27T18:20:16.163016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:16.163083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471711971986898:2050] [node 3] ICP22 created new session: [6:7577471716266954292:2050]
2025-11-27T18:20:16.163138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471716266954292:2050] [node 3] ICS09 handshake done sender: [6:7577471711971986984:2120] self: [6:7577471711971986987:0] peer: [3:7577471708806378878:0] socket: 419 qp: -1
2025-11-27T18:20:16.163178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471716266954292:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:16.163224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:16.162153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471709757529232:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.162195Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471709757529232:2134] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.158650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471711069462238:2050] [node 3] ICP22 created new session: [5:7577471715364429635:2050]
2025-11-27T18:20:16.162513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471715364429635:2050] [node 3] ICS09 handshake done sender: [5:7577471711069462329:2121] self: [5:7577471715364429631:0] peer: [3:7577471708806378870:0] socket: 451 qp: -1
2025-11-27T18:20:16.162575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471715364429635:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:16.162641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.166482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471709757529232:2134] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:16.166764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471716266954294:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-27T18:20:16.166492Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471709757529134:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(50882894ns) {HandleHandshakeStatus:395(50670545ns) {LogHandshakeFail:472(15301ns) {UpdateErrorStateLog:545(5394ns)} DropOutgoingHandshake:457(1074ns) ProcessPendingSessionEvents:495(49769178ns) {unordered DropSessionEvent:512 num# 100 sum# 49694764ns max# 7727914ns}}}
2025-11-27T18:20:16.167250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471711971986988:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:16.166570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471709757529134:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:16.166649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:16.166706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471709757529134:2048] [node 1] ICP22 created new session: [4:7577471714052496549:2048]
2025-11-27T18:20:16.168147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471711971986961:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:16.166751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471714052496549:2048] [node 1] ICS09 handshake done sender: [4:7577471709757529242:2138] self: [4:7577471709757529245:0] peer: [1:7577471711977844564:0] socket: 445 qp: -1
2025-11-27T18:20:16.168345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:16.166795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471714052496549:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:16.166803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471709757529215:2116] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.168737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471711971986961:2109] [node 4] ICP22 created new session: [6:7577471716266954295:2109]
2025-11-27T18:20:16.166824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:16.173575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471716266954295:2109] [node 4] ICS09 handshake done sender: [6:7577471711971986988:2122] self: [6:7577471716266954291:0] peer: [4:7577471709757529231:0] socket: 456 qp: -1
2025-11-27T18:20:16.173647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471716266954295:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:16.167139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:16.173881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:16.167566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471709757529215:2116] [node 6] ICP22 created new session: [4:7577471714052496551:2116]
2025-11-27T18:20:16.167747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471714052496551:2116] [node 6] ICS09 handshake done sender: [4:7577471709757529232:2134] self: [4:7577471709757529231:0] peer: [6:7577471716266954291:0] socket: 455 qp: -1
2025-11-27T18:20:16.168022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471714052496551:2116] [node 6] ICS10 traffic start
2025-11-27T18:20:16.168075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:16.239048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:16.239321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471715038652644:2048] [node 1] ICS01 socket: 409 reason# EndOfStream
2025-11-27T18:20:16.239374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471710743685247:2048] [node 1] ICP30 unregister session Session# [2:7577471715038652644:2048] VirtualId# [2:7577471710743685336:0]
2025-11-27T18:20:16.239415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471715038652644:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.239508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:16.239565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:16.240462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:16.240690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471708806378890:2048] [node 1] ICS07 socket disconnect 410 reason# EndOfStream
2025-11-27T18:20:16.240733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471708806378890:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.240826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:16.240889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471708806378890:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:16.243316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:16.243425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471708806378770:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28863) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:16.243476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471708806378890:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:16.243553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471714052496549:2048] [node 1] ICS01 socket: 445 reason# EndOfStream
2025-11-27T18:20:16.243792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471709757529134:2048] [node 1] ICP30 unregister session Session# [4:7577471714052496549:2048] VirtualId# [4:7577471709757529245:0]
2025-11-27T18:20:16.243844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471714052496549:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.243937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:16.243984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:16.244598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471708806378770:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28863) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:16.244706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471708806378890:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:16.244767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471708806378770:2048] [node 1] ICP30 unregister session Session# [3:7577471708806378890:2048] VirtualId# [3:7577471708806378881:0]
2025-11-27T18:20:16.244894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:16.244933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471708806378770:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28863) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.244947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28863) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.244982Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:16.245194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471711069462325:2048] [node 1] ICS01 socket: 411 reason# EndOfStream
2025-11-27T18:20:16.245272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471711069462236:2048] [node 1] ICP30 unregister session Session# [5:7577471711069462325:2048] VirtualId# [5:7577471711069462319:0]
2025-11-27T18:20:16.245306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471711069462325:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.245380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:16.245427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:16.246227Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:16.246420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471716266954289:2048] [node 1] ICS01 socket: 447 reason# EndOfStream
2025-11-27T18:20:16.246469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471711971986896:2048] [node 1] ICP30 unregister session Session# [6:7577471716266954289:2048] VirtualId# [6:7577471711971986985:0]
2025-11-27T18:20:16.246497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471716266954289:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.246591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:16.246660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:16.256531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471708806378770:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:16.344438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:16.344666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471708806378895:2049] [node 2] ICS07 socket disconnect 432 reason# EndOfStream
2025-11-27T18:20:16.344707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471708806378895:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.344783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:16.344829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471708806378895:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:16.347229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471708806378771:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:16.347277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471708806378895:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:16.347248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:16.347479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471709757529237:2049] [node 2] ICS07 socket disconnect 385 reason# EndOfStream
2025-11-27T18:20:16.347517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471709757529237:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.347594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:16.347637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471709757529237:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:16.348382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471708806378771:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:16.348454Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471709757529135:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:16.348465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471708806378895:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:16.348568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471709757529237:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:16.348651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471709757529135:2049] [node 2] ICP30 unregister session Session# [4:7577471709757529237:2049] VirtualId# [4:7577471709757529226:0]
2025-11-27T18:20:16.348548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471708806378771:2049] [node 2] ICP30 unregister session Session# [3:7577471708806378895:2049] VirtualId# [3:7577471708806378886:0]
2025-11-27T18:20:16.348698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:16.348738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471708806378771:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.348746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:16.348753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.348778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471709757529135:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.348790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.348868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:16.349155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471711069462320:2049] [node 2] ICS07 socket disconnect 388 reason# EndOfStream
2025-11-27T18:20:16.349194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471711069462320:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.349273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:16.349313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471711069462320:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:16.349977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471711069462237:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:16.350044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471711069462320:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:16.350102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:16.350125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471711069462237:2049] [node 2] ICP30 unregister session Session# [5:7577471711069462320:2049] VirtualId# [5:7577471711069462314:0]
2025-11-27T18:20:16.350238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:16.350275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471711069462237:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.350293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.350303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471711971986980:2049] [node 2] ICS07 socket disconnect 434 reason# EndOfStream
2025-11-27T18:20:16.350339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471711971986980:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.350426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:16.350471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471711971986980:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:16.351199Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471711971986897:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:16.351324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471711971986980:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:16.351363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471711971986897:2049] [node 2] ICP30 unregister session Session# [6:7577471711971986980:2049] VirtualId# [6:7577471711971986976:0]
2025-11-27T18:20:16.351458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:16.351517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471711971986897:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.351531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11860) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.358685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471709757529135:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:16.359711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471711069462237:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:16.361761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471711971986897:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:16.414004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:16.414183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471709757529244:2050] [node 3] ICS01 socket: 424 reason# EndOfStream
2025-11-27T18:20:16.414227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471709757529136:2050] [node 3] ICP30 unregister session Session# [4:7577471709757529244:2050] VirtualId# [4:7577471709757529238:0]
2025-11-27T18:20:16.414261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471709757529244:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.414336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:16.414385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:16.415131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:16.415359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471715364429635:2050] [node 3] ICS01 socket: 451 reason# EndOfStream
2025-11-27T18:20:16.415412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471711069462238:2050] [node 3] ICP30 unregister session Session# [5:7577471715364429635:2050] VirtualId# [5:7577471715364429631:0]
2025-11-27T18:20:16.415464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471715364429635:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.415539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:16.415604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:16.416194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:16.416437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471716266954292:2050] [node 3] ICS01 socket: 419 reason# EndOfStream
2025-11-27T18:20:16.416491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471711971986898:2050] [node 3] ICP30 unregister session Session# [6:7577471716266954292:2050] VirtualId# [6:7577471711971986987:0]
2025-11-27T18:20:16.416520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471716266954292:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.416599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:16.416686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:16.488514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:16.488761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471715364429633:2108] [node 4] ICS01 socket: 399 reason# EndOfStream
2025-11-27T18:20:16.488807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471711069462301:2108] [node 4] ICP30 unregister session Session# [5:7577471715364429633:2108] VirtualId# [5:7577471711069462331:0]
2025-11-27T18:20:16.488842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471715364429633:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.488914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:16.488956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:16.489727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:16.489964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471716266954295:2109] [node 4] ICS01 socket: 456 reason# EndOfStream
2025-11-27T18:20:16.490003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471711971986961:2109] [node 4] ICP30 unregister session Session# [6:7577471716266954295:2109] VirtualId# [6:7577471716266954291:0]
2025-11-27T18:20:16.490032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471716266954295:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.490096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:16.490142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:16.554020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:16.554304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471716266954286:2107] [node 5] ICS07 socket disconnect 449 reason# EndOfStream
2025-11-27T18:20:16.554348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471716266954286:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:16.554422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:16.554461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471716266954286:2107] [node 5] ICS15 start handshake
2025-11-27T18:20:16.555195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471711971986957:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23892) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:16.555262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471716266954286:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:16.555332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471711971986957:2107] [node 5] ICP30 unregister session Session# [6:7577471716266954286:2107] VirtualId# [6:7577471711971986981:0]
2025-11-27T18:20:16.555448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:16.555480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471711971986957:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23892) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.555492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23892) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:16.565539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471711971986957:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:16.680696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471715261766728:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:16.680757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471715261766729:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:16.703994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471713781373153:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:16.704264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471713781373154:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:16.727377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471716269911806:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:16.727388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471716269911805:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:16.748415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471715930604012:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:16.748564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471715930604013:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:16.748794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471715930604014:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:16.772525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471712580479404:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:16.772571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471712580479405:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:16.772628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471712580479406:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:16.798009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471714601635325:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:16.798059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471714601635324:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:16.798070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471714601635326:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:16.867170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471714601635385:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:16.870487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471715261766789:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:16.912967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471715930604086:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:16.915491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471715261766793:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:16.916882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471713781373220:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:16.915448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471716269911869:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:16.923372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471712580479467:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:16.915894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471715930604087:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:16.921797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471716269911870:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:16.915700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471713781373219:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:16.913462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471715261766792:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:16.923413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471714601635389:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:16.925738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471713781373221:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:16.921820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471716269911871:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:16.923374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471712580479468:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:16.946689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471713781373154:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19622915ns) {Configure:104(19618926ns) {ProcessConfigured:135(407779ns) {StartInitialHandshake:157(405894ns) {DropHandshakes:470(1116ns) {DropIncomingHandshake:442(341ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:477(1788ns) {SwitchToState:179(777ns)}}}}}
2025-11-27T18:20:16.946694Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471715261766728:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(65958249ns) {Configure:104(65942955ns) {ProcessConfigured:135(885110ns) {StartInitialHandshake:157(882610ns) {DropHandshakes:470(1343ns) {DropIncomingHandshake:442(421ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(1643ns) {SwitchToState:179(868ns)}}}}}
2025-11-27T18:20:16.946727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471714601635389:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18158151ns) {Configure:104(18139667ns) {ProcessConfigured:135(1325978ns) {StartInitialHandshake:157(1324669ns) {DropHandshakes:470(669ns) {DropIncomingHandshake:442(178ns) DropOutgoingHandshake:457(142ns)} GenerateSessionVirtualId:988(590ns) PrepareNewSessionHandshake:477(1260ns) {SwitchToState:179(584ns)}}}}}
2025-11-27T18:20:16.947406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471712580479468:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17049692ns) {Configure:104(17045866ns) {ProcessConfigured:135(1066414ns) {StartInitialHandshake:157(1064826ns) {DropHandshakes:470(994ns) {DropIncomingHandshake:442(312ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(378ns) PrepareNewSessionHandshake:477(1005ns) {SwitchToState:179(560ns)}}}}}
2025-11-27T18:20:16.947287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471715930604014:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(23151844ns) {Configure:104(23144428ns) {ProcessConfigured:135(1491118ns) {StartInitialHandshake:157(1489390ns) {DropHandshakes:470(530ns) {DropIncomingHandshake:442(165ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(178ns) PrepareNewSessionHandshake:477(7212ns) {SwitchToState:179(548ns)}}}}}
2025-11-27T18:20:16.946742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471715261766789:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17098835ns) {Configure:104(17093156ns) {ProcessConfigured:135(156342ns) {StartInitialHandshake:157(154396ns) {DropHandshakes:470(4238ns) {DropIncomingHandshake:442(350ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(569ns) PrepareNewSessionHandshake:477(1694ns) {SwitchToState:179(812ns)}}}}}
2025-11-27T18:20:16.946702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471713781373153:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(39411350ns) {Configure:104(31623275ns) {ProcessConfigured:135(4475056ns) {StartInitialHandshake:157(4473120ns) {DropHandshakes:470(1142ns) {DropIncomingHandshake:442(337ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(531ns) PrepareNewSessionHandshake:477(2443ns) {SwitchToState:179(570ns)}}}}}
2025-11-27T18:20:16.947149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471714601635385:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20795491ns) {Configure:104(20787957ns) {ProcessConfigured:135(801234ns) {StartInitialHandshake:157(799434ns) {DropHandshakes:470(1005ns) {DropIncomingHandshake:442(275ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(451ns) PrepareNewSessionHandshake:477(1536ns) {SwitchToState:179(769ns)}}}}}
2025-11-27T18:20:16.948023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471715930604012:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31633594ns) {Configure:104(31628332ns) {ProcessConfigured:135(2082905ns) {StartInitialHandshake:157(2081328ns) {DropHandshakes:470(1182ns) {DropIncomingHandshake:442(404ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(464ns) PrepareNewSessionHandshake:477(1074ns) {SwitchToState:179(69ns)}}}}}
2025-11-27T18:20:16.947524Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471713781373220:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17016125ns) {Configure:104(17009628ns) {ProcessConfigured:135(2508588ns) {StartInitialHandshake:157(2505773ns) {DropHandshakes:470(1344ns) {DropIncomingHandshake:442(386ns) DropOutgoingHandshake:457(209ns)} GenerateSessionVirtualId:988(436ns) PrepareNewSessionHandshake:477(2679ns) {SwitchToState:179(1299ns)}}}}}
2025-11-27T18:20:16.948800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471715930604086:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17321011ns) {Configure:104(17315268ns) {ProcessConfigured:135(1043605ns) {StartInitialHandshake:157(1041833ns) {DropHandshakes:470(688ns) {DropIncomingHandshake:442(144ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(311ns) PrepareNewSessionHandshake:477(879ns) {SwitchToState:179(200ns)}}}}}
2025-11-27T18:20:16.952974Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471715930604087:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16932565ns) {Configure:104(16927157ns) {ProcessConfigured:135(1846501ns) {StartInitialHandshake:157(1844547ns) {DropHandshakes:470(1115ns) {DropIncomingHandshake:442(294ns) DropOutgoingHandshake:457(278ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:477(1979ns) {SwitchToState:179(783ns)}}}}}
2025-11-27T18:20:16.947657Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471712580479404:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20914097ns) {Configure:104(20910353ns) {ProcessConfigured:135(289555ns) {StartInitialHandshake:157(287659ns) {DropHandshakes:470(1480ns) {DropIncomingHandshake:442(300ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(619ns) PrepareNewSessionHandshake:477(1965ns) {SwitchToState:179(846ns)}}}}}
2025-11-27T18:20:16.946829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471715261766729:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(65428283ns) {Configure:104(55041507ns) {ProcessConfigured:135(1321135ns) {StartInitialHandshake:157(1318897ns) {DropHandshakes:470(1153ns) {DropIncomingHandshake:442(388ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(344ns) PrepareNewSessionHandshake:477(1557ns) {SwitchToState:179(779ns)}}}}}
2025-11-27T18:20:16.948458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471714601635325:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21148536ns) {Configure:104(21144990ns) {ProcessConfigured:135(1557910ns) {StartInitialHandshake:157(1556407ns) {DropHandshakes:470(687ns) {DropIncomingHandshake:442(247ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(915ns) PrepareNewSessionHandshake:477(1397ns) {SwitchToState:179(634ns)}}}}}
2025-11-27T18:20:16.953730Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471716269911870:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18963907ns) {Configure:104(18956749ns) {ProcessConfigured:135(1576175ns) {StartInitialHandshake:157(1574468ns) {DropHandshakes:470(237975ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(989ns)} GenerateSessionVirtualId:988(718ns) PrepareNewSessionHandshake:477(1150ns) {SwitchToState:179(486ns)}}}}}
2025-11-27T18:20:16.947695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471712580479405:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18475965ns) {Configure:104(18468121ns) {ProcessConfigured:135(737540ns) {StartInitialHandshake:157(735268ns) {DropHandshakes:470(1071ns) {DropIncomingHandshake:442(358ns) DropOutgoingHandshake:457(92ns)} GenerateSessionVirtualId:988(439ns) PrepareNewSessionHandshake:477(3112ns) {SwitchToState:179(1893ns)}}}}}
2025-11-27T18:20:16.948509Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471714601635324:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17770751ns) {Configure:104(17764410ns) {ProcessConfigured:135(1024910ns) {StartInitialHandshake:157(1023576ns) {DropHandshakes:470(309458ns) {DropIncomingHandshake:442(955ns) DropOutgoingHandshake:457(92ns)} GenerateSessionVirtualId:988(855ns) PrepareNewSessionHandshake:477(989ns) {SwitchToState:179(462ns)}}}}}
2025-11-27T18:20:16.947534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471715261766792:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17091553ns) {Configure:104(17082456ns) {ProcessConfigured:135(757458ns) {StartInitialHandshake:157(755608ns) {DropHandshakes:470(1170ns) {DropIncomingHandshake:442(376ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(684ns) PrepareNewSessionHandshake:477(1407ns) {SwitchToState:179(486ns)}}}}}
2025-11-27T18:20:16.957156Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471716269911869:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17994392ns) {Configure:104(17989320ns) {ProcessConfigured:135(830403ns) {StartInitialHandshake:157(828475ns) {DropHandshakes:470(214905ns) {DropIncomingHandshake:442(466ns) DropOutgoingHandshake:457(793ns)} GenerateSessionVirtualId:988(647ns) PrepareNewSessionHandshake:477(1258ns) {SwitchToState:179(638ns)}}}}}
2025-11-27T18:20:16.950827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471712580479467:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17572782ns) {Configure:104(17567558ns) {ProcessConfigured:135(1627457ns) {StartInitialHandshake:157(1625311ns) {DropHandshakes:470(2837ns) {DropIncomingHandshake:442(936ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(751ns) PrepareNewSessionHandshake:477(1240ns) {SwitchToState:179(524ns)}}}}}
2025-11-27T18:20:16.957479Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471716269911871:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18997142ns) {Configure:104(18993420ns) {ProcessConfigured:135(1521451ns) {StartInitialHandshake:157(1519631ns) {DropHandshakes:470(3773ns) {DropIncomingHandshake:442(2901ns) DropOutgoingHandshake:457(279ns)} GenerateSessionVirtualId:988(587ns) PrepareNewSessionHandshake:477(1172ns) {SwitchToState:179(318ns)}}}}}
2025-11-27T18:20:16.955475Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471712580479406:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21279012ns) {Configure:104(21275127ns) {ProcessConfigured:135(1759401ns) {StartInitialHandshake:157(1756553ns) {DropHandshakes:470(566ns) {DropIncomingHandshake:442(280ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(1049ns) {SwitchToState:179(363ns)}}}}}
2025-11-27T18:20:16.957545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471716269911806:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20409529ns) {Configure:104(20404060ns) {ProcessConfigured:135(1842825ns) {StartInitialHandshake:157(1838109ns) {DropHandshakes:470(220851ns) {DropIncomingHandshake:442(398ns) DropOutgoingHandshake:457(417ns)} GenerateSessionVirtualId:988(661ns) PrepareNewSessionHandshake:477(5164ns) {SwitchToState:179(1335ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.965177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471712580479480:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24366
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.965467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471714601635403:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33289
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.965158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471715261766806:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3187
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.967137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471715930604099:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41508
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.966360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471714601635405:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33289
2025-11-27T18:20:16.971092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471715261766808:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3187
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.971553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471712580479482:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24366
2025-11-27T18:20:16.970045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471716269911885:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51444
2025-11-27T18:20:16.971266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471713781373235:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:628
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.972244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471714601635405:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.972381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471715261766809:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3187
2025-11-27T18:20:16.970062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471716269911884:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51444
2025-11-27T18:20:16.972443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471715930604099:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.971999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471713781373236:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:628
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.973040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471712580479482:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.973342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471714601635403:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.973700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471715261766810:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3187
2025-11-27T18:20:16.973793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471712580479404:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:16.973328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471716269911884:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.973468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471714601635324:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:16.972990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471713781373235:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.974877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471715261766803:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.973802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471714601635326:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:16.974707Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471712580479480:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.974279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471715930604102:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41508
2025-11-27T18:20:16.973750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471716269911885:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.973338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471713781373237:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:628
2025-11-27T18:20:16.974283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471714601635404:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33289
No RDMA context found for GID: 2025-11-27T18:20:16.974901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471716269911878:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:16.975802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471712580479405:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:16.974915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471715261766803:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.974483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471715930604013:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:16.974711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471713781373236:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.977201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471714601635404:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.975001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471715261766802:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.977418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471715930604102:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.975698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471713781373238:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:628
2025-11-27T18:20:16.974936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471716269911878:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.975035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471715261766802:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.977510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471714601635389:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:16.979149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471716269911878:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:16.976554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471713781373237:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.976580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471715261766809:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.976684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471713781373223:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.976714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471713781373223:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.977322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471713781373153:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:16.977457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471713781373226:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.977487Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471713781373226:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.977702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471713781373239:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:628
2025-11-27T18:20:16.977724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471713781373238:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.979145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471713781373239:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.980207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471713781373223:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:16.980347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471713781373226:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:16.979244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471715930604097:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.977152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471715261766810:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.979281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471715930604097:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.977154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471715261766808:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:16.977175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471715261766806:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.978575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471715261766801:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.981616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471716269911869:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.981667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471713781373221:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.981856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471715930604097:2125] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:16.982035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:16.982362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471714601635410:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33289
2025-11-27T18:20:16.982698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471713781373219:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.978609Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471715261766801:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.983064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471715930604105:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41508
2025-11-27T18:20:16.983052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471712580479486:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24366
2025-11-27T18:20:16.982664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471714601635409:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33289
2025-11-27T18:20:16.982422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471716269911869:2109] [node 6] ICP22 created new session: [3:7577471716269911890:2109]
2025-11-27T18:20:16.979017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471715261766803:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:16.983527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:16.983080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471712580479487:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24366
2025-11-27T18:20:16.982743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471716269911890:2109] [node 6] ICS09 handshake done sender: [3:7577471716269911878:2112] self: [3:7577471716269911876:0] peer: [6:7577471714601635406:0] socket: 426 qp: -1
2025-11-27T18:20:16.979353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471715261766802:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:16.984377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471714601635326:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62664) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471714601635405:2118] held: no
2025-11-27T18:20:16.984445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471712580479405:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29698) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471712580479480:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.986362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471715930604099:2126] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:16.985743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471714601635324:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471714601635403:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.983581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471713781373219:2107] [node 4] ICP22 created new session: [2:7577471713781373248:2107]
2025-11-27T18:20:16.990008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471712580479482:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:16.989930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471715930604013:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:16.983122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471716269911890:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:16.981670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471715261766793:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.984277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:16.986365Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471714601635403:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:16.981984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:16.983580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:16.996778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471712580479404:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:16.989224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471715930604087:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.982009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471715261766801:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:16.996837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471714601635411:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33289
2025-11-27T18:20:16.984344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471713781373221:2109] [node 5] ICP22 created new session: [2:7577471713781373249:2109]
2025-11-27T18:20:16.994338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:16.997236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:16.982371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471715261766792:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.982723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471715261766793:2108] [node 5] ICP22 created new session: [1:7577471715261766818:2108]
2025-11-27T18:20:16.984836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471713781373248:2107] [node 4] ICS09 handshake done sender: [2:7577471713781373223:2113] self: [2:7577471713781373222:0] peer: [4:7577471715930604103:0] socket: 390 qp: -1
2025-11-27T18:20:16.996203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:16.998579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471712580479404:2048] [node 1] ICP22 created new session: [5:7577471712580479491:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.004402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471714601635404:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:16.985122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471713781373249:2109] [node 5] ICS09 handshake done sender: [2:7577471713781373226:2114] self: [2:7577471713781373224:0] peer: [5:7577471712580479485:0] socket: 391 qp: -1
2025-11-27T18:20:16.996219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471715930604013:2049] [node 2] ICP22 created new session: [4:7577471715930604107:2049]
2025-11-27T18:20:17.003801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471712580479491:2048] [node 1] ICS09 handshake done sender: [5:7577471712580479482:2116] self: [5:7577471712580479484:0] peer: [1:7577471715261766796:0] socket: 413 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.985161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471713781373249:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:17.004410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471714601635405:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:16.984214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:16.996718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471715930604107:2049] [node 2] ICS09 handshake done sender: [4:7577471715930604099:2126] self: [4:7577471715930604103:0] peer: [2:7577471713781373222:0] socket: 396 qp: -1
2025-11-27T18:20:17.013965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471716269911894:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51444
2025-11-27T18:20:17.004755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471712580479491:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:16.984247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471715261766728:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:16.986102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471713781373248:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.013924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471714601635326:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.004803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:17.015510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471716269911893:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51444
2025-11-27T18:20:16.996726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471715930604087:2110] [node 6] ICP22 created new session: [4:7577471715930604108:2110]
2025-11-27T18:20:16.984279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471715261766792:2109] [node 6] ICP22 created new session: [1:7577471715261766819:2109]
2025-11-27T18:20:16.986351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:17.014389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471714601635389:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.006229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471712580479480:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:17.015548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:17.015944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471716269911894:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.016630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471716269911806:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.013897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471712580479489:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24366
2025-11-27T18:20:16.988452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:17.015272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471712580479405:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:16.984646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471715261766819:2109] [node 6] ICS09 handshake done sender: [1:7577471715261766803:2114] self: [1:7577471715261766800:0] peer: [6:7577471714601635407:0] socket: 395 qp: -1
2025-11-27T18:20:16.995494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471713781373247:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:628
2025-11-27T18:20:17.015968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471714601635326:2050] [node 3] ICP22 created new session: [6:7577471718896602712:2050]
2025-11-27T18:20:17.015561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471712580479489:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.984673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471715261766818:2108] [node 5] ICS09 handshake done sender: [1:7577471715261766802:2112] self: [1:7577471715261766796:0] peer: [5:7577471712580479484:0] socket: 394 qp: -1
2025-11-27T18:20:16.996860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471713781373153:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471713781373237:2120] held: no
2025-11-27T18:20:16.984687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:17.016603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471718896602712:2050] [node 3] ICS09 handshake done sender: [6:7577471714601635405:2118] self: [6:7577471714601635406:0] peer: [3:7577471716269911876:0] socket: 427 qp: -1
2025-11-27T18:20:17.016331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:17.016820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.985147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471715261766818:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:17.016860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471712580479468:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:16.996783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471715930604108:2110] [node 6] ICS09 handshake done sender: [4:7577471715930604097:2125] self: [4:7577471715930604091:0] peer: [6:7577471714601635408:0] socket: 416 qp: -1
2025-11-27T18:20:17.017069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471718896602712:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:16.986090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471715261766819:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:16.996820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471715930604108:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:17.021459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471716269911893:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.023141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471713781373230:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.017483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471714601635389:2108] [node 4] ICP22 created new session: [6:7577471718896602713:2108]
2025-11-27T18:20:17.023276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471716269911805:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.016861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471712580479405:2049] [node 2] ICP22 created new session: [5:7577471716875446791:2049]
2025-11-27T18:20:17.023188Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471713781373230:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:16.986297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471715261766728:2048] [node 2] ICP22 created new session: [1:7577471715261766820:2048]
2025-11-27T18:20:16.996891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:17.018779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471718896602711:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33289
2025-11-27T18:20:17.017426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471712580479488:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24366
2025-11-27T18:20:16.997026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471715930604107:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:16.988445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471715261766820:2048] [node 2] ICS09 handshake done sender: [1:7577471715261766801:2113] self: [1:7577471715261766795:0] peer: [2:7577471713781373242:0] socket: 402 qp: -1
2025-11-27T18:20:17.023822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471720564879192:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51444
2025-11-27T18:20:17.030280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471713781373230:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:17.018808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471718896602713:2108] [node 4] ICS09 handshake done sender: [6:7577471714601635404:2117] self: [6:7577471714601635408:0] peer: [4:7577471715930604091:0] socket: 418 qp: -1
2025-11-27T18:20:17.029714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471720564879192:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:16.988451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:17.018719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471716875446791:2049] [node 2] ICS09 handshake done sender: [5:7577471712580479480:2115] self: [5:7577471712580479485:0] peer: [2:7577471713781373224:0] socket: 401 qp: -1
2025-11-27T18:20:17.032732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471713781373154:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.018994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471716875446791:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:16.997067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:16.988616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:17.032792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:17.018857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471718896602713:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:17.016395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471715930604110:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41508
2025-11-27T18:20:16.988728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471715261766820:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:17.018968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:17.021421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:17.033125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471713781373154:2049] [node 3] ICP22 created new session: [2:7577471718076340551:2049]
2025-11-27T18:20:17.017411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471720225571410:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41508
2025-11-27T18:20:16.994379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:17.036679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471716269911880:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.025859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471712580479488:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.034089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471718076340551:2049] [node 3] ICS09 handshake done sender: [2:7577471713781373230:2116] self: [2:7577471713781373229:0] peer: [3:7577471720564879191:0] socket: 399 qp: -1
2025-11-27T18:20:17.019079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:17.036725Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471716269911880:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.023398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471715930604110:2129] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.022794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471714601635412:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33289
2025-11-27T18:20:17.031437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471712580479406:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.029354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471719556734122:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3187
2025-11-27T18:20:17.036726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471718076340551:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:17.043185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471716269911806:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29698) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471716269911894:2119] held: no
2025-11-27T18:20:17.032368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471716875446790:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24366
2025-11-27T18:20:17.025701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471718896602711:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.024469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471715930604095:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.030140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471715261766798:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.036795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:17.034025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471712580479477:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.045220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471720564879195:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51444
2025-11-27T18:20:17.030169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471715261766798:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.047999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471716269911880:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:17.034057Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471712580479477:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.037183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471713781373153:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21460065ns) {HandleHandshakeStatus:395(21457215ns) {LogHandshakeFail:472(11475ns) {UpdateErrorStateLog:545(4667ns)} DropOutgoingHandshake:457(1117ns) ProcessPendingSessionEvents:495(19920590ns) {unordered DropSessionEvent:512 num# 100 sum# 10466125ns max# 6649230ns}}}
2025-11-27T18:20:17.029785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471714601635412:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.031389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471719556734122:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.040189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471713781373231:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.032732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471715261766799:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.031490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471714601635385:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.040223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471713781373231:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.032553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471714601635325:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.032758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471715261766799:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-27T18:20:17.042346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471713781373221:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(37301306ns) {HandleHandshakeStatus:329(37299408ns) {DropIncomingHandshake:442(15712ns) DropOutgoingHandshake:457(501ns) SwitchToState:179(373ns) ProcessPendingSessionEvents:495(23644397ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23261443ns max# 5993208ns}}}
2025-11-27T18:20:17.033073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471714601635324:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18645374ns) {HandleHandshakeStatus:395(18643250ns) {LogHandshakeFail:472(10731ns) {UpdateErrorStateLog:545(3186ns)} DropOutgoingHandshake:457(302ns) ProcessPendingSessionEvents:495(18575213ns) {unordered DropSessionEvent:512 num# 100 sum# 3951561ns max# 1391711ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.033131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471714601635324:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.043167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471713781373237:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:17.024502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471715930604095:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.037170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471715261766799:2111] [node 4] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:17.034878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471716875446790:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.043352Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471713781373219:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(41825195ns) {HandleHandshakeStatus:329(41822781ns) {DropIncomingHandshake:442(10158ns) DropOutgoingHandshake:457(214ns) SwitchToState:179(157ns) ProcessPendingSessionEvents:495(35052549ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 34949919ns max# 6061263ns}}}
2025-11-27T18:20:17.034853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:17.029906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471715930604012:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.035122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471714601635324:2048] [node 1] ICP22 created new session: [6:7577471718896602718:2048]
2025-11-27T18:20:17.034126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471715930604095:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:17.043499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471713781373231:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:17.036690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471718896602718:2048] [node 1] ICS09 handshake done sender: [6:7577471714601635403:2116] self: [6:7577471714601635407:0] peer: [1:7577471715261766800:0] socket: 408 qp: -1
2025-11-27T18:20:17.039018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471715930604086:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.039239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471712580479477:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:17.038961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471715261766798:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.047265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471713781373153:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.040243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:17.039798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471718896602718:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:17.052045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471716269911894:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:17.047835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:17.041997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471715261766729:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.043142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471712580479467:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.039850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:17.042754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471715930604086:2108] [node 5] ICP22 created new session: [4:7577471720225571414:2108]
2025-11-27T18:20:17.046652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471715261766789:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.052532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471716269911871:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.046675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:17.048430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471713781373153:2048] [node 1] ICP22 created new session: [2:7577471718076340554:2048]
2025-11-27T18:20:17.050421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471714601635385:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11871) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471718896602711:2123] held: no
2025-11-27T18:20:17.047001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:17.046735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471712580479467:2109] [node 6] ICP22 created new session: [5:7577471716875446799:2109]
2025-11-27T18:20:17.052946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:17.049319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471713781373220:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.047108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:17.047377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471716875446799:2109] [node 6] ICS09 handshake done sender: [5:7577471712580479477:2113] self: [5:7577471712580479470:0] peer: [6:7577471718896602716:0] socket: 392 qp: -1
2025-11-27T18:20:17.047154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471715261766789:2107] [node 4] ICP22 created new session: [1:7577471719556734127:2107]
2025-11-27T18:20:17.053251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471716269911871:2108] [node 5] ICP22 created new session: [3:7577471720564879198:2108]
2025-11-27T18:20:17.050328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471718076340554:2048] [node 1] ICS09 handshake done sender: [2:7577471713781373237:2120] self: [2:7577471713781373242:0] peer: [1:7577471715261766795:0] socket: 420 qp: -1
2025-11-27T18:20:17.043989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471720225571414:2108] [node 5] ICS09 handshake done sender: [4:7577471715930604095:2123] self: [4:7577471715930604092:0] peer: [5:7577471716875446792:0] socket: 400 qp: -1
2025-11-27T18:20:17.048210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471712580479468:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471712580479489:2120] held: no
2025-11-27T18:20:17.054196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471720564879196:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51444
2025-11-27T18:20:17.044038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471720225571414:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:17.047879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471715261766729:2049] [node 3] ICP22 created new session: [1:7577471719556734126:2049]
2025-11-27T18:20:17.051075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471718896602721:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33289
2025-11-27T18:20:17.050388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471718076340554:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:17.044878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:17.048450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471716875446799:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:17.054264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471720564879198:2108] [node 5] ICS09 handshake done sender: [3:7577471716269911880:2114] self: [3:7577471716269911872:0] peer: [5:7577471716875446794:0] socket: 421 qp: -1
2025-11-27T18:20:17.053167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471718896602720:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33289
2025-11-27T18:20:17.048463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471719556734126:2049] [node 3] ICS09 handshake done sender: [1:7577471715261766798:2110] self: [1:7577471715261766794:0] peer: [3:7577471720564879193:0] socket: 397 qp: -1
2025-11-27T18:20:17.050405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.051019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:17.055654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471714601635412:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:17.048873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471719556734127:2107] [node 4] ICS09 handshake done sender: [1:7577471715261766799:2111] self: [1:7577471715261766797:0] peer: [4:7577471720225571411:0] socket: 407 qp: -1
2025-11-27T18:20:17.054304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471716269911806:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.050997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471715930604013:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(44002354ns) {HandleHandshakeStatus:329(44000115ns) {DropIncomingHandshake:442(5082ns) DropOutgoingHandshake:457(9739ns) SwitchToState:179(653ns) ProcessPendingSessionEvents:495(27772063ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27667168ns max# 6754502ns}}}
2025-11-27T18:20:17.056845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471714601635325:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29698) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471714601635412:2122] held: no
2025-11-27T18:20:17.050435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:17.052434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471716875446796:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24366
2025-11-27T18:20:17.051094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471715930604013:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:17.048888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471719556734126:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:17.061184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471718896602711:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:17.050448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471713781373220:2110] [node 6] ICP22 created new session: [2:7577471718076340555:2110]
2025-11-27T18:20:17.055601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471720564879198:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:17.050335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471719556734127:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:17.062230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471714601635385:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.050554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471718076340555:2110] [node 6] ICS09 handshake done sender: [2:7577471713781373231:2117] self: [2:7577471713781373227:0] peer: [6:7577471718896602717:0] socket: 393 qp: -1
2025-11-27T18:20:17.055635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:17.050402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:17.063808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:17.051508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471718076340555:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:17.051522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471720225571413:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41508
2025-11-27T18:20:17.055641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:17.064368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471714601635385:2105] [node 5] ICP22 created new session: [6:7577471718896602722:2105]
2025-11-27T18:20:17.050945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:17.051547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:17.055687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471716269911869:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(55346142ns) {HandleHandshakeStatus:329(55341595ns) {DropIncomingHandshake:442(16103ns) DropOutgoingHandshake:457(1249ns) SwitchToState:179(413ns) ProcessPendingSessionEvents:495(50615183ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 50455039ns max# 6072246ns}}}
2025-11-27T18:20:17.054999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471715930604110:2129] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:17.064701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471718896602722:2105] [node 5] ICS09 handshake done sender: [6:7577471718896602711:2123] self: [6:7577471718896602716:0] peer: [5:7577471712580479470:0] socket: 411 qp: -1
2025-11-27T18:20:17.056375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471716269911806:2049] [node 2] ICP22 created new session: [3:7577471720564879199:2049]
2025-11-27T18:20:17.065361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471718896602722:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:17.056380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471716269911805:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471716269911893:2118] held: no
2025-11-27T18:20:17.052017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471715261766793:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(50890103ns) {HandleHandshakeStatus:329(50887572ns) {DropIncomingHandshake:442(17218ns) DropOutgoingHandshake:457(629ns) SwitchToState:179(284ns) ProcessPendingSessionEvents:495(37027899ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 35943151ns max# 6074419ns}}}
2025-11-27T18:20:17.059649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471720564879199:2049] [node 2] ICS09 handshake done sender: [3:7577471716269911894:2119] self: [3:7577471720564879191:0] peer: [2:7577471713781373229:0] socket: 428 qp: -1
2025-11-27T18:20:17.052491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471715261766792:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(55068217ns) {HandleHandshakeStatus:329(55064410ns) {DropIncomingHandshake:442(10094ns) DropOutgoingHandshake:457(200ns) SwitchToState:179(546ns) ProcessPendingSessionEvents:495(41125969ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 41014471ns max# 7682016ns}}}
2025-11-27T18:20:17.059701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471720564879199:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:17.060067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471715261766728:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(64184054ns) {HandleHandshakeStatus:329(64178427ns) {DropIncomingHandshake:442(23414ns) DropOutgoingHandshake:457(352ns) SwitchToState:179(470ns) ProcessPendingSessionEvents:495(51548597ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 51383118ns max# 6067834ns}}}
2025-11-27T18:20:17.060965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:17.059986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471715930604012:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.062946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471716269911893:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:17.059690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471712580479489:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:17.061466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:17.061478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471715930604087:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(59884985ns) {HandleHandshakeStatus:329(59882600ns) {DropIncomingHandshake:442(2546ns) DropOutgoingHandshake:457(1205ns) SwitchToState:179(4898ns) ProcessPendingSessionEvents:495(44834361ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 44701890ns max# 7255657ns}}}
2025-11-27T18:20:17.064318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471716875446798:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24366
2025-11-27T18:20:17.062288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471715930604012:2048] [node 1] ICP22 created new session: [4:7577471720225571416:2048]
2025-11-27T18:20:17.064342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471712580479404:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(53613177ns) {HandleHandshakeStatus:329(53610673ns) {DropIncomingHandshake:442(609ns) DropOutgoingHandshake:457(4866ns) SwitchToState:179(1010ns) ProcessPendingSessionEvents:495(43642811ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 43502928ns max# 7269452ns}}}
2025-11-27T18:20:17.063522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471720225571416:2048] [node 1] ICS09 handshake done sender: [4:7577471715930604110:2129] self: [4:7577471720225571411:0] peer: [1:7577471715261766797:0] socket: 412 qp: -1
2025-11-27T18:20:17.064404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471712580479468:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.063816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471720225571416:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:17.065331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471712580479406:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62664) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471712580479488:2119] held: no
2025-11-27T18:20:17.064080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:17.065407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:17.066044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471712580479468:2108] [node 4] ICP22 created new session: [5:7577471716875446801:2108]
2025-11-27T18:20:17.066101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471716875446801:2108] [node 4] ICS09 handshake done sender: [5:7577471712580479489:2120] self: [5:7577471716875446792:0] peer: [4:7577471715930604092:0] socket: 447 qp: -1
2025-11-27T18:20:17.066139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471716875446801:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:17.066184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:17.066494Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471712580479404:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:17.066939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:17.066624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471716269911805:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.067616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471714601635325:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.069426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.069478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471714601635325:2049] [node 2] ICP22 created new session: [6:7577471718896602724:2049]
2025-11-27T18:20:17.069612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471713781373154:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(31743962ns) {HandleHandshakeStatus:329(31741054ns) {DropIncomingHandshake:442(13789ns) DropOutgoingHandshake:457(770ns) SwitchToState:179(551ns) ProcessPendingSessionEvents:495(24438332ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24306675ns max# 3067087ns}}}
2025-11-27T18:20:17.066715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:17.067544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471716269911805:2048] [node 1] ICP22 created new session: [3:7577471720564879203:2048]
2025-11-27T18:20:17.069526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471714601635389:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(49410989ns) {HandleHandshakeStatus:329(49408033ns) {DropIncomingHandshake:442(788ns) DropOutgoingHandshake:457(4646ns) SwitchToState:179(361ns) ProcessPendingSessionEvents:495(32396297ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32249830ns max# 2567883ns}}}
2025-11-27T18:20:17.067605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471720564879203:2048] [node 1] ICS09 handshake done sender: [3:7577471716269911893:2118] self: [3:7577471720564879193:0] peer: [1:7577471715261766794:0] socket: 404 qp: -1
2025-11-27T18:20:17.068756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471720564879203:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:17.069914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471718896602724:2049] [node 2] ICS09 handshake done sender: [6:7577471714601635412:2122] self: [6:7577471718896602717:0] peer: [2:7577471713781373227:0] socket: 422 qp: -1
2025-11-27T18:20:17.071065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471715261766789:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(21074140ns) {HandleHandshakeStatus:329(19624301ns) {DropIncomingHandshake:442(13334ns) DropOutgoingHandshake:457(331ns) SwitchToState:179(462ns) ProcessPendingSessionEvents:495(15741986ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15385147ns max# 2074965ns}}}
2025-11-27T18:20:17.070254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471720225571418:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41508
2025-11-27T18:20:17.069516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:17.069952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471718896602724:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:17.070751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:17.071383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471715930604012:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:17.071657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471712580479488:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:17.072118Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471713781373220:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(22897025ns) {HandleHandshakeStatus:329(22491969ns) {DropIncomingHandshake:442(1033148ns) DropOutgoingHandshake:457(1067ns) SwitchToState:179(300ns) ProcessPendingSessionEvents:495(16702042ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16594424ns max# 2059335ns}}}
2025-11-27T18:20:17.073370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471712580479406:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.073430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:17.073629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471715261766729:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(29073437ns) {HandleHandshakeStatus:329(29070872ns) {DropIncomingHandshake:442(22549ns) DropOutgoingHandshake:457(515ns) SwitchToState:179(532ns) ProcessPendingSessionEvents:495(18210554ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17756390ns max# 2151141ns}}}
2025-11-27T18:20:17.073612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471720225571418:2132] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.073874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471715930604014:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.073852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471712580479406:2050] [node 3] ICP22 created new session: [5:7577471716875446803:2050]
2025-11-27T18:20:17.075607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471716875446803:2050] [node 3] ICS09 handshake done sender: [5:7577471712580479488:2119] self: [5:7577471716875446794:0] peer: [3:7577471716269911872:0] socket: 446 qp: -1
2025-11-27T18:20:17.075665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471716875446803:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:17.075707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:17.075851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471716269911881:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.075892Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471716269911881:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.075948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471716269911871:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18836253ns) {HandleHandshakeStatus:329(18653317ns) {DropIncomingHandshake:442(12525ns) DropOutgoingHandshake:457(266ns) SwitchToState:179(394ns) ProcessPendingSessionEvents:495(9999786ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9905261ns max# 1493703ns}}}
2025-11-27T18:20:17.076163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471715930604086:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(32751149ns) {HandleHandshakeStatus:329(32748915ns) {DropIncomingHandshake:442(14230ns) DropOutgoingHandshake:457(1744ns) SwitchToState:179(371ns) ProcessPendingSessionEvents:495(20898651ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20260755ns max# 1689588ns}}}
2025-11-27T18:20:17.076246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471712580479467:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(29725560ns) {HandleHandshakeStatus:329(29266390ns) {DropIncomingHandshake:442(17198ns) DropOutgoingHandshake:457(271ns) SwitchToState:179(406ns) ProcessPendingSessionEvents:495(20336068ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18075368ns max# 1975948ns}}}
2025-11-27T18:20:17.077528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471716269911881:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.077766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471720225571420:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41508
2025-11-27T18:20:17.077895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471716269911870:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.078001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:17.078054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471716269911870:2107] [node 4] ICP22 created new session: [3:7577471720564879206:2107]
2025-11-27T18:20:17.078102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471720564879206:2107] [node 4] ICS09 handshake done sender: [3:7577471716269911881:2115] self: [3:7577471716269911873:0] peer: [4:7577471720225571419:0] socket: 398 qp: -1
2025-11-27T18:20:17.078139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471720564879206:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:17.078125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471720225571418:2132] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:17.078176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:17.078435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471715930604014:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.078598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:17.078667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471715930604014:2050] [node 3] ICP22 created new session: [4:7577471720225571421:2050]
2025-11-27T18:20:17.078730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471720225571421:2050] [node 3] ICS09 handshake done sender: [4:7577471720225571418:2132] self: [4:7577471720225571419:0] peer: [3:7577471716269911873:0] socket: 403 qp: -1
2025-11-27T18:20:17.078765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471720225571421:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:17.078842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:17.080124Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471715930604014:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:17.149223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:17.149447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471718076340554:2048] [node 1] ICS01 socket: 420 reason# EndOfStream
2025-11-27T18:20:17.149497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471713781373153:2048] [node 1] ICP30 unregister session Session# [2:7577471718076340554:2048] VirtualId# [2:7577471713781373242:0]
2025-11-27T18:20:17.149544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471718076340554:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.149629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:17.149686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:17.150637Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:17.150870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471720564879203:2048] [node 1] ICS01 socket: 404 reason# EndOfStream
2025-11-27T18:20:17.150917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471716269911805:2048] [node 1] ICP30 unregister session Session# [3:7577471720564879203:2048] VirtualId# [3:7577471720564879193:0]
2025-11-27T18:20:17.150943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471720564879203:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.151025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:17.151083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:17.153144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:17.153391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471720225571416:2048] [node 1] ICS07 socket disconnect 412 reason# EndOfStream
2025-11-27T18:20:17.153451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471720225571416:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.153548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:17.153620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471720225571416:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:17.154069Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:17.154238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471712580479491:2048] [node 1] ICS07 socket disconnect 413 reason# EndOfStream
2025-11-27T18:20:17.154271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471712580479491:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.154343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:17.154388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471712580479491:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:17.154415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471715930604012:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29452) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.154488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471720225571416:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.154535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471715930604012:2048] [node 1] ICP30 unregister session Session# [4:7577471720225571416:2048] VirtualId# [4:7577471720225571411:0]
2025-11-27T18:20:17.154749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:17.154792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471715930604012:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.154813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.155111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:17.155273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471712580479404:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29452) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.155328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471712580479491:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.155417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471712580479404:2048] [node 1] ICP30 unregister session Session# [5:7577471712580479491:2048] VirtualId# [5:7577471712580479484:0]
2025-11-27T18:20:17.155444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471718896602718:2048] [node 1] ICS01 socket: 408 reason# EndOfStream
2025-11-27T18:20:17.155484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471714601635324:2048] [node 1] ICP30 unregister session Session# [6:7577471718896602718:2048] VirtualId# [6:7577471714601635407:0]
2025-11-27T18:20:17.155529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471718896602718:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.155575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:17.155622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471712580479404:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.155639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.155623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:17.155674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:17.164627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471715930604012:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:17.165644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471712580479404:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:17.237121Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:17.237399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471720564879199:2049] [node 2] ICS01 socket: 428 reason# EndOfStream
2025-11-27T18:20:17.237438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471716269911806:2049] [node 2] ICP30 unregister session Session# [3:7577471720564879199:2049] VirtualId# [3:7577471720564879191:0]
2025-11-27T18:20:17.237467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471720564879199:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.237573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:17.237630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:17.240020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:17.240199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471715930604107:2049] [node 2] ICS07 socket disconnect 396 reason# EndOfStream
2025-11-27T18:20:17.240249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471715930604107:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.240345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:17.240396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471715930604107:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:17.241124Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471715930604013:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29698) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.241242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471715930604107:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.241304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471715930604013:2049] [node 2] ICP30 unregister session Session# [4:7577471715930604107:2049] VirtualId# [4:7577471715930604103:0]
2025-11-27T18:20:17.241331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:17.241432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:17.241493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471715930604013:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29698) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.241516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29698) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.241542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471716875446791:2049] [node 2] ICS01 socket: 401 reason# EndOfStream
2025-11-27T18:20:17.241582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471712580479405:2049] [node 2] ICP30 unregister session Session# [5:7577471716875446791:2049] VirtualId# [5:7577471712580479485:0]
2025-11-27T18:20:17.241615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471716875446791:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.241687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:17.241738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:17.242462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:17.242652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471718896602724:2049] [node 2] ICS01 socket: 422 reason# EndOfStream
2025-11-27T18:20:17.242718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471714601635325:2049] [node 2] ICP30 unregister session Session# [6:7577471718896602724:2049] VirtualId# [6:7577471718896602717:0]
2025-11-27T18:20:17.242753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471718896602724:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.242829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:17.242892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:17.251650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471715930604013:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:17.292100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:17.292355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471720225571421:2050] [node 3] ICS07 socket disconnect 403 reason# EndOfStream
2025-11-27T18:20:17.292387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471720225571421:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.292461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:17.292502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471720225571421:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:17.293395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:17.293328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471715930604014:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62664) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.293528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471720225571421:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.293585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471715930604014:2050] [node 3] ICP30 unregister session Session# [4:7577471720225571421:2050] VirtualId# [4:7577471720225571419:0]
2025-11-27T18:20:17.293636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471716875446803:2050] [node 3] ICS01 socket: 446 reason# EndOfStream
2025-11-27T18:20:17.293684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:17.293681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471712580479406:2050] [node 3] ICP30 unregister session Session# [5:7577471716875446803:2050] VirtualId# [5:7577471716875446794:0]
2025-11-27T18:20:17.293715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471716875446803:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.293724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471715930604014:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62664) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.293745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62664) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.293781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:17.293834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:17.294833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:17.295055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471718896602712:2050] [node 3] ICS01 socket: 427 reason# EndOfStream
2025-11-27T18:20:17.295094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471714601635326:2050] [node 3] ICP30 unregister session Session# [6:7577471718896602712:2050] VirtualId# [6:7577471714601635406:0]
2025-11-27T18:20:17.295129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471718896602712:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.295223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:17.295280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:17.303925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471715930604014:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:17.357237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:17.357538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471716875446801:2108] [node 4] ICS01 socket: 447 reason# EndOfStream
2025-11-27T18:20:17.357588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471712580479468:2108] [node 4] ICP30 unregister session Session# [5:7577471716875446801:2108] VirtualId# [5:7577471716875446792:0]
2025-11-27T18:20:17.357625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471716875446801:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.357731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:17.357782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:17.358290Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:17.358498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471718896602713:2108] [node 4] ICS07 socket disconnect 418 reason# EndOfStream
2025-11-27T18:20:17.358535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471718896602713:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.358603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:17.358677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471718896602713:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:17.359373Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471714601635389:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9378) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.359458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471718896602713:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.359506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471714601635389:2108] [node 4] ICP30 unregister session Session# [6:7577471718896602713:2108] VirtualId# [6:7577471714601635408:0]
2025-11-27T18:20:17.359608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:17.359676Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471714601635389:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9378) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.359692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9378) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.369420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471714601635389:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:17.445525Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:17.445784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471718896602722:2105] [node 5] ICS01 socket: 411 reason# EndOfStream
2025-11-27T18:20:17.445832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471714601635385:2105] [node 5] ICP30 unregister session Session# [6:7577471718896602722:2105] VirtualId# [6:7577471718896602716:0]
2025-11-27T18:20:17.445875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471718896602722:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.445975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:17.446030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:17.547952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471717211599361:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:17.548024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471717211599362:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:17.569503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471719078493039:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:17.569577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471719078493040:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:17.588167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471717626330864:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:17.588237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471717626330865:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:17.606324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471719105039820:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:17.606401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471719105039821:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:17.606448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471719105039822:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:17.625160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471719134800327:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:17.625223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471719134800328:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:17.625285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471719134800329:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:17.646327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471720317345978:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:17.646421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471720317345979:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:17.646468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471720317345980:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:17.727948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471720317346040:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:17.754926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471717626330928:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:17.752567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471719078493103:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:17.755275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471719134800392:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:17.755255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471719105039892:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:17.755592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471717626330929:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:17.755608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471720317346042:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:17.755433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471719078493104:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:17.755995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471719134800393:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:17.756237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471719078493105:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:17.756416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471717626330930:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:17.756807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471717211599361:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(57967983ns) {Configure:104(57949143ns) {ProcessConfigured:135(86668ns) {StartInitialHandshake:157(83844ns) {DropHandshakes:470(2235ns) {DropIncomingHandshake:442(549ns) DropOutgoingHandshake:457(295ns)} GenerateSessionVirtualId:988(790ns) PrepareNewSessionHandshake:477(1461ns) {SwitchToState:179(433ns)}}}}}
2025-11-27T18:20:17.757516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471717211599426:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:17.758174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471717211599427:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:17.758896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471717211599428:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:17.758841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471719105039895:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.770063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471717626330944:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25727
2025-11-27T18:20:17.770256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471719134800404:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.772291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471719078493116:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30192
2025-11-27T18:20:17.772680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471719078493117:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30192
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.773077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471719078493121:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30192
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.774818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471719105039904:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-11-27T18:20:17.775011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471719078493122:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30192
2025-11-27T18:20:17.774909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471719105039905:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.775302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471719105039909:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.780707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471717211599440:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44867
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.781953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471719134800404:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.782046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471717626330945:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25727
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.782459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471717626330946:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25727
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.782561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471717211599440:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.782645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471717211599441:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44867
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.782662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471717626330944:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.782735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471717626330947:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25727
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.782914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471717626330946:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.782965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471717211599442:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44867
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.783088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471717211599443:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44867
2025-11-27T18:20:17.783253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471717211599441:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.783526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471717211599444:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44867
No RDMA context found for GID: 2025-11-27T18:20:17.784043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471717626330864:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-27T18:20:17.784375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471717211599443:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.784382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471719078493123:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30192
2025-11-27T18:20:17.784519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471717211599430:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.784546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471717211599430:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.784583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471719078493117:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.784785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471719078493121:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.784918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471719078493116:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471717626330947:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471719105039905:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471717626330945:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471719105039822:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.785250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471717211599444:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471717626330865:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.785369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471719105039904:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471719078493122:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471719078493123:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471717626330936:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471719105039909:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471717211599442:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785565Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471717626330936:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.785689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471719078493109:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.785689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471719105039820:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.785714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471719078493109:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.785754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471719105039821:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.785884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471717211599430:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:17.786627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471719078493039:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.786810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471717626330952:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25727
2025-11-27T18:20:17.786830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471719078493111:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.786856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471719078493111:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.787020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471717211599432:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.787052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471717211599432:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.787204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471719105039913:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-11-27T18:20:17.787781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471717211599362:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.787810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471717626330936:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:17.787853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:17.787914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471717211599362:2049] [node 3] ICP22 created new session: [1:7577471717211599451:2049]
2025-11-27T18:20:17.787966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471717211599451:2049] [node 3] ICS09 handshake done sender: [1:7577471717211599430:2111] self: [1:7577471717211599429:0] peer: [3:7577471717626330948:0] socket: 400 qp: -1
2025-11-27T18:20:17.788197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471717626330928:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.788053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471717211599451:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:17.788151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:17.788311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471719105039905:2125] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:17.788340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:17.788398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471717626330928:2107] [node 4] ICP22 created new session: [3:7577471717626330954:2107]
2025-11-27T18:20:17.788670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471717626330954:2107] [node 4] ICS09 handshake done sender: [3:7577471717626330936:2113] self: [3:7577471717626330935:0] peer: [4:7577471719105039910:0] socket: 410 qp: -1
2025-11-27T18:20:17.788743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471717626330954:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:17.788990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:17.789224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471719078493109:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:17.789287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471719105039822:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.789520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:17.789568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471719105039822:2050] [node 3] ICP22 created new session: [4:7577471719105039914:2050]
2025-11-27T18:20:17.789649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471719105039914:2050] [node 3] ICS09 handshake done sender: [4:7577471719105039905:2125] self: [4:7577471719105039910:0] peer: [3:7577471717626330935:0] socket: 414 qp: -1
2025-11-27T18:20:17.789701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471719105039914:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:17.789745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:17.789766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471719078493040:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.789832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:17.789981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471719078493040:2049] [node 3] ICP22 created new session: [2:7577471719078493131:2049]
2025-11-27T18:20:17.790027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471719078493131:2049] [node 3] ICS09 handshake done sender: [2:7577471719078493109:2112] self: [2:7577471719078493108:0] peer: [3:7577471717626330951:0] socket: 416 qp: -1
2025-11-27T18:20:17.790053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471719078493131:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:17.790079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:17.791638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471717211599425:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.791672Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471717211599425:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.791854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471717626330944:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:17.793003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471717211599432:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.793475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471719078493111:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:17.794561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471719134800409:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-11-27T18:20:17.794633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471717211599426:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.794705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:17.794761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471717211599426:2107] [node 4] ICP22 created new session: [1:7577471717211599456:2107]
2025-11-27T18:20:17.795051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471717211599456:2107] [node 4] ICS09 handshake done sender: [1:7577471717211599432:2112] self: [1:7577471717211599431:0] peer: [4:7577471719105039911:0] socket: 403 qp: -1
2025-11-27T18:20:17.795117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471717211599456:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:17.795155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:17.796418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471717626330864:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.796195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471717211599425:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:17.796690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:17.796749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471717626330864:2048] [node 1] ICP22 created new session: [3:7577471717626330957:2048]
2025-11-27T18:20:17.796812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471717626330957:2048] [node 1] ICS09 handshake done sender: [3:7577471717626330944:2116] self: [3:7577471717626330948:0] peer: [1:7577471717211599429:0] socket: 406 qp: -1
2025-11-27T18:20:17.796831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471719078493103:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.796851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471717626330957:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:17.796915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:17.796936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:17.796984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471719078493103:2107] [node 4] ICP22 created new session: [2:7577471719078493134:2107]
2025-11-27T18:20:17.797021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471719078493134:2107] [node 4] ICS09 handshake done sender: [2:7577471719078493111:2113] self: [2:7577471719078493110:0] peer: [4:7577471719105039912:0] socket: 419 qp: -1
2025-11-27T18:20:17.797078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471719078493134:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:17.797152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:17.797352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471717211599361:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.797690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:17.797740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471717211599361:2048] [node 2] ICP22 created new session: [1:7577471717211599458:2048]
2025-11-27T18:20:17.797824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471717211599458:2048] [node 2] ICS09 handshake done sender: [1:7577471717211599425:2110] self: [1:7577471717211599424:0] peer: [2:7577471719078493127:0] socket: 396 qp: -1
2025-11-27T18:20:17.797878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471717211599458:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:17.797916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:17.798248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471717626330864:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.799646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471719078493133:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30192
2025-11-27T18:20:17.800133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471719105039820:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471719105039904:2124] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.800381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471719134800410:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-11-27T18:20:17.800437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471719078493116:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.800819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471719078493039:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.800791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471720317346056:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-27T18:20:17.800907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:17.800958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471719078493039:2048] [node 1] ICP22 created new session: [2:7577471719078493138:2048]
2025-11-27T18:20:17.801003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471719078493138:2048] [node 1] ICS09 handshake done sender: [2:7577471719078493116:2116] self: [2:7577471719078493127:0] peer: [1:7577471717211599424:0] socket: 401 qp: -1
2025-11-27T18:20:17.801036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471719078493138:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:17.801069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.802694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471719105039916:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-11-27T18:20:17.802706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471717626330865:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61557) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471717626330945:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.803192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471719105039917:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-11-27T18:20:17.803211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471720317346057:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-27T18:20:17.803267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471719134800409:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.803598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471719134800411:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-11-27T18:20:17.803682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471719134800327:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.803680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471717626330956:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25727
2025-11-27T18:20:17.804060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471719105039821:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61557) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471719105039909:2126] held: no
2025-11-27T18:20:17.804355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471717211599434:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.804390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471717211599434:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.805586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471717626330945:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:17.805855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471717626330865:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.805939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:17.805991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471717626330865:2049] [node 2] ICP22 created new session: [3:7577471717626330961:2049]
2025-11-27T18:20:17.806029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471717626330961:2049] [node 2] ICS09 handshake done sender: [3:7577471717626330945:2117] self: [3:7577471717626330951:0] peer: [2:7577471719078493108:0] socket: 426 qp: -1
2025-11-27T18:20:17.805998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471720317346058:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-27T18:20:17.806060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471717626330961:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:17.806111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:17.807511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471719105039909:2126] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:17.807815Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471719078493039:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:17.807855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471720317346057:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.807982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471720317345978:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.808060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471719105039904:2124] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:17.808035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471717211599434:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:17.810358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471719105039821:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.810907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:17.811106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471719134800410:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.811091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471719105039821:2049] [node 2] ICP22 created new session: [4:7577471719105039920:2049]
2025-11-27T18:20:17.811454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471720317346060:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.811671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471719134800413:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-11-27T18:20:17.811873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471720317346056:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.810903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471717211599436:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.812015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471720317345979:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.812074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471719105039920:2049] [node 2] ICS09 handshake done sender: [4:7577471719105039909:2126] self: [4:7577471719105039912:0] peer: [2:7577471719078493110:0] socket: 428 qp: -1
2025-11-27T18:20:17.810939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471717211599436:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.812188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471719105039920:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:17.812270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:17.811693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471717211599427:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.811761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.811819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471717211599427:2108] [node 5] ICP22 created new session: [1:7577471717211599462:2108]
2025-11-27T18:20:17.812383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471719078493115:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.811864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471717211599462:2108] [node 5] ICS09 handshake done sender: [1:7577471717211599434:2113] self: [1:7577471717211599433:0] peer: [5:7577471719134800412:0] socket: 446 qp: -1
2025-11-27T18:20:17.812415Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471719078493115:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.811895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471717211599462:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:17.812460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471719134800414:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-11-27T18:20:17.811950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:17.812578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471719134800411:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.812628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471719134800328:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.812670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471719134800409:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:17.813525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471719134800327:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.813579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:17.813640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471719134800327:2048] [node 1] ICP22 created new session: [5:7577471719134800416:2048]
2025-11-27T18:20:17.813689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471719134800416:2048] [node 1] ICS09 handshake done sender: [5:7577471719134800409:2118] self: [5:7577471719134800412:0] peer: [1:7577471717211599433:0] socket: 447 qp: -1
2025-11-27T18:20:17.813716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471719134800416:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:17.813747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:17.813756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471720317346058:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.813769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471719078493113:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.813830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471719105039820:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.813797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471719078493113:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.813933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:17.814447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471719105039820:2048] [node 1] ICP22 created new session: [4:7577471719105039922:2048]
2025-11-27T18:20:17.814605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471719105039922:2048] [node 1] ICS09 handshake done sender: [4:7577471719105039904:2124] self: [4:7577471719105039911:0] peer: [1:7577471717211599431:0] socket: 413 qp: -1
2025-11-27T18:20:17.814646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471719105039922:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:17.814668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.814818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471720317345980:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.814894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471719078493115:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:17.814905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471720317346062:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-27T18:20:17.815112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471717211599436:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:17.815524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471719078493105:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.815609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:17.815661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471719078493105:2109] [node 6] ICP22 created new session: [2:7577471719078493142:2109]
2025-11-27T18:20:17.815706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471719078493142:2109] [node 6] ICS09 handshake done sender: [2:7577471719078493115:2115] self: [2:7577471719078493114:0] peer: [6:7577471720317346061:0] socket: 433 qp: -1
2025-11-27T18:20:17.815733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471719078493142:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:17.815779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:17.815804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471717626330940:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.815836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471717626330940:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.815887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471720317346056:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:17.816358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471720317345979:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.816404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:17.816453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471720317345979:2049] [node 2] ICP22 created new session: [6:7577471720317346065:2049]
2025-11-27T18:20:17.816495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471720317346065:2049] [node 2] ICS09 handshake done sender: [6:7577471720317346056:2115] self: [6:7577471720317346061:0] peer: [2:7577471719078493114:0] socket: 437 qp: -1
2025-11-27T18:20:17.816530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471720317346065:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:17.816562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:17.816794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471717211599428:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.816782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471717626330940:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:17.816886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:17.816939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471717211599428:2109] [node 6] ICP22 created new session: [1:7577471717211599465:2109]
2025-11-27T18:20:17.817334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471717211599465:2109] [node 6] ICS09 handshake done sender: [1:7577471717211599436:2114] self: [1:7577471717211599435:0] peer: [6:7577471720317346059:0] socket: 423 qp: -1
2025-11-27T18:20:17.817455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471717211599465:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:17.817477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471717626330930:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.817525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:17.817543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:17.817581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471717626330930:2109] [node 6] ICP22 created new session: [3:7577471717626330964:2109]
2025-11-27T18:20:17.817618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471717626330964:2109] [node 6] ICS09 handshake done sender: [3:7577471717626330940:2115] self: [3:7577471717626330939:0] peer: [6:7577471720317346063:0] socket: 415 qp: -1
2025-11-27T18:20:17.817666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471717626330964:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:17.817699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:17.818136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471719134800329:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.818223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471719078493113:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.820597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471719134800418:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.820823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471717626330938:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.820818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471720317346064:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-27T18:20:17.820856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471717626330938:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.822328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471719134800410:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:17.822324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471719078493104:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.822548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471717626330938:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:17.822446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:17.822495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471719078493104:2108] [node 5] ICP22 created new session: [2:7577471719078493144:2108]
2025-11-27T18:20:17.822848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471719078493144:2108] [node 5] ICS09 handshake done sender: [2:7577471719078493113:2114] self: [2:7577471719078493112:0] peer: [5:7577471719134800415:0] socket: 432 qp: -1
2025-11-27T18:20:17.822918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471719078493144:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:17.822942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471719134800328:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.822959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:17.823547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:17.823647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471719134800328:2049] [node 2] ICP22 created new session: [5:7577471719134800422:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.823727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471719134800422:2049] [node 2] ICS09 handshake done sender: [5:7577471719134800410:2119] self: [5:7577471719134800415:0] peer: [2:7577471719078493112:0] socket: 430 qp: -1
2025-11-27T18:20:17.823698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471717626330929:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.823773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471719134800422:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:17.823792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:17.823815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:17.823835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471717626330929:2108] [node 5] ICP22 created new session: [3:7577471717626330967:2108]
2025-11-27T18:20:17.823784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471719105039924:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-11-27T18:20:17.823888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471717626330967:2108] [node 5] ICS09 handshake done sender: [3:7577471717626330938:2114] self: [3:7577471717626330937:0] peer: [5:7577471719134800419:0] socket: 408 qp: -1
2025-11-27T18:20:17.823921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471717626330967:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:17.823992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:17.826144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471720317346057:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:17.826198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471719134800414:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.826428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471720317345978:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471720317346057:2116] held: no
2025-11-27T18:20:17.826986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471720317345978:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.827032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:17.827069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471720317345978:2048] [node 1] ICP22 created new session: [6:7577471720317346068:2048]
2025-11-27T18:20:17.827116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471720317346068:2048] [node 1] ICS09 handshake done sender: [6:7577471720317346057:2116] self: [6:7577471720317346059:0] peer: [1:7577471717211599435:0] socket: 431 qp: -1
2025-11-27T18:20:17.827148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471720317346068:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:17.827183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:17.828308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471720317346060:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.829061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471720317346067:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.829752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471719134800421:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-11-27T18:20:17.829983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471720317346042:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.830067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471719134800329:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32612) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471719134800411:2120] held: no
2025-11-27T18:20:17.830119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471720317346058:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.830208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471717626330969:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25727
2025-11-27T18:20:17.830328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471719105039925:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-11-27T18:20:17.830513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471719105039903:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.830518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471720317345980:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.830539Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471719105039903:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.830587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:17.830657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471720317345980:2050] [node 3] ICP22 created new session: [6:7577471720317346073:2050]
2025-11-27T18:20:17.830710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471720317346073:2050] [node 3] ICS09 handshake done sender: [6:7577471720317346058:2117] self: [6:7577471720317346063:0] peer: [3:7577471717626330939:0] socket: 449 qp: -1
2025-11-27T18:20:17.830743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471720317346073:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:17.830789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:17.831428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471719134800411:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:17.831444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471719105039924:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.831721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471719134800392:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.831789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471719134800329:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.831833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471719105039903:2123] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:17.831852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:17.831896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471719134800329:2050] [node 3] ICP22 created new session: [5:7577471719134800426:2050]
2025-11-27T18:20:17.831936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471719134800426:2050] [node 3] ICS09 handshake done sender: [5:7577471719134800411:2120] self: [5:7577471719134800419:0] peer: [3:7577471717626330937:0] socket: 448 qp: -1
2025-11-27T18:20:17.831977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471719134800426:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:17.832010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:17.832116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471719105039895:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.832188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:17.832237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471719105039895:2109] [node 6] ICP22 created new session: [4:7577471719105039928:2109]
2025-11-27T18:20:17.832290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471719105039928:2109] [node 6] ICS09 handshake done sender: [4:7577471719105039903:2123] self: [4:7577471719105039902:0] peer: [6:7577471720317346072:0] socket: 452 qp: -1
2025-11-27T18:20:17.832329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471719105039928:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:17.832387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:17.833213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471719105039901:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.833235Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471719105039901:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.834293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471719105039901:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:17.834539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471719105039892:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.834583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:17.834620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471719105039892:2108] [node 5] ICP22 created new session: [4:7577471719105039931:2108]
2025-11-27T18:20:17.834665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471719105039931:2108] [node 5] ICS09 handshake done sender: [4:7577471719105039901:2122] self: [4:7577471719105039900:0] peer: [5:7577471719134800424:0] socket: 450 qp: -1
2025-11-27T18:20:17.834699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471719105039931:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:17.834721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.836005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471719134800428:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-11-27T18:20:17.836098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471720317346075:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.836526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471719134800414:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:17.836566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471720317346076:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-27T18:20:17.836843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471720317346060:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:17.836862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471719105039925:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.837014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471719105039895:2109] [node 6] ICP18 handshake (actor [4:7577471719105039925:2131]) is too old
2025-11-27T18:20:17.837484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471720317346042:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.837574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:17.837635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471720317346042:2109] [node 4] ICP22 created new session: [6:7577471720317346077:2109]
2025-11-27T18:20:17.837647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471719134800392:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.837693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471720317346077:2109] [node 4] ICS09 handshake done sender: [6:7577471720317346060:2118] self: [6:7577471720317346072:0] peer: [4:7577471719105039902:0] socket: 455 qp: -1
2025-11-27T18:20:17.837730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471720317346077:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:17.837711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:17.837757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:17.837755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471719134800392:2108] [node 4] ICP22 created new session: [5:7577471719134800429:2108]
2025-11-27T18:20:17.837801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471719134800429:2108] [node 4] ICS09 handshake done sender: [5:7577471719134800414:2122] self: [5:7577471719134800424:0] peer: [4:7577471719105039900:0] socket: 454 qp: -1
2025-11-27T18:20:17.837821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471719134800429:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:17.837841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:17.838447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471719134800392:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:17.840541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471720317346042:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:17.840719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471720317346076:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.840839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471720317346040:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:17.841083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471719134800403:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:17.841107Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471719134800403:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:17.842095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471719134800403:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:17.842384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471719134800393:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:17.842381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471720317346080:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-27T18:20:17.842428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:17.842456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471719134800393:2110] [node 6] ICP22 created new session: [5:7577471719134800432:2110]
2025-11-27T18:20:17.842493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471719134800432:2110] [node 6] ICS09 handshake done sender: [5:7577471719134800403:2116] self: [5:7577471719134800402:0] peer: [6:7577471720317346079:0] socket: 463 qp: -1
2025-11-27T18:20:17.842522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471719134800432:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:17.842566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:17.842786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471720317346076:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:17.842965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471720317346040:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:17.843005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:17.843073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471720317346040:2107] [node 5] ICP22 created new session: [6:7577471720317346081:2107]
2025-11-27T18:20:17.843121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471720317346081:2107] [node 5] ICS09 handshake done sender: [6:7577471720317346076:2123] self: [6:7577471720317346079:0] peer: [5:7577471719134800402:0] socket: 464 qp: -1
2025-11-27T18:20:17.843154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471720317346081:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:17.843183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:17.861843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:17.862008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471719078493138:2048] [node 1] ICS07 socket disconnect 401 reason# EndOfStream
2025-11-27T18:20:17.862032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471719078493138:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.862111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:17.862154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471719078493138:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:17.863213Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:17.863406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471717626330957:2048] [node 1] ICS07 socket disconnect 406 reason# EndOfStream
2025-11-27T18:20:17.863434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471717626330957:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.863501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:17.863554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471717626330957:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:17.866253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471719078493039:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:17.866288Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:17.866286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471719078493138:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:17.866519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471719105039922:2048] [node 1] ICS01 socket: 413 reason# EndOfStream
2025-11-27T18:20:17.866563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471719105039820:2048] [node 1] ICP30 unregister session Session# [4:7577471719105039922:2048] VirtualId# [4:7577471719105039911:0]
2025-11-27T18:20:17.866602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471719105039922:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.866669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:17.866721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:17.867164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471719078493039:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.867229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471719078493138:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.867264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471719078493039:2048] [node 1] ICP30 unregister session Session# [2:7577471719078493138:2048] VirtualId# [2:7577471719078493127:0]
2025-11-27T18:20:17.867243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471717626330864:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:17.867277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471717626330957:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:17.867362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:17.867406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471719078493039:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.867421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.867554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:17.867800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471719134800416:2048] [node 1] ICS07 socket disconnect 447 reason# EndOfStream
2025-11-27T18:20:17.867833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471719134800416:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.867901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:17.867951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471719134800416:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:17.868034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471717626330864:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.868085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471717626330957:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.868132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471717626330864:2048] [node 1] ICP30 unregister session Session# [3:7577471717626330957:2048] VirtualId# [3:7577471717626330948:0]
2025-11-27T18:20:17.868175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:17.868243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471717626330864:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.868266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.868591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471719134800327:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.868625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471719134800416:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.868643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471719134800327:2048] [node 1] ICP30 unregister session Session# [5:7577471719134800416:2048] VirtualId# [5:7577471719134800412:0]
2025-11-27T18:20:17.868698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:17.868730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:17.868781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471719134800327:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.868791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17327) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.868924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471720317346068:2048] [node 1] ICS01 socket: 431 reason# EndOfStream
2025-11-27T18:20:17.868973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471720317345978:2048] [node 1] ICP30 unregister session Session# [6:7577471720317346068:2048] VirtualId# [6:7577471720317346059:0]
2025-11-27T18:20:17.869017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471720317346068:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.869102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:17.869143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:17.877333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471719078493039:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:17.878297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471719134800327:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:17.878326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471717626330864:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:17.925072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:17.925312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471717626330961:2049] [node 2] ICS01 socket: 426 reason# EndOfStream
2025-11-27T18:20:17.925367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471717626330865:2049] [node 2] ICP30 unregister session Session# [3:7577471717626330961:2049] VirtualId# [3:7577471717626330951:0]
2025-11-27T18:20:17.925404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471717626330961:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.925498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:17.925554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:17.928708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:17.928880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471719105039920:2049] [node 2] ICS01 socket: 428 reason# EndOfStream
2025-11-27T18:20:17.928914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471719105039821:2049] [node 2] ICP30 unregister session Session# [4:7577471719105039920:2049] VirtualId# [4:7577471719105039912:0]
2025-11-27T18:20:17.928936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471719105039920:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.928992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:17.929039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:17.929784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:17.930052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471719134800422:2049] [node 2] ICS07 socket disconnect 430 reason# EndOfStream
2025-11-27T18:20:17.930092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471719134800422:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.930189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:17.930248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471719134800422:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:17.930649Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:17.930867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471720317346065:2049] [node 2] ICS07 socket disconnect 437 reason# EndOfStream
2025-11-27T18:20:17.930907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471720317346065:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.930989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:17.931032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471720317346065:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:17.931147Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471719134800328:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61557) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.931208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471719134800422:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.931269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471719134800328:2049] [node 2] ICP30 unregister session Session# [5:7577471719134800422:2049] VirtualId# [5:7577471719134800415:0]
2025-11-27T18:20:17.931469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:17.931507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471719134800328:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.931522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.932129Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471720317345979:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61557) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.932185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471720317346065:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.932219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471720317345979:2049] [node 2] ICP30 unregister session Session# [6:7577471720317346065:2049] VirtualId# [6:7577471720317346061:0]
2025-11-27T18:20:17.932324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:17.932357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471720317345979:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.932375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.941808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471719134800328:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:17.941804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471720317345979:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:17.959441Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:17.959628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471719105039914:2050] [node 3] ICS07 socket disconnect 414 reason# EndOfStream
2025-11-27T18:20:17.959654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471719105039914:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.959728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:17.959772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471719105039914:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:17.960546Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:17.960608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471719105039822:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.960665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471719105039914:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.960699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471719105039822:2050] [node 3] ICP30 unregister session Session# [4:7577471719105039914:2050] VirtualId# [4:7577471719105039910:0]
2025-11-27T18:20:17.960758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471719134800426:2050] [node 3] ICS01 socket: 448 reason# EndOfStream
2025-11-27T18:20:17.960796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:17.960813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471719134800329:2050] [node 3] ICP30 unregister session Session# [5:7577471719134800426:2050] VirtualId# [5:7577471719134800419:0]
2025-11-27T18:20:17.960839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471719134800426:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.960835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471719105039822:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.960849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.960903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:17.960951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:17.961601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:17.961797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471720317346073:2050] [node 3] ICS07 socket disconnect 449 reason# EndOfStream
2025-11-27T18:20:17.961832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471720317346073:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:17.961896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:17.961943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471720317346073:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:17.962651Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471720317345980:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:17.962698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471720317346073:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:17.962732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471720317345980:2050] [node 3] ICP30 unregister session Session# [6:7577471720317346073:2050] VirtualId# [6:7577471720317346063:0]
2025-11-27T18:20:17.962836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:17.962871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471720317345980:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.962884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:17.970501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471719105039822:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:17.972521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471720317345980:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:18.006676Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:18.006897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471719134800429:2108] [node 4] ICS07 socket disconnect 454 reason# EndOfStream
2025-11-27T18:20:18.006931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471719134800429:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.007023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:18.007072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471719134800429:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:18.007869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:18.008061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471719134800392:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8643) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:18.008088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471720317346077:2109] [node 4] ICS07 socket disconnect 455 reason# EndOfStream
2025-11-27T18:20:18.008116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471719134800429:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:18.008122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471720317346077:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.008154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471719134800392:2108] [node 4] ICP30 unregister session Session# [5:7577471719134800429:2108] VirtualId# [5:7577471719134800424:0]
2025-11-27T18:20:18.008183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:18.008224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471720317346077:2109] [node 4] ICS15 start handshake
2025-11-27T18:20:18.008283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:18.008321Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471719134800392:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8643) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.008336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8643) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.009090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471720317346042:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8643) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:18.009139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471720317346077:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:18.009174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471720317346042:2109] [node 4] ICP30 unregister session Session# [6:7577471720317346077:2109] VirtualId# [6:7577471720317346072:0]
2025-11-27T18:20:18.009306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:18.009345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471720317346042:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8643) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.009377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8643) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.019656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471720317346042:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:20:18.035699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:18.035905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471720317346081:2107] [node 5] ICS07 socket disconnect 464 reason# EndOfStream
2025-11-27T18:20:18.035930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471720317346081:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.036008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:18.036070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471720317346081:2107] [node 5] ICS15 start handshake
2025-11-27T18:20:18.036852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471720317346040:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:18.036892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471720317346081:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:18.036919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471720317346040:2107] [node 5] ICP30 unregister session Session# [6:7577471720317346081:2107] VirtualId# [6:7577471720317346079:0]
2025-11-27T18:20:18.037018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:18.037045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471720317346040:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.037055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.047304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471720317346040:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:18.115310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471724084131067:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:18.115369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471724084131068:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:18.136624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471724040567872:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:18.136659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471724040567873:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:18.156815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471723096749799:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:18.156903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471723096749800:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:18.177114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471724383616487:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:18.177166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471724383616488:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:18.177216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471724383616489:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:18.197561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471723507175919:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:18.197600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471723507175920:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:18.197690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471723507175921:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:18.218937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471722752972192:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:18.218978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471722752972193:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:18.219021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471722752972194:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:18.289946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471722752972251:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:18.306223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471724084131130:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:18.306344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471724040567936:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:18.321928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471723507175982:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:18.313045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471724040567937:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:18.322098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471724084131131:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:18.322414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471724383616551:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:18.322088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471723096749862:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:18.322611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471724040567938:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:18.322559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471724084131132:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:18.322477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471722752972254:2106] [node 4] ICP01 ready to work
2025-11-27T18:20:18.323215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471723096749863:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:18.323332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471723507175983:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:18.323514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471724383616552:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:18.323259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471723096749864:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.338086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471724040567952:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64610
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.339063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471724040567953:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64610
2025-11-27T18:20:18.339717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471724040567954:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64610
No RDMA context found for GID: No RDMA context found for GID: 2025-11-27T18:20:18.340007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471723507175995:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59984
::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:18.341924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471724383616566:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5727
2025-11-27T18:20:18.342118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471723096749876:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6386
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-27T18:20:18.343848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471723096749879:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6386
2025-11-27T18:20:18.344490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471724383616568:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5727
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.344579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471724084131146:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35656
2025-11-27T18:20:18.344564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471723096749880:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6386
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.346692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471724084131147:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35656
2025-11-27T18:20:18.347131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471724383616567:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5727
2025-11-27T18:20:18.347098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471723096749881:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6386
2025-11-27T18:20:18.348233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471724084131149:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35656
2025-11-27T18:20:18.349158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471724084131147:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.350315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471723507175995:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.350834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471724040567955:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64610
2025-11-27T18:20:18.350564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471723096749879:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.350844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471723096749880:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.351222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471724040567953:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.351151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471724383616567:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.351606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471724040567954:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.351653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471724383616488:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.351641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471724040567956:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64610
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.352568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471724040567955:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.353418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471724040567945:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.353488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471724084131148:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35656
2025-11-27T18:20:18.353458Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471724040567945:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.354630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471723096749881:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.354720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471724084131150:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35656
2025-11-27T18:20:18.354976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471724084131149:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.355161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471724084131146:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.355199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471724383616568:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.355333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471723096749800:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.355603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471724084131148:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.355381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471724040567945:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:18.355625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471724383616489:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.355662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471724040567952:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.355763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471724383616570:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5727
2025-11-27T18:20:18.355815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471724040567872:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.356023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471724383616566:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.356029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471723096749876:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.356034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471724084131150:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.356196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471724040567942:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.356225Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471724040567942:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.356378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471724084131134:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.356201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471724383616487:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.356320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471723096749871:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.356658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471724383616567:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:18.356666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471724040567956:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.356413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471724084131134:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.356356Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471723096749871:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.357435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471724040567936:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.359681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471724084131138:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.356636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471723096749799:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.359712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471724084131138:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.360449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471724383616488:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.359702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:18.360442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471724084131136:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.360569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471723096749871:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:18.359764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471724040567936:2107] [node 4] ICP22 created new session: [2:7577471724040567962:2107]
2025-11-27T18:20:18.360561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471724040567962:2107] [node 4] ICS09 handshake done sender: [2:7577471724040567945:2113] self: [2:7577471724040567943:0] peer: [4:7577471724383616569:0] socket: 407 qp: -1
2025-11-27T18:20:18.360531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:18.360921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471724040567962:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:18.360478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471724084131136:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.360914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471724383616488:2049] [node 2] ICP22 created new session: [4:7577471724383616573:2049]
2025-11-27T18:20:18.361853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471724084131138:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:18.361461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:18.360975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471724383616573:2049] [node 2] ICS09 handshake done sender: [4:7577471724383616567:2117] self: [4:7577471724383616569:0] peer: [2:7577471724040567943:0] socket: 436 qp: -1
2025-11-27T18:20:18.362521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471724084131134:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:18.361090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471724383616573:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:18.361623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:18.362722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471723096749862:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.362824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:18.363345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471724084131136:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:18.363013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471723096749862:2107] [node 4] ICP22 created new session: [3:7577471723096749889:2107]
2025-11-27T18:20:18.363369Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471724040567942:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:18.363598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471723096749889:2107] [node 4] ICS09 handshake done sender: [3:7577471723096749871:2112] self: [3:7577471723096749869:0] peer: [4:7577471724383616571:0] socket: 434 qp: -1
2025-11-27T18:20:18.363784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471724084131130:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.364278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471723096749889:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:18.364619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:18.369284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:18.369592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471724084131130:2107] [node 4] ICP22 created new session: [1:7577471724084131159:2107]
2025-11-27T18:20:18.369652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471724084131067:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.369891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:18.369913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471724084131159:2107] [node 4] ICS09 handshake done sender: [1:7577471724084131138:2112] self: [1:7577471724084131137:0] peer: [4:7577471724383616572:0] socket: 406 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.369975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471724084131159:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:18.370068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:18.370151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471724084131067:2048] [node 2] ICP22 created new session: [1:7577471724084131160:2048]
2025-11-27T18:20:18.370534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471723096749887:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6386
2025-11-27T18:20:18.370270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471724084131160:2048] [node 2] ICS09 handshake done sender: [1:7577471724084131134:2110] self: [1:7577471724084131133:0] peer: [2:7577471724040567961:0] socket: 402 qp: -1
2025-11-27T18:20:18.370472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471724084131160:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:18.370509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:18.370860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471724040567873:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.370918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:18.370967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471724040567873:2049] [node 3] ICP22 created new session: [2:7577471724040567968:2049]
2025-11-27T18:20:18.371106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471724040567968:2049] [node 3] ICS09 handshake done sender: [2:7577471724040567942:2112] self: [2:7577471724040567941:0] peer: [3:7577471723096749882:0] socket: 405 qp: -1
2025-11-27T18:20:18.371145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471724040567968:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:18.371188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471724383616489:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471724383616568:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.371180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:18.371704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471723096749888:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6386
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:18.372512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471723507175999:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59984
::ffff:127.1.0.0
2025-11-27T18:20:18.372675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471723096749876:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.372730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471724383616575:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5727
2025-11-27T18:20:18.372937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471723096749879:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:18.373620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471724084131068:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.373720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471724383616576:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5727
2025-11-27T18:20:18.373691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:18.373893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471724084131068:2049] [node 3] ICP22 created new session: [1:7577471724084131163:2049]
2025-11-27T18:20:18.374299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471724084131163:2049] [node 3] ICS09 handshake done sender: [1:7577471724084131136:2111] self: [1:7577471724084131135:0] peer: [3:7577471723096749885:0] socket: 404 qp: -1
2025-11-27T18:20:18.374010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471723096749799:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.374127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.374429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471724084131163:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:18.374400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471723096749799:2048] [node 1] ICP22 created new session: [3:7577471723096749892:2048]
2025-11-27T18:20:18.374980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471723507176000:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59984
2025-11-27T18:20:18.374472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:18.374519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471723096749892:2048] [node 1] ICS09 handshake done sender: [3:7577471723096749876:2114] self: [3:7577471723096749885:0] peer: [1:7577471724084131135:0] socket: 410 qp: -1
2025-11-27T18:20:18.375447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471724040567965:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64610
2025-11-27T18:20:18.374706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471723096749892:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:18.374740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:18.376610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471723096749800:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.376869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:18.376926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471723096749800:2049] [node 2] ICP22 created new session: [3:7577471723096749894:2049]
2025-11-27T18:20:18.377536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471723096749894:2049] [node 2] ICS09 handshake done sender: [3:7577471723096749879:2116] self: [3:7577471723096749882:0] peer: [2:7577471724040567941:0] socket: 416 qp: -1
2025-11-27T18:20:18.377827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471723096749894:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:18.377861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:18.378480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471724040567872:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18571) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471724040567952:2116] held: no
2025-11-27T18:20:18.378528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471724383616487:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18571) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471724383616566:2116] held: no
2025-11-27T18:20:18.379269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471724383616568:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.379468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471723507176001:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59984
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.379568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471723507176002:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59984
2025-11-27T18:20:18.379666Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471723096749800:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:18.379713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471724383616566:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:18.379643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471722752972270:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47436
2025-11-27T18:20:18.380223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471724040567952:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.380260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471724383616489:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.380316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:18.381192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471722752972271:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47436
2025-11-27T18:20:18.380554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471724383616489:2050] [node 3] ICP22 created new session: [4:7577471724383616579:2050]
2025-11-27T18:20:18.381068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471724383616579:2050] [node 3] ICS09 handshake done sender: [4:7577471724383616568:2118] self: [4:7577471724383616571:0] peer: [3:7577471723096749869:0] socket: 438 qp: -1
2025-11-27T18:20:18.381272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471724383616579:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:18.381503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471724383616487:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.381507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:18.381540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:18.381591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471724383616487:2048] [node 1] ICP22 created new session: [4:7577471724383616581:2048]
2025-11-27T18:20:18.381638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471724383616581:2048] [node 1] ICS09 handshake done sender: [4:7577471724383616566:2116] self: [4:7577471724383616572:0] peer: [1:7577471724084131137:0] socket: 430 qp: -1
2025-11-27T18:20:18.381667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471724383616581:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:18.381701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:18.381939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471722752972270:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.382225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471724040567872:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.382227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471722752972192:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.382738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471723507176001:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.382299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:18.382345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471724040567872:2048] [node 1] ICP22 created new session: [2:7577471724040567971:2048]
2025-11-27T18:20:18.383035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471724084131145:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.383105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471723507175919:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.383076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471724084131145:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.382399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471724040567971:2048] [node 1] ICS09 handshake done sender: [2:7577471724040567952:2116] self: [2:7577471724040567961:0] peer: [1:7577471724084131133:0] socket: 409 qp: -1
2025-11-27T18:20:18.382438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471724040567971:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.382487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:18.384666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471722752972273:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47436
2025-11-27T18:20:18.384543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471724084131140:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.384578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471724084131140:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.385190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471722752972274:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47436
2025-11-27T18:20:18.385393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471724084131145:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:18.386451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471724084131131:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.387012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471723507176000:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.387326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:18.387390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471724084131131:2109] [node 6] ICP22 created new session: [1:7577471724084131169:2109]
2025-11-27T18:20:18.387532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471724084131169:2109] [node 6] ICS09 handshake done sender: [1:7577471724084131145:2114] self: [1:7577471724084131144:0] peer: [6:7577471722752972272:0] socket: 420 qp: -1
2025-11-27T18:20:18.387572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471724084131169:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:18.387609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:18.387629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471723096749872:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.387632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471722752972270:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:18.387657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471723096749872:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.388037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471722752972273:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.387375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471723507175921:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.388377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471724383616560:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.388407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471724383616560:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.387374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471723507176002:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.387622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471723507175982:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.389155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471722752972192:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.389448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471723096749872:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:18.389346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471722752972275:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47436
2025-11-27T18:20:18.388130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471723507175999:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.389353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:18.389952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471724084131140:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:18.394105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471724383616560:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:18.388401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471723507175920:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.389831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471722752972192:2048] [node 1] ICP22 created new session: [6:7577471722752972276:2048]
2025-11-27T18:20:18.393837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471724040567946:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.391311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471723096749863:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.393999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471724084131132:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.392129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471722752972276:2048] [node 1] ICS09 handshake done sender: [6:7577471722752972270:2115] self: [6:7577471722752972272:0] peer: [1:7577471724084131144:0] socket: 433 qp: -1
2025-11-27T18:20:18.393885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471724040567946:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.392256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:18.395138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:18.394441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471723096749863:2108] [node 5] ICP22 created new session: [3:7577471723096749898:2108]
2025-11-27T18:20:18.392173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471722752972276:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:18.395865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471724383616551:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.394863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471723096749898:2108] [node 5] ICS09 handshake done sender: [3:7577471723096749872:2113] self: [3:7577471723096749870:0] peer: [5:7577471723507176004:0] socket: 425 qp: -1
2025-11-27T18:20:18.395533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471724084131132:2108] [node 5] ICP22 created new session: [1:7577471724084131171:2108]
2025-11-27T18:20:18.394243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:18.397645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471722752972254:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.395594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471724084131171:2108] [node 5] ICS09 handshake done sender: [1:7577471724084131140:2113] self: [1:7577471724084131139:0] peer: [5:7577471723507176003:0] socket: 403 qp: -1
2025-11-27T18:20:18.397839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:18.399103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471724040567946:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:18.395168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471723096749898:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:18.398690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471724383616551:2108] [node 5] ICP22 created new session: [4:7577471724383616584:2108]
2025-11-27T18:20:18.395686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471724084131171:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:18.398741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471722752972275:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.396808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:18.398983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471724383616584:2108] [node 5] ICS09 handshake done sender: [4:7577471724383616560:2114] self: [4:7577471724383616559:0] peer: [5:7577471723507176005:0] socket: 431 qp: -1
2025-11-27T18:20:18.395639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:18.399113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471724383616584:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:18.400855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471724040567937:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.399749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:18.401035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471722752972194:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.400749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471724383616565:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.400936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:18.401214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471724040567937:2108] [node 5] ICP22 created new session: [2:7577471724040567974:2108]
2025-11-27T18:20:18.401348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471723507175921:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471723507176000:2117] held: no
2025-11-27T18:20:18.401276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471724040567974:2108] [node 5] ICS09 handshake done sender: [2:7577471724040567946:2114] self: [2:7577471724040567944:0] peer: [5:7577471723507176006:0] socket: 411 qp: -1
2025-11-27T18:20:18.401355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471724040567974:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:18.401492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:18.400779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471724383616565:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.404421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471724383616565:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.404675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471723507176007:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59984
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.404863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471723096749878:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.404882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471724383616552:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.404896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471723096749878:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.404927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:18.404925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471723507176008:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59984
2025-11-27T18:20:18.405142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471722752972271:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.405001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471724383616552:2110] [node 6] ICP22 created new session: [4:7577471724383616587:2110]
2025-11-27T18:20:18.405171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471723507175919:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18571) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471723507176001:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.405909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471724383616587:2110] [node 6] ICS09 handshake done sender: [4:7577471724383616565:2115] self: [4:7577471724383616564:0] peer: [6:7577471722752972278:0] socket: 457 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.405975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471724383616587:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:18.405943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471722752972281:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47436
2025-11-27T18:20:18.406009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:18.406073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471723096749900:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6386
2025-11-27T18:20:18.406697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471723096749878:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:18.407089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471723096749900:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.407098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471723096749864:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.407114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471722752972193:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.407137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:18.407181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471722752972192:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16018368ns) {HandleHandshakeStatus:329(16016911ns) {DropIncomingHandshake:442(422ns) DropOutgoingHandshake:457(3913ns) SwitchToState:179(389ns) ProcessPendingSessionEvents:495(9552534ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9374295ns max# 1487353ns}}}
2025-11-27T18:20:18.407177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471723096749864:2109] [node 6] ICP22 created new session: [3:7577471723096749902:2109]
2025-11-27T18:20:18.407215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471722752972273:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:18.407217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471723096749902:2109] [node 6] ICS09 handshake done sender: [3:7577471723096749878:2115] self: [3:7577471723096749877:0] peer: [6:7577471722752972280:0] socket: 428 qp: -1
2025-11-27T18:20:18.408227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471723096749902:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:18.408302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:18.408746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471722752972192:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:18.408827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471723507176001:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:18.408833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471722752972254:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.408830Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471723507175920:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25340) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471723507175999:2116] held: no
2025-11-27T18:20:18.408918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471724040567951:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.408929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:18.408943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471724040567951:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.408988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471722752972254:2106] [node 4] ICP22 created new session: [6:7577471722752972284:2106]
2025-11-27T18:20:18.409878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471722752972284:2106] [node 4] ICS09 handshake done sender: [6:7577471722752972273:2117] self: [6:7577471722752972278:0] peer: [4:7577471724383616564:0] socket: 458 qp: -1
No RDMA context found for GID: 2025-11-27T18:20:18.409930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471722752972284:2106] [node 4] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:20:18.409971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:18.410052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471723507176009:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59984
2025-11-27T18:20:18.410223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471723096749864:2109] [node 6] ICP18 handshake (actor [3:7577471723096749900:2121]) is too old
2025-11-27T18:20:18.410208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471724040567951:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:18.410275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471723507176000:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:18.410392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471724040567938:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.410832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:18.410874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471723507175919:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.411057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471724040567938:2109] [node 6] ICP22 created new session: [2:7577471724040567977:2109]
2025-11-27T18:20:18.411181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471724040567977:2109] [node 6] ICS09 handshake done sender: [2:7577471724040567951:2115] self: [2:7577471724040567950:0] peer: [6:7577471722752972283:0] socket: 455 qp: -1
2025-11-27T18:20:18.411198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:18.411216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471724040567977:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:18.411260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471723507175919:2048] [node 1] ICP22 created new session: [5:7577471723507176013:2048]
2025-11-27T18:20:18.411252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:18.411303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471723507176013:2048] [node 1] ICS09 handshake done sender: [5:7577471723507176001:2118] self: [5:7577471723507176003:0] peer: [1:7577471724084131139:0] socket: 412 qp: -1
2025-11-27T18:20:18.411340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471723507176013:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:18.411366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471723507175921:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.411372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:18.411442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:18.411489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471723507175921:2050] [node 3] ICP22 created new session: [5:7577471723507176015:2050]
2025-11-27T18:20:18.411529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471723507176015:2050] [node 3] ICS09 handshake done sender: [5:7577471723507176000:2117] self: [5:7577471723507176004:0] peer: [3:7577471723096749870:0] socket: 440 qp: -1
2025-11-27T18:20:18.411562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471723507176015:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:18.411619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.412659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471723507176010:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59984
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.412892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471722752972282:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47436
2025-11-27T18:20:18.413388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471723507176002:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.413922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471723507175982:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.413991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:18.413966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471724383616589:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5727
2025-11-27T18:20:18.414039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471723507175982:2108] [node 4] ICP22 created new session: [5:7577471723507176017:2108]
2025-11-27T18:20:18.414204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471723507176017:2108] [node 4] ICS09 handshake done sender: [5:7577471723507176002:2119] self: [5:7577471723507176005:0] peer: [4:7577471724383616559:0] socket: 435 qp: -1
2025-11-27T18:20:18.414254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471723507176017:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:18.414282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:18.415027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471722752972194:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471722752972275:2119] held: no
2025-11-27T18:20:18.415383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471723507175999:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:18.415592Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471722752972193:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25340) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471722752972271:2116] held: no
2025-11-27T18:20:18.416007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471723507175920:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.416071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:18.416120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471723507175920:2049] [node 2] ICP22 created new session: [5:7577471723507176019:2049]
2025-11-27T18:20:18.416184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471723507176019:2049] [node 2] ICS09 handshake done sender: [5:7577471723507175999:2116] self: [5:7577471723507176006:0] peer: [2:7577471724040567944:0] socket: 432 qp: -1
2025-11-27T18:20:18.416222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471723507176019:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:18.416296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.416467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471722752972275:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:18.416520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471722752972286:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47436
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.417040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471722752972194:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.417074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471722752972287:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47436
2025-11-27T18:20:18.417088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:18.417110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471722752972271:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:18.417129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471722752972194:2050] [node 3] ICP22 created new session: [6:7577471722752972288:2050]
2025-11-27T18:20:18.417166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471722752972288:2050] [node 3] ICS09 handshake done sender: [6:7577471722752972275:2119] self: [6:7577471722752972280:0] peer: [3:7577471723096749877:0] socket: 461 qp: -1
2025-11-27T18:20:18.417202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471722752972288:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:18.417237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:18.417480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471722752972193:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.417556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:18.417603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471722752972193:2049] [node 2] ICP22 created new session: [6:7577471722752972290:2049]
2025-11-27T18:20:18.417653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471722752972290:2049] [node 2] ICS09 handshake done sender: [6:7577471722752972271:2116] self: [6:7577471722752972283:0] peer: [2:7577471724040567950:0] socket: 456 qp: -1
2025-11-27T18:20:18.417686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471722752972290:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:18.417715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:18.419420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471722752972287:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.419562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471722752972251:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:18.419739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471723507175997:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:18.419774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471723507175997:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:18.420495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471723507175997:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:18.420633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471722752972293:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47436
2025-11-27T18:20:18.420703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471723507175983:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:18.420768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:18.420851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471722752972287:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:18.420858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471723507175983:2109] [node 6] ICP22 created new session: [5:7577471723507176022:2109]
2025-11-27T18:20:18.420914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471723507176022:2109] [node 6] ICS09 handshake done sender: [5:7577471723507175997:2115] self: [5:7577471723507175996:0] peer: [6:7577471722752972292:0] socket: 423 qp: -1
2025-11-27T18:20:18.420943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471723507176022:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:18.421089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:18.421162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471722752972251:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:18.421211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:18.421301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471722752972251:2105] [node 5] ICP22 created new session: [6:7577471722752972294:2105]
2025-11-27T18:20:18.421366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471722752972294:2105] [node 5] ICS09 handshake done sender: [6:7577471722752972287:2123] self: [6:7577471722752972292:0] peer: [5:7577471723507175996:0] socket: 417 qp: -1
2025-11-27T18:20:18.421402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471722752972294:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:18.421439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:18.468431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:18.468670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471724040567971:2048] [node 1] ICS01 socket: 409 reason# EndOfStream
2025-11-27T18:20:18.468724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471724040567872:2048] [node 1] ICP30 unregister session Session# [2:7577471724040567971:2048] VirtualId# [2:7577471724040567961:0]
2025-11-27T18:20:18.468765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471724040567971:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.468846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:18.468900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:18.469685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:18.469891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471723096749892:2048] [node 1] ICS07 socket disconnect 410 reason# EndOfStream
2025-11-27T18:20:18.469937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471723096749892:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.470013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:18.470088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471723096749892:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:18.471875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:18.472265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471724383616581:2048] [node 1] ICS01 socket: 430 reason# EndOfStream
2025-11-27T18:20:18.472301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471724383616487:2048] [node 1] ICP30 unregister session Session# [4:7577471724383616581:2048] VirtualId# [4:7577471724383616572:0]
2025-11-27T18:20:18.472278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471723096749799:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18571) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:18.472322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471724383616581:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.472383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:18.472318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471723096749892:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:18.472423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:18.473358Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471723096749799:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18571) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:18.473423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471723096749892:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:18.473480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471723096749799:2048] [node 1] ICP30 unregister session Session# [3:7577471723096749892:2048] VirtualId# [3:7577471723096749885:0]
2025-11-27T18:20:18.473586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:18.473625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471723096749799:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18571) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.473632Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:18.473640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18571) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.473842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471723507176013:2048] [node 1] ICS01 socket: 412 reason# EndOfStream
2025-11-27T18:20:18.473888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471723507175919:2048] [node 1] ICP30 unregister session Session# [5:7577471723507176013:2048] VirtualId# [5:7577471723507176003:0]
2025-11-27T18:20:18.473926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471723507176013:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.473989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:18.474044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:18.474683Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:18.474882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471722752972276:2048] [node 1] ICS07 socket disconnect 433 reason# EndOfStream
2025-11-27T18:20:18.474921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471722752972276:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.474982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:18.475013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471722752972276:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:18.475678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471722752972192:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18571) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:18.475781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471722752972276:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:18.475838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471722752972192:2048] [node 1] ICP30 unregister session Session# [6:7577471722752972276:2048] VirtualId# [6:7577471722752972272:0]
2025-11-27T18:20:18.475954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:18.475989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471722752972192:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18571) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.476003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18571) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.483513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471723096749799:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:18.485553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471722752972192:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:18.530895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:18.531142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471723096749894:2049] [node 2] ICS07 socket disconnect 416 reason# EndOfStream
2025-11-27T18:20:18.531183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471723096749894:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.531265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:18.531321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471723096749894:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:18.533976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:18.534116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471723096749800:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25340) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:18.534169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471723096749894:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:18.534182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471724383616573:2049] [node 2] ICS07 socket disconnect 436 reason# EndOfStream
2025-11-27T18:20:18.534206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471724383616573:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.534281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:18.534311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471724383616573:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:18.535235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471724383616488:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25340) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:18.535235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471723096749800:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25340) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:18.535500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:18.535392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471724383616573:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:18.535392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471723096749894:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:18.535439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471724383616488:2049] [node 2] ICP30 unregister session Session# [4:7577471724383616573:2049] VirtualId# [4:7577471724383616569:0]
2025-11-27T18:20:18.535632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471723096749800:2049] [node 2] ICP30 unregister session Session# [3:7577471723096749894:2049] VirtualId# [3:7577471723096749882:0]
2025-11-27T18:20:18.535614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:18.535673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471724383616488:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.535689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.535715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471723507176019:2049] [node 2] ICS01 socket: 432 reason# EndOfStream
2025-11-27T18:20:18.535753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471723507175920:2049] [node 2] ICP30 unregister session Session# [5:7577471723507176019:2049] VirtualId# [5:7577471723507176006:0]
2025-11-27T18:20:18.535742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:18.535782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471723096749800:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.535775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471723507176019:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.535796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25340) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.535841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:18.535890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:18.536735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:18.536859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471722752972290:2049] [node 2] ICS01 socket: 456 reason# EndOfStream
2025-11-27T18:20:18.536887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471722752972193:2049] [node 2] ICP30 unregister session Session# [6:7577471722752972290:2049] VirtualId# [6:7577471722752972283:0]
2025-11-27T18:20:18.536907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471722752972290:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.536951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:18.536980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:18.545969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471724383616488:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:18.583040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:18.583253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471724383616579:2050] [node 3] ICS01 socket: 438 reason# EndOfStream
2025-11-27T18:20:18.583298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471724383616489:2050] [node 3] ICP30 unregister session Session# [4:7577471724383616579:2050] VirtualId# [4:7577471724383616571:0]
2025-11-27T18:20:18.583346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471724383616579:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.583429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:18.583475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:18.584142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:18.584360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471723507176015:2050] [node 3] ICS01 socket: 440 reason# EndOfStream
2025-11-27T18:20:18.584409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471723507175921:2050] [node 3] ICP30 unregister session Session# [5:7577471723507176015:2050] VirtualId# [5:7577471723507176004:0]
2025-11-27T18:20:18.584438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471723507176015:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.584514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:18.584554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:18.585090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:18.585320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471722752972288:2050] [node 3] ICS01 socket: 461 reason# EndOfStream
2025-11-27T18:20:18.585375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471722752972194:2050] [node 3] ICP30 unregister session Session# [6:7577471722752972288:2050] VirtualId# [6:7577471722752972280:0]
2025-11-27T18:20:18.585408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471722752972288:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.585475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:18.585530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:18.659386Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:18.659629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471723507176017:2108] [node 4] ICS07 socket disconnect 435 reason# EndOfStream
2025-11-27T18:20:18.659670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471723507176017:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.659766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:18.659849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471723507176017:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:18.660381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:18.660602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471722752972284:2106] [node 4] ICS07 socket disconnect 458 reason# EndOfStream
2025-11-27T18:20:18.660646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471722752972284:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.660657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471723507175982:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24342) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:18.660722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:18.660745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471723507176017:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:18.660769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471722752972284:2106] [node 4] ICS15 start handshake
2025-11-27T18:20:18.660810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471723507175982:2108] [node 4] ICP30 unregister session Session# [5:7577471723507176017:2108] VirtualId# [5:7577471723507176005:0]
2025-11-27T18:20:18.660967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:18.661010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471723507175982:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24342) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.661028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24342) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.661529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471722752972254:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24342) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:18.661645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471722752972284:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:18.661697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471722752972254:2106] [node 4] ICP30 unregister session Session# [6:7577471722752972284:2106] VirtualId# [6:7577471722752972278:0]
2025-11-27T18:20:18.661795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:18.661850Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471722752972254:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24342) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.661868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24342) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.671942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471722752972254:2106] [node 4] ICP33 wake up from error state
2025-11-27T18:20:18.709583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:18.709825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471722752972294:2105] [node 5] ICS07 socket disconnect 417 reason# EndOfStream
2025-11-27T18:20:18.709866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471722752972294:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:18.709988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:18.710049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471722752972294:2105] [node 5] ICS15 start handshake
2025-11-27T18:20:18.710806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471722752972251:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20714) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:18.710857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471722752972294:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:18.710942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471722752972251:2105] [node 5] ICP30 unregister session Session# [6:7577471722752972294:2105] VirtualId# [6:7577471722752972292:0]
2025-11-27T18:20:18.711067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:18.711115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471722752972251:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20714) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.711130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20714) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:18.721139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471722752972251:2105] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:18.812341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471724088041914:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:18.812428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471724088041915:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:18.832459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471723337227921:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:18.832522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471723337227922:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:18.851877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471722052955358:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:18.851954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471722052955359:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:18.870443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471720721984453:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:18.870665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471720721984454:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:18.870738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471720721984455:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:18.888789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471722720486520:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:18.888819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471722720486522:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:18.888833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471722720486521:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:18.910692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471720992760239:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:18.910735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471720992760241:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:18.910776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471720992760240:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:18.963931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471720992760301:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:19.011567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471723337227987:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:19.018662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471726347922744:2131] [node 6] ICP01 ready to work
2025-11-27T18:20:19.030714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471725016951812:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:19.031071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471727015453881:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:19.018304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471728383009292:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:19.032128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471726347922746:2113] [node 5] ICP01 ready to work
2025-11-27T18:20:19.031684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471727015453880:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:19.018807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471727632195285:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:19.032406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471726347922745:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:19.031934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471727632195284:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:19.031647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471725016951813:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:19.032751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471725287727599:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:19.020120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471724088041914:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56901862ns) {Configure:104(56890730ns) {ProcessConfigured:135(8257617ns) {StartInitialHandshake:157(8254849ns) {DropHandshakes:470(2960ns) {DropIncomingHandshake:442(754ns) DropOutgoingHandshake:457(357ns)} GenerateSessionVirtualId:988(1715ns) PrepareNewSessionHandshake:477(8001ns) {SwitchToState:179(1926ns)}}}}}
2025-11-27T18:20:19.025960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471724088041995:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:19.032839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471728383009297:2118] [node 6] ICP01 ready to work
2025-11-27T18:20:19.038212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471720992760301:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(69563804ns) {Configure:104(69559553ns) {ProcessConfigured:135(804306ns) {StartInitialHandshake:157(801859ns) {DropHandshakes:470(2771ns) {DropIncomingHandshake:442(694ns) DropOutgoingHandshake:457(345ns)} GenerateSessionVirtualId:988(916ns) PrepareNewSessionHandshake:477(1560ns) {SwitchToState:179(730ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.043900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471728383009307:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10008
2025-11-27T18:20:19.050044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471728383009307:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.051382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471727632195292:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52011
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:19.052015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471727632195293:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52011
::ffff:127.1.0.0
2025-11-27T18:20:19.052484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471726347922753:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29450
2025-11-27T18:20:19.052507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471727632195292:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.052567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727015453888:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30197
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.052651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471727632195293:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.052978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471723337227921:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.053000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471728383009310:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10008
2025-11-27T18:20:19.052983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727015453889:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30197
No RDMA context found for GID: 2025-11-27T18:20:19.053425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471725287727613:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7484
::ffff:127.1.0.0
2025-11-27T18:20:19.053805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471728383009294:2128] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.054078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727015453888:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.054054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471726347922753:2140] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.053819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471728383009309:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10008
2025-11-27T18:20:19.054796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471727632195301:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52011
2025-11-27T18:20:19.053851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471728383009294:2128] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.054457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727015453889:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.056364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471728383009294:2128] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.056418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471722720486520:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.058796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471725016951823:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42061
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.058808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471725287727614:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7484
2025-11-27T18:20:19.058835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471722052955358:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.059046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471728383009311:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10008
2025-11-27T18:20:19.059057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471726347922761:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29450
2025-11-27T18:20:19.059107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471728383009313:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10008
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.059334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471728383009300:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.059216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471727632195304:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52011
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.059286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471726347922763:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29450
2025-11-27T18:20:19.059377Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471728383009300:2130] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.059984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471725287727613:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.059646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471724088041914:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.063458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471727632195306:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52011
2025-11-27T18:20:19.062030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471720992760240:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.059792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:19.063465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471727632195305:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52011
2025-11-27T18:20:19.064888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471725016951823:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.059866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471728383009296:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.065433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471726347922761:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.065588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471727632195289:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.065535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471720721984454:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.059888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471728383009296:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.065631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471727632195289:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.069559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471725287727614:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.061947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471724088041914:2048] [node 2] ICP22 created new session: [1:7577471728383009314:2048]
2025-11-27T18:20:19.068519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471727632195301:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.069930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471726347922763:2144] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.069537Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471727632195291:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.070156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471720992760241:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.062648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471728383009314:2048] [node 2] ICS09 handshake done sender: [1:7577471728383009294:2128] self: [1:7577471728383009293:0] peer: [2:7577471727632195302:0] socket: 408 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.063450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471728383009309:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.070826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471726347922752:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.069566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471727632195291:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.070872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471726347922752:2139] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.063513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471728383009314:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.073494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471725016951829:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42061
2025-11-27T18:20:19.069799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471727632195292:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:19.073479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471726347922764:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29450
2025-11-27T18:20:19.063560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.070619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471727632195289:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:19.073302Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471727632195305:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.074297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727015453899:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30197
2025-11-27T18:20:19.073362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471723337227921:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.066891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471728383009296:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:19.074495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471725287727617:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7484
2025-11-27T18:20:19.068465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471728383009300:2130] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:19.074881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471726347922753:2140] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:19.073509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:19.069959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471728383009310:2135] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.070011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471724088041915:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.074889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471726347922752:2139] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:19.073596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471723337227921:2048] [node 1] ICP22 created new session: [2:7577471727632195309:2048]
2025-11-27T18:20:19.076177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471727015453888:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.073939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471727632195304:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.075840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471725287727613:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:19.075648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471722052955358:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.070051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:19.074054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471727632195309:2048] [node 1] ICS09 handshake done sender: [2:7577471727632195292:2116] self: [2:7577471727632195302:0] peer: [1:7577471728383009293:0] socket: 417 qp: -1
2025-11-27T18:20:19.077199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471720992760240:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.075705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:19.070196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471724088041915:2049] [node 3] ICP22 created new session: [1:7577471728383009319:2049]
2025-11-27T18:20:19.077243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471722720486520:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.074079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471727632195285:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.076486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471722052955358:2048] [node 1] ICP22 created new session: [3:7577471726347922768:2048]
2025-11-27T18:20:19.077529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471725287727618:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7484
2025-11-27T18:20:19.070746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471728383009319:2049] [node 3] ICS09 handshake done sender: [1:7577471728383009296:2129] self: [1:7577471728383009295:0] peer: [3:7577471726347922762:0] socket: 409 qp: -1
2025-11-27T18:20:19.077380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:19.074139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:19.077988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:19.076873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471726347922744:2131] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.071002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471728383009319:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:19.074204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471727632195309:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:19.076901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471726347922768:2048] [node 1] ICS09 handshake done sender: [3:7577471726347922753:2140] self: [3:7577471726347922762:0] peer: [1:7577471728383009295:0] socket: 421 qp: -1
2025-11-27T18:20:19.077593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471722720486520:2048] [node 1] ICP22 created new session: [5:7577471727015453900:2048]
2025-11-27T18:20:19.071064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:19.078056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471727015453900:2048] [node 1] ICS09 handshake done sender: [5:7577471727015453888:2114] self: [5:7577471727015453897:0] peer: [1:7577471728383009298:0] socket: 419 qp: -1
2025-11-27T18:20:19.078256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471720992760240:2049] [node 2] ICP22 created new session: [6:7577471725287727619:2049]
2025-11-27T18:20:19.074244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471727632195285:2110] [node 6] ICP22 created new session: [2:7577471727632195310:2110]
2025-11-27T18:20:19.077304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:19.071299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471728383009292:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.074433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:19.079039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471725287727619:2049] [node 2] ICS09 handshake done sender: [6:7577471725287727613:2115] self: [6:7577471725287727615:0] peer: [2:7577471727632195286:0] socket: 426 qp: -1
2025-11-27T18:20:19.077386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471726347922768:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:19.074507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471727632195310:2110] [node 6] ICS09 handshake done sender: [2:7577471727632195289:2113] self: [2:7577471727632195286:0] peer: [6:7577471725287727615:0] socket: 422 qp: -1
2025-11-27T18:20:19.079069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471725287727619:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:19.077558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471726347922744:2131] [node 6] ICP22 created new session: [3:7577471726347922769:2131]
2025-11-27T18:20:19.074897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471727632195310:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:19.078008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471726347922769:2131] [node 6] ICS09 handshake done sender: [3:7577471726347922752:2139] self: [3:7577471726347922748:0] peer: [6:7577471725287727616:0] socket: 440 qp: -1
2025-11-27T18:20:19.080537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:19.078357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471727015453900:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:19.074963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:19.073321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:19.081963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471725287727614:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:19.078240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.078984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471726347922769:2131] [node 6] ICS10 traffic start
2025-11-27T18:20:19.073907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471728383009292:2108] [node 5] ICP22 created new session: [1:7577471728383009321:2108]
2025-11-27T18:20:19.074205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471728383009321:2108] [node 5] ICS09 handshake done sender: [1:7577471728383009300:2130] self: [1:7577471728383009298:0] peer: [5:7577471727015453897:0] socket: 410 qp: -1
2025-11-27T18:20:19.079033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:19.074341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471728383009321:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:19.083914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471720992760241:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.074385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:19.083968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:19.084286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471720992760241:2050] [node 3] ICP22 created new session: [6:7577471725287727622:2050]
2025-11-27T18:20:19.079026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:19.085529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471725016951830:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42061
2025-11-27T18:20:19.085103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471725287727622:2050] [node 3] ICS09 handshake done sender: [6:7577471725287727614:2116] self: [6:7577471725287727616:0] peer: [3:7577471726347922748:0] socket: 442 qp: -1
2025-11-27T18:20:19.085140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471725287727622:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:19.085177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.087623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471724088041914:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(22973655ns) {HandleHandshakeStatus:329(22970181ns) {DropIncomingHandshake:442(1223ns) DropOutgoingHandshake:457(1144ns) SwitchToState:179(343ns) ProcessPendingSessionEvents:495(16668071ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15814926ns max# 2975382ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.087658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471727632195291:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:19.087807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471725016951830:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.088022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471728383009313:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.088040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471728383009311:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.088065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471725016951831:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42061
2025-11-27T18:20:19.088678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471725016951832:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42061
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.088661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727015453903:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30197
2025-11-27T18:20:19.089447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471725016951833:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42061
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.090194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471723337227987:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.090159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471724088041914:2048] [node 2] ICP18 handshake (actor [1:7577471728383009309:2134]) is too old
2025-11-27T18:20:19.090170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471725016951833:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.090278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:19.090632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727015453904:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30197
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.090339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471725016951823:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:19.091115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471724088041915:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17356353ns) {HandleHandshakeStatus:329(17353725ns) {DropIncomingHandshake:442(189ns) DropOutgoingHandshake:457(621ns) SwitchToState:179(291ns) ProcessPendingSessionEvents:495(13413733ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13303658ns max# 1500186ns}}}
2025-11-27T18:20:19.091359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727015453903:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.091569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471726347922774:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29450
2025-11-27T18:20:19.090911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471723337227987:2107] [node 4] ICP22 created new session: [2:7577471727632195318:2107]
2025-11-27T18:20:19.091545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471720721984454:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.091573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471727632195318:2107] [node 4] ICS09 handshake done sender: [2:7577471727632195291:2115] self: [2:7577471727632195288:0] peer: [4:7577471725016951828:0] socket: 424 qp: -1
2025-11-27T18:20:19.091558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471722720486522:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.091598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:19.091764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471727632195318:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:19.091725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471725016951831:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.091811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:19.093505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471726347922760:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.091954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471720721984454:2049] [node 2] ICP22 created new session: [4:7577471725016951836:2049]
2025-11-27T18:20:19.093560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471726347922760:2142] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.097127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727015453907:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30197
2025-11-27T18:20:19.093176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471725016951836:2049] [node 2] ICS09 handshake done sender: [4:7577471725016951823:2113] self: [4:7577471725016951828:0] peer: [2:7577471727632195288:0] socket: 433 qp: -1
2025-11-27T18:20:19.096280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471726347922759:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.093224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471725016951836:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:19.096311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471726347922759:2141] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.093278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:19.097037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471726347922760:2142] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:19.093418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471720721984455:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.099266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471728383009292:2108] [node 5] ICP18 handshake (actor [1:7577471728383009313:2137]) is too old
2025-11-27T18:20:19.100976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471725016951829:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.100969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471726347922759:2141] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.101424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471727015453903:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:19.101431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471720721984453:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.102451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471725287727625:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7484
2025-11-27T18:20:19.101718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471726347922746:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.102741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471728383009301:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.102780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471728383009301:2131] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.101792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.102588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471722720486522:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.102116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471726347922746:2113] [node 5] ICP22 created new session: [3:7577471726347922777:2113]
2025-11-27T18:20:19.102644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:19.103287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471725016951840:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42061
2025-11-27T18:20:19.102910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471726347922777:2113] [node 5] ICS09 handshake done sender: [3:7577471726347922760:2142] self: [3:7577471726347922758:0] peer: [5:7577471727015453905:0] socket: 438 qp: -1
2025-11-27T18:20:19.102764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471722720486522:2050] [node 3] ICP22 created new session: [5:7577471727015453908:2050]
2025-11-27T18:20:19.103109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471726347922777:2113] [node 5] ICS10 traffic start
2025-11-27T18:20:19.103154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:19.102807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727015453906:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30197
2025-11-27T18:20:19.105872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471725016951831:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:19.102815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471727015453908:2050] [node 3] ICS09 handshake done sender: [5:7577471727015453903:2119] self: [5:7577471727015453905:0] peer: [3:7577471726347922758:0] socket: 462 qp: -1
2025-11-27T18:20:19.107209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471720721984455:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.102855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471727015453908:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:19.107360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:19.103269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:19.103185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471726347922745:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.103373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:19.103427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471726347922745:2107] [node 4] ICP22 created new session: [3:7577471726347922779:2107]
2025-11-27T18:20:19.104950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471726347922779:2107] [node 4] ICS09 handshake done sender: [3:7577471726347922759:2141] self: [3:7577471726347922757:0] peer: [4:7577471725016951838:0] socket: 413 qp: -1
2025-11-27T18:20:19.107420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471720721984455:2050] [node 3] ICP22 created new session: [4:7577471725016951842:2050]
2025-11-27T18:20:19.108076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727015453906:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.105235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471726347922779:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:19.107657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471725016951842:2050] [node 3] ICS09 handshake done sender: [4:7577471725016951831:2118] self: [4:7577471725016951838:0] peer: [3:7577471726347922757:0] socket: 435 qp: -1
2025-11-27T18:20:19.105282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:19.114155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471727015453880:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.107885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471725016951842:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:19.108075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:19.115209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727015453904:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.117435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471724088041915:2049] [node 3] ICP18 handshake (actor [1:7577471728383009311:2136]) is too old
2025-11-27T18:20:19.117858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471722720486521:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.118747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471725287727626:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7484
2025-11-27T18:20:19.118762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471725016951825:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.118795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471725287727625:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.119050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471728383009301:2131] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:19.119162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471723337227987:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17425683ns) {HandleHandshakeStatus:329(17422576ns) {DropIncomingHandshake:442(15057ns) DropOutgoingHandshake:457(561ns) SwitchToState:179(861ns) ProcessPendingSessionEvents:495(14181323ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14078461ns max# 2300532ns}}}
2025-11-27T18:20:19.118800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471725016951825:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.119060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471720992760239:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.119288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471720721984454:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16134637ns) {HandleHandshakeStatus:329(16132159ns) {DropIncomingHandshake:442(899ns) DropOutgoingHandshake:457(8767ns) SwitchToState:179(264ns) ProcessPendingSessionEvents:495(12016084ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11559091ns max# 2339739ns}}}
2025-11-27T18:20:19.120685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471724088041995:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.120347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471727632195297:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.120340Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471720721984454:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:19.120856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:19.120382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471727632195297:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.120907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471724088041995:2107] [node 4] ICP22 created new session: [1:7577471728383009326:2107]
2025-11-27T18:20:19.121814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471725287727626:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.120918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471728383009303:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.122632Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471726347922746:2113] [node 5] ICP35 event processing took too much time PendingConnection:279(16652100ns) {HandleHandshakeStatus:329(16649684ns) {DropIncomingHandshake:442(18381ns) DropOutgoingHandshake:457(537ns) SwitchToState:179(152ns) ProcessPendingSessionEvents:495(12796293ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12149105ns max# 5104878ns}}}
2025-11-27T18:20:19.123032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471720992760301:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.120945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471728383009303:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.123205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471726347922774:2146] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.121335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471728383009326:2107] [node 4] ICS09 handshake done sender: [1:7577471728383009301:2131] self: [1:7577471728383009299:0] peer: [4:7577471725016951841:0] socket: 454 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.121465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471728383009326:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:19.124434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471727632195297:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:19.121504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:19.124514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727015453914:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30197
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.123265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471725016951844:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42061
2025-11-27T18:20:19.125400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471728383009303:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:19.124603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727015453896:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.124522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471720721984453:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471725016951829:2116] held: no
2025-11-27T18:20:19.124641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471727015453896:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.125499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471727632195284:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.125775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:19.125477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727015453913:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30197
2025-11-27T18:20:19.128806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471727632195284:2109] [node 5] ICP22 created new session: [2:7577471727632195321:2109]
2025-11-27T18:20:19.128713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471727015453904:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:19.128954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471727632195321:2109] [node 5] ICS09 handshake done sender: [2:7577471727632195297:2119] self: [2:7577471727632195296:0] peer: [5:7577471727015453912:0] socket: 469 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.130196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471725016951825:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:19.129086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471722720486521:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11211) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471727015453904:2120] held: no
2025-11-27T18:20:19.129212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471727632195321:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:19.129854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471728383009297:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.129416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:19.129917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:19.130905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471725287727629:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7484
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.132561Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471720992760239:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471725287727625:2119] held: no
2025-11-27T18:20:19.129967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471728383009297:2118] [node 6] ICP22 created new session: [1:7577471728383009329:2118]
2025-11-27T18:20:19.132660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471727015453896:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:19.130014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471728383009329:2118] [node 6] ICS09 handshake done sender: [1:7577471728383009303:2132] self: [1:7577471728383009302:0] peer: [6:7577471725287727627:0] socket: 431 qp: -1
2025-11-27T18:20:19.133189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471722720486521:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.132833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471722052955359:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.130045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471728383009329:2118] [node 6] ICS10 traffic start
2025-11-27T18:20:19.130077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:19.132804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471725287727630:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7484
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.133339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:19.133578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471725016951829:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:19.133338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471727632195295:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.133395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471722720486521:2049] [node 2] ICP22 created new session: [5:7577471727015453916:2049]
2025-11-27T18:20:19.134876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471725287727631:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7484
2025-11-27T18:20:19.134620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471727015453916:2049] [node 2] ICS09 handshake done sender: [5:7577471727015453904:2120] self: [5:7577471727015453912:0] peer: [2:7577471727632195296:0] socket: 470 qp: -1
2025-11-27T18:20:19.134661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471727015453916:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:19.134698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:19.133372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471727632195295:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.135081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471725016951813:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.135262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:19.135950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471727015453881:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.135546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471725016951813:2108] [node 5] ICP22 created new session: [4:7577471725016951847:2108]
2025-11-27T18:20:19.136134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:19.136577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471725016951847:2108] [node 5] ICS09 handshake done sender: [4:7577471725016951825:2114] self: [4:7577471725016951824:0] peer: [5:7577471727015453911:0] socket: 425 qp: -1
2025-11-27T18:20:19.136649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471727632195295:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:19.136681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471727015453881:2109] [node 6] ICP22 created new session: [5:7577471727015453918:2109]
2025-11-27T18:20:19.136738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471725016951847:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:19.136890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471727015453918:2109] [node 6] ICS09 handshake done sender: [5:7577471727015453896:2117] self: [5:7577471727015453895:0] peer: [6:7577471725287727628:0] socket: 444 qp: -1
2025-11-27T18:20:19.136935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471727015453918:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:19.136973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:19.136965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471725287727626:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:19.136975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471725287727625:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.137072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471720721984453:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.137093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471720992760301:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.137121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:19.137919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471727015453906:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:19.137407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471723337227922:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.137527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:19.137594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471726347922782:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29450
2025-11-27T18:20:19.137209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:19.137306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:19.137657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471723337227922:2049] [node 3] ICP22 created new session: [2:7577471727632195324:2049]
2025-11-27T18:20:19.137365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471720992760301:2106] [node 5] ICP22 created new session: [6:7577471725287727632:2106]
2025-11-27T18:20:19.137339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471720721984453:2048] [node 1] ICP22 created new session: [4:7577471725016951849:2048]
2025-11-27T18:20:19.137978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471727632195324:2049] [node 3] ICS09 handshake done sender: [2:7577471727632195295:2118] self: [2:7577471727632195294:0] peer: [3:7577471726347922781:0] socket: 467 qp: -1
2025-11-27T18:20:19.137884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471725287727632:2106] [node 5] ICS09 handshake done sender: [6:7577471725287727626:2120] self: [6:7577471725287727628:0] peer: [5:7577471727015453895:0] socket: 476 qp: -1
2025-11-27T18:20:19.137942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471725016951849:2048] [node 1] ICS09 handshake done sender: [4:7577471725016951829:2116] self: [4:7577471725016951841:0] peer: [1:7577471728383009299:0] socket: 455 qp: -1
2025-11-27T18:20:19.137930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471720992760239:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.138408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471727632195324:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:19.137972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:19.137993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471725016951849:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:19.138453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:19.138035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:19.139036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471726347922774:2146] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:19.138020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471720992760239:2048] [node 1] ICP22 created new session: [6:7577471725287727633:2048]
2025-11-27T18:20:19.138038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471725287727632:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:19.141014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471727015453880:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.138220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:19.138379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471725287727633:2048] [node 1] ICS09 handshake done sender: [6:7577471725287727625:2119] self: [6:7577471725287727627:0] peer: [1:7577471728383009302:0] socket: 471 qp: -1
2025-11-27T18:20:19.141859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:19.138513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471725287727633:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:19.141910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471727015453880:2108] [node 4] ICP22 created new session: [5:7577471727015453920:2108]
2025-11-27T18:20:19.138947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:19.141968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471727015453920:2108] [node 4] ICS09 handshake done sender: [5:7577471727015453906:2121] self: [5:7577471727015453911:0] peer: [4:7577471725016951824:0] socket: 428 qp: -1
2025-11-27T18:20:19.141948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471722052955359:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11211) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471726347922774:2146] held: no
2025-11-27T18:20:19.142242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471727015453920:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:19.142905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471722052955359:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.142398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:19.143137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:19.143195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471722052955359:2049] [node 2] ICP22 created new session: [3:7577471726347922783:2049]
2025-11-27T18:20:19.143898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471726347922783:2049] [node 2] ICS09 handshake done sender: [3:7577471726347922774:2146] self: [3:7577471726347922781:0] peer: [2:7577471727632195294:0] socket: 468 qp: -1
2025-11-27T18:20:19.144062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471726347922783:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:19.144110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:19.144790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471727015453880:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:19.144848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471725287727629:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.145006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471725287727599:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.145298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471725016951827:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.145333Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471725016951827:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.146734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471725016951827:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.146982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471725016951812:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.146958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471725287727637:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7484
2025-11-27T18:20:19.147085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:19.147153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471725016951812:2109] [node 6] ICP22 created new session: [4:7577471725016951852:2109]
2025-11-27T18:20:19.147187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471725287727629:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:19.147197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471725016951852:2109] [node 6] ICS09 handshake done sender: [4:7577471725016951827:2115] self: [4:7577471725016951826:0] peer: [6:7577471725287727636:0] socket: 436 qp: -1
2025-11-27T18:20:19.147239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471725016951852:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:19.147287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:19.147517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471725287727599:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471725287727629:2121] held: no
2025-11-27T18:20:19.148183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471725287727599:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.148227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:19.148282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471725287727599:2109] [node 4] ICP22 created new session: [6:7577471725287727638:2109]
2025-11-27T18:20:19.148327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471725287727638:2109] [node 4] ICS09 handshake done sender: [6:7577471725287727629:2121] self: [6:7577471725287727636:0] peer: [4:7577471725016951826:0] socket: 448 qp: -1
2025-11-27T18:20:19.148364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471725287727638:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:19.148396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:19.190421Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:19.190645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471727632195309:2048] [node 1] ICS07 socket disconnect 417 reason# EndOfStream
2025-11-27T18:20:19.190681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471727632195309:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.190772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:19.190820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471727632195309:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:19.191764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:19.191976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471726347922768:2048] [node 1] ICS07 socket disconnect 421 reason# EndOfStream
2025-11-27T18:20:19.192028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471726347922768:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.192105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:19.192146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471726347922768:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:19.193299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471723337227921:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:19.193353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471727632195309:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:19.195488Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:19.194475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471722052955358:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:19.194528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471726347922768:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:19.194512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471723337227921:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.195454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471727632195309:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.195736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471723337227921:2048] [node 1] ICP30 unregister session Session# [2:7577471727632195309:2048] VirtualId# [2:7577471727632195302:0]
2025-11-27T18:20:19.195798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:19.195849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471725016951849:2048] [node 1] ICS01 socket: 455 reason# EndOfStream
2025-11-27T18:20:19.195830Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471723337227921:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.195877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.195935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471720721984453:2048] [node 1] ICP30 unregister session Session# [4:7577471725016951849:2048] VirtualId# [4:7577471725016951841:0]
2025-11-27T18:20:19.195980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471725016951849:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.196046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:19.196094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:19.196259Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471722052955358:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.196359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471726347922768:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.196496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471722052955358:2048] [node 1] ICP30 unregister session Session# [3:7577471726347922768:2048] VirtualId# [3:7577471726347922762:0]
2025-11-27T18:20:19.196557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:19.196586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471722052955358:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.196600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.197068Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:19.197296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471727015453900:2048] [node 1] ICS07 socket disconnect 419 reason# EndOfStream
2025-11-27T18:20:19.197332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471727015453900:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.197391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:19.197434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471727015453900:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:19.198172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471722720486520:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.198192Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:19.198264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471727015453900:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.198301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471722720486520:2048] [node 1] ICP30 unregister session Session# [5:7577471727015453900:2048] VirtualId# [5:7577471727015453897:0]
2025-11-27T18:20:19.198367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471725287727633:2048] [node 1] ICS01 socket: 471 reason# EndOfStream
2025-11-27T18:20:19.198389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:19.198414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471720992760239:2048] [node 1] ICP30 unregister session Session# [6:7577471725287727633:2048] VirtualId# [6:7577471725287727627:0]
2025-11-27T18:20:19.198417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471722720486520:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.198451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471725287727633:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.198426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6183) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.198509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:19.198562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:19.206472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471722052955358:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:19.208471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471722720486520:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:19.264025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:19.264283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471726347922783:2049] [node 2] ICS01 socket: 468 reason# EndOfStream
2025-11-27T18:20:19.264331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471722052955359:2049] [node 2] ICP30 unregister session Session# [3:7577471726347922783:2049] VirtualId# [3:7577471726347922781:0]
2025-11-27T18:20:19.264387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471726347922783:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.264494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:19.264582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:19.266446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:19.266663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471725016951836:2049] [node 2] ICS07 socket disconnect 433 reason# EndOfStream
2025-11-27T18:20:19.266699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471725016951836:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.266784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:19.266851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471725016951836:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:19.267553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471720721984454:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11211) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.267530Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:19.267611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471725016951836:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.267655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471720721984454:2049] [node 2] ICP30 unregister session Session# [4:7577471725016951836:2049] VirtualId# [4:7577471725016951828:0]
2025-11-27T18:20:19.267703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:19.267754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471720721984454:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11211) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.267769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11211) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.267776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471727015453916:2049] [node 2] ICS01 socket: 470 reason# EndOfStream
2025-11-27T18:20:19.267830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471722720486521:2049] [node 2] ICP30 unregister session Session# [5:7577471727015453916:2049] VirtualId# [5:7577471727015453912:0]
2025-11-27T18:20:19.267887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471727015453916:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.267960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:19.268022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:19.268636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:19.268784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471725287727619:2049] [node 2] ICS07 socket disconnect 426 reason# EndOfStream
2025-11-27T18:20:19.268809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471725287727619:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.268864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:19.268892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471725287727619:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:19.269446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471720992760240:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11211) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.269508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471725287727619:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.269561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471720992760240:2049] [node 2] ICP30 unregister session Session# [6:7577471725287727619:2049] VirtualId# [6:7577471725287727615:0]
2025-11-27T18:20:19.269641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:19.269672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471720992760240:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11211) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.269682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11211) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.277186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471720721984454:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:19.279712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471720992760240:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:19.338481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:19.338666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471725016951842:2050] [node 3] ICS01 socket: 435 reason# EndOfStream
2025-11-27T18:20:19.338702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471720721984455:2050] [node 3] ICP30 unregister session Session# [4:7577471725016951842:2050] VirtualId# [4:7577471725016951838:0]
2025-11-27T18:20:19.338728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471725016951842:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.338811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:19.338860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:19.339823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:19.340040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471727015453908:2050] [node 3] ICS01 socket: 462 reason# EndOfStream
2025-11-27T18:20:19.340094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471722720486522:2050] [node 3] ICP30 unregister session Session# [5:7577471727015453908:2050] VirtualId# [5:7577471727015453905:0]
2025-11-27T18:20:19.340130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471727015453908:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.340201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:19.340257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:19.340880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:19.341054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471725287727622:2050] [node 3] ICS07 socket disconnect 442 reason# EndOfStream
2025-11-27T18:20:19.341092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471725287727622:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.341168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:19.341372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471725287727622:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:19.342217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471720992760241:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2675) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.342362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471725287727622:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.342417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471720992760241:2050] [node 3] ICP30 unregister session Session# [6:7577471725287727622:2050] VirtualId# [6:7577471725287727616:0]
2025-11-27T18:20:19.342464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:19.342502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471720992760241:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2675) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.342517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2675) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.351885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471720992760241:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:19.422064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:19.422294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471727015453920:2108] [node 4] ICS07 socket disconnect 428 reason# EndOfStream
2025-11-27T18:20:19.422333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471727015453920:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.422413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:19.422479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471727015453920:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:19.423146Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:19.423239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471727015453880:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19876) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.423344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471727015453920:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.423375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471725287727638:2109] [node 4] ICS01 socket: 448 reason# EndOfStream
2025-11-27T18:20:19.423404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471727015453880:2108] [node 4] ICP30 unregister session Session# [5:7577471727015453920:2108] VirtualId# [5:7577471727015453911:0]
2025-11-27T18:20:19.423416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471725287727599:2109] [node 4] ICP30 unregister session Session# [6:7577471725287727638:2109] VirtualId# [6:7577471725287727636:0]
2025-11-27T18:20:19.423461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471725287727638:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.423504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:19.423542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:19.423559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471727015453880:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19876) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.423597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:19.423595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19876) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.498240Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:19.498487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471725287727632:2106] [node 5] ICS07 socket disconnect 476 reason# EndOfStream
2025-11-27T18:20:19.498531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471725287727632:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.498632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:19.498692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471725287727632:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:19.499349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471720992760301:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62837) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.499450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471725287727632:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.499500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471720992760301:2106] [node 5] ICP30 unregister session Session# [6:7577471725287727632:2106] VirtualId# [6:7577471725287727628:0]
2025-11-27T18:20:19.499602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:19.499643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471720992760301:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62837) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.499664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62837) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:19.584154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471726523781026:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:19.584310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471726523781027:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:19.602927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471725443473171:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:19.603302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471725443473172:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:19.624205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471729005434713:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:19.624207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471729005434714:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:19.643631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471728728147736:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:19.643680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471728728147737:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:19.643685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471728728147738:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:19.666125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471727142421927:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:19.666360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471727142421928:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:19.666367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471727142421929:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:19.690209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471727632124690:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:19.690255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471727632124691:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:19.691061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471727632124692:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:19.780998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471725443473236:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:19.784382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471728728147800:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:19.776510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471726523781089:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:19.786695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471727632124761:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:19.785726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471727142421990:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:19.786603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471726523781091:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:19.786705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471725443473238:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:19.785888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471728728147801:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:19.787802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471729005434795:2125] [node 6] ICP01 ready to work
2025-11-27T18:20:19.787352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471727142421991:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:19.786838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471726523781090:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:19.787392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471725443473237:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:19.787914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471729005434794:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:19.788465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471727632124760:2106] [node 4] ICP01 ready to work
2025-11-27T18:20:19.788498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471729005434796:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.800634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471725443473250:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29241
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.803917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471728728147812:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41000
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.804332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471728728147814:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41000
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.808019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471729005434805:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29469
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.810321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727142422003:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35948
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.809798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471729005434806:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29469
2025-11-27T18:20:19.811098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471728728147812:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.810096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471729005434807:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29469
2025-11-27T18:20:19.810220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471725443473251:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29241
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.811963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471726523781104:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21553
2025-11-27T18:20:19.811691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471725443473250:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.811240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471728728147737:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.810960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471729005434806:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.812217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471725443473245:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.811971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471726523781103:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21553
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.811299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471729005434808:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29469
2025-11-27T18:20:19.813885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471728728147814:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:19.812255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471725443473245:2114] [node 4] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-27T18:20:19.813829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471726523781096:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.811621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471729005434805:2132] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.813837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471726523781103:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.812263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471725443473171:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.813924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471728728147817:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41000
2025-11-27T18:20:19.813006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471725443473251:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.815524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727142422005:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35948
2025-11-27T18:20:19.812210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471729005434807:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.815200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471728728147736:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.816639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727142422004:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35948
2025-11-27T18:20:19.813867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471726523781096:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.815007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471725443473252:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29241
2025-11-27T18:20:19.813019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471729005434714:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.818045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727142422003:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.815758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471726523781107:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21553
2025-11-27T18:20:19.815083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471725443473245:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:19.818679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471729005434808:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.818454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727142422005:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.815839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471726523781096:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.815750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471725443473244:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.819226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727142422008:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35948
2025-11-27T18:20:19.815781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471725443473244:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.816196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471726523781099:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.819238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727142422006:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35948
2025-11-27T18:20:19.818882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471725443473236:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.820048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471728728147820:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41000
2025-11-27T18:20:19.819648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727142422004:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.816223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471726523781099:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.820670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471728728147817:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.819683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471727142421928:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.819035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:19.821440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471729005434818:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29469
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.818940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471726523781099:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:19.819204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471725443473244:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:19.820846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471728728147738:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.820132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471727142421927:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.820145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471727142421990:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.819025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471726523781104:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.821983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471729005434801:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.821771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471728728147821:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41000
2025-11-27T18:20:19.819290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471725443473236:2107] [node 4] ICP22 created new session: [2:7577471725443473259:2107]
2025-11-27T18:20:19.822007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471729005434801:2128] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.821806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471728728147812:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:19.819315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471725443473252:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.819628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471726523781026:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.820495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727142422008:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.819871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471725443473259:2107] [node 4] ICS09 handshake done sender: [2:7577471725443473245:2114] self: [2:7577471725443473239:0] peer: [4:7577471728728147818:0] socket: 414 qp: -1
2025-11-27T18:20:19.819675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:19.821882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471728728147806:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.825640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471729005434805:2132] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:19.820544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727142422006:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.821480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471727142421929:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.825731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471729005434802:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.820491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471725443473259:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:19.819950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471726523781026:2048] [node 2] ICP22 created new session: [1:7577471726523781111:2048]
2025-11-27T18:20:19.821915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471728728147806:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.825758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471729005434802:2129] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.820683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:19.824794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471728728147737:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.820296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471726523781111:2048] [node 2] ICS09 handshake done sender: [1:7577471726523781096:2111] self: [1:7577471726523781093:0] peer: [2:7577471725443473255:0] socket: 417 qp: -1
2025-11-27T18:20:19.820814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471725443473172:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.821424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471725443473243:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:19.820499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471726523781111:2048] [node 2] ICS10 traffic start
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.824840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:19.825826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471729005434801:2128] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:19.821455Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471725443473243:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.820792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471726523781089:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.825107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471728728147737:2049] [node 2] ICP22 created new session: [4:7577471728728147823:2049]
2025-11-27T18:20:19.827572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471729005434794:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.831080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471727632124777:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19799
2025-11-27T18:20:19.821548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:19.830448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727142422016:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35948
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.820675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:19.820823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471726523781107:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.827623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:19.825652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471728728147823:2049] [node 2] ICS09 handshake done sender: [4:7577471728728147812:2115] self: [4:7577471728728147818:0] peer: [2:7577471725443473239:0] socket: 421 qp: -1
2025-11-27T18:20:19.832215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727142422015:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35948
2025-11-27T18:20:19.821796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471725443473172:2049] [node 3] ICP22 created new session: [2:7577471725443473264:2049]
2025-11-27T18:20:19.827838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471729005434714:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.820860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:19.825750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471728728147814:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:19.822021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471725443473264:2049] [node 3] ICS09 handshake done sender: [2:7577471725443473244:2113] self: [2:7577471725443473242:0] peer: [3:7577471729005434815:0] socket: 415 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.827877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:19.821423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471726523781089:2107] [node 4] ICP22 created new session: [1:7577471726523781114:2107]
2025-11-27T18:20:19.822377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471725443473260:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29241
2025-11-27T18:20:19.825777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471728728147823:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:20:19.828111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471729005434714:2049] [node 2] ICP22 created new session: [3:7577471729005434822:2049]
::ffff:127.1.0.0
2025-11-27T18:20:19.821785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471726523781114:2107] [node 4] ICS09 handshake done sender: [1:7577471726523781099:2113] self: [1:7577471726523781094:0] peer: [4:7577471728728147819:0] socket: 419 qp: -1
2025-11-27T18:20:19.833990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727142422018:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35948
2025-11-27T18:20:19.826000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:19.828128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471729005434794:2107] [node 4] ICP22 created new session: [3:7577471729005434821:2107]
2025-11-27T18:20:19.828450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471728728147736:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12628) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471728728147814:2116] held: no
2025-11-27T18:20:19.821888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471726523781114:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:19.828181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471729005434821:2107] [node 4] ICS09 handshake done sender: [3:7577471729005434801:2128] self: [3:7577471729005434797:0] peer: [4:7577471728728147822:0] socket: 448 qp: -1
2025-11-27T18:20:19.829178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471728728147806:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:19.834405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471727142422003:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:19.823093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471725443473264:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.821928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:19.828767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471729005434821:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:19.823127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:19.822853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471726523781095:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.834881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471727632124778:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19799
2025-11-27T18:20:19.832266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471728728147800:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.834448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471727142422008:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:19.827367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471725443473250:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:19.828883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471729005434822:2049] [node 2] ICS09 handshake done sender: [3:7577471729005434805:2132] self: [3:7577471729005434815:0] peer: [2:7577471725443473242:0] socket: 423 qp: -1
2025-11-27T18:20:19.832434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471728728147736:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.828821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471725443473243:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:19.822899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471726523781095:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.834868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471727142422017:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35948
2025-11-27T18:20:19.832649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:19.829196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471726523781115:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21553
2025-11-27T18:20:19.835589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471727142421929:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.829051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471725443473171:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.829471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471726523781095:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:19.835758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:19.828946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:19.831651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471726523781090:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.832813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471728728147800:2108] [node 5] ICP22 created new session: [4:7577471728728147828:2108]
2025-11-27T18:20:19.829922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:19.835988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471727142421929:2050] [node 3] ICP22 created new session: [5:7577471727142422019:2050]
2025-11-27T18:20:19.829212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471729005434822:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:19.830053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471725443473171:2048] [node 1] ICP22 created new session: [2:7577471725443473268:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.831687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:19.833157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:19.836123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471727142421928:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.837249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471727632124778:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.829385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:19.832402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471726523781090:2108] [node 5] ICP22 created new session: [1:7577471726523781121:2108]
2025-11-27T18:20:19.836540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471727142422019:2050] [node 3] ICS09 handshake done sender: [5:7577471727142422008:2118] self: [5:7577471727142422014:0] peer: [3:7577471729005434799:0] socket: 455 qp: -1
2025-11-27T18:20:19.833191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471728728147736:2048] [node 1] ICP22 created new session: [4:7577471728728147829:2048]
2025-11-27T18:20:19.830772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471729005434802:2129] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:19.830486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471725443473268:2048] [node 1] ICS09 handshake done sender: [2:7577471725443473250:2116] self: [2:7577471725443473255:0] peer: [1:7577471726523781093:0] socket: 420 qp: -1
2025-11-27T18:20:19.832691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471726523781121:2108] [node 5] ICS09 handshake done sender: [1:7577471726523781095:2110] self: [1:7577471726523781092:0] peer: [5:7577471727142422011:0] socket: 418 qp: -1
2025-11-27T18:20:19.836745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471727142422019:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:19.832719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471726523781121:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:19.838012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471727632124690:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.833523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471728728147829:2048] [node 1] ICS09 handshake done sender: [4:7577471728728147814:2116] self: [4:7577471728728147819:0] peer: [1:7577471726523781094:0] socket: 425 qp: -1
2025-11-27T18:20:19.833184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471729005434796:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.833983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:19.830794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471725443473268:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:19.834039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471728728147828:2108] [node 5] ICS09 handshake done sender: [4:7577471728728147806:2112] self: [4:7577471728728147804:0] peer: [5:7577471727142422012:0] socket: 427 qp: -1
2025-11-27T18:20:19.836967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.839258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471726523781106:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.834056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:19.831115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:19.834218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471728728147829:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:19.837018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471727142421928:2049] [node 2] ICP22 created new session: [5:7577471727142422020:2049]
2025-11-27T18:20:19.834223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471728728147828:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:19.834510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471729005434796:2108] [node 5] ICP22 created new session: [3:7577471729005434826:2108]
2025-11-27T18:20:19.834389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471725443473238:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.834252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:19.839288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471726523781106:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.840086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471727632124779:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19799
2025-11-27T18:20:19.835281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471729005434826:2108] [node 5] ICS09 handshake done sender: [3:7577471729005434802:2129] self: [3:7577471729005434799:0] peer: [5:7577471727142422014:0] socket: 449 qp: -1
2025-11-27T18:20:19.834694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:19.837097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:19.835023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:19.841360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471727632124777:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.835314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471729005434826:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:19.836976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471728728147738:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7539) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471728728147817:2118] held: no
2025-11-27T18:20:19.842424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471727632124691:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.835084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471725443473238:2109] [node 5] ICP22 created new session: [2:7577471725443473270:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.835559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:19.837585Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471727142422004:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:19.837760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471727142422020:2049] [node 2] ICS09 handshake done sender: [5:7577471727142422003:2114] self: [5:7577471727142422010:0] peer: [2:7577471725443473240:0] socket: 426 qp: -1
2025-11-27T18:20:19.835734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471729005434823:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29469
2025-11-27T18:20:19.835787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471725443473270:2109] [node 5] ICS09 handshake done sender: [2:7577471725443473243:2112] self: [2:7577471725443473240:0] peer: [5:7577471727142422010:0] socket: 416 qp: -1
2025-11-27T18:20:19.842479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471726523781106:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:19.837782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471727142422020:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:19.838840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471728728147826:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41000
2025-11-27T18:20:19.835908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471725443473270:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:19.842729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471727632124779:2126] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.839250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471729005434823:2138] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.838225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:19.842851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471727632124692:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.841334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471729005434713:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.836583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:19.842852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471727632124780:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19799
2025-11-27T18:20:19.841295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471727142422005:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:19.842920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471727142421927:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12628) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471727142422004:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.842963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471726523781098:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.843114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471725443473236:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17372946ns) {HandleHandshakeStatus:329(16992581ns) {DropIncomingHandshake:442(589ns) DropOutgoingHandshake:457(744ns) SwitchToState:179(514ns) ProcessPendingSessionEvents:495(14195078ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14075264ns max# 1228585ns}}}
2025-11-27T18:20:19.843066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471729005434813:2136] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.843012Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471726523781098:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.843063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471727632124783:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19799
2025-11-27T18:20:19.843102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471728728147817:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:19.843028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471726523781026:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(16314630ns) {HandleHandshakeStatus:329(16311250ns) {DropIncomingHandshake:442(402ns) DropOutgoingHandshake:457(412ns) SwitchToState:179(289ns) ProcessPendingSessionEvents:495(13959270ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13325989ns max# 1417321ns}}}
2025-11-27T18:20:19.844338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471727142421929:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:19.843092Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471729005434813:2136] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.844400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471725443473254:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.844435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471725443473254:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.845635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471728728147738:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.846061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471727142421927:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.846112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471726523781091:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.845965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471727632124778:2125] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:19.845755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:19.846163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:19.845816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471728728147738:2050] [node 3] ICP22 created new session: [4:7577471728728147832:2050]
2025-11-27T18:20:19.846211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471726523781091:2109] [node 6] ICP22 created new session: [1:7577471726523781125:2109]
2025-11-27T18:20:19.846189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471729005434813:2136] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:19.847200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471726523781098:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:19.846336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471727632124780:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.847442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471726523781125:2109] [node 6] ICS09 handshake done sender: [1:7577471726523781106:2116] self: [1:7577471726523781105:0] peer: [6:7577471727632124781:0] socket: 477 qp: -1
2025-11-27T18:20:19.846149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:19.847206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471727632124690:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.845857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471728728147832:2050] [node 3] ICS09 handshake done sender: [4:7577471728728147817:2118] self: [4:7577471728728147822:0] peer: [3:7577471729005434797:0] socket: 451 qp: -1
2025-11-27T18:20:19.846208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471727142421990:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.845891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471728728147832:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:19.847680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471726523781125:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:19.847904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471726523781026:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:19.847856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:19.846210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471727142421927:2048] [node 1] ICP22 created new session: [5:7577471727142422023:2048]
2025-11-27T18:20:19.846170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:19.847540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471727142422023:2048] [node 1] ICS09 handshake done sender: [5:7577471727142422004:2115] self: [5:7577471727142422011:0] peer: [1:7577471726523781092:0] socket: 436 qp: -1
2025-11-27T18:20:19.848096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471727632124690:2048] [node 1] ICP22 created new session: [6:7577471727632124786:2048]
2025-11-27T18:20:19.847732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471727142422023:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:19.848036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:19.847775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:19.848103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471727632124760:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.848661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471727632124786:2048] [node 1] ICS09 handshake done sender: [6:7577471727632124778:2125] self: [6:7577471727632124781:0] peer: [1:7577471726523781105:0] socket: 478 qp: -1
2025-11-27T18:20:19.847851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.848798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471727632124786:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:19.848414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471727142421990:2108] [node 4] ICP22 created new session: [5:7577471727142422025:2108]
2025-11-27T18:20:19.848965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:19.848780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471727142422025:2108] [node 4] ICS09 handshake done sender: [5:7577471727142422005:2116] self: [5:7577471727142422012:0] peer: [4:7577471728728147804:0] socket: 444 qp: -1
2025-11-27T18:20:19.849037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471729005434795:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.849041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471728728147809:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.849030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471727632124785:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19799
2025-11-27T18:20:19.849101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:19.849079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471728728147809:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.849149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471729005434795:2125] [node 6] ICP22 created new session: [3:7577471729005434832:2125]
2025-11-27T18:20:19.849018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471726523781027:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.849158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:19.849935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471726523781027:2049] [node 3] ICP22 created new session: [1:7577471726523781127:2049]
2025-11-27T18:20:19.849945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471729005434832:2125] [node 6] ICS09 handshake done sender: [3:7577471729005434813:2136] self: [3:7577471729005434812:0] peer: [6:7577471727632124784:0] socket: 422 qp: -1
2025-11-27T18:20:19.849211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471727142422025:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:19.849984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471726523781127:2049] [node 3] ICS09 handshake done sender: [1:7577471726523781098:2112] self: [1:7577471726523781097:0] peer: [3:7577471729005434829:0] socket: 469 qp: -1
2025-11-27T18:20:19.849994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471729005434832:2125] [node 6] ICS10 traffic start
2025-11-27T18:20:19.850542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471726523781127:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:19.850416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:19.850668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471727632124779:2126] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:19.849994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:19.850728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:19.850970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471725443473254:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:19.850956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471728728147809:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.852009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471725443473237:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.852160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471728728147801:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.852030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471729005434831:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29469
2025-11-27T18:20:19.852202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:19.855204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471728728147801:2109] [node 6] ICP22 created new session: [4:7577471728728147836:2109]
2025-11-27T18:20:19.855692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471728728147836:2109] [node 6] ICS09 handshake done sender: [4:7577471728728147809:2114] self: [4:7577471728728147803:0] peer: [6:7577471727632124787:0] socket: 441 qp: -1
2025-11-27T18:20:19.855748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471728728147836:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.855720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471727632124692:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.855951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:19.855752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:19.856057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471725443473237:2110] [node 6] ICP22 created new session: [2:7577471725443473273:2110]
2025-11-27T18:20:19.857029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:19.857184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471727632124790:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19799
2025-11-27T18:20:19.858326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471728728147835:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41000
2025-11-27T18:20:19.857831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471725443473273:2110] [node 6] ICS09 handshake done sender: [2:7577471725443473254:2119] self: [2:7577471725443473253:0] peer: [6:7577471727632124782:0] socket: 466 qp: -1
2025-11-27T18:20:19.858645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471729005434713:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12628) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471729005434823:2138] held: no
2025-11-27T18:20:19.858030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471725443473273:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:19.858066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:19.857215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471727632124692:2050] [node 3] ICP22 created new session: [6:7577471727632124793:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.857918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471727632124793:2050] [node 3] ICS09 handshake done sender: [6:7577471727632124779:2126] self: [6:7577471727632124784:0] peer: [3:7577471729005434812:0] socket: 443 qp: -1
2025-11-27T18:20:19.857960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471727632124793:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.858670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:19.860783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471727632124792:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19799
2025-11-27T18:20:19.860857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471727632124791:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19799
2025-11-27T18:20:19.862134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471727632124691:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14706) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471727632124777:2124] held: no
2025-11-27T18:20:19.861951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471729005434823:2138] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:19.862217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471727632124692:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:19.862254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471729005434713:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.862427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:19.862477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471729005434713:2048] [node 1] ICP22 created new session: [3:7577471729005434834:2048]
2025-11-27T18:20:19.862529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471729005434834:2048] [node 1] ICS09 handshake done sender: [3:7577471729005434823:2138] self: [3:7577471729005434829:0] peer: [1:7577471726523781097:0] socket: 472 qp: -1
2025-11-27T18:20:19.862809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471729005434834:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:19.862851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:19.863719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471728728147835:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.863852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471728728147801:2109] [node 6] ICP18 handshake (actor [4:7577471728728147835:2122]) is too old
2025-11-27T18:20:19.863921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471727632124777:2124] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:19.863931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471727632124780:2127] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:19.864627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471727632124691:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.864698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471727632124760:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.864716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:19.864774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471727632124792:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.864779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471727632124691:2049] [node 2] ICP22 created new session: [6:7577471727632124795:2049]
2025-11-27T18:20:19.864780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:19.864820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471727632124795:2049] [node 2] ICS09 handshake done sender: [6:7577471727632124777:2124] self: [6:7577471727632124782:0] peer: [2:7577471725443473253:0] socket: 467 qp: -1
2025-11-27T18:20:19.864824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471727632124760:2106] [node 4] ICP22 created new session: [6:7577471727632124796:2106]
2025-11-27T18:20:19.864844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471727632124795:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:19.864880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471727632124796:2106] [node 4] ICS09 handshake done sender: [6:7577471727632124780:2127] self: [6:7577471727632124787:0] peer: [4:7577471728728147803:0] socket: 437 qp: -1
2025-11-27T18:20:19.864882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:19.864922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471727632124796:2106] [node 4] ICS10 traffic start
2025-11-27T18:20:19.864950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:19.865059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471727632124761:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:19.865303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471727142422009:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:19.865353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471727142422009:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:19.865722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471727632124760:2106] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:19.866603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471727142422009:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:19.866741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471727632124800:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19799
2025-11-27T18:20:19.866766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471727142421991:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:19.866949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:19.867019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471727142421991:2109] [node 6] ICP22 created new session: [5:7577471727142422030:2109]
2025-11-27T18:20:19.867023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471727632124792:2132] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:19.867067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471727142422030:2109] [node 6] ICS09 handshake done sender: [5:7577471727142422009:2119] self: [5:7577471727142422007:0] peer: [6:7577471727632124799:0] socket: 452 qp: -1
2025-11-27T18:20:19.867097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471727142422030:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:19.867128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471727632124761:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:19.867220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:19.867144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:19.867502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471727632124761:2105] [node 5] ICP22 created new session: [6:7577471727632124801:2105]
2025-11-27T18:20:19.867709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471727632124801:2105] [node 5] ICS09 handshake done sender: [6:7577471727632124792:2132] self: [6:7577471727632124799:0] peer: [5:7577471727142422007:0] socket: 486 qp: -1
2025-11-27T18:20:19.867757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471727632124801:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:19.867839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:19.869686Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471727632124761:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:19.911155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:19.911379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471725443473268:2048] [node 1] ICS07 socket disconnect 420 reason# EndOfStream
2025-11-27T18:20:19.911418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471725443473268:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.911494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:19.911543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471725443473268:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:19.912499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:19.912636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471729005434834:2048] [node 1] ICS01 socket: 472 reason# EndOfStream
2025-11-27T18:20:19.912667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471729005434713:2048] [node 1] ICP30 unregister session Session# [3:7577471729005434834:2048] VirtualId# [3:7577471729005434829:0]
2025-11-27T18:20:19.912697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471729005434834:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.912753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:19.912791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:19.913771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471725443473171:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12628) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:19.913810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471725443473268:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:19.915134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471725443473171:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12628) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.916185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471725443473268:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.916255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471725443473171:2048] [node 1] ICP30 unregister session Session# [2:7577471725443473268:2048] VirtualId# [2:7577471725443473255:0]
2025-11-27T18:20:19.916308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:19.916372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471725443473171:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.916394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.916400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:19.916613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471728728147829:2048] [node 1] ICS01 socket: 425 reason# EndOfStream
2025-11-27T18:20:19.916655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471728728147736:2048] [node 1] ICP30 unregister session Session# [4:7577471728728147829:2048] VirtualId# [4:7577471728728147819:0]
2025-11-27T18:20:19.916688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471728728147829:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.916772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:19.916815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:19.917508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:19.917648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471727142422023:2048] [node 1] ICS01 socket: 436 reason# EndOfStream
2025-11-27T18:20:19.917678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471727142421927:2048] [node 1] ICP30 unregister session Session# [5:7577471727142422023:2048] VirtualId# [5:7577471727142422011:0]
2025-11-27T18:20:19.917695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471727142422023:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.917742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:19.917773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:19.918502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:19.918698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471727632124786:2048] [node 1] ICS07 socket disconnect 478 reason# EndOfStream
2025-11-27T18:20:19.918732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471727632124786:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.918809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:19.918862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471727632124786:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:19.919535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471727632124690:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12628) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.919709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471727632124786:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.919768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471727632124690:2048] [node 1] ICP30 unregister session Session# [6:7577471727632124786:2048] VirtualId# [6:7577471727632124781:0]
2025-11-27T18:20:19.919848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:19.919872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471727632124690:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.919883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12628) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.929382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471727632124690:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:19.965287Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:19.965534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471729005434822:2049] [node 2] ICS07 socket disconnect 423 reason# EndOfStream
2025-11-27T18:20:19.965572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471729005434822:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.965661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:19.965718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471729005434822:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:19.968278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:19.968291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471729005434714:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14706) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:19.968323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471729005434822:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:19.968492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471728728147823:2049] [node 2] ICS01 socket: 421 reason# EndOfStream
2025-11-27T18:20:19.968541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471728728147737:2049] [node 2] ICP30 unregister session Session# [4:7577471728728147823:2049] VirtualId# [4:7577471728728147818:0]
2025-11-27T18:20:19.968595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471728728147823:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.968673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:19.968749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:19.968896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471729005434714:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14706) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.968959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471729005434822:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.968991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471729005434714:2049] [node 2] ICP30 unregister session Session# [3:7577471729005434822:2049] VirtualId# [3:7577471729005434815:0]
2025-11-27T18:20:19.969236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:19.969280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471729005434714:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.969293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.969545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:19.969723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471727142422020:2049] [node 2] ICS07 socket disconnect 426 reason# EndOfStream
2025-11-27T18:20:19.969749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471727142422020:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.969810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:19.969836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471727142422020:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:19.970433Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:19.970489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471727142421928:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14706) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:19.970585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471727142422020:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:19.970607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471727632124795:2049] [node 2] ICS01 socket: 467 reason# EndOfStream
2025-11-27T18:20:19.970626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471727142421928:2049] [node 2] ICP30 unregister session Session# [5:7577471727142422020:2049] VirtualId# [5:7577471727142422010:0]
2025-11-27T18:20:19.970634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471727632124691:2049] [node 2] ICP30 unregister session Session# [6:7577471727632124795:2049] VirtualId# [6:7577471727632124782:0]
2025-11-27T18:20:19.970659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471727632124795:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:19.970744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:19.970745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:19.970807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:19.970797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471727142421928:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.970813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:19.979570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471729005434714:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:19.980560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471727142421928:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:20.054695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:20.054957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471728728147832:2050] [node 3] ICS01 socket: 451 reason# EndOfStream
2025-11-27T18:20:20.055007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471728728147738:2050] [node 3] ICP30 unregister session Session# [4:7577471728728147832:2050] VirtualId# [4:7577471728728147822:0]
2025-11-27T18:20:20.055061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471728728147832:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.055150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:20.055200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:20.055539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:20.055745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471727142422019:2050] [node 3] ICS07 socket disconnect 455 reason# EndOfStream
2025-11-27T18:20:20.055772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471727142422019:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.055838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:20.055875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471727142422019:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:20.056276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:20.056486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471727632124793:2050] [node 3] ICS07 socket disconnect 443 reason# EndOfStream
2025-11-27T18:20:20.056516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471727632124793:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.056580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:20.056523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471727142421929:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7539) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.056634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471727632124793:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:20.056589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471727142422019:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.056621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471727142421929:2050] [node 3] ICP30 unregister session Session# [5:7577471727142422019:2050] VirtualId# [5:7577471727142422014:0]
2025-11-27T18:20:20.056842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:20.056915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471727142421929:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7539) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.056938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7539) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.057379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471727632124692:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7539) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.057457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471727632124793:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.057505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471727632124692:2050] [node 3] ICP30 unregister session Session# [6:7577471727632124793:2050] VirtualId# [6:7577471727632124784:0]
2025-11-27T18:20:20.057608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:20.057659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471727632124692:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7539) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.057678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7539) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.066636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471727142421929:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:20.067603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471727632124692:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:20.100742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:20.100993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471727142422025:2108] [node 4] ICS07 socket disconnect 444 reason# EndOfStream
2025-11-27T18:20:20.101028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471727142422025:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.101101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:20.101144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471727142422025:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:20.101829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471727142421990:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10400) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.101876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471727142422025:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.101911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471727142421990:2108] [node 4] ICP30 unregister session Session# [5:7577471727142422025:2108] VirtualId# [5:7577471727142422012:0]
2025-11-27T18:20:20.101885Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:20.102023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:20.102060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471727632124796:2106] [node 4] ICS07 socket disconnect 437 reason# EndOfStream
2025-11-27T18:20:20.102055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471727142421990:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10400) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.102084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471727632124796:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.102067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10400) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.102132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:20.102160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471727632124796:2106] [node 4] ICS15 start handshake
2025-11-27T18:20:20.102724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471727632124760:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10400) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.102756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471727632124796:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.102844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471727632124760:2106] [node 4] ICP30 unregister session Session# [6:7577471727632124796:2106] VirtualId# [6:7577471727632124787:0]
2025-11-27T18:20:20.103002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:20.103035Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471727632124760:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10400) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.103045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10400) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.111676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471727142421990:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:20.112850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471727632124760:2106] [node 4] ICP33 wake up from error state
2025-11-27T18:20:20.164960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:20.165136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471727632124801:2105] [node 5] ICS07 socket disconnect 486 reason# EndOfStream
2025-11-27T18:20:20.165166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471727632124801:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.165234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:20.165291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471727632124801:2105] [node 5] ICS15 start handshake
2025-11-27T18:20:20.165924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471727632124761:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27788) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.165970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471727632124801:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.166048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471727632124761:2105] [node 5] ICP30 unregister session Session# [6:7577471727632124801:2105] VirtualId# [6:7577471727632124799:0]
2025-11-27T18:20:20.166149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:20.166176Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471727632124761:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.166191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:20.277406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471732520413188:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:20.277624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471732520413189:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:20.297722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471732901231746:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:20.297789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471732901231747:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:20.318296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471732349944421:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:20.318357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471732349944420:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:20.339281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471730947869973:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:20.339314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471730947869975:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:20.339753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471730947869974:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:20.360873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471732499503556:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:20.360909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471732499503557:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:20.361017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471732499503558:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:20.381196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471732460474539:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:20.381293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471732460474540:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:20.381447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471732460474541:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:20.467181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471730947870036:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:20.471223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471732349944483:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:20.473194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471732460474600:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:20.466987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471732520413251:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:20.473643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471732901231811:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:20.473913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471732349944485:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:20.473160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471732520413252:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:20.474318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471732499503619:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:20.474423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471732460474602:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:20.473909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471732901231812:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:20.474360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471730947870037:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:20.474363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471732349944484:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:20.473904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471732520413253:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:20.474829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471732499503620:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:20.474122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471732901231813:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:20.492152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471732520413189:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16316281ns) {Configure:104(16310683ns) {ProcessConfigured:135(2415407ns) {StartInitialHandshake:157(2413579ns) {DropHandshakes:470(1380ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(403ns)} GenerateSessionVirtualId:988(659ns) PrepareNewSessionHandshake:477(2279ns) {SwitchToState:179(1192ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.493180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471732460474614:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39478
2025-11-27T18:20:20.493595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471732349944496:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28992
2025-11-27T18:20:20.493902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471732499503628:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8534
2025-11-27T18:20:20.494465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471732460474614:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.495429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471732460474602:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.495476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471732349944496:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.495450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471732499503628:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.495797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471730947870042:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.495838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471730947870042:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.499890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471732499503619:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.500628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471732901231825:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22324
2025-11-27T18:20:20.496877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471730947870051:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51058
2025-11-27T18:20:20.500644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471732349944500:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28992
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.500620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471732520413266:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21877
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.500912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471732499503632:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8534
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.499967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471730947870042:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:20.502047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471732520413266:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.501482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471732901231825:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.503107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471732460474619:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39478
2025-11-27T18:20:20.503167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471732349944500:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.503905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471732520413268:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21877
2025-11-27T18:20:20.503626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471732901231828:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22324
2025-11-27T18:20:20.501095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471730947870053:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51058
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.501461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471732499503637:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8534
2025-11-27T18:20:20.504337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471732349944491:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.501524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471730947870052:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51058
2025-11-27T18:20:20.502346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471732499503635:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8534
2025-11-27T18:20:20.504361Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471732349944491:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.506658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471732520413261:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.506891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471732901231828:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.503149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471730947870051:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.503095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471732499503636:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8534
2025-11-27T18:20:20.505565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471732349944501:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28992
2025-11-27T18:20:20.507775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471732901231746:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.507495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471732460474620:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39478
2025-11-27T18:20:20.506681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471732520413261:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.503141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471732499503632:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.503632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471730947870046:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.508000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471732901231819:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.505842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471732349944492:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.508225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471732520413262:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.503916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471732499503635:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.503649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471730947870036:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.505870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471732349944492:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.508003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471732901231822:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.508266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471732520413262:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.505887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471732349944502:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28992
2025-11-27T18:20:20.503670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471730947870046:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.508025Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471732901231822:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.513012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471732460474614:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:20.504331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471732499503637:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.508808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471732520413257:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.508027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471732901231819:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.513575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471732460474619:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.506474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471732349944491:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:20.503901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:20.508828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471732520413257:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.504338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471732499503636:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.504347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471730947870036:2109] [node 6] ICP22 created new session: [4:7577471730947870056:2109]
2025-11-27T18:20:20.512034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471732901231820:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.511823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471732520413268:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.513893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471732460474602:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.506873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471732349944501:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.504433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471732499503558:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.512046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471732901231819:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:20.504312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471730947869975:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.511984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471732520413261:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.507755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471732349944421:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.514111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471732460474600:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.505908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471732499503557:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.504617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471730947870056:2109] [node 6] ICS09 handshake done sender: [4:7577471730947870042:2110] self: [4:7577471730947870040:0] peer: [6:7577471732460474618:0] socket: 420 qp: -1
2025-11-27T18:20:20.515999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:20.512061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471732901231820:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.514182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471732520413257:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:20.512178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471732349944492:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:20.507706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471732499503556:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.504620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471730947870052:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.517452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471732460474602:2108] [node 4] ICP22 created new session: [6:7577471732460474623:2108]
2025-11-27T18:20:20.514120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471732901231822:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:20.512375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471732349944483:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.515073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471732499503624:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.514728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471732520413252:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.504658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471730947870056:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.518957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471732460474623:2108] [node 4] ICS09 handshake done sender: [6:7577471732460474614:2115] self: [6:7577471732460474618:0] peer: [4:7577471730947870040:0] socket: 425 qp: -1
2025-11-27T18:20:20.515051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471732520413262:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:20.514734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471732901231820:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:20.512647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471732349944502:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.505273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:20.515098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471732499503624:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.515053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471732901231831:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22324
2025-11-27T18:20:20.519501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471732460474623:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:20.505485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471730947870053:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.516275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471732520413188:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.515467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471732901231812:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.516310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:20.505936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471730947869974:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.516384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471732520413251:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.507703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471730947869973:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.516429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:20.515681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471732901231747:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.513810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471732349944485:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.515783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471732499503644:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8534
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.520323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:20.516479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471732520413252:2108] [node 5] ICP22 created new session: [1:7577471732520413274:2108]
2025-11-27T18:20:20.508000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471730947870046:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:20.518052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471732499503556:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30037) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471732499503637:2119] held: no
2025-11-27T18:20:20.513833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:20.515860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471732901231813:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.523864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471732460474624:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39478
2025-11-27T18:20:20.514419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471732349944483:2107] [node 4] ICP22 created new session: [3:7577471732349944508:2107]
2025-11-27T18:20:20.515904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:20.510701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471730947870037:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.517010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471732520413188:2048] [node 2] ICP22 created new session: [1:7577471732520413275:2048]
2025-11-27T18:20:20.510736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:20.515994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:20.520276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471732499503645:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8534
2025-11-27T18:20:20.522819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471732499503646:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8534
2025-11-27T18:20:20.525177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471732460474627:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39478
2025-11-27T18:20:20.515370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:20.517057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471732901231747:2049] [node 3] ICP22 created new session: [2:7577471732901231836:2049]
2025-11-27T18:20:20.517678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:20.511816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471730947870037:2108] [node 5] ICP22 created new session: [4:7577471730947870063:2108]
2025-11-27T18:20:20.523400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471732499503647:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8534
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.525493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471732460474626:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39478
2025-11-27T18:20:20.512579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471730947870063:2108] [node 5] ICS09 handshake done sender: [4:7577471730947870046:2113] self: [4:7577471730947870039:0] peer: [5:7577471732499503639:0] socket: 422 qp: -1
2025-11-27T18:20:20.518018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471732520413275:2048] [node 2] ICS09 handshake done sender: [1:7577471732520413257:2110] self: [1:7577471732520413256:0] peer: [2:7577471732901231830:0] socket: 450 qp: -1
2025-11-27T18:20:20.515397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471732349944485:2108] [node 5] ICP22 created new session: [3:7577471732349944510:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.517065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:20.523838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471732499503632:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:20.515809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471732349944508:2107] [node 4] ICS09 handshake done sender: [3:7577471732349944491:2110] self: [3:7577471732349944486:0] peer: [4:7577471730947870055:0] socket: 428 qp: -1
2025-11-27T18:20:20.518062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471732520413251:2107] [node 4] ICP22 created new session: [1:7577471732520413277:2107]
2025-11-27T18:20:20.524209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471732499503624:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:20.526313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471732460474624:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.517118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471732901231813:2107] [node 4] ICP22 created new session: [2:7577471732901231837:2107]
2025-11-27T18:20:20.512603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471730947870063:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:20.518095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471732520413274:2108] [node 5] ICS09 handshake done sender: [1:7577471732520413261:2112] self: [1:7577471732520413255:0] peer: [5:7577471732499503641:0] socket: 442 qp: -1
2025-11-27T18:20:20.516075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471732349944508:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:20.526813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471732460474541:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.517759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471732901231837:2107] [node 4] ICS09 handshake done sender: [2:7577471732901231822:2115] self: [2:7577471732901231818:0] peer: [4:7577471730947870059:0] socket: 443 qp: -1
2025-11-27T18:20:20.513526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:20.527069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471732460474619:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:20.518063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471732901231836:2049] [node 3] ICS09 handshake done sender: [2:7577471732901231820:2113] self: [2:7577471732901231815:0] peer: [3:7577471732349944506:0] socket: 451 qp: -1
2025-11-27T18:20:20.524629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471732499503628:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:20.518959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471732520413275:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:20.515427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471730947870062:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51058
2025-11-27T18:20:20.527309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471732460474600:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.518076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471732901231812:2108] [node 5] ICP22 created new session: [2:7577471732901231838:2108]
2025-11-27T18:20:20.525431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471732499503558:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.516421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:20.519005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471732520413274:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:20.519524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:20.518106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471732901231837:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:20.528822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471732460474628:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39478
2025-11-27T18:20:20.521639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471730947869975:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16497) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471730947870051:2115] held: no
2025-11-27T18:20:20.519829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:20.529897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:20.518148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:20.525438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471732499503636:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:20.521930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471730947870066:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51058
2025-11-27T18:20:20.529947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471732460474600:2106] [node 5] ICP22 created new session: [6:7577471732460474630:2106]
2025-11-27T18:20:20.519506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471732901231836:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:20.520447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471732520413277:2107] [node 4] ICS09 handshake done sender: [1:7577471732520413262:2113] self: [1:7577471732520413258:0] peer: [4:7577471730947870060:0] socket: 446 qp: -1
2025-11-27T18:20:20.517703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471732349944510:2108] [node 5] ICS09 handshake done sender: [3:7577471732349944492:2111] self: [3:7577471732349944487:0] peer: [5:7577471732499503640:0] socket: 427 qp: -1
2025-11-27T18:20:20.525475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:20.523292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471730947870051:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:20.519535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471732901231838:2108] [node 5] ICS09 handshake done sender: [2:7577471732901231819:2112] self: [2:7577471732901231814:0] peer: [5:7577471732499503643:0] socket: 441 qp: -1
2025-11-27T18:20:20.531107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471732460474630:2106] [node 5] ICS09 handshake done sender: [6:7577471732460474619:2116] self: [6:7577471732460474621:0] peer: [5:7577471732499503623:0] socket: 456 qp: -1
2025-11-27T18:20:20.521619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471732520413277:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:20.518062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471732349944510:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:20.526381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471732499503558:2050] [node 3] ICP22 created new session: [5:7577471732499503649:2050]
2025-11-27T18:20:20.521933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:20.524634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471730947870067:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51058
2025-11-27T18:20:20.531306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471732460474630:2106] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.518992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:20.526398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471732499503620:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.519552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:20.524476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471732520413278:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21877
2025-11-27T18:20:20.526873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471730947869975:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.522461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471732349944512:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28992
2025-11-27T18:20:20.519575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471732901231838:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:20.526444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:20.519846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:20.522501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471732349944421:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471732349944501:2117] held: no
2025-11-27T18:20:20.526598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471732499503619:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.522270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471732901231831:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.527010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471730947870053:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:20.525645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471732520413279:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21877
2025-11-27T18:20:20.531344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:20.533667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471732460474626:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.533812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471732460474539:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.528880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471732349944501:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:20.526666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471732499503649:2050] [node 3] ICS09 handshake done sender: [5:7577471732499503632:2116] self: [5:7577471732499503640:0] peer: [3:7577471732349944487:0] socket: 436 qp: -1
2025-11-27T18:20:20.529946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471732520413279:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.526803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:20.529895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471732349944515:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28992
2025-11-27T18:20:20.527101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:20.523227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471732901231834:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22324
2025-11-27T18:20:20.526804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471732499503620:2109] [node 6] ICP22 created new session: [5:7577471732499503651:2109]
2025-11-27T18:20:20.530489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471732349944495:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.530588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471732520413278:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.527334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471730947870052:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:20.526842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471732499503649:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:20.526378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471732901231828:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:20.530509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471732349944495:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.526986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471732499503637:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:20.526681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471732901231842:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22324
2025-11-27T18:20:20.533685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471732520413284:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21877
2025-11-27T18:20:20.527143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471732901231746:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.527073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:20.527659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471730947869975:2050] [node 3] ICP22 created new session: [4:7577471730947870068:2050]
2025-11-27T18:20:20.534380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471732520413260:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.527297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471732499503619:2108] [node 4] ICP22 created new session: [5:7577471732499503652:2108]
2025-11-27T18:20:20.527194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.531119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471732349944421:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.534400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471732520413260:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.528405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471730947870068:2050] [node 3] ICS09 handshake done sender: [4:7577471730947870051:2115] self: [4:7577471730947870055:0] peer: [3:7577471732349944486:0] socket: 429 qp: -1
2025-11-27T18:20:20.528798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471732499503652:2108] [node 4] ICS09 handshake done sender: [5:7577471732499503628:2112] self: [5:7577471732499503639:0] peer: [4:7577471730947870039:0] socket: 426 qp: -1
2025-11-27T18:20:20.527239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471732901231746:2048] [node 1] ICP22 created new session: [2:7577471732901231845:2048]
2025-11-27T18:20:20.537220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471732460474633:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39478
2025-11-27T18:20:20.531581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:20.536774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471732520413263:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.528443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471730947870068:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:20.527305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471732901231845:2048] [node 1] ICS09 handshake done sender: [2:7577471732901231828:2118] self: [2:7577471732901231830:0] peer: [1:7577471732520413256:0] socket: 453 qp: -1
2025-11-27T18:20:20.528823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471732499503651:2109] [node 6] ICS09 handshake done sender: [5:7577471732499503624:2110] self: [5:7577471732499503623:0] peer: [6:7577471732460474621:0] socket: 452 qp: -1
2025-11-27T18:20:20.531632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471732349944421:2049] [node 2] ICP22 created new session: [3:7577471732349944516:2049]
2025-11-27T18:20:20.536811Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471732520413263:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.538241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471732460474624:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:20.528870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471732499503651:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:20.529395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:20.531938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471732349944516:2049] [node 2] ICS09 handshake done sender: [3:7577471732349944501:2117] self: [3:7577471732349944506:0] peer: [2:7577471732901231815:0] socket: 454 qp: -1
2025-11-27T18:20:20.527336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471732901231845:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:20.538853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471732460474541:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16497) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471732460474624:2118] held: no
2025-11-27T18:20:20.538037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471732520413260:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:20.528980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471730947869973:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.528907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:20.532990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471732349944516:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:20.528345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:20.529363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471732499503652:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:20.538934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471732520413263:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:20.533159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471732349944515:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.529930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: 2025-11-27T18:20:20.533054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471732901231842:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:20.531912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471732499503557:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.539367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471732520413253:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.533394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:20.533706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471732349944495:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:20.538793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471732901231747:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(16225846ns) {HandleHandshakeStatus:329(16223061ns) {DropIncomingHandshake:442(17486ns) DropOutgoingHandshake:457(410ns) SwitchToState:179(132ns) ProcessPendingSessionEvents:495(9932832ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9064427ns max# 1321998ns}}}
2025-11-27T18:20:20.529604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:20.531959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:20.540789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471732460474628:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.540211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471732901231813:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16338606ns) {HandleHandshakeStatus:329(16336052ns) {DropIncomingHandshake:442(674ns) DropOutgoingHandshake:457(555ns) SwitchToState:179(222ns) ProcessPendingSessionEvents:495(11215239ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11121951ns max# 1253461ns}}}
2025-11-27T18:20:20.533967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471732349944420:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.533042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471732499503557:2049] [node 2] ICP22 created new session: [5:7577471732499503656:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.540115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471732520413251:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17318037ns) {HandleHandshakeStatus:329(17316392ns) {DropIncomingHandshake:442(234ns) DropOutgoingHandshake:457(270ns) SwitchToState:179(1019ns) ProcessPendingSessionEvents:495(10413808ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10294135ns max# 1176526ns}}}
2025-11-27T18:20:20.530635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471730947869973:2048] [node 1] ICP22 created new session: [4:7577471730947870070:2048]
2025-11-27T18:20:20.530925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471730947870070:2048] [node 1] ICS09 handshake done sender: [4:7577471730947870053:2117] self: [4:7577471730947870060:0] peer: [1:7577471732520413258:0] socket: 448 qp: -1
2025-11-27T18:20:20.534310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471732349944484:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.533213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471732499503656:2049] [node 2] ICS09 handshake done sender: [5:7577471732499503636:2118] self: [5:7577471732499503643:0] peer: [2:7577471732901231814:0] socket: 444 qp: -1
2025-11-27T18:20:20.531106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471730947869974:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.534355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:20.533815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471732499503656:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:20.531131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:20.536397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471732349944484:2109] [node 6] ICP22 created new session: [3:7577471732349944520:2109]
2025-11-27T18:20:20.534492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:20.531136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471730947870070:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:20.540816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471732901231847:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22324
2025-11-27T18:20:20.534592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471732499503556:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.536916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471732349944520:2109] [node 6] ICS09 handshake done sender: [3:7577471732349944495:2114] self: [3:7577471732349944488:0] peer: [6:7577471732460474629:0] socket: 481 qp: -1
2025-11-27T18:20:20.531161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:20.540407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471732520413189:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.541334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471732901231812:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(16446994ns) {HandleHandshakeStatus:329(16444200ns) {DropIncomingHandshake:442(168ns) DropOutgoingHandshake:457(244ns) SwitchToState:179(283ns) ProcessPendingSessionEvents:495(9061914ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8982293ns max# 1226633ns}}}
2025-11-27T18:20:20.536949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471732349944520:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:20.541580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471732460474602:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(19267925ns) {HandleHandshakeStatus:329(19265943ns) {DropIncomingHandshake:442(961ns) DropOutgoingHandshake:457(517833ns) SwitchToState:179(462ns) ProcessPendingSessionEvents:495(12675260ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11753129ns max# 1484596ns}}}
2025-11-27T18:20:20.536412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:20.541928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471732901231813:2107] [node 4] ICP18 handshake (actor [2:7577471732901231831:2119]) is too old
2025-11-27T18:20:20.540590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:20.531541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471730947869974:2049] [node 2] ICP22 created new session: [4:7577471730947870071:2049]
2025-11-27T18:20:20.542449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471732460474541:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.536466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471732499503556:2048] [node 1] ICP22 created new session: [5:7577471732499503658:2048]
2025-11-27T18:20:20.537286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:20.540584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:20.543023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:20.539518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471732349944483:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(19519903ns) {HandleHandshakeStatus:329(19517653ns) {DropIncomingHandshake:442(12875ns) DropOutgoingHandshake:457(190ns) SwitchToState:179(91ns) ProcessPendingSessionEvents:495(11346863ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10070498ns max# 956670ns}}}
2025-11-27T18:20:20.531926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471730947870071:2049] [node 2] ICS09 handshake done sender: [4:7577471730947870052:2116] self: [4:7577471730947870059:0] peer: [2:7577471732901231818:0] socket: 445 qp: -1
2025-11-27T18:20:20.537097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471732499503658:2048] [node 1] ICS09 handshake done sender: [5:7577471732499503637:2119] self: [5:7577471732499503641:0] peer: [1:7577471732520413255:0] socket: 447 qp: -1
2025-11-27T18:20:20.540635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471732520413253:2109] [node 6] ICP22 created new session: [1:7577471732520413288:2109]
2025-11-27T18:20:20.543058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471732460474541:2050] [node 3] ICP22 created new session: [6:7577471732460474635:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.532977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471730947870071:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:20.540638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471732520413189:2049] [node 3] ICP22 created new session: [1:7577471732520413289:2049]
2025-11-27T18:20:20.540931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471732349944485:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(19980273ns) {HandleHandshakeStatus:329(19451027ns) {DropIncomingHandshake:442(1499ns) DropOutgoingHandshake:457(465ns) SwitchToState:179(387ns) ProcessPendingSessionEvents:495(12518900ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12117608ns max# 1522693ns}}}
2025-11-27T18:20:20.543392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471732460474635:2050] [node 3] ICS09 handshake done sender: [6:7577471732460474624:2118] self: [6:7577471732460474629:0] peer: [3:7577471732349944488:0] socket: 485 qp: -1
2025-11-27T18:20:20.533131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:20.541014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471732349944485:2108] [node 5] ICP18 handshake (actor [3:7577471732349944502:2118]) is too old
2025-11-27T18:20:20.537131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471732499503658:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:20.543747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471732460474635:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:20.534295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471730947870036:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(21247800ns) {HandleHandshakeStatus:329(21245417ns) {DropIncomingHandshake:442(198ns) DropOutgoingHandshake:457(591ns) SwitchToState:179(502ns) ProcessPendingSessionEvents:495(17330044ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17248789ns max# 1819723ns}}}
2025-11-27T18:20:20.536504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471730947870037:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17603409ns) {HandleHandshakeStatus:329(17601509ns) {DropIncomingHandshake:442(448ns) DropOutgoingHandshake:457(430ns) SwitchToState:179(665ns) ProcessPendingSessionEvents:495(13639925ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13543173ns max# 1421216ns}}}
2025-11-27T18:20:20.541076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471732349944522:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28992
2025-11-27T18:20:20.540743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471732520413252:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18458258ns) {HandleHandshakeStatus:329(18455537ns) {DropIncomingHandshake:442(14189ns) DropOutgoingHandshake:457(383ns) SwitchToState:179(289ns) ProcessPendingSessionEvents:495(10250962ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10175612ns max# 1186815ns}}}
2025-11-27T18:20:20.537162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:20.543939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:20.540840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471732520413288:2109] [node 6] ICS09 handshake done sender: [1:7577471732520413260:2111] self: [1:7577471732520413254:0] peer: [6:7577471732460474632:0] socket: 489 qp: -1
2025-11-27T18:20:20.545808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471732460474634:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39478
2025-11-27T18:20:20.546534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471732460474539:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30037) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471732460474626:2119] held: no
2025-11-27T18:20:20.546551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471732460474540:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:20.540915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471732520413289:2049] [node 3] ICS09 handshake done sender: [1:7577471732520413263:2114] self: [1:7577471732520413259:0] peer: [3:7577471732349944519:0] socket: 431 qp: -1
2025-11-27T18:20:20.540966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471732520413288:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:20.541045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:20.541111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471732520413289:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:20.541512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:20.541814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471732520413251:2107] [node 4] ICP18 handshake (actor [1:7577471732520413278:2117]) is too old
2025-11-27T18:20:20.547169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471732901231821:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:20.546532Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471732349944420:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30037) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471732349944515:2120] held: no
2025-11-27T18:20:20.547206Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471732901231821:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:20.547508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471732349944515:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:20.547700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471732460474626:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:20.548056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471732349944420:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.549432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471732460474539:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.549640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:20.549956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:20.549893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471732349944420:2048] [node 1] ICP22 created new session: [3:7577471732349944523:2048]
2025-11-27T18:20:20.549941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471732349944523:2048] [node 1] ICS09 handshake done sender: [3:7577471732349944515:2120] self: [3:7577471732349944519:0] peer: [1:7577471732520413259:0] socket: 492 qp: -1
2025-11-27T18:20:20.549997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471732901231821:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:20.550163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471732349944523:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:20.550017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471732460474539:2048] [node 1] ICP22 created new session: [6:7577471732460474638:2048]
2025-11-27T18:20:20.550069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471732460474638:2048] [node 1] ICS09 handshake done sender: [6:7577471732460474626:2119] self: [6:7577471732460474632:0] peer: [1:7577471732520413254:0] socket: 490 qp: -1
2025-11-27T18:20:20.550210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:20.550256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471732460474638:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:20.550286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471732901231811:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:20.550399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:20.550349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:20.550588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471732460474639:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39478
2025-11-27T18:20:20.550399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471732901231811:2109] [node 6] ICP22 created new session: [2:7577471732901231850:2109]
2025-11-27T18:20:20.550446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471732901231850:2109] [node 6] ICS09 handshake done sender: [2:7577471732901231821:2114] self: [2:7577471732901231816:0] peer: [6:7577471732460474637:0] socket: 459 qp: -1
2025-11-27T18:20:20.550480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471732901231850:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:20.550994Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471732460474540:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471732460474628:2121] held: no
2025-11-27T18:20:20.550506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:20.552427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471732460474628:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:20.552536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471732460474540:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:20.552685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:20.552729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471732460474540:2049] [node 2] ICP22 created new session: [6:7577471732460474641:2049]
2025-11-27T18:20:20.552757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471732460474641:2049] [node 2] ICS09 handshake done sender: [6:7577471732460474628:2121] self: [6:7577471732460474637:0] peer: [2:7577471732901231816:0] socket: 493 qp: -1
2025-11-27T18:20:20.552779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471732460474641:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:20.552817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:20.589151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:20.589354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471732901231845:2048] [node 1] ICS07 socket disconnect 453 reason# EndOfStream
2025-11-27T18:20:20.589381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471732901231845:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.589447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:20.589507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471732901231845:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:20.590336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:20.590545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471732349944523:2048] [node 1] ICS01 socket: 492 reason# EndOfStream
2025-11-27T18:20:20.590587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471732349944420:2048] [node 1] ICP30 unregister session Session# [3:7577471732349944523:2048] VirtualId# [3:7577471732349944519:0]
2025-11-27T18:20:20.590621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471732349944523:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.590691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:20.590733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:20.591489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471732901231746:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30037) Socket error# Connection reset by peer state# SendExRequestHeader processed# 0 remain# 8 incoming: [0:0:0] held: no
2025-11-27T18:20:20.591552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471732901231845:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:20.592362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471732901231746:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30037) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.592627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471732901231845:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.592655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471732901231746:2048] [node 1] ICP30 unregister session Session# [2:7577471732901231845:2048] VirtualId# [2:7577471732901231830:0]
2025-11-27T18:20:20.592743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:20.592780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471732901231746:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30037) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.592805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30037) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.592895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:20.593065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471730947870070:2048] [node 1] ICS07 socket disconnect 448 reason# EndOfStream
2025-11-27T18:20:20.593097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471730947870070:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.593172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:20.593242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471730947870070:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:20.593602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:20.593763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471732499503658:2048] [node 1] ICS01 socket: 447 reason# EndOfStream
2025-11-27T18:20:20.593794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471732499503556:2048] [node 1] ICP30 unregister session Session# [5:7577471732499503658:2048] VirtualId# [5:7577471732499503641:0]
2025-11-27T18:20:20.593815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471732499503658:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.593902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:20.593962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:20.593934Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471730947869973:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30037) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.594120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471730947870070:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.594174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471730947869973:2048] [node 1] ICP30 unregister session Session# [4:7577471730947870070:2048] VirtualId# [4:7577471730947870060:0]
2025-11-27T18:20:20.594292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:20.594328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471730947869973:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30037) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.594346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30037) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.594677Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:20.594798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471732460474638:2048] [node 1] ICS01 socket: 490 reason# EndOfStream
2025-11-27T18:20:20.594824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471732460474539:2048] [node 1] ICP30 unregister session Session# [6:7577471732460474638:2048] VirtualId# [6:7577471732460474632:0]
2025-11-27T18:20:20.594844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471732460474638:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.594903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:20.594946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:20.604219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471730947869973:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:20.651652Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:20.651807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471732349944516:2049] [node 2] ICS01 socket: 454 reason# EndOfStream
2025-11-27T18:20:20.651838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471732349944421:2049] [node 2] ICP30 unregister session Session# [3:7577471732349944516:2049] VirtualId# [3:7577471732349944506:0]
2025-11-27T18:20:20.651860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471732349944516:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.651919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:20.651956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:20.654242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:20.654384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471730947870071:2049] [node 2] ICS07 socket disconnect 445 reason# EndOfStream
2025-11-27T18:20:20.654401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471730947870071:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.654453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:20.654483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471730947870071:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:20.655301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471730947869974:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.655361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471730947870071:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.655394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471730947869974:2049] [node 2] ICP30 unregister session Session# [4:7577471730947870071:2049] VirtualId# [4:7577471730947870059:0]
2025-11-27T18:20:20.655496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:20.655450Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:20.655529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471730947869974:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.655547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.655598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471732499503656:2049] [node 2] ICS07 socket disconnect 444 reason# EndOfStream
2025-11-27T18:20:20.655622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471732499503656:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.655684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:20.655734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471732499503656:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:20.656366Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:20.656507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471732460474641:2049] [node 2] ICS01 socket: 493 reason# EndOfStream
2025-11-27T18:20:20.656497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471732499503557:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.656536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471732499503656:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.656537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471732460474540:2049] [node 2] ICP30 unregister session Session# [6:7577471732460474641:2049] VirtualId# [6:7577471732460474637:0]
2025-11-27T18:20:20.656573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471732460474641:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.656635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471732499503557:2049] [node 2] ICP30 unregister session Session# [5:7577471732499503656:2049] VirtualId# [5:7577471732499503643:0]
2025-11-27T18:20:20.656626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:20.656671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:20.656764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:20.656845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471732499503557:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.656867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.665652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471730947869974:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:20.666687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471732499503557:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:20.680186Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:20.680457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471730947870068:2050] [node 3] ICS01 socket: 429 reason# EndOfStream
2025-11-27T18:20:20.680508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471730947869975:2050] [node 3] ICP30 unregister session Session# [4:7577471730947870068:2050] VirtualId# [4:7577471730947870055:0]
2025-11-27T18:20:20.680555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471730947870068:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.680660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:20.680718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:20.681109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:20.681343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471732499503649:2050] [node 3] ICS01 socket: 436 reason# EndOfStream
2025-11-27T18:20:20.681415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471732499503558:2050] [node 3] ICP30 unregister session Session# [5:7577471732499503649:2050] VirtualId# [5:7577471732499503640:0]
2025-11-27T18:20:20.681452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471732499503649:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.681535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:20.681618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:20.682104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:20.682246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471732460474635:2050] [node 3] ICS01 socket: 485 reason# EndOfStream
2025-11-27T18:20:20.682282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471732460474541:2050] [node 3] ICP30 unregister session Session# [6:7577471732460474635:2050] VirtualId# [6:7577471732460474629:0]
2025-11-27T18:20:20.682299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471732460474635:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.682346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:20.682376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:20.762189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:20.762393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471732499503652:2108] [node 4] ICS07 socket disconnect 426 reason# EndOfStream
2025-11-27T18:20:20.762428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471732499503652:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.762510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:20.762556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471732499503652:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:20.763156Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:20.763350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471732460474623:2108] [node 4] ICS07 socket disconnect 425 reason# EndOfStream
2025-11-27T18:20:20.763352Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471732499503619:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29383) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.763379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471732460474623:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.763455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471732499503652:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.763477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:20.763521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471732460474623:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:20.763508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471732499503619:2108] [node 4] ICP30 unregister session Session# [5:7577471732499503652:2108] VirtualId# [5:7577471732499503639:0]
2025-11-27T18:20:20.763566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:20.763610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471732499503619:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.763625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.764348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471732460474602:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29383) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.764456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471732460474623:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.764502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471732460474602:2108] [node 4] ICP30 unregister session Session# [6:7577471732460474623:2108] VirtualId# [6:7577471732460474618:0]
2025-11-27T18:20:20.764621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:20.764655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471732460474602:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.764670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.774226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471732460474602:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:20.809883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:20.810055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471732460474630:2106] [node 5] ICS07 socket disconnect 456 reason# EndOfStream
2025-11-27T18:20:20.810077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471732460474630:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:20.810137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:20.810168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471732460474630:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:20.810895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471732460474600:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22049) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:20.810951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471732460474630:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:20.810983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471732460474600:2106] [node 5] ICP30 unregister session Session# [6:7577471732460474630:2106] VirtualId# [6:7577471732460474621:0]
2025-11-27T18:20:20.811074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:20.811111Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471732460474600:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22049) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.811121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22049) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:20.821386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471732460474600:2106] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:20.924568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471730596511742:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:20.924610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471730596511743:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:20.942301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471731741590555:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:20.942355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471731741590556:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:20.960675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471730687512897:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:20.960731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471730687512898:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:20.979660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471733570053956:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:20.979722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471733570053957:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:20.979766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471733570053958:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:21.000572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471733385642709:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:21.000672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471733385642710:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:21.000722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471733385642711:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:21.018249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471736520647729:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:21.018284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471736520647730:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:21.018316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471736520647731:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:21.068496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471736520647790:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:21.135281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471734982480258:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:21.136623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471734891479101:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:21.136854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471737680610068:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:21.137294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471737865021333:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:21.137339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471734982480259:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:21.137363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471736036557914:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:21.137485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471734891479102:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:21.137854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471737680610069:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:21.137831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471736036557915:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:21.138365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471734982480260:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:21.138443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471734891479103:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:21.138994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471736036557916:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:21.139805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471736520647790:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(70933491ns) {Configure:104(70928947ns) {ProcessConfigured:135(142284ns) {StartInitialHandshake:157(139131ns) {DropHandshakes:470(2135ns) {DropIncomingHandshake:442(522ns) DropOutgoingHandshake:457(291ns)} GenerateSessionVirtualId:988(999ns) PrepareNewSessionHandshake:477(5688ns) {SwitchToState:179(955ns)}}}}}
2025-11-27T18:20:21.140983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471736520647795:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:21.141489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471737865021336:2118] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.150946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471734891479114:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10293
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.151942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471734982480271:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14446
2025-11-27T18:20:21.151924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471737680610080:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34900
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.152114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737865021345:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57908
2025-11-27T18:20:21.152556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737865021346:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57908
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.152677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471734982480275:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14446
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.154792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471734891479118:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10293
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.154853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471734982480276:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14446
2025-11-27T18:20:21.154840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737865021350:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57908
2025-11-27T18:20:21.155050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471734982480277:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14446
2025-11-27T18:20:21.162461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471737680610080:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.163102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471734891479119:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10293
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.163264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471734891479120:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10293
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.163947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471734891479121:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10293
2025-11-27T18:20:21.164189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471736036557930:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45662
2025-11-27T18:20:21.164334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471734891479114:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.164698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471736036557931:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45662
2025-11-27T18:20:21.164732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471734891479118:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.164830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471736036557932:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45662
2025-11-27T18:20:21.164863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471734891479120:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.164785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471734982480271:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.164904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471734982480276:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.164948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471736036557933:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45662
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.165061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471730687512898:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.165066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471736036557934:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45662
2025-11-27T18:20:21.165191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471734891479119:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.165360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471736036557931:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.165488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471736036557933:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.165541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737865021345:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.165570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471734891479121:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.165597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471736036557920:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.165620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471736036557920:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.165649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471734982480275:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.165746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471734982480277:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.165841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737865021350:2135] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.165877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471730687512897:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.166024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737865021346:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.166227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471733570053957:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.166184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471734891479107:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.166210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471734891479107:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.166334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471733570053958:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.166410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471733570053956:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.166490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471736036557930:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.166725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471736036557934:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.166820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471734982480266:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.166846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471736036557932:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.166848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471734982480266:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.166958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471734891479109:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.166982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471736036557922:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.166992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471734891479109:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.167008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471736036557922:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.168802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471734982480266:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:21.168898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471731741590555:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.169139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471734891479107:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.169317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737865021354:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57908
2025-11-27T18:20:21.169402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471736036557920:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:21.169411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471734982480258:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.169522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:21.169570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471734982480258:2107] [node 4] ICP22 created new session: [3:7577471734982480285:2107]
2025-11-27T18:20:21.169643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471734982480285:2107] [node 4] ICS09 handshake done sender: [3:7577471734982480266:2114] self: [3:7577471734982480265:0] peer: [4:7577471737865021352:0] socket: 448 qp: -1
2025-11-27T18:20:21.169683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471734982480285:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:21.169753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:21.169787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471734891479109:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:21.171884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471731741590556:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.172113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471734891479105:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.172124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:21.172131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737865021355:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57908
2025-11-27T18:20:21.172199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471731741590556:2049] [node 3] ICP22 created new session: [2:7577471736036557942:2049]
2025-11-27T18:20:21.172145Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471734891479105:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.172308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471736036557942:2049] [node 3] ICS09 handshake done sender: [2:7577471736036557920:2111] self: [2:7577471736036557919:0] peer: [3:7577471734982480278:0] socket: 427 qp: -1
2025-11-27T18:20:21.172367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471736036557942:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.172437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:21.172728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471734982480283:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14446
2025-11-27T18:20:21.172803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471730596511743:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.172876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:21.172923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471730596511743:2049] [node 3] ICP22 created new session: [1:7577471734891479129:2049]
2025-11-27T18:20:21.172962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471734891479129:2049] [node 3] ICS09 handshake done sender: [1:7577471734891479107:2111] self: [1:7577471734891479106:0] peer: [3:7577471734982480280:0] socket: 434 qp: -1
2025-11-27T18:20:21.172990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471734891479129:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:21.173071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.175054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471734982480284:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14446
2025-11-27T18:20:21.175268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471737865021350:2135] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:21.175815Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471733570053958:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28216) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471737865021350:2135] held: no
2025-11-27T18:20:21.176131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471734982480271:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:21.176443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471734982480275:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:21.176525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471733570053958:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.176659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471734891479101:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.176857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471730687512898:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.176874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:21.177019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:21.176996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471737680610085:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34900
2025-11-27T18:20:21.177049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471734891479101:2107] [node 4] ICP22 created new session: [1:7577471734891479131:2107]
2025-11-27T18:20:21.177081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:21.177123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471734891479131:2107] [node 4] ICS09 handshake done sender: [1:7577471734891479109:2112] self: [1:7577471734891479108:0] peer: [4:7577471737865021353:0] socket: 435 qp: -1
2025-11-27T18:20:21.177158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471734891479131:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:21.177160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471730687512898:2049] [node 2] ICP22 created new session: [3:7577471734982480288:2049]
2025-11-27T18:20:21.177189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:21.177223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471734982480288:2049] [node 2] ICS09 handshake done sender: [3:7577471734982480271:2117] self: [3:7577471734982480278:0] peer: [2:7577471736036557919:0] socket: 431 qp: -1
2025-11-27T18:20:21.177285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471734982480288:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:21.177315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:21.177098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471733570053958:2050] [node 3] ICP22 created new session: [4:7577471737865021357:2050]
2025-11-27T18:20:21.177146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471737865021357:2050] [node 3] ICS09 handshake done sender: [4:7577471737865021350:2135] self: [4:7577471737865021352:0] peer: [3:7577471734982480265:0] socket: 463 qp: -1
2025-11-27T18:20:21.177179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471737865021357:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:21.177338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471736036557922:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:21.177208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:21.178517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471736036557914:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.178639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:21.178689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471736036557914:2107] [node 4] ICP22 created new session: [2:7577471736036557944:2107]
2025-11-27T18:20:21.178741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471736036557944:2107] [node 4] ICS09 handshake done sender: [2:7577471736036557922:2112] self: [2:7577471736036557921:0] peer: [4:7577471737865021351:0] socket: 428 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.178800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471736036557944:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:21.178835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:21.178912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737865021356:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57908
2025-11-27T18:20:21.179856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471737865021346:2134] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:21.179913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471730687512897:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.179962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:21.180034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471730687512897:2048] [node 1] ICP22 created new session: [3:7577471734982480290:2048]
2025-11-27T18:20:21.180216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471734982480290:2048] [node 1] ICS09 handshake done sender: [3:7577471734982480275:2118] self: [3:7577471734982480280:0] peer: [1:7577471734891479106:0] socket: 441 qp: -1
2025-11-27T18:20:21.180282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471734982480290:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:21.180317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:21.181007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471733570053956:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.181106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471734891479105:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:21.181166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.181231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471733570053956:2048] [node 1] ICP22 created new session: [4:7577471737865021359:2048]
2025-11-27T18:20:21.181300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471737865021359:2048] [node 1] ICS09 handshake done sender: [4:7577471737865021346:2134] self: [4:7577471737865021353:0] peer: [1:7577471734891479108:0] socket: 437 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.181339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471737865021359:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:21.181523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471737680610086:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34900
2025-11-27T18:20:21.181408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:21.181736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471730596511742:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.181729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471736520647807:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53797
2025-11-27T18:20:21.181795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:21.181840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471730596511742:2048] [node 2] ICP22 created new session: [1:7577471734891479136:2048]
2025-11-27T18:20:21.181878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471734891479136:2048] [node 2] ICS09 handshake done sender: [1:7577471734891479105:2110] self: [1:7577471734891479104:0] peer: [2:7577471736036557939:0] socket: 432 qp: -1
2025-11-27T18:20:21.181908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471734891479136:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:21.181941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.182273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471737680610087:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34900
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.182778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471736036557947:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45662
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.183751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471736520647808:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53797
2025-11-27T18:20:21.185857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471736036557930:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:21.185859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471730687512898:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:21.186389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471730687512897:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:21.187265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471737865021345:2133] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:21.188272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471733570053956:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:21.188347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471731741590555:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.188437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:21.188490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471731741590555:2048] [node 1] ICP22 created new session: [2:7577471736036557949:2048]
2025-11-27T18:20:21.188452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471737680610087:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.188548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471733570053957:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24242) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471737865021345:2133] held: no
2025-11-27T18:20:21.188544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471736036557949:2048] [node 1] ICS09 handshake done sender: [2:7577471736036557930:2115] self: [2:7577471736036557939:0] peer: [1:7577471734891479104:0] socket: 454 qp: -1
2025-11-27T18:20:21.188584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471736036557949:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:21.188589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471737680610085:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.188653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:21.188776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471733385642710:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.188856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471733385642711:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.188975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471736520647809:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53797
2025-11-27T18:20:21.189095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471733570053957:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.189152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:21.189183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471733570053957:2049] [node 2] ICP22 created new session: [4:7577471737865021362:2049]
2025-11-27T18:20:21.189205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471737865021362:2049] [node 2] ICS09 handshake done sender: [4:7577471737865021345:2133] self: [4:7577471737865021351:0] peer: [2:7577471736036557921:0] socket: 436 qp: -1
2025-11-27T18:20:21.189227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471737865021362:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:21.189248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:21.190226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471734982480268:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.190256Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471734982480268:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.191079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471736520647809:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.191704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471734982480268:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:21.191734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471736520647731:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.192159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471734982480270:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.192185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471734982480270:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.192187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471736520647808:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.192176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471737680610090:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34900
2025-11-27T18:20:21.192244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471737680610091:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34900
2025-11-27T18:20:21.192333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471736520647730:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.192300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471736036557924:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.192336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471736036557924:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.193522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471734982480259:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.193574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:21.193620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471734982480259:2109] [node 5] ICP22 created new session: [3:7577471734982480294:2109]
2025-11-27T18:20:21.193668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471734982480294:2109] [node 5] ICS09 handshake done sender: [3:7577471734982480268:2115] self: [3:7577471734982480267:0] peer: [5:7577471737680610089:0] socket: 456 qp: -1
2025-11-27T18:20:21.193705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471734982480294:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:21.193737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:21.193729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471737680610085:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.194028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471733385642711:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.194064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471736520647812:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53797
2025-11-27T18:20:21.194109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:21.194157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471733385642711:2050] [node 3] ICP22 created new session: [5:7577471737680610092:2050]
2025-11-27T18:20:21.194202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471737680610092:2050] [node 3] ICS09 handshake done sender: [5:7577471737680610085:2116] self: [5:7577471737680610089:0] peer: [3:7577471734982480267:0] socket: 462 qp: -1
2025-11-27T18:20:21.194234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471737680610092:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:21.194204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471731741590555:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:21.194265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:21.194664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471736036557926:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.194678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471736520647807:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.194694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471736036557926:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.195207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471736520647729:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.195431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471734891479113:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.195461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471734891479113:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.196059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471736036557924:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:21.196550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471736036557926:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:21.196372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471734891479113:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:21.197038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471734891479103:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.197098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471736036557915:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.197140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:21.197207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471734891479103:2109] [node 6] ICP22 created new session: [1:7577471734891479139:2109]
2025-11-27T18:20:21.197217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.197297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471736036557915:2108] [node 5] ICP22 created new session: [2:7577471736036557953:2108]
2025-11-27T18:20:21.197345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471736036557953:2108] [node 5] ICS09 handshake done sender: [2:7577471736036557924:2113] self: [2:7577471736036557923:0] peer: [5:7577471737680610088:0] socket: 479 qp: -1
2025-11-27T18:20:21.197329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471734891479139:2109] [node 6] ICS09 handshake done sender: [1:7577471734891479113:2114] self: [1:7577471734891479112:0] peer: [6:7577471736520647813:0] socket: 460 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.197379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471736036557953:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:21.197379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471734891479139:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:21.197359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471736520647814:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53797
2025-11-27T18:20:21.197419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:21.197430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471736520647815:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53797
2025-11-27T18:20:21.197409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:21.197669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471734982480270:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:21.198701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471736520647812:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.198880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471737680610086:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.198905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471734982480260:2110] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:21.198983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:21.199082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471734982480260:2110] [node 6] ICP22 created new session: [3:7577471734982480297:2110]
2025-11-27T18:20:21.199130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471734982480297:2110] [node 6] ICS09 handshake done sender: [3:7577471734982480270:2116] self: [3:7577471734982480269:0] peer: [6:7577471736520647810:0] socket: 429 qp: -1
2025-11-27T18:20:21.199160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471734982480297:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:21.199212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471736520647816:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53797
2025-11-27T18:20:21.199230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:21.199255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471736036557916:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.199296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:21.199333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471736036557916:2109] [node 6] ICP22 created new session: [2:7577471736036557955:2109]
2025-11-27T18:20:21.199365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471736036557955:2109] [node 6] ICS09 handshake done sender: [2:7577471736036557926:2114] self: [2:7577471736036557925:0] peer: [6:7577471736520647811:0] socket: 482 qp: -1
2025-11-27T18:20:21.199386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471736036557955:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:21.199406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:21.199691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471736520647808:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:21.199866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471737680610090:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.200066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471733385642709:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.200165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471737680610094:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34900
2025-11-27T18:20:21.200842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471737680610068:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.200945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471734891479111:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.200975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471734891479111:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.201045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471736520647809:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:21.201673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471734891479111:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:21.201411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737865021365:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57908
2025-11-27T18:20:21.202135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471736520647730:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.202203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:21.202301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471736520647730:2049] [node 2] ICP22 created new session: [6:7577471736520647817:2049]
2025-11-27T18:20:21.202421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471736520647817:2049] [node 2] ICS09 handshake done sender: [6:7577471736520647808:2116] self: [6:7577471736520647811:0] peer: [2:7577471736036557925:0] socket: 483 qp: -1
2025-11-27T18:20:21.202466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471736520647817:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:21.202509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:21.202466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471737680610087:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:21.202486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471734891479102:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.202573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:21.202627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471734891479102:2108] [node 5] ICP22 created new session: [1:7577471734891479142:2108]
2025-11-27T18:20:21.202670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471734891479142:2108] [node 5] ICS09 handshake done sender: [1:7577471734891479111:2113] self: [1:7577471734891479110:0] peer: [5:7577471737680610096:0] socket: 458 qp: -1
2025-11-27T18:20:21.202697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471734891479142:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:21.202726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:21.202863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471733385642710:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24242) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471737680610087:2118] held: no
2025-11-27T18:20:21.203021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737865021342:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.203052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471737865021342:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.203469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471733385642710:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.203535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:21.203575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471733385642710:2049] [node 2] ICP22 created new session: [5:7577471737680610098:2049]
2025-11-27T18:20:21.203625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471737680610098:2049] [node 2] ICS09 handshake done sender: [5:7577471737680610087:2118] self: [5:7577471737680610088:0] peer: [2:7577471736036557923:0] socket: 480 qp: -1
2025-11-27T18:20:21.203664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471737680610098:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:21.203691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:21.204089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471737865021342:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:21.204304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471736520647807:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:21.204525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471737865021333:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.204594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:21.204608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471736520647729:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471736520647807:2115] held: no
2025-11-27T18:20:21.204641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471737865021333:2108] [node 5] ICP22 created new session: [4:7577471737865021367:2108]
2025-11-27T18:20:21.204686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471737865021367:2108] [node 5] ICS09 handshake done sender: [4:7577471737865021342:2131] self: [4:7577471737865021341:0] peer: [5:7577471737680610097:0] socket: 444 qp: -1
2025-11-27T18:20:21.204717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471737865021367:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:21.204752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:21.205144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471736520647729:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.205177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:21.205214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471736520647729:2048] [node 1] ICP22 created new session: [6:7577471736520647819:2048]
2025-11-27T18:20:21.205270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471736520647819:2048] [node 1] ICS09 handshake done sender: [6:7577471736520647807:2115] self: [6:7577471736520647813:0] peer: [1:7577471734891479112:0] socket: 478 qp: -1
2025-11-27T18:20:21.205302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471736520647819:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:21.205331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:21.206145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471736520647795:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.206998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471736520647731:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.207037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:21.207076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471736520647731:2050] [node 3] ICP22 created new session: [6:7577471736520647822:2050]
2025-11-27T18:20:21.207038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471733385642709:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471737680610086:2117] held: no
2025-11-27T18:20:21.207112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471736520647822:2050] [node 3] ICS09 handshake done sender: [6:7577471736520647809:2117] self: [6:7577471736520647810:0] peer: [3:7577471734982480269:0] socket: 442 qp: -1
2025-11-27T18:20:21.207143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471736520647822:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:21.207195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.209202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471737680610100:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34900
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.209513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471737680610101:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34900
2025-11-27T18:20:21.210023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471737680610086:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:21.210247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471737680610090:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:21.210407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471733385642709:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.210425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471736520647730:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:21.210466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:21.210494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471733385642709:2048] [node 1] ICP22 created new session: [5:7577471737680610103:2048]
2025-11-27T18:20:21.210518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471737680610103:2048] [node 1] ICS09 handshake done sender: [5:7577471737680610086:2117] self: [5:7577471737680610096:0] peer: [1:7577471734891479110:0] socket: 467 qp: -1
2025-11-27T18:20:21.210539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471737680610103:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:21.210558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:21.211106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471737680610068:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.211164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:21.211209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471737680610068:2108] [node 4] ICP22 created new session: [5:7577471737680610105:2108]
2025-11-27T18:20:21.211235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471737680610105:2108] [node 4] ICS09 handshake done sender: [5:7577471737680610090:2119] self: [5:7577471737680610097:0] peer: [4:7577471737865021341:0] socket: 451 qp: -1
2025-11-27T18:20:21.211255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471737680610105:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:21.211284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.211624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471736520647824:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53797
2025-11-27T18:20:21.212898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737865021365:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.213047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737865021344:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.213078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471737865021344:2132] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.215005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471737865021333:2108] [node 5] ICP18 handshake (actor [4:7577471737865021365:2139]) is too old
2025-11-27T18:20:21.215101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737865021369:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57908
2025-11-27T18:20:21.215232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471737865021344:2132] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.215449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471736520647827:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53797
2025-11-27T18:20:21.215861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471737865021336:2118] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.215926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:21.215973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471737865021336:2118] [node 6] ICP22 created new session: [4:7577471737865021371:2118]
2025-11-27T18:20:21.215973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471736520647824:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.216016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471737865021371:2118] [node 6] ICS09 handshake done sender: [4:7577471737865021344:2132] self: [4:7577471737865021343:0] peer: [6:7577471736520647821:0] socket: 484 qp: -1
2025-11-27T18:20:21.216055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471737865021371:2118] [node 6] ICS10 traffic start
2025-11-27T18:20:21.216061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471736520647812:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:21.216091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:21.216333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471736520647795:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.216366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:21.216394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471736520647795:2109] [node 4] ICP22 created new session: [6:7577471736520647828:2109]
2025-11-27T18:20:21.216435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471736520647828:2109] [node 4] ICS09 handshake done sender: [6:7577471736520647812:2118] self: [6:7577471736520647821:0] peer: [4:7577471737865021343:0] socket: 485 qp: -1
2025-11-27T18:20:21.216462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471736520647828:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:21.216500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:21.217094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737865021369:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.217226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471737865021336:2118] [node 6] ICP18 handshake (actor [4:7577471737865021369:2140]) is too old
2025-11-27T18:20:21.217998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471736520647790:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.219436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471737680610079:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.219463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471737680610079:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.220050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471737680610079:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.220272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471737680610069:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.220334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:21.220316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471736520647831:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53797
2025-11-27T18:20:21.220372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471737680610069:2109] [node 6] ICP22 created new session: [5:7577471737680610108:2109]
2025-11-27T18:20:21.220405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471737680610108:2109] [node 6] ICS09 handshake done sender: [5:7577471737680610079:2114] self: [5:7577471737680610078:0] peer: [6:7577471736520647830:0] socket: 440 qp: -1
2025-11-27T18:20:21.220435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471737680610108:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:21.220465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:21.220730Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471736520647824:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:21.220921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471736520647790:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.220969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:21.221036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471736520647790:2107] [node 5] ICP22 created new session: [6:7577471736520647832:2107]
2025-11-27T18:20:21.221078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471736520647832:2107] [node 5] ICS09 handshake done sender: [6:7577471736520647824:2122] self: [6:7577471736520647830:0] peer: [5:7577471737680610078:0] socket: 446 qp: -1
2025-11-27T18:20:21.221108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471736520647832:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:21.221137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:21.239760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:21.239992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471736036557949:2048] [node 1] ICS07 socket disconnect 454 reason# EndOfStream
2025-11-27T18:20:21.240031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471736036557949:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.240116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:21.240174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471736036557949:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:21.240975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:21.241165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471734982480290:2048] [node 1] ICS07 socket disconnect 441 reason# EndOfStream
2025-11-27T18:20:21.241195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471734982480290:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.241280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:21.241349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471734982480290:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:21.242985Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471731741590555:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:21.243282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471736036557949:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:21.243524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:21.243720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471737865021359:2048] [node 1] ICS07 socket disconnect 437 reason# EndOfStream
2025-11-27T18:20:21.243766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471737865021359:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.243848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:21.243888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471737865021359:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:21.244094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471730687512897:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:21.244125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471734982480290:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:21.244206Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471731741590555:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.244282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471736036557949:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.244336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471731741590555:2048] [node 1] ICP30 unregister session Session# [2:7577471736036557949:2048] VirtualId# [2:7577471736036557939:0]
2025-11-27T18:20:21.244391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:21.244424Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471731741590555:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.244442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.244620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471733570053956:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.244662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471737865021359:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.244708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471733570053956:2048] [node 1] ICP30 unregister session Session# [4:7577471737865021359:2048] VirtualId# [4:7577471737865021353:0]
2025-11-27T18:20:21.244728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:21.244747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:21.244767Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471733570053956:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.244775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.244998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471737680610103:2048] [node 1] ICS01 socket: 467 reason# EndOfStream
2025-11-27T18:20:21.244990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471730687512897:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.245061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471733385642709:2048] [node 1] ICP30 unregister session Session# [5:7577471737680610103:2048] VirtualId# [5:7577471737680610096:0]
2025-11-27T18:20:21.245065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471734982480290:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.245091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471737680610103:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.245097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471730687512897:2048] [node 1] ICP30 unregister session Session# [3:7577471734982480290:2048] VirtualId# [3:7577471734982480280:0]
2025-11-27T18:20:21.245171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:21.245184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:21.245215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471730687512897:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.245221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:21.245227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.245758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:21.245950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471736520647819:2048] [node 1] ICS01 socket: 478 reason# EndOfStream
2025-11-27T18:20:21.245996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471736520647729:2048] [node 1] ICP30 unregister session Session# [6:7577471736520647819:2048] VirtualId# [6:7577471736520647813:0]
2025-11-27T18:20:21.246028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471736520647819:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.246104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:21.246167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:21.254440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471733570053956:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:21.254440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471731741590555:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:21.255456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471730687512897:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:21.284308Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:21.284579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471734982480288:2049] [node 2] ICS07 socket disconnect 431 reason# EndOfStream
2025-11-27T18:20:21.284605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471734982480288:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.284668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:21.284705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471734982480288:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:21.286644Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:21.286851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471737865021362:2049] [node 2] ICS01 socket: 436 reason# EndOfStream
2025-11-27T18:20:21.286896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471733570053957:2049] [node 2] ICP30 unregister session Session# [4:7577471737865021362:2049] VirtualId# [4:7577471737865021351:0]
2025-11-27T18:20:21.286932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471737865021362:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.287006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:21.287039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:21.287377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471730687512898:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24242) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:21.287404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471734982480288:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:21.287695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:21.287876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471737680610098:2049] [node 2] ICS01 socket: 480 reason# EndOfStream
2025-11-27T18:20:21.287975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471733385642710:2049] [node 2] ICP30 unregister session Session# [5:7577471737680610098:2049] VirtualId# [5:7577471737680610088:0]
2025-11-27T18:20:21.288008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471737680610098:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.288026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471730687512898:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24242) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.288081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:21.288085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471734982480288:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.288118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471730687512898:2049] [node 2] ICP30 unregister session Session# [3:7577471734982480288:2049] VirtualId# [3:7577471734982480278:0]
2025-11-27T18:20:21.288125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:21.288217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:21.288267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471730687512898:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24242) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.288282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24242) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.288895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:21.289082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471736520647817:2049] [node 2] ICS07 socket disconnect 483 reason# EndOfStream
2025-11-27T18:20:21.289131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471736520647817:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.289239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:21.289323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471736520647817:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:21.289949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471736520647730:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24242) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.289982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471736520647817:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.290011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471736520647730:2049] [node 2] ICP30 unregister session Session# [6:7577471736520647817:2049] VirtualId# [6:7577471736520647811:0]
2025-11-27T18:20:21.290109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:21.290144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471736520647730:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24242) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.290156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24242) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.299512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471736520647730:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:21.312357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:21.312621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471737865021357:2050] [node 3] ICS01 socket: 463 reason# EndOfStream
2025-11-27T18:20:21.312675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471733570053958:2050] [node 3] ICP30 unregister session Session# [4:7577471737865021357:2050] VirtualId# [4:7577471737865021352:0]
2025-11-27T18:20:21.312722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471737865021357:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.312800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:21.312855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:21.313592Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:21.313804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471737680610092:2050] [node 3] ICS07 socket disconnect 462 reason# EndOfStream
2025-11-27T18:20:21.313837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471737680610092:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.313911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:21.313960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471737680610092:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:21.314497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:21.314724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471736520647822:2050] [node 3] ICS07 socket disconnect 442 reason# EndOfStream
2025-11-27T18:20:21.314774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471736520647822:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.314807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471733385642711:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28216) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.314843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:21.314888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471736520647822:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:21.314860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471737680610092:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.314910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471733385642711:2050] [node 3] ICP30 unregister session Session# [5:7577471737680610092:2050] VirtualId# [5:7577471737680610089:0]
2025-11-27T18:20:21.315030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:21.315071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471733385642711:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.315087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.315644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471736520647731:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28216) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.315691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471736520647822:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.315721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471736520647731:2050] [node 3] ICP30 unregister session Session# [6:7577471736520647822:2050] VirtualId# [6:7577471736520647810:0]
2025-11-27T18:20:21.315812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:21.315847Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471736520647731:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.315860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28216) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.325135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471733385642711:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:21.326147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471736520647731:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:21.357132Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:21.357377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471737680610105:2108] [node 4] ICS07 socket disconnect 451 reason# EndOfStream
2025-11-27T18:20:21.357416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471737680610105:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.357504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:21.357597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471737680610105:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:21.358140Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:21.358348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471737680610068:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13538) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.358374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471736520647828:2109] [node 4] ICS07 socket disconnect 485 reason# EndOfStream
2025-11-27T18:20:21.358387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471737680610105:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.358405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471736520647828:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.358414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471737680610068:2108] [node 4] ICP30 unregister session Session# [5:7577471737680610105:2108] VirtualId# [5:7577471737680610097:0]
2025-11-27T18:20:21.358448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:21.358469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471737680610068:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13538) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.358473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:21.358479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13538) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.358517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471736520647828:2109] [node 4] ICS15 start handshake
2025-11-27T18:20:21.359257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471736520647795:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13538) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.359296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471736520647828:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.359341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471736520647795:2109] [node 4] ICP30 unregister session Session# [6:7577471736520647828:2109] VirtualId# [6:7577471736520647821:0]
2025-11-27T18:20:21.359425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:21.359449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471736520647795:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13538) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.359458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13538) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.369134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471736520647795:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:20:21.384622Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:21.384842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471736520647832:2107] [node 5] ICS07 socket disconnect 446 reason# EndOfStream
2025-11-27T18:20:21.384887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471736520647832:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.384954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:21.384998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471736520647832:2107] [node 5] ICS15 start handshake
2025-11-27T18:20:21.385780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471736520647790:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21640) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.385831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471736520647832:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.385863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471736520647790:2107] [node 5] ICP30 unregister session Session# [6:7577471736520647832:2107] VirtualId# [6:7577471736520647830:0]
2025-11-27T18:20:21.385965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:21.386002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471736520647790:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.386019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.395777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471736520647790:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:21.457846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471733835535167:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:21.457917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471733835535166:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:21.475804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471733715208850:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:21.475804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471733715208849:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:21.495393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471735659465828:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:21.495435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471735659465829:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:21.516768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471737744996414:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:21.516796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471737744996413:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:21.516837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471737744996415:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:21.539020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471734756175861:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:21.539017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471734756175860:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:21.539104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471734756175862:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:21.561415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471737702402112:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:21.561475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471737702402114:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:21.561479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471737702402113:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:21.633468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471737702402174:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:21.673308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471733835535230:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:21.683938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471737744996476:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:21.687409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471733715208915:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:21.686765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471734756175932:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:21.686804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471735659465891:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:21.685763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471737702402175:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:21.687559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471735659465892:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:21.687578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471733715208914:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:21.688003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471733835535231:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:21.688092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471737744996477:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:21.688150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471733835535232:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:21.688158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471733715208916:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:21.688164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471735659465893:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:21.688275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471734756175933:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.700132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471733835535246:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52038
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.700692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471733835535247:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52038
2025-11-27T18:20:21.701091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737744996489:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57643
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.706920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471735659465902:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65324
2025-11-27T18:20:21.706966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471733835535248:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52038
2025-11-27T18:20:21.708543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471733835535247:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.707228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471735659465903:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65324
2025-11-27T18:20:21.708520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471735659465904:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65324
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.710483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471735659465904:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.710558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471734756175947:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26187
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.711121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471735659465903:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.711124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471733835535246:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.711456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737744996489:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.711549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471735659465828:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.711557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471733835535249:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52038
2025-11-27T18:20:21.711587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737744996492:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57643
2025-11-27T18:20:21.711637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471737744996413:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.711658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471733835535248:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.711729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471735659465902:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.711706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471734756175947:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.712074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471733835535238:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.712127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471733835535238:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.712953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471733835535236:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.713398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471733715208925:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33395
2025-11-27T18:20:21.712985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471733835535236:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.714258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471733715208926:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33395
2025-11-27T18:20:21.713527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471733835535238:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:21.714453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471733715208927:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33395
2025-11-27T18:20:21.714977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471733715208928:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33395
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:21.716312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471733835535230:2107] [node 4] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:21.716310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471733715208925:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.715958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737744996494:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57643
2025-11-27T18:20:21.716369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471733715208926:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.716426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:21.716484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471733835535230:2107] [node 4] ICP22 created new session: [1:7577471733835535255:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.716954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471733715208931:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33395
2025-11-27T18:20:21.717274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471733835535255:2107] [node 4] ICS09 handshake done sender: [1:7577471733835535238:2113] self: [1:7577471733835535237:0] peer: [4:7577471737744996493:0] socket: 440 qp: -1
2025-11-27T18:20:21.717681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471734756175949:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26187
2025-11-27T18:20:21.717722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471733835535255:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:21.716969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471733715208927:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.718106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737744996492:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.717770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:21.717686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471733715208928:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.718110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471733835535249:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.718561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471737744996489:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:21.718624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471733715208849:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.718399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471733835535236:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:21.718700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471735659465914:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65324
2025-11-27T18:20:21.718841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471733715208931:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.719048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471735659465913:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65324
2025-11-27T18:20:21.720882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471733835535167:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.721053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:21.721051Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471737744996413:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471737744996489:2114] held: no
2025-11-27T18:20:21.721385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471735659465904:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:21.721117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471733835535167:2049] [node 3] ICP22 created new session: [1:7577471733835535259:2049]
2025-11-27T18:20:21.721381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471737744996415:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.721710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471735659465828:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.721387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471733835535259:2049] [node 3] ICS09 handshake done sender: [1:7577471733835535236:2112] self: [1:7577471733835535235:0] peer: [3:7577471735659465910:0] socket: 439 qp: -1
2025-11-27T18:20:21.721963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:21.721501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471733835535259:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:21.722019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471735659465828:2048] [node 1] ICP22 created new session: [3:7577471735659465915:2048]
2025-11-27T18:20:21.721962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:21.722182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471735659465915:2048] [node 1] ICS09 handshake done sender: [3:7577471735659465904:2116] self: [3:7577471735659465910:0] peer: [1:7577471733835535235:0] socket: 447 qp: -1
2025-11-27T18:20:21.722221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471735659465915:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:21.723783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471737744996413:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.723923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:21.724182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471734756175949:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.722548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:21.724439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471734756175861:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.723968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471737744996413:2048] [node 1] ICP22 created new session: [4:7577471737744996497:2048]
2025-11-27T18:20:21.724006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471737744996497:2048] [node 1] ICS09 handshake done sender: [4:7577471737744996489:2114] self: [4:7577471737744996493:0] peer: [1:7577471733835535237:0] socket: 444 qp: -1
2025-11-27T18:20:21.724056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471737744996497:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:21.729742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471733715208922:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:21.724085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
::ffff:127.1.0.0
2025-11-27T18:20:21.729790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471733715208922:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.729921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471734756175950:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26187
2025-11-27T18:20:21.729973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471735659465899:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.730008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471735659465899:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.730029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471733835535234:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.730071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471733835535234:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.731312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471733835535256:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52038
2025-11-27T18:20:21.731580Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471733715208922:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:21.731823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471733715208914:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.731814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471734756175950:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.731977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:21.732051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471733715208914:2108] [node 5] ICP22 created new session: [2:7577471733715208941:2108]
2025-11-27T18:20:21.732490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471733715208941:2108] [node 5] ICS09 handshake done sender: [2:7577471733715208922:2114] self: [2:7577471733715208921:0] peer: [5:7577471734756175951:0] socket: 470 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.732534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471733715208941:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.732586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:21.732528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471733835535167:2049] [node 3] ICP18 handshake (actor [1:7577471733835535249:2119]) is too old
2025-11-27T18:20:21.732624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471734756175953:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26187
2025-11-27T18:20:21.732662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471734756175952:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26187
2025-11-27T18:20:21.732723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471734756175932:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.732748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471735659465899:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:21.733094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471733835535256:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.733752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471734756175861:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29570) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471734756175949:2125] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.733630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737744996485:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.733669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471737744996485:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.735563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737744996499:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57643
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.735805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471737702402191:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19027
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.736182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471737744996492:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:21.735609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471735659465891:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.735774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:21.736275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471734756175949:2125] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:21.736277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471733715208943:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33395
2025-11-27T18:20:21.735826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471735659465891:2107] [node 4] ICP22 created new session: [3:7577471735659465919:2107]
2025-11-27T18:20:21.736493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471733835535234:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:21.736584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471737744996415:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.736047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471735659465919:2107] [node 4] ICS09 handshake done sender: [3:7577471735659465899:2112] self: [3:7577471735659465898:0] peer: [4:7577471737744996495:0] socket: 462 qp: -1
2025-11-27T18:20:21.736760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:21.736092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471735659465919:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:21.737234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471734756175861:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.736145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:21.736805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471737744996415:2050] [node 3] ICP22 created new session: [4:7577471737744996502:2050]
2025-11-27T18:20:21.737636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:21.737073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471737744996502:2050] [node 3] ICS09 handshake done sender: [4:7577471737744996492:2116] self: [4:7577471737744996495:0] peer: [3:7577471735659465898:0] socket: 464 qp: -1
2025-11-27T18:20:21.737681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471734756175861:2049] [node 2] ICP22 created new session: [5:7577471734756175956:2049]
2025-11-27T18:20:21.737674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471733835535166:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.737643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471737744996502:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.737718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471734756175956:2049] [node 2] ICS09 handshake done sender: [5:7577471734756175949:2125] self: [5:7577471734756175951:0] peer: [2:7577471733715208921:0] socket: 471 qp: -1
2025-11-27T18:20:21.737712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:21.737727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:21.737758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471734756175956:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:21.737772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471733835535166:2048] [node 2] ICP22 created new session: [1:7577471733835535263:2048]
2025-11-27T18:20:21.737854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471737744996485:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:21.737786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:21.737898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471734756175955:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26187
2025-11-27T18:20:21.737998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471733835535263:2048] [node 2] ICS09 handshake done sender: [1:7577471733835535234:2111] self: [1:7577471733835535233:0] peer: [2:7577471733715208937:0] socket: 437 qp: -1
2025-11-27T18:20:21.738041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471733835535263:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:21.738071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471735659465913:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.738207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471733715208927:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:21.738087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.739460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471733715208849:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.739405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471734756175957:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26187
2025-11-27T18:20:21.739718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:21.739933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471733715208849:2048] [node 1] ICP22 created new session: [2:7577471733715208945:2048]
2025-11-27T18:20:21.740319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471737744996476:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.740349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:21.740441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471737744996476:2108] [node 5] ICP22 created new session: [4:7577471737744996504:2108]
2025-11-27T18:20:21.740611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471737744996504:2108] [node 5] ICS09 handshake done sender: [4:7577471737744996485:2113] self: [4:7577471737744996484:0] peer: [5:7577471734756175954:0] socket: 433 qp: -1
2025-11-27T18:20:21.740441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471733715208945:2048] [node 1] ICS09 handshake done sender: [2:7577471733715208927:2118] self: [2:7577471733715208937:0] peer: [1:7577471733835535233:0] socket: 457 qp: -1
2025-11-27T18:20:21.740652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471737744996504:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:21.740685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:21.740494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471733715208945:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:21.740941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471734756175953:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.740534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:21.741885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471734756175950:2126] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.741959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471733835535166:2048] [node 2] ICP18 handshake (actor [1:7577471733835535256:2120]) is too old
2025-11-27T18:20:21.742426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471735659465829:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.742485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471737702402191:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.742786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471737702402192:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19027
2025-11-27T18:20:21.743031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471734756175932:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.742833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471737702402112:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.743110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:21.743169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471734756175932:2108] [node 4] ICP22 created new session: [5:7577471734756175960:2108]
2025-11-27T18:20:21.743154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471733715208918:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.743211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471734756175960:2108] [node 4] ICS09 handshake done sender: [5:7577471734756175950:2126] self: [5:7577471734756175954:0] peer: [4:7577471737744996484:0] socket: 477 qp: -1
2025-11-27T18:20:21.743313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471734756175960:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:21.743178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471733715208918:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.743349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:21.743378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471733835535245:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.743407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471733835535245:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.745049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471733835535245:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:21.745241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471733835535232:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.745844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471737702402192:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.745493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:21.745542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471733835535232:2109] [node 6] ICP22 created new session: [1:7577471733835535266:2109]
2025-11-27T18:20:21.745873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471733835535266:2109] [node 6] ICS09 handshake done sender: [1:7577471733835535245:2115] self: [1:7577471733835535244:0] peer: [6:7577471737702402193:0] socket: 485 qp: -1
2025-11-27T18:20:21.746121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471733715208918:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:21.745910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471733835535266:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.746029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:21.746396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471734756175860:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.746485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471737702402194:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19027
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.746791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471737702402114:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.746974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471735659465923:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65324
2025-11-27T18:20:21.747046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737744996501:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57643
2025-11-27T18:20:21.747258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471733715208850:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.747970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471735659465913:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:21.747314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:21.747923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471734756175955:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.748076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471733715208850:2049] [node 3] ICP22 created new session: [2:7577471733715208949:2049]
2025-11-27T18:20:21.748196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471734756175862:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.748127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471733715208949:2049] [node 3] ICS09 handshake done sender: [2:7577471733715208918:2112] self: [2:7577471733715208917:0] peer: [3:7577471735659465922:0] socket: 469 qp: -1
2025-11-27T18:20:21.748224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471733715208949:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:21.748602Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471737702402112:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471737702402191:2116] held: no
2025-11-27T18:20:21.748264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: 2025-11-27T18:20:21.748796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471735659465829:2049] [node 2] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:21.748851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:21.748892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471735659465829:2049] [node 2] ICP22 created new session: [3:7577471735659465924:2049]
2025-11-27T18:20:21.748908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471735659465906:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.748929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471735659465906:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.749453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471737702402195:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19027
2025-11-27T18:20:21.748942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471735659465924:2049] [node 2] ICS09 handshake done sender: [3:7577471735659465913:2118] self: [3:7577471735659465922:0] peer: [2:7577471733715208917:0] socket: 472 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.748975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471735659465924:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:21.750949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737744996506:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57643
2025-11-27T18:20:21.749000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:21.751048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471733835535240:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.751086Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471733835535240:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.751245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471737702402191:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:21.752244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471737744996476:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:21.752513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471733835535240:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:21.752467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471737702402112:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.752554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737744996501:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.752580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:21.752706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471737702402112:2048] [node 1] ICP22 created new session: [6:7577471737702402200:2048]
2025-11-27T18:20:21.752758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471737702402200:2048] [node 1] ICS09 handshake done sender: [6:7577471737702402191:2116] self: [6:7577471737702402193:0] peer: [1:7577471733835535244:0] socket: 486 qp: -1
2025-11-27T18:20:21.752770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471737702402197:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19027
2025-11-27T18:20:21.752827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471733835535231:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.752794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471737702402200:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.752968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471735659465901:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.752943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:21.752996Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471735659465901:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.752827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.752985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471733835535231:2108] [node 5] ICP22 created new session: [1:7577471733835535269:2108]
2025-11-27T18:20:21.753479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471737744996414:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.753322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471733835535269:2108] [node 5] ICS09 handshake done sender: [1:7577471733835535240:2114] self: [1:7577471733835535239:0] peer: [5:7577471734756175962:0] socket: 478 qp: -1
2025-11-27T18:20:21.753458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471737702402197:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.754040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471737702402198:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19027
2025-11-27T18:20:21.753438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471734756175965:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26187
2025-11-27T18:20:21.754868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471734756175860:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471734756175953:2128] held: no
2025-11-27T18:20:21.753369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471733835535269:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:21.753624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:21.756798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471734756175953:2128] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:21.756847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471737702402174:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.757000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471735659465906:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:21.757319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471734756175860:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.757491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:21.757469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471733715208924:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.757603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471734756175860:2048] [node 1] ICP22 created new session: [5:7577471734756175967:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.757515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471733715208924:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.757800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471734756175967:2048] [node 1] ICS09 handshake done sender: [5:7577471734756175953:2128] self: [5:7577471734756175962:0] peer: [1:7577471733835535239:0] socket: 480 qp: -1
2025-11-27T18:20:21.757848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471734756175967:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:21.757857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471734756175966:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26187
2025-11-27T18:20:21.757886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:21.757909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471737702402198:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.757978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471735659465893:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.758014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:21.758055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471735659465893:2109] [node 6] ICP22 created new session: [3:7577471735659465929:2109]
2025-11-27T18:20:21.758108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471734756175946:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.758104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471735659465929:2109] [node 6] ICS09 handshake done sender: [3:7577471735659465906:2117] self: [3:7577471735659465905:0] peer: [6:7577471737702402196:0] socket: 438 qp: -1
2025-11-27T18:20:21.758136Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471734756175946:2123] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.758146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471735659465929:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:21.758193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471737702402175:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.758156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471735659465927:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65324
2025-11-27T18:20:21.758427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471737702402203:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19027
2025-11-27T18:20:21.758832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471737744996491:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.758876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471737744996491:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.758172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:21.758864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471733715208924:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:21.758886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471737702402195:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.760973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471733715208915:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.760478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471737702402192:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:21.758385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471735659465901:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:21.761017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:21.760995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471737702402113:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:21.761474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471733715208915:2107] [node 4] ICP22 created new session: [2:7577471733715208952:2107]
2025-11-27T18:20:21.765375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471737702402114:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.765360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471733715208952:2107] [node 4] ICS09 handshake done sender: [2:7577471733715208924:2115] self: [2:7577471733715208923:0] peer: [4:7577471737744996508:0] socket: 434 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.765414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471733715208952:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:21.765381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471737744996491:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:21.765536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:21.765639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:21.765657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737744996509:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57643
2025-11-27T18:20:21.765600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471737702402114:2050] [node 3] ICP22 created new session: [6:7577471737702402206:2050]
2025-11-27T18:20:21.765669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471735659465927:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.765648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471737702402206:2050] [node 3] ICS09 handshake done sender: [6:7577471737702402192:2117] self: [6:7577471737702402196:0] peer: [3:7577471735659465905:0] socket: 445 qp: -1
2025-11-27T18:20:21.765724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471737702402206:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:21.765721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471735659465892:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.765862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:21.766322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471733715208930:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:21.765959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:21.766023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471735659465892:2108] [node 5] ICP22 created new session: [3:7577471735659465931:2108]
2025-11-27T18:20:21.766356Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471733715208930:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:21.766329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471735659465931:2108] [node 5] ICS09 handshake done sender: [3:7577471735659465901:2113] self: [3:7577471735659465900:0] peer: [5:7577471734756175964:0] socket: 483 qp: -1
2025-11-27T18:20:21.766536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471737744996477:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.766326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471734756175955:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:21.766581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:21.766361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471735659465931:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:21.766546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:21.766625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471737744996477:2109] [node 6] ICP22 created new session: [4:7577471737744996511:2109]
2025-11-27T18:20:21.767433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471737744996511:2109] [node 6] ICS09 handshake done sender: [4:7577471737744996491:2115] self: [4:7577471737744996490:0] peer: [6:7577471737702402204:0] socket: 495 qp: -1
2025-11-27T18:20:21.767613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471737744996511:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:21.767703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:21.769316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471737744996414:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29570) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471737744996501:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.769727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471734756175862:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.769367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471733715208930:2120] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:21.769789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:21.770707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471737702402207:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19027
2025-11-27T18:20:21.770488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471734756175946:2123] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:21.770501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471734756175862:2050] [node 3] ICP22 created new session: [5:7577471734756175970:2050]
2025-11-27T18:20:21.770755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471734756175970:2050] [node 3] ICS09 handshake done sender: [5:7577471734756175955:2129] self: [5:7577471734756175964:0] peer: [3:7577471735659465900:0] socket: 453 qp: -1
2025-11-27T18:20:21.770965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471733715208916:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.770928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471734756175970:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:21.770999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:21.771107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:21.771151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471733715208916:2110] [node 6] ICP22 created new session: [2:7577471733715208955:2110]
2025-11-27T18:20:21.771235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471737744996501:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:21.771231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471733715208955:2110] [node 6] ICS09 handshake done sender: [2:7577471733715208930:2120] self: [2:7577471733715208929:0] peer: [6:7577471737702402205:0] socket: 492 qp: -1
2025-11-27T18:20:21.771337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471733715208955:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:21.771803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:21.773614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471737744996414:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.773655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:21.773700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471737744996414:2049] [node 2] ICP22 created new session: [4:7577471737744996513:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.773811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471737744996513:2049] [node 2] ICS09 handshake done sender: [4:7577471737744996501:2119] self: [4:7577471737744996508:0] peer: [2:7577471733715208923:0] socket: 488 qp: -1
2025-11-27T18:20:21.773844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471737744996513:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:21.773900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:21.773881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471737702402209:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19027
2025-11-27T18:20:21.774012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471735659465893:2109] [node 6] ICP18 handshake (actor [3:7577471735659465927:2121]) is too old
2025-11-27T18:20:21.774028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471734756175933:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:21.774090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:21.774143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471734756175933:2109] [node 6] ICP22 created new session: [5:7577471734756175972:2109]
2025-11-27T18:20:21.774188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471734756175972:2109] [node 6] ICS09 handshake done sender: [5:7577471734756175946:2123] self: [5:7577471734756175945:0] peer: [6:7577471737702402202:0] socket: 490 qp: -1
2025-11-27T18:20:21.774218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471734756175972:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:21.774264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:21.775334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471737702402198:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:21.775768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471737702402113:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29570) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471737702402195:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.776197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471737702402210:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19027
2025-11-27T18:20:21.776337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471737702402175:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.776388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:21.776429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471737702402175:2107] [node 4] ICP22 created new session: [6:7577471737702402212:2107]
2025-11-27T18:20:21.776486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471737702402212:2107] [node 4] ICS09 handshake done sender: [6:7577471737702402198:2121] self: [6:7577471737702402204:0] peer: [4:7577471737744996490:0] socket: 461 qp: -1
2025-11-27T18:20:21.776526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471737744996515:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57643
2025-11-27T18:20:21.776560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471737702402212:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:21.776611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:21.776918Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471737702402197:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:21.778819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471737702402174:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.778913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:21.778978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471737702402174:2105] [node 5] ICP22 created new session: [6:7577471737702402214:2105]
2025-11-27T18:20:21.779050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471737702402214:2105] [node 5] ICS09 handshake done sender: [6:7577471737702402197:2120] self: [6:7577471737702402202:0] peer: [5:7577471734756175945:0] socket: 452 qp: -1
2025-11-27T18:20:21.779091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471737702402214:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:21.779120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:21.779251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471737702402195:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:21.779802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471737702402174:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:21.779804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471737702402113:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:21.779837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:21.779893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471737702402113:2049] [node 2] ICP22 created new session: [6:7577471737702402216:2049]
2025-11-27T18:20:21.779933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471737702402216:2049] [node 2] ICS09 handshake done sender: [6:7577471737702402195:2119] self: [6:7577471737702402205:0] peer: [2:7577471733715208929:0] socket: 494 qp: -1
2025-11-27T18:20:21.779971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471737702402216:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:21.779999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:21.813432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:21.813656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471733715208945:2048] [node 1] ICS07 socket disconnect 457 reason# EndOfStream
2025-11-27T18:20:21.813708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471733715208945:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.813776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:21.813831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471733715208945:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:21.814592Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:21.814795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471735659465915:2048] [node 1] ICS07 socket disconnect 447 reason# EndOfStream
2025-11-27T18:20:21.814826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471735659465915:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.814908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:21.814966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471735659465915:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:21.816012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471735659465828:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18123) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.816109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471735659465915:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.816219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471733715208849:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18123) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:21.816270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471733715208945:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:21.816244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471735659465828:2048] [node 1] ICP30 unregister session Session# [3:7577471735659465915:2048] VirtualId# [3:7577471735659465910:0]
2025-11-27T18:20:21.816807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:21.816843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471735659465828:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18123) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.816862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18123) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.817857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471733715208849:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18123) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.817900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471733715208945:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.817927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471733715208849:2048] [node 1] ICP30 unregister session Session# [2:7577471733715208945:2048] VirtualId# [2:7577471733715208937:0]
2025-11-27T18:20:21.818290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:21.818337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471733715208849:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18123) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.818353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18123) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.818456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:21.818676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471737744996497:2048] [node 1] ICS01 socket: 444 reason# EndOfStream
2025-11-27T18:20:21.818721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471737744996413:2048] [node 1] ICP30 unregister session Session# [4:7577471737744996497:2048] VirtualId# [4:7577471737744996493:0]
2025-11-27T18:20:21.818751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471737744996497:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.818817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:21.818862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:21.819603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:21.819820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471734756175967:2048] [node 1] ICS01 socket: 480 reason# EndOfStream
2025-11-27T18:20:21.819859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471734756175860:2048] [node 1] ICP30 unregister session Session# [5:7577471734756175967:2048] VirtualId# [5:7577471734756175962:0]
2025-11-27T18:20:21.819888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471734756175967:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.819948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:21.819985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:21.820649Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:21.820842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471737702402200:2048] [node 1] ICS01 socket: 486 reason# EndOfStream
2025-11-27T18:20:21.820866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471737702402112:2048] [node 1] ICP30 unregister session Session# [6:7577471737702402200:2048] VirtualId# [6:7577471737702402193:0]
2025-11-27T18:20:21.820886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471737702402200:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.820938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:21.820984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:21.826835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471735659465828:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:21.827903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471733715208849:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:21.867271Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:21.867480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471735659465924:2049] [node 2] ICS07 socket disconnect 472 reason# EndOfStream
2025-11-27T18:20:21.867517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471735659465924:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.867603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:21.867671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471735659465924:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:21.869775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:21.869930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471737744996513:2049] [node 2] ICS01 socket: 488 reason# EndOfStream
2025-11-27T18:20:21.869968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471737744996414:2049] [node 2] ICP30 unregister session Session# [4:7577471737744996513:2049] VirtualId# [4:7577471737744996508:0]
2025-11-27T18:20:21.869999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471737744996513:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.869967Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471735659465829:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29570) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:21.870067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:21.870001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471735659465924:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:21.870102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:21.870848Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471735659465829:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29570) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.870943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471735659465924:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.870974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471735659465829:2049] [node 2] ICP30 unregister session Session# [3:7577471735659465924:2049] VirtualId# [3:7577471735659465922:0]
2025-11-27T18:20:21.871063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:21.871106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471735659465829:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29570) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.871123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29570) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.871260Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:21.871464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471734756175956:2049] [node 2] ICS01 socket: 471 reason# EndOfStream
2025-11-27T18:20:21.871517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471734756175861:2049] [node 2] ICP30 unregister session Session# [5:7577471734756175956:2049] VirtualId# [5:7577471734756175951:0]
2025-11-27T18:20:21.871542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471734756175956:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.871599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:21.871660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:21.872313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:21.872505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471737702402216:2049] [node 2] ICS01 socket: 494 reason# EndOfStream
2025-11-27T18:20:21.872551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471737702402113:2049] [node 2] ICP30 unregister session Session# [6:7577471737702402216:2049] VirtualId# [6:7577471737702402205:0]
2025-11-27T18:20:21.872576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471737702402216:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.872627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:21.872661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:21.918507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:21.918689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471737744996502:2050] [node 3] ICS07 socket disconnect 464 reason# EndOfStream
2025-11-27T18:20:21.918725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471737744996502:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.918802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:21.918834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471737744996502:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:21.919646Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:21.919844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471734756175970:2050] [node 3] ICS07 socket disconnect 453 reason# EndOfStream
2025-11-27T18:20:21.919771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471737744996415:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11519) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.919880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471734756175970:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.919903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471737744996502:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.919939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471737744996415:2050] [node 3] ICP30 unregister session Session# [4:7577471737744996502:2050] VirtualId# [4:7577471737744996495:0]
2025-11-27T18:20:21.919973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:21.920020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471734756175970:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:21.920083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:21.920130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471737744996415:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.920152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.920751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:21.920800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471734756175862:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11519) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.920911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471734756175970:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.920941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471734756175862:2050] [node 3] ICP30 unregister session Session# [5:7577471734756175970:2050] VirtualId# [5:7577471734756175964:0]
2025-11-27T18:20:21.920950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471737702402206:2050] [node 3] ICS07 socket disconnect 445 reason# EndOfStream
2025-11-27T18:20:21.921031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:21.920983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471737702402206:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.921070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471734756175862:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.921055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:21.921093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471737702402206:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:21.921083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.921755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471737702402114:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11519) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.921792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471737702402206:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.921816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471737702402114:2050] [node 3] ICP30 unregister session Session# [6:7577471737702402206:2050] VirtualId# [6:7577471737702402196:0]
2025-11-27T18:20:21.921905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:21.921926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471737702402114:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.921933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.930347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471737744996415:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:21.931368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471734756175862:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:21.931320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471737702402114:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:21.989333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:21.989585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471734756175960:2108] [node 4] ICS07 socket disconnect 477 reason# EndOfStream
2025-11-27T18:20:21.989618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471734756175960:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.989721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:21.989774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471734756175960:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:21.991131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471734756175932:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11233) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.991051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:21.991243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471734756175960:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.991352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471734756175932:2108] [node 4] ICP30 unregister session Session# [5:7577471734756175960:2108] VirtualId# [5:7577471734756175954:0]
2025-11-27T18:20:21.991548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:21.991719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471734756175932:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.991746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.991589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471737702402212:2107] [node 4] ICS07 socket disconnect 461 reason# EndOfStream
2025-11-27T18:20:21.991630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471737702402212:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:21.991718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:21.991765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471737702402212:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:21.992669Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471737702402175:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11233) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:21.992792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471737702402212:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:21.992845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471737702402175:2107] [node 4] ICP30 unregister session Session# [6:7577471737702402212:2107] VirtualId# [6:7577471737702402204:0]
2025-11-27T18:20:21.992948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:21.992986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471737702402175:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.993000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:21.998902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471737702402175:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:22.000933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471734756175932:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:22.057449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:22.057711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471737702402214:2105] [node 5] ICS07 socket disconnect 452 reason# EndOfStream
2025-11-27T18:20:22.057748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471737702402214:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.057840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:22.057910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471737702402214:2105] [node 5] ICS15 start handshake
2025-11-27T18:20:22.058848Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471737702402174:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19302) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:22.058904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471737702402214:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:22.058978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471737702402174:2105] [node 5] ICP30 unregister session Session# [6:7577471737702402214:2105] VirtualId# [6:7577471737702402202:0]
2025-11-27T18:20:22.059108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:22.059150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471737702402174:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19302) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:22.059170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19302) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:22.157382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471741556727072:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:22.157578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471741556727073:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:22.178635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471739915690589:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:22.178980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471739915690590:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:22.200446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471739602772639:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:22.200495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471739602772640:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:22.221327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471741156656818:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:22.221379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471741156656820:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:22.221487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471741156656819:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:22.241671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471740576354330:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:22.241717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471740576354331:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:22.241893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471740576354332:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:22.263044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471740008881377:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:22.263076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471740008881378:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:22.263118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471740008881379:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:22.328085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471740008881438:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:22.359599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471739915690654:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:22.352963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471741556727136:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:22.378912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471741156656881:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:22.359586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471741556727135:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:22.380241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471739915690655:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:22.359608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471741556727137:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:22.380402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471740576354394:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:22.380640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471741156656882:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:22.380818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471740008881450:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:22.380621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471739602772704:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:22.380636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471739602772703:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:22.381808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471740576354395:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:22.382443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471739915690656:2111] [node 6] ICP01 ready to work
2025-11-27T18:20:22.382554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471739602772705:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.392098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471739915690665:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.393823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471739915690670:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.396898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471739602772712:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37884
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.397221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471741156656896:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19722
2025-11-27T18:20:22.397686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471740576354407:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35967
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.398632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471739602772712:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.399411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471739915690672:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
2025-11-27T18:20:22.400099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471739602772718:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37884
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.401497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471739915690665:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.400596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471741156656896:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.401007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471741156656819:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.403300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471739915690662:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.403282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471740576354410:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35967
2025-11-27T18:20:22.406711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471741556727144:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35690
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.403321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471740576354407:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.403340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471739915690662:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.406862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471741156656887:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.406352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471740576354394:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.407557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471739915690662:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:22.406934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471741556727146:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35690
2025-11-27T18:20:22.406900Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471741156656887:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.409091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471740576354410:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.408055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471739915690670:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:22.409033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471741556727148:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35690
::ffff:127.1.0.0
2025-11-27T18:20:22.409420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471739602772721:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37884
2025-11-27T18:20:22.409427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471739915690673:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
2025-11-27T18:20:22.409569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471741156656898:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19722
2025-11-27T18:20:22.409643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471740576354332:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.409960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471739915690654:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.410101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471741556727146:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.410634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471739602772709:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.410183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.410146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471741156656887:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:22.410138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471741556727144:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.410406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471739915690654:2107] [node 4] ICP22 created new session: [2:7577471739915690678:2107]
2025-11-27T18:20:22.410665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471739602772718:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.412790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471740576354413:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35967
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.411889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471739915690678:2107] [node 4] ICS09 handshake done sender: [2:7577471739915690662:2114] self: [2:7577471739915690661:0] peer: [4:7577471741156656897:0] socket: 451 qp: -1
2025-11-27T18:20:22.410686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471739602772709:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.410446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471741556727148:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.412079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471739915690678:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:22.411899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471739602772721:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.411877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471741556727155:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35690
2025-11-27T18:20:22.412340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471739915690672:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.412459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471739602772640:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.413548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471741156656819:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25783) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471741156656896:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.414200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471740008881464:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1891
2025-11-27T18:20:22.412347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471741556727151:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35690
2025-11-27T18:20:22.413628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471741156656881:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.413523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471739602772709:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:22.412379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
::ffff:127.1.0.0
2025-11-27T18:20:22.417393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471739602772704:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.417464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:22.413245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471741556727155:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.412936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471739915690673:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.417095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471740576354414:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35967
2025-11-27T18:20:22.417082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:22.417701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471739602772704:2108] [node 5] ICP22 created new session: [3:7577471739602772726:2108]
2025-11-27T18:20:22.417705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471740576354407:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:22.417151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471741156656881:2108] [node 5] ICP22 created new session: [4:7577471741156656901:2108]
2025-11-27T18:20:22.414257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471741556727151:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.418364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471739602772726:2108] [node 5] ICS09 handshake done sender: [3:7577471739602772709:2111] self: [3:7577471739602772707:0] peer: [5:7577471740576354412:0] socket: 461 qp: -1
2025-11-27T18:20:22.414245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471739915690660:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.418670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471739602772726:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:22.417839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471741156656901:2108] [node 5] ICS09 handshake done sender: [4:7577471741156656887:2111] self: [4:7577471741156656886:0] peer: [5:7577471740576354411:0] socket: 445 qp: -1
2025-11-27T18:20:22.414281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471739915690660:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.417716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471740576354415:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35967
2025-11-27T18:20:22.418703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:22.418355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471741156656901:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:22.417915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471740576354394:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.418404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:22.420026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471739915690660:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:22.418090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:22.421134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471740008881464:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.418886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471741156656896:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:22.421661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471740008881377:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.418400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471740576354394:2108] [node 4] ICP22 created new session: [5:7577471740576354417:2108]
2025-11-27T18:20:22.421579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471739915690590:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.420558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471741156656819:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.419023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471740576354417:2108] [node 4] ICS09 handshake done sender: [5:7577471740576354407:2115] self: [5:7577471740576354411:0] peer: [4:7577471741156656886:0] socket: 447 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.422294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471741556727142:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.421633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:22.420591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:22.420275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471740576354417:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.422338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471741556727142:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.423749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471739602772728:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37884
2025-11-27T18:20:22.421676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471739915690590:2049] [node 3] ICP22 created new session: [2:7577471739915690684:2049]
2025-11-27T18:20:22.420913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471741156656819:2049] [node 2] ICP22 created new session: [4:7577471741156656903:2049]
2025-11-27T18:20:22.420318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.421347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471741156656903:2049] [node 2] ICS09 handshake done sender: [4:7577471741156656896:2115] self: [4:7577471741156656897:0] peer: [2:7577471739915690661:0] socket: 453 qp: -1
2025-11-27T18:20:22.422450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471739915690684:2049] [node 3] ICS09 handshake done sender: [2:7577471739915690660:2113] self: [2:7577471739915690659:0] peer: [3:7577471739602772724:0] socket: 450 qp: -1
2025-11-27T18:20:22.421270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471740576354332:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64659) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471740576354410:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.423600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471739915690684:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:22.424728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471740576354415:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.421384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471741156656903:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:22.430186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471740576354414:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.430535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471740576354416:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35967
2025-11-27T18:20:22.421578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:22.423735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:22.424453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471741156656900:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19722
2025-11-27T18:20:22.430689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471739602772640:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25783) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471739602772718:2116] held: no
2025-11-27T18:20:22.431000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471741556727142:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:22.430621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471740008881466:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1891
2025-11-27T18:20:22.431695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471741156656900:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.431935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471741156656905:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19722
2025-11-27T18:20:22.432040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471740008881467:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1891
2025-11-27T18:20:22.432266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471741556727136:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.433211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471740008881466:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.432080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471741156656818:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.433204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:22.433850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471740008881468:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1891
2025-11-27T18:20:22.433872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471740576354331:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.434096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471740576354330:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.433880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471741556727136:2109] [node 6] ICP22 created new session: [1:7577471741556727162:2109]
2025-11-27T18:20:22.434023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471740008881378:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.434821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471739915690671:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.435728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471741156656905:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.435775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471740576354410:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:22.435819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471739602772718:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:22.434629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471740008881464:2124] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:22.434757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471741556727162:2109] [node 6] ICS09 handshake done sender: [1:7577471741556727142:2111] self: [1:7577471741556727140:0] peer: [6:7577471740008881465:0] socket: 480 qp: -1
2025-11-27T18:20:22.434859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471739915690671:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.434975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471741556727162:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:22.436497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471741156656820:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.435026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:22.436058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471740008881377:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.436866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471740576354332:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.437414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471739915690664:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.437292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:22.437447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471739915690664:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.436291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471741556727141:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.436841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:22.439162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471739915690654:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(21541727ns) {HandleHandshakeStatus:329(21539143ns) {DropIncomingHandshake:442(22386ns) DropOutgoingHandshake:457(399ns) SwitchToState:179(227ns) ProcessPendingSessionEvents:495(17954340ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12962313ns max# 1661696ns}}}
2025-11-27T18:20:22.437429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471740576354332:2050] [node 3] ICP22 created new session: [5:7577471740576354422:2050]
2025-11-27T18:20:22.436319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471741556727141:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.439988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471739602772640:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.437007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471740008881377:2048] [node 1] ICP22 created new session: [6:7577471740008881471:2048]
2025-11-27T18:20:22.439799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471740576354422:2050] [node 3] ICS09 handshake done sender: [5:7577471740576354410:2117] self: [5:7577471740576354412:0] peer: [3:7577471739602772707:0] socket: 463 qp: -1
2025-11-27T18:20:22.440173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471741556727143:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:22.440044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-11-27T18:20:22.439932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471740576354422:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:22.440200Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471741556727143:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.437413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471740008881471:2048] [node 1] ICS09 handshake done sender: [6:7577471740008881464:2124] self: [6:7577471740008881465:0] peer: [1:7577471741556727140:0] socket: 481 qp: -1
2025-11-27T18:20:22.440213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471739602772640:2049] [node 2] ICP22 created new session: [3:7577471739602772733:2049]
2025-11-27T18:20:22.440873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471739602772733:2049] [node 2] ICS09 handshake done sender: [3:7577471739602772718:2116] self: [3:7577471739602772724:0] peer: [2:7577471739915690659:0] socket: 456 qp: -1
2025-11-27T18:20:22.439967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:22.439170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471740008881471:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:22.441515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471739602772733:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:22.439814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.441908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:22.441745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471740008881470:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1891
2025-11-27T18:20:22.441937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471739602772716:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.441327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471741556727141:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:22.441965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471739602772716:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.442987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471739915690671:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:22.442968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471741156656910:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19722
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.443139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471740008881468:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.443151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471741156656881:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(20244266ns) {HandleHandshakeStatus:329(20241785ns) {DropIncomingHandshake:442(262ns) DropOutgoingHandshake:457(338ns) SwitchToState:179(438ns) ProcessPendingSessionEvents:495(13534893ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13431198ns max# 4654879ns}}}
2025-11-27T18:20:22.443739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471739602772730:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37884
2025-11-27T18:20:22.443694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471741156656900:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.443572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471741556727143:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:22.444181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471741556727135:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.444865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471739602772731:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37884
2025-11-27T18:20:22.444397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:22.444450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471741556727135:2107] [node 4] ICP22 created new session: [1:7577471741556727167:2107]
2025-11-27T18:20:22.445223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471741556727137:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.445345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471739602772716:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:22.445235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471741556727167:2107] [node 4] ICS09 handshake done sender: [1:7577471741556727141:2110] self: [1:7577471741556727138:0] peer: [4:7577471741156656908:0] socket: 476 qp: -1
2025-11-27T18:20:22.445280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:22.445378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471741556727167:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:22.445421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:22.445617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471741556727137:2108] [node 5] ICP22 created new session: [1:7577471741556727168:2108]
2025-11-27T18:20:22.445641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471740008881450:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.445863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471739602772730:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.445956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471740008881470:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.445343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471741156656818:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.446033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471741556727168:2108] [node 5] ICS09 handshake done sender: [1:7577471741556727143:2112] self: [1:7577471741556727139:0] peer: [5:7577471740576354421:0] socket: 479 qp: -1
2025-11-27T18:20:22.446613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471740008881474:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1891
2025-11-27T18:20:22.445407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:22.446516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471741556727168:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:22.445451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471741156656818:2048] [node 1] ICP22 created new session: [4:7577471741156656911:2048]
2025-11-27T18:20:22.446550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:22.446659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471740008881475:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1891
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.448175Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471739915690664:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:22.445497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471741156656911:2048] [node 1] ICS09 handshake done sender: [4:7577471741156656900:2117] self: [4:7577471741156656908:0] peer: [1:7577471741556727138:0] socket: 477 qp: -1
2025-11-27T18:20:22.449253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471740008881438:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.449890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471739915690656:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.445531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471741156656911:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:22.449826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471739602772703:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.445568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:22.449341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471740008881466:2125] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:22.449961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:22.449992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:22.450016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471739915690656:2111] [node 6] ICP22 created new session: [2:7577471739915690688:2111]
2025-11-27T18:20:22.450246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471739915690655:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.449892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471740576354425:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35967
2025-11-27T18:20:22.450292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471740008881378:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.450272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471739915690688:2111] [node 6] ICS09 handshake done sender: [2:7577471739915690671:2118] self: [2:7577471739915690669:0] peer: [6:7577471740008881469:0] socket: 488 qp: -1
2025-11-27T18:20:22.450355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471739602772703:2107] [node 4] ICP22 created new session: [3:7577471739602772736:2107]
2025-11-27T18:20:22.450573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:22.450522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:22.450628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471739602772731:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.450726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471740008881377:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:22.450820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471739915690655:2109] [node 5] ICP22 created new session: [2:7577471739915690689:2109]
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:22.450963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471739915690688:2111] [node 6] ICS10 traffic start
2025-11-27T18:20:22.457393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471739602772736:2107] [node 4] ICS09 handshake done sender: [3:7577471739602772716:2115] self: [3:7577471739602772713:0] peer: [4:7577471741156656909:0] socket: 487 qp: -1
2025-11-27T18:20:22.457676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471740008881378:2049] [node 2] ICP22 created new session: [6:7577471740008881478:2049]
2025-11-27T18:20:22.451005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:22.457459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471739602772736:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:22.457831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471739915690689:2109] [node 5] ICS09 handshake done sender: [2:7577471739915690664:2115] self: [2:7577471739915690663:0] peer: [5:7577471740576354420:0] socket: 483 qp: -1
2025-11-27T18:20:22.459499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:22.458129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471739915690689:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:22.458014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471740008881478:2049] [node 2] ICS09 handshake done sender: [6:7577471740008881466:2125] self: [6:7577471740008881469:0] peer: [2:7577471739915690669:0] socket: 489 qp: -1
2025-11-27T18:20:22.459506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471739602772639:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.459848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471740576354426:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35967
2025-11-27T18:20:22.458171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:22.458150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471740008881478:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:22.459977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471740576354424:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35967
2025-11-27T18:20:22.459119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:22.459888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471740008881475:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.460853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471741556727147:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.460897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471741556727147:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.460951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471740576354409:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.460993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471740576354409:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.463120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471740008881379:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.463519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471741156656895:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.463195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471740576354330:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27275) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471740576354414:2119] held: no
2025-11-27T18:20:22.463555Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471741156656895:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.464444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471740576354424:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.464615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471739602772720:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.464645Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471739602772720:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.464745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471739602772703:2107] [node 4] ICP18 handshake (actor [3:7577471739602772730:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.466732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471740576354415:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:22.466799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471739602772720:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:22.468582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471739602772705:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.468558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471741156656913:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19722
2025-11-27T18:20:22.469520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471741156656895:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:22.468750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:22.469529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471741156656820:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64659) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471741156656905:2118] held: no
2025-11-27T18:20:22.469201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471740576354414:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:22.468791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471739602772705:2109] [node 6] ICP22 created new session: [3:7577471739602772740:2109]
2025-11-27T18:20:22.469591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471740576354331:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25783) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471740576354415:2120] held: no
2025-11-27T18:20:22.469365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471739602772740:2109] [node 6] ICS09 handshake done sender: [3:7577471739602772720:2117] self: [3:7577471739602772719:0] peer: [6:7577471740008881481:0] socket: 504 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.469774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471741556727135:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(19669434ns) {HandleHandshakeStatus:329(19561634ns) {DropIncomingHandshake:442(13350ns) DropOutgoingHandshake:457(389ns) SwitchToState:179(279ns) ProcessPendingSessionEvents:495(18020037ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17919744ns max# 6688888ns}}}
2025-11-27T18:20:22.470534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471739915690692:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
2025-11-27T18:20:22.469497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471739602772740:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:22.469542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:22.469950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471741556727147:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:22.470717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471741156656882:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.470566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471740576354330:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.471044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:22.470624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:22.471390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471741156656882:2109] [node 6] ICP22 created new session: [4:7577471741156656916:2109]
2025-11-27T18:20:22.470669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471740576354330:2048] [node 1] ICP22 created new session: [5:7577471740576354428:2048]
2025-11-27T18:20:22.471609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471741156656916:2109] [node 6] ICS09 handshake done sender: [4:7577471741156656895:2114] self: [4:7577471741156656894:0] peer: [6:7577471740008881476:0] socket: 491 qp: -1
2025-11-27T18:20:22.471640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471741156656916:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:22.471158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471740576354428:2048] [node 1] ICS09 handshake done sender: [5:7577471740576354414:2119] self: [5:7577471740576354421:0] peer: [1:7577471741556727139:0] socket: 482 qp: -1
2025-11-27T18:20:22.471347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471741556727073:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.471561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471740576354428:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:22.471595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:22.471628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:22.471751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:22.471643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471741556727073:2049] [node 3] ICP22 created new session: [1:7577471741556727173:2049]
2025-11-27T18:20:22.471695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471740576354409:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.471733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471741556727173:2049] [node 3] ICS09 handshake done sender: [1:7577471741556727147:2115] self: [1:7577471741556727145:0] peer: [3:7577471739602772737:0] socket: 495 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.472120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471740576354331:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.471774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471741556727173:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:22.472089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471741156656905:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:22.471803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:22.472599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471741156656915:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19722
2025-11-27T18:20:22.472465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:22.473021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471739602772742:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37884
2025-11-27T18:20:22.472647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471740576354331:2049] [node 2] ICP22 created new session: [5:7577471740576354431:2049]
2025-11-27T18:20:22.473767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471741156656820:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.473966Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471739602772639:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27275) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471739602772731:2121] held: no
2025-11-27T18:20:22.473968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471740008881379:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64659) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471740008881475:2130] held: no
2025-11-27T18:20:22.473812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:22.474342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471740008881378:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20485936ns) {HandleHandshakeStatus:329(20484065ns) {DropIncomingHandshake:442(1588ns) DropOutgoingHandshake:457(5674ns) SwitchToState:179(253ns) ProcessPendingSessionEvents:495(9640543ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9550338ns max# 783767ns}}}
2025-11-27T18:20:22.473073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471740576354431:2049] [node 2] ICS09 handshake done sender: [5:7577471740576354415:2120] self: [5:7577471740576354420:0] peer: [2:7577471739915690663:0] socket: 484 qp: -1
2025-11-27T18:20:22.474019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471741156656820:2050] [node 3] ICP22 created new session: [4:7577471741156656918:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.475172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471741156656918:2050] [node 3] ICS09 handshake done sender: [4:7577471741156656905:2118] self: [4:7577471741156656909:0] peer: [3:7577471739602772713:0] socket: 440 qp: -1
2025-11-27T18:20:22.475162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471739915690692:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.473103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471740576354431:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.473242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:22.474340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471740576354395:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.475168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:22.475342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471740576354395:2110] [node 6] ICP22 created new session: [5:7577471740576354433:2110]
2025-11-27T18:20:22.475770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471740576354433:2110] [node 6] ICS09 handshake done sender: [5:7577471740576354409:2116] self: [5:7577471740576354408:0] peer: [6:7577471740008881477:0] socket: 493 qp: -1
2025-11-27T18:20:22.475442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471740008881482:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1891
2025-11-27T18:20:22.475807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471740576354433:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:22.476266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471741156656918:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:22.475545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471740008881483:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1891
2025-11-27T18:20:22.476637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471739602772731:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:22.477481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471739915690589:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:22.476018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:22.476316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.478376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471739602772639:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.478185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471741156656915:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.478567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471741556727150:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:22.478754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471740008881468:2127] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:22.478417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:22.478879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471740008881484:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1891
2025-11-27T18:20:22.478688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471739602772639:2048] [node 1] ICP22 created new session: [3:7577471739602772743:2048]
2025-11-27T18:20:22.478604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471741556727150:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:22.479891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471739602772743:2048] [node 1] ICS09 handshake done sender: [3:7577471739602772731:2121] self: [3:7577471739602772737:0] peer: [1:7577471741556727145:0] socket: 496 qp: -1
2025-11-27T18:20:22.480079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471739602772743:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:22.480834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471740008881475:2130] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:22.480312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:22.481244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471741156656882:2109] [node 6] ICP18 handshake (actor [4:7577471741156656915:2121]) is too old
2025-11-27T18:20:22.481125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471741556727150:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:22.482173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471739915690694:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46948
2025-11-27T18:20:22.482487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471741556727072:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:22.482362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471740008881450:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.482617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:22.482410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:22.482768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471741556727072:2048] [node 2] ICP22 created new session: [1:7577471741556727176:2048]
2025-11-27T18:20:22.482675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471740008881450:2108] [node 4] ICP22 created new session: [6:7577471740008881485:2108]
2025-11-27T18:20:22.482726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471739915690692:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:22.482996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471740008881485:2108] [node 4] ICS09 handshake done sender: [6:7577471740008881468:2127] self: [6:7577471740008881476:0] peer: [4:7577471741156656894:0] socket: 492 qp: -1
2025-11-27T18:20:22.483080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471741556727176:2048] [node 2] ICS09 handshake done sender: [1:7577471741556727150:2117] self: [1:7577471741556727149:0] peer: [2:7577471739915690693:0] socket: 438 qp: -1
2025-11-27T18:20:22.483158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471739915690589:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.483027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471740008881485:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:22.483128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471741556727176:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:22.483562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:22.483061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:22.483272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:22.483696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471739915690589:2048] [node 1] ICP22 created new session: [2:7577471739915690695:2048]
2025-11-27T18:20:22.483157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471740008881438:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32652) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471740008881470:2128] held: no
2025-11-27T18:20:22.484130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471739915690695:2048] [node 1] ICS09 handshake done sender: [2:7577471739915690692:2121] self: [2:7577471739915690693:0] peer: [1:7577471741556727149:0] socket: 452 qp: -1
2025-11-27T18:20:22.484167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471739915690695:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:22.484205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:22.484841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471740008881379:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.484913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:22.484954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471740008881379:2050] [node 3] ICP22 created new session: [6:7577471740008881487:2050]
2025-11-27T18:20:22.484981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471740008881487:2050] [node 3] ICS09 handshake done sender: [6:7577471740008881475:2130] self: [6:7577471740008881481:0] peer: [3:7577471739602772719:0] socket: 506 qp: -1
2025-11-27T18:20:22.485001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471740008881487:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:22.485032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:22.485067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471739915690589:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:22.486252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471740008881450:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:22.486474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471740008881470:2128] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:22.486991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471740008881438:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:22.487077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:22.487123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471740008881438:2106] [node 5] ICP22 created new session: [6:7577471740008881489:2106]
2025-11-27T18:20:22.487170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471740008881489:2106] [node 5] ICS09 handshake done sender: [6:7577471740008881470:2128] self: [6:7577471740008881477:0] peer: [5:7577471740576354408:0] socket: 497 qp: -1
2025-11-27T18:20:22.487204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471740008881489:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:22.487236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:22.522723Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:22.522944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471739915690695:2048] [node 1] ICS07 socket disconnect 452 reason# EndOfStream
2025-11-27T18:20:22.522970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471739915690695:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.523058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:22.523099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471739915690695:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:22.523764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:22.523897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471739602772743:2048] [node 1] ICS01 socket: 496 reason# EndOfStream
2025-11-27T18:20:22.523927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471739602772639:2048] [node 1] ICP30 unregister session Session# [3:7577471739602772743:2048] VirtualId# [3:7577471739602772737:0]
2025-11-27T18:20:22.523950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471739602772743:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.524010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:22.524057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:22.525123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471739915690589:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27275) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:22.525160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471739915690695:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:22.526165Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471739915690589:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27275) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:22.526217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471739915690695:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:22.526626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471739915690589:2048] [node 1] ICP30 unregister session Session# [2:7577471739915690695:2048] VirtualId# [2:7577471739915690693:0]
2025-11-27T18:20:22.526727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:22.526756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471739915690589:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27275) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:22.526767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27275) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:22.526868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:22.527142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471741156656911:2048] [node 1] ICS01 socket: 477 reason# EndOfStream
2025-11-27T18:20:22.527191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471741156656818:2048] [node 1] ICP30 unregister session Session# [4:7577471741156656911:2048] VirtualId# [4:7577471741156656908:0]
2025-11-27T18:20:22.527218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471741156656911:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.527279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:22.527323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:22.527982Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:22.528217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471740576354428:2048] [node 1] ICS01 socket: 482 reason# EndOfStream
2025-11-27T18:20:22.528263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471740576354330:2048] [node 1] ICP30 unregister session Session# [5:7577471740576354428:2048] VirtualId# [5:7577471740576354421:0]
2025-11-27T18:20:22.528291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471740576354428:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.528374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:22.528439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:22.528786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:22.529001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471740008881471:2048] [node 1] ICS07 socket disconnect 481 reason# EndOfStream
2025-11-27T18:20:22.529041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471740008881471:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.529101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:22.529165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471740008881471:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:22.529857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471740008881377:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27275) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:22.529926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471740008881471:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:22.530021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471740008881377:2048] [node 1] ICP30 unregister session Session# [6:7577471740008881471:2048] VirtualId# [6:7577471740008881465:0]
2025-11-27T18:20:22.530127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:22.530166Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471740008881377:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27275) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:22.530180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27275) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:22.539631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471740008881377:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:22.572091Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:22.572303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471739602772733:2049] [node 2] ICS01 socket: 456 reason# EndOfStream
2025-11-27T18:20:22.572353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471739602772640:2049] [node 2] ICP30 unregister session Session# [3:7577471739602772733:2049] VirtualId# [3:7577471739602772724:0]
2025-11-27T18:20:22.572383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471739602772733:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.572465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:22.572504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:22.574546Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:22.574716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471741156656903:2049] [node 2] ICS01 socket: 453 reason# EndOfStream
2025-11-27T18:20:22.574753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471741156656819:2049] [node 2] ICP30 unregister session Session# [4:7577471741156656903:2049] VirtualId# [4:7577471741156656897:0]
2025-11-27T18:20:22.574773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471741156656903:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.574827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:22.574873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:22.575652Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:22.575830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471740576354431:2049] [node 2] ICS01 socket: 484 reason# EndOfStream
2025-11-27T18:20:22.575866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471740576354331:2049] [node 2] ICP30 unregister session Session# [5:7577471740576354431:2049] VirtualId# [5:7577471740576354420:0]
2025-11-27T18:20:22.575888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471740576354431:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.575961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:22.575999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:22.576739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:22.576970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471740008881478:2049] [node 2] ICS07 socket disconnect 489 reason# EndOfStream
2025-11-27T18:20:22.577022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471740008881478:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.577095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:22.577136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471740008881478:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:22.577858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471740008881378:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25783) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:22.577965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471740008881478:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:22.577999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471740008881378:2049] [node 2] ICP30 unregister session Session# [6:7577471740008881478:2049] VirtualId# [6:7577471740008881469:0]
2025-11-27T18:20:22.578102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:22.578146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471740008881378:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25783) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:22.578165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25783) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:22.587710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471740008881378:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:22.650490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:22.650718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471741156656918:2050] [node 3] ICS01 socket: 440 reason# EndOfStream
2025-11-27T18:20:22.650771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471741156656820:2050] [node 3] ICP30 unregister session Session# [4:7577471741156656918:2050] VirtualId# [4:7577471741156656909:0]
2025-11-27T18:20:22.650816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471741156656918:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.650903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:22.650956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:22.651629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:22.651812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471740576354422:2050] [node 3] ICS01 socket: 463 reason# EndOfStream
2025-11-27T18:20:22.651854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471740576354332:2050] [node 3] ICP30 unregister session Session# [5:7577471740576354422:2050] VirtualId# [5:7577471740576354412:0]
2025-11-27T18:20:22.651883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471740576354422:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.651937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:22.651970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:22.652647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:22.652851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471740008881487:2050] [node 3] ICS01 socket: 506 reason# EndOfStream
2025-11-27T18:20:22.652892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471740008881379:2050] [node 3] ICP30 unregister session Session# [6:7577471740008881487:2050] VirtualId# [6:7577471740008881481:0]
2025-11-27T18:20:22.652921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471740008881487:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.652995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:22.653040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:22.712952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:22.713196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471740576354417:2108] [node 4] ICS01 socket: 447 reason# EndOfStream
2025-11-27T18:20:22.713248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471740576354394:2108] [node 4] ICP30 unregister session Session# [5:7577471740576354417:2108] VirtualId# [5:7577471740576354411:0]
2025-11-27T18:20:22.713302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471740576354417:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.713375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:22.713431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:22.713994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:22.714107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471740008881485:2108] [node 4] ICS07 socket disconnect 492 reason# EndOfStream
2025-11-27T18:20:22.714130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471740008881485:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.714177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:22.714212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471740008881485:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:22.714893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471740008881450:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2637) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:22.714951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471740008881485:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:22.715036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471740008881450:2108] [node 4] ICP30 unregister session Session# [6:7577471740008881485:2108] VirtualId# [6:7577471740008881476:0]
2025-11-27T18:20:22.715118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:22.715141Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471740008881450:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2637) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:22.715151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2637) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:22.724907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471740008881450:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:22.764090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:22.764340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471740008881489:2106] [node 5] ICS01 socket: 497 reason# EndOfStream
2025-11-27T18:20:22.764405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471740008881438:2106] [node 5] ICP30 unregister session Session# [6:7577471740008881489:2106] VirtualId# [6:7577471740008881477:0]
2025-11-27T18:20:22.764442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471740008881489:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:22.764523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:22.764569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:22.871056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471738451740608:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:22.871129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471738451740609:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:22.893651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471741861501004:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:22.893654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471741861501003:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:22.914191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471741617787224:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:22.914225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471741617787223:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:22.937321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471738385733009:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:22.937361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471738385733010:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:22.937390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471738385733011:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:22.957192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471738026050352:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:22.957228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471738026050353:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:22.957250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471738026050354:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:22.977586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471741824753646:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:22.977600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471741824753645:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:22.977601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471741824753644:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:23.068700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471742746707965:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:23.089318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471746156468362:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:23.087943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471746119721000:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:23.102666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471742680700368:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:23.105576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471742746707969:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:23.105482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471742321017711:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:23.102626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471745912754583:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:23.102962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471746156468365:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:23.109621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471745912754584:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:23.110212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471746119721005:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:23.105695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471742680700369:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:23.110209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471742321017712:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:23.110739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471745912754585:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:23.109662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471746156468364:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:23.109626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471742746707968:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:23.117828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471738385733010:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(74571760ns) {Configure:104(74562852ns) {ProcessConfigured:135(395183ns) {StartInitialHandshake:157(392136ns) {DropHandshakes:470(3716ns) {DropIncomingHandshake:442(1359ns) DropOutgoingHandshake:457(502ns)} GenerateSessionVirtualId:988(1118ns) PrepareNewSessionHandshake:477(3022ns) {SwitchToState:179(1282ns)}}}}}
2025-11-27T18:20:23.121279Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471738451740608:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(67756150ns) {Configure:104(67751622ns) {ProcessConfigured:135(544281ns) {StartInitialHandshake:157(541525ns) {DropHandshakes:470(1410ns) {DropIncomingHandshake:442(434ns) DropOutgoingHandshake:457(178ns)} GenerateSessionVirtualId:988(480ns) PrepareNewSessionHandshake:477(1939ns) {SwitchToState:179(1033ns)}}}}}
2025-11-27T18:20:23.125612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471741617787224:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(73016874ns) {Configure:104(73005368ns) {ProcessConfigured:135(703704ns) {StartInitialHandshake:157(702103ns) {DropHandshakes:470(671ns) {DropIncomingHandshake:442(182ns) DropOutgoingHandshake:457(146ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:477(214653ns) {SwitchToState:179(2197ns)}}}}}
2025-11-27T18:20:23.125478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471738385733011:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17931709ns) {Configure:104(17922129ns) {ProcessConfigured:135(719932ns) {StartInitialHandshake:157(718035ns) {DropHandshakes:470(1033ns) {DropIncomingHandshake:442(306ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(2436ns) PrepareNewSessionHandshake:477(1785ns) {SwitchToState:179(973ns)}}}}}
2025-11-27T18:20:23.126092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471741617787223:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(80514287ns) {Configure:104(80507716ns) {ProcessConfigured:135(1074048ns) {StartInitialHandshake:157(1072273ns) {DropHandshakes:470(1132ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(465ns) PrepareNewSessionHandshake:477(1709ns) {SwitchToState:179(779ns)}}}}}
2025-11-27T18:20:23.128654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471741861501004:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16547006ns) {Configure:104(16540022ns) {ProcessConfigured:135(561208ns) {StartInitialHandshake:157(559709ns) {DropHandshakes:470(1101ns) {DropIncomingHandshake:442(446ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:477(1536ns) {SwitchToState:179(547ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.138374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471746119721020:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57956
2025-11-27T18:20:23.139009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471746119721019:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57956
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:23.142766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471742680700383:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55815
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.143922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471742746707981:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13393
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.144210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471742746707982:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13393
2025-11-27T18:20:23.144713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745912754598:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:20:23.144209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471746156468376:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30270
2025-11-27T18:20:23.144213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471742680700384:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55815
2025-11-27T18:20:23.145617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471746119721019:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.146052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471742321017726:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60415
2025-11-27T18:20:23.144783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742746707981:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.145112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471746156468377:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30270
2025-11-27T18:20:23.144803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471742680700383:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.145580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745912754595:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.146556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745912754599:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:20:23.145932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471746156468378:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30270
2025-11-27T18:20:23.145798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471742746707983:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13393
2025-11-27T18:20:23.146572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471741824753644:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.147721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745912754595:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.147832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471746156468376:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.148750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745912754599:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.146015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742746707982:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.147954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471746156468377:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.148412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742746707976:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.151538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471746156468378:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.149478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745912754598:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.152083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471746156468380:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30270
2025-11-27T18:20:23.148468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471742746707976:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.152919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471746119721020:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.152715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471742680700384:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.149834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745912754600:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:20:23.152704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471741861501003:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.153664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471738385733010:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.151994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745912754601:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:20:23.154752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471741824753645:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.149766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471742746707986:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13393
2025-11-27T18:20:23.152454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745912754600:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.153005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471741617787223:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.157445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745912754601:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.158192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471741617787224:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.151825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742746707983:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.158391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471746156468370:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.158439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471746156468370:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.158818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471742321017726:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.154765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742746707971:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.158702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471746156468367:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.154763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471742746707976:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:23.161076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471738026050354:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.162372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745912754592:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.154783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742746707973:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.158735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471746156468380:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.158740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471746156468367:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.154804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471742746707973:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.159538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471746156468373:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.159568Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471746156468373:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.154805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471742746707971:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.162418Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471745912754592:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.162481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471746156468370:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:23.163928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471746119721023:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57956
2025-11-27T18:20:23.163938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745912754609:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:20:23.159524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471742746707971:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:23.159559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742746707986:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.162558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471746156468373:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.164925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471746119721024:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57956
2025-11-27T18:20:23.165302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745912754610:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29472
2025-11-27T18:20:23.162753Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471746156468367:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:23.165886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471741824753644:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471746119721019:2116] held: no
2025-11-27T18:20:23.165848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471745912754598:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:23.160895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471742746707973:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:23.164197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471746156468362:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.161643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471742746707969:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.167264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471742680700387:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55815
2025-11-27T18:20:23.167263Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471746119721019:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:23.165891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471745912754592:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:23.164422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:23.168531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471746119721020:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:23.167285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471741617787223:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.161966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:23.173618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471742321017728:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60415
2025-11-27T18:20:23.173423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471741824753645:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.164479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471746156468362:2107] [node 4] ICP22 created new session: [2:7577471746156468391:2107]
2025-11-27T18:20:23.167307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471745912754601:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:23.162192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471742746707969:2109] [node 6] ICP22 created new session: [1:7577471742746707993:2109]
2025-11-27T18:20:23.167333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:23.173668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:23.164980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471746156468391:2107] [node 4] ICS09 handshake done sender: [2:7577471746156468367:2111] self: [2:7577471746156468366:0] peer: [4:7577471742680700386:0] socket: 460 qp: -1
2025-11-27T18:20:23.174794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471738385733010:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15990) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471742680700384:2116] held: no
2025-11-27T18:20:23.162262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471738451740608:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.162603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471738451740609:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.167648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471741617787223:2048] [node 1] ICP22 created new session: [3:7577471745912754612:2048]
2025-11-27T18:20:23.165001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471746156468365:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.174876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471742680700384:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:23.173744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471741824753645:2049] [node 2] ICP22 created new session: [6:7577471746119721025:2049]
2025-11-27T18:20:23.176938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471742321017726:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:23.167691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471745912754584:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.173811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471746119721025:2049] [node 2] ICS09 handshake done sender: [6:7577471746119721020:2117] self: [6:7577471746119721022:0] peer: [2:7577471746156468368:0] socket: 464 qp: -1
2025-11-27T18:20:23.162655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:23.165160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471741861501004:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.178840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471738026050354:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8307) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471742321017726:2115] held: no
2025-11-27T18:20:23.173950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471746119721025:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:23.162744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:23.167744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:23.165209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471746156468391:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:23.174580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:23.162768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471742746707993:2109] [node 6] ICS09 handshake done sender: [1:7577471742746707976:2112] self: [1:7577471742746707974:0] peer: [6:7577471746119721021:0] socket: 450 qp: -1
2025-11-27T18:20:23.167921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471745912754584:2108] [node 5] ICP22 created new session: [3:7577471745912754613:2108]
2025-11-27T18:20:23.165245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471746156468387:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30270
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.162807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471742746707993:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:23.175054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471741824753644:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.167925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471745912754612:2048] [node 1] ICS09 handshake done sender: [3:7577471745912754598:2116] self: [3:7577471745912754606:0] peer: [1:7577471742746707972:0] socket: 467 qp: -1
2025-11-27T18:20:23.181977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471738385733010:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.165288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:23.175513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:23.182025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:23.168521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471745912754612:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.182324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471738385733010:2049] [node 2] ICP22 created new session: [4:7577471742680700390:2049]
2025-11-27T18:20:23.165421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:23.162862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:23.175798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471741824753644:2048] [node 1] ICP22 created new session: [6:7577471746119721027:2048]
2025-11-27T18:20:23.169280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471745912754613:2108] [node 5] ICS09 handshake done sender: [3:7577471745912754592:2113] self: [3:7577471745912754590:0] peer: [5:7577471742321017727:0] socket: 479 qp: -1
2025-11-27T18:20:23.184293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471738026050354:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.173622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:23.166103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471746156468365:2109] [node 6] ICP22 created new session: [2:7577471746156468393:2109]
2025-11-27T18:20:23.162946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471738451740608:2048] [node 2] ICP22 created new session: [1:7577471742746707995:2048]
2025-11-27T18:20:23.182846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471742680700390:2049] [node 2] ICS09 handshake done sender: [4:7577471742680700384:2116] self: [4:7577471742680700386:0] peer: [2:7577471746156468366:0] socket: 474 qp: -1
2025-11-27T18:20:23.176585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471746119721027:2048] [node 1] ICS09 handshake done sender: [6:7577471746119721019:2116] self: [6:7577471746119721021:0] peer: [1:7577471742746707974:0] socket: 458 qp: -1
2025-11-27T18:20:23.166636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:23.163019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471738451740609:2049] [node 3] ICP22 created new session: [1:7577471742746707996:2049]
2025-11-27T18:20:23.173643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471745912754613:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:23.182848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471742680700388:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55815
2025-11-27T18:20:23.163071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471742746707995:2048] [node 2] ICS09 handshake done sender: [1:7577471742746707971:2110] self: [1:7577471742746707970:0] peer: [2:7577471746156468384:0] socket: 446 qp: -1
2025-11-27T18:20:23.166977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471741861501004:2049] [node 3] ICP22 created new session: [2:7577471746156468394:2049]
2025-11-27T18:20:23.183541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471742680700390:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:23.163648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471742746707995:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:23.183584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:23.167147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471746156468393:2109] [node 6] ICS09 handshake done sender: [2:7577471746156468370:2112] self: [2:7577471746156468368:0] peer: [6:7577471746119721022:0] socket: 461 qp: -1
2025-11-27T18:20:23.184338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.176619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471746119721027:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:23.173708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.163972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471742746707996:2049] [node 3] ICS09 handshake done sender: [1:7577471742746707973:2111] self: [1:7577471742746707972:0] peer: [3:7577471745912754606:0] socket: 448 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.184498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471738026050354:2050] [node 3] ICP22 created new session: [5:7577471742321017734:2050]
2025-11-27T18:20:23.177238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:23.185846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471742680700388:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.167292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471746156468393:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:23.174692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471741617787224:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.164012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471742746707996:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:23.184791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471742321017729:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60415
2025-11-27T18:20:23.167369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471746156468394:2049] [node 3] ICS09 handshake done sender: [2:7577471746156468373:2114] self: [2:7577471746156468372:0] peer: [3:7577471745912754608:0] socket: 477 qp: -1
2025-11-27T18:20:23.190152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471738385733009:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.174908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:23.191172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471746119721031:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57956
2025-11-27T18:20:23.185321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471742321017734:2050] [node 3] ICS09 handshake done sender: [5:7577471742321017726:2115] self: [5:7577471742321017727:0] peer: [3:7577471745912754590:0] socket: 481 qp: -1
2025-11-27T18:20:23.164043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:23.167725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471746156468394:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:23.191195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471746119721030:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57956
2025-11-27T18:20:23.176125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471741617787224:2049] [node 2] ICP22 created new session: [3:7577471745912754616:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.192602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471742680700392:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55815
2025-11-27T18:20:23.185668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471742321017734:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:23.167942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:23.164229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:23.176429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471745912754616:2049] [node 2] ICS09 handshake done sender: [3:7577471745912754601:2119] self: [3:7577471745912754608:0] peer: [2:7577471746156468372:0] socket: 483 qp: -1
2025-11-27T18:20:23.167944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:23.197690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742746707977:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.189427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471742321017730:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60415
2025-11-27T18:20:23.180643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471746156468377:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:23.176930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471745912754616:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:23.197661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471746119721030:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.197732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471742746707977:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.189464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:23.176965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:23.182848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471741861501003:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.199185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471742680700377:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.198971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471746119721000:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.191730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471742321017731:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60415
2025-11-27T18:20:23.199453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471738451740608:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(22374498ns) {HandleHandshakeStatus:329(22371107ns) {DropIncomingHandshake:442(421ns) DropOutgoingHandshake:457(306ns) SwitchToState:179(2873ns) ProcessPendingSessionEvents:495(19281529ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19158775ns max# 4362472ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.183231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:23.199194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471742680700392:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.199200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471746119721031:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.191757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471742321017730:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.199225Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471742680700377:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.201426Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471742746707969:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(27818249ns) {HandleHandshakeStatus:329(27815589ns) {DropIncomingHandshake:442(13248ns) DropOutgoingHandshake:457(1416ns) SwitchToState:179(395ns) ProcessPendingSessionEvents:495(24131245ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23990356ns max# 4841915ns}}}
2025-11-27T18:20:23.183504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471741861501003:2048] [node 1] ICP22 created new session: [2:7577471746156468399:2048]
2025-11-27T18:20:23.200302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471741824753646:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.194265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471742321017731:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.201592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742746707985:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.202885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745912754593:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.184386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471746156468399:2048] [node 1] ICS09 handshake done sender: [2:7577471746156468377:2116] self: [2:7577471746156468384:0] peer: [1:7577471742746707970:0] socket: 455 qp: -1
2025-11-27T18:20:23.199479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471738385733011:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.202929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471745912754593:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.201644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471742746707985:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.194283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471742321017729:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.184647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471746156468399:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:23.205449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471742680700377:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.203669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471738451740609:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(27598033ns) {HandleHandshakeStatus:329(27596748ns) {DropIncomingHandshake:442(12476ns) DropOutgoingHandshake:457(589ns) SwitchToState:179(112ns) ProcessPendingSessionEvents:495(24866284ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24017407ns max# 4082885ns}}}
2025-11-27T18:20:23.197771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471742321017733:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60415
2025-11-27T18:20:23.209836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471742680700368:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.210553Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471745912754584:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(30175046ns) {HandleHandshakeStatus:329(30005232ns) {DropIncomingHandshake:442(11601ns) DropOutgoingHandshake:457(270ns) SwitchToState:179(242ns) ProcessPendingSessionEvents:495(22914491ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22199863ns max# 3885909ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.210261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:23.211323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745912754603:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.184792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: 2025-11-27T18:20:23.197915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471738026050352:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-27T18:20:23.211118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471742680700368:2108] [node 5] ICP22 created new session: [4:7577471742680700396:2108]
2025-11-27T18:20:23.204028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471742746708002:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13393
2025-11-27T18:20:23.201025Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471746156468362:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(26148910ns) {HandleHandshakeStatus:329(26006084ns) {DropIncomingHandshake:442(11808ns) DropOutgoingHandshake:457(2315ns) SwitchToState:179(899ns) ProcessPendingSessionEvents:495(22830712ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21985356ns max# 4509953ns}}}
2025-11-27T18:20:23.219744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471741824753645:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(29147462ns) {HandleHandshakeStatus:329(29144948ns) {DropIncomingHandshake:442(1742ns) DropOutgoingHandshake:457(6758ns) SwitchToState:179(679ns) ProcessPendingSessionEvents:495(22317950ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22203215ns max# 4199754ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.211355Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471745912754603:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.198680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471742321017712:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.211680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471742680700396:2108] [node 5] ICS09 handshake done sender: [4:7577471742680700377:2113] self: [4:7577471742680700376:0] peer: [5:7577471742321017737:0] socket: 445 qp: -1
2025-11-27T18:20:23.209936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471741861501004:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(27028348ns) {HandleHandshakeStatus:329(27025641ns) {DropIncomingHandshake:442(11595ns) DropOutgoingHandshake:457(379ns) SwitchToState:179(401ns) ProcessPendingSessionEvents:495(17767590ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17654879ns max# 3937045ns}}}
2025-11-27T18:20:23.221712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471746119721034:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57956
2025-11-27T18:20:23.216316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471742746707977:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:23.220590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471745912754593:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:23.200775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471742321017720:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.215246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471742680700396:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:23.219302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471742746707965:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.221115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471741617787223:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(30589042ns) {HandleHandshakeStatus:329(30586215ns) {DropIncomingHandshake:442(1293ns) DropOutgoingHandshake:457(5354ns) SwitchToState:179(172ns) ProcessPendingSessionEvents:495(22967094ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22841405ns max# 3935967ns}}}
2025-11-27T18:20:23.211031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471746156468365:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(32570191ns) {HandleHandshakeStatus:329(32567184ns) {DropIncomingHandshake:442(335ns) DropOutgoingHandshake:457(317ns) SwitchToState:179(252ns) ProcessPendingSessionEvents:495(23883099ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23747709ns max# 4074840ns}}}
2025-11-27T18:20:23.200807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471742321017720:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.216263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:23.219359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:23.223457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471745912754603:2120] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:23.225085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471746119721000:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65515) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471746119721030:2120] held: no
2025-11-27T18:20:23.221051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471742680700398:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55815
2025-11-27T18:20:23.219411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471742746707965:2107] [node 4] ICP22 created new session: [1:7577471742746708005:2107]
2025-11-27T18:20:23.205428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471742321017720:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.225445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471741617787224:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(27810051ns) {HandleHandshakeStatus:329(27806788ns) {DropIncomingHandshake:442(1649ns) DropOutgoingHandshake:457(173141ns) SwitchToState:179(454ns) ProcessPendingSessionEvents:495(22751407ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22611586ns max# 3785489ns}}}
2025-11-27T18:20:23.225522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471745912754583:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.225593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:23.220651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471742746708005:2107] [node 4] ICS09 handshake done sender: [1:7577471742746707977:2113] self: [1:7577471742746707975:0] peer: [4:7577471742680700393:0] socket: 452 qp: -1
2025-11-27T18:20:23.215247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471742321017711:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.226291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471745912754583:2107] [node 4] ICP22 created new session: [3:7577471745912754622:2107]
2025-11-27T18:20:23.234584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471746119721035:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57956
2025-11-27T18:20:23.235100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471742680700388:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:23.216209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:23.221091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471742746708005:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:23.226422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471745912754585:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.221766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:23.216803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471742321017743:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60415
2025-11-27T18:20:23.226495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.218735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471742321017711:2109] [node 6] ICP22 created new session: [5:7577471742321017741:2109]
2025-11-27T18:20:23.244682Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471738385733009:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471742680700388:2118] held: no
2025-11-27T18:20:23.222817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471742746707985:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:23.226749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471745912754585:2109] [node 6] ICP22 created new session: [3:7577471745912754623:2109]
2025-11-27T18:20:23.227275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471742746707968:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.219028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471742321017741:2109] [node 6] ICS09 handshake done sender: [5:7577471742321017720:2113] self: [5:7577471742321017719:0] peer: [6:7577471746119721032:0] socket: 501 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.233461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:23.226790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471745912754622:2107] [node 4] ICS09 handshake done sender: [3:7577471745912754593:2114] self: [3:7577471745912754591:0] peer: [4:7577471742680700394:0] socket: 504 qp: -1
2025-11-27T18:20:23.219730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471742321017741:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:23.235680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471742746707968:2108] [node 5] ICP22 created new session: [1:7577471742746708007:2108]
2025-11-27T18:20:23.241745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471742746708007:2108] [node 5] ICS09 handshake done sender: [1:7577471742746707985:2117] self: [1:7577471742746707984:0] peer: [5:7577471742321017738:0] socket: 459 qp: -1
2025-11-27T18:20:23.245968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471742680700400:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55815
2025-11-27T18:20:23.220815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:23.227223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471745912754623:2109] [node 6] ICS09 handshake done sender: [3:7577471745912754603:2120] self: [3:7577471745912754602:0] peer: [6:7577471746119721033:0] socket: 506 qp: -1
2025-11-27T18:20:23.245330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471746119721036:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57956
2025-11-27T18:20:23.241983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471742746708007:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:23.223286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471742321017740:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60415
2025-11-27T18:20:23.227073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471745912754622:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:23.245800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471746119721035:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.253980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471741861501003:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(43235861ns) {HandleHandshakeStatus:329(43233628ns) {DropIncomingHandshake:442(1158ns) DropOutgoingHandshake:457(5059ns) SwitchToState:179(177ns) ProcessPendingSessionEvents:495(36261764ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 34386844ns max# 6432522ns}}}
2025-11-27T18:20:23.242580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:23.228061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471745912754623:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:23.228098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:23.224242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471742321017729:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:23.228123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: 2025-11-27T18:20:23.225723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471742321017731:2119] [node 4] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:23.226645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471738026050352:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.226674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471742321017712:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.226806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:23.227205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471738026050352:2048] [node 1] ICP22 created new session: [5:7577471742321017745:2048]
2025-11-27T18:20:23.228134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:23.254924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471742680700401:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55815
2025-11-27T18:20:23.233596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471742321017745:2048] [node 1] ICS09 handshake done sender: [5:7577471742321017729:2117] self: [5:7577471742321017738:0] peer: [1:7577471742746707984:0] socket: 478 qp: -1
2025-11-27T18:20:23.234647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471742321017712:2108] [node 4] ICP22 created new session: [5:7577471742321017746:2108]
2025-11-27T18:20:23.262065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471738385733009:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.264035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:23.262013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471746119721000:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(20461214ns) {HandleHandshakeStatus:395(20457539ns) {LogHandshakeFail:472(14429ns) {UpdateErrorStateLog:545(3178ns)} DropOutgoingHandshake:457(1967ns) ProcessPendingSessionEvents:495(19444828ns) {unordered DropSessionEvent:512 num# 100 sum# 19384073ns max# 6675185ns}}}
2025-11-27T18:20:23.234656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471742321017745:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:23.234704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:23.264469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471738385733009:2048] [node 1] ICP22 created new session: [4:7577471742680700402:2048]
2025-11-27T18:20:23.241575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471742321017746:2108] [node 4] ICS09 handshake done sender: [5:7577471742321017731:2119] self: [5:7577471742321017737:0] peer: [4:7577471742680700376:0] socket: 484 qp: -1
2025-11-27T18:20:23.241936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471742321017746:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:23.264670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471742680700402:2048] [node 1] ICS09 handshake done sender: [4:7577471742680700388:2118] self: [4:7577471742680700393:0] peer: [1:7577471742746707975:0] socket: 463 qp: -1
2025-11-27T18:20:23.242498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:23.264707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471742680700402:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:23.264751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:23.265499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471738385733011:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8307) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471742680700392:2119] held: no
2025-11-27T18:20:23.265492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471746119721031:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:23.265505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471741824753646:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8307) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471746119721031:2121] held: no
2025-11-27T18:20:23.266193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471742321017711:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(43178818ns) {HandleHandshakeStatus:329(42836623ns) {DropIncomingHandshake:442(20509ns) DropOutgoingHandshake:457(4261ns) SwitchToState:179(1282ns) ProcessPendingSessionEvents:495(30651313ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29351492ns max# 5746781ns}}}
2025-11-27T18:20:23.266059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471746119721005:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.266474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471742321017733:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.266619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471742680700392:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:23.266489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471742746707965:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(35372444ns) {HandleHandshakeStatus:329(35369389ns) {DropIncomingHandshake:442(12969ns) DropOutgoingHandshake:457(437ns) SwitchToState:179(678ns) ProcessPendingSessionEvents:495(30138006ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 30018080ns max# 6401367ns}}}
2025-11-27T18:20:23.266461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471745912754585:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(29049546ns) {HandleHandshakeStatus:329(29046521ns) {DropIncomingHandshake:442(22885ns) DropOutgoingHandshake:457(533ns) SwitchToState:179(480ns) ProcessPendingSessionEvents:495(13100267ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13014595ns max# 3629350ns}}}
2025-11-27T18:20:23.266640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471742680700368:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(44510870ns) {HandleHandshakeStatus:329(44507583ns) {DropIncomingHandshake:442(472ns) DropOutgoingHandshake:457(645ns) SwitchToState:179(672ns) ProcessPendingSessionEvents:495(37909973ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36821915ns max# 5742129ns}}}
2025-11-27T18:20:23.267213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471746119721030:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:23.267432Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471745912754583:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(34024096ns) {HandleHandshakeStatus:329(34021022ns) {DropIncomingHandshake:442(21333ns) DropOutgoingHandshake:457(534ns) SwitchToState:179(474ns) ProcessPendingSessionEvents:495(24394455ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24307682ns max# 6773009ns}}}
2025-11-27T18:20:23.268116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471738026050352:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(34409322ns) {HandleHandshakeStatus:329(34406971ns) {DropIncomingHandshake:442(1333ns) DropOutgoingHandshake:457(6537ns) SwitchToState:179(313ns) ProcessPendingSessionEvents:495(16375069ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15902860ns max# 4126466ns}}}
2025-11-27T18:20:23.268460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471742746707968:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(37671297ns) {HandleHandshakeStatus:329(37669171ns) {DropIncomingHandshake:442(11206ns) DropOutgoingHandshake:457(281ns) SwitchToState:179(388ns) ProcessPendingSessionEvents:495(20571348ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20382985ns max# 3748345ns}}}
2025-11-27T18:20:23.267954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471741824753646:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.268020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471738385733011:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.268620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471742321017712:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(38113835ns) {HandleHandshakeStatus:329(38013937ns) {DropIncomingHandshake:442(425ns) DropOutgoingHandshake:457(6266ns) SwitchToState:179(333ns) ProcessPendingSessionEvents:495(19241099ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18734657ns max# 4380633ns}}}
2025-11-27T18:20:23.268578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:23.268458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471742680700379:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.269725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471738026050353:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.268699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471741824753646:2050] [node 3] ICP22 created new session: [6:7577471746119721038:2050]
2025-11-27T18:20:23.268484Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471742680700379:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.268728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471746119721000:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.268498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471742680700368:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:23.268771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:23.268552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:23.269998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471746156468382:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.269705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471746119721038:2050] [node 3] ICS09 handshake done sender: [6:7577471746119721031:2121] self: [6:7577471746119721033:0] peer: [3:7577471745912754602:0] socket: 507 qp: -1
2025-11-27T18:20:23.270041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471746156468382:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.268740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471738385733011:2050] [node 3] ICP22 created new session: [4:7577471742680700404:2050]
2025-11-27T18:20:23.270000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471742680700404:2050] [node 3] ICS09 handshake done sender: [4:7577471742680700392:2119] self: [4:7577471742680700394:0] peer: [3:7577471745912754591:0] socket: 505 qp: -1
2025-11-27T18:20:23.269888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471746119721038:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:23.270043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471742680700404:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:23.269900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471746119721000:2106] [node 5] ICP22 created new session: [6:7577471746119721039:2106]
2025-11-27T18:20:23.270189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:23.270489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471742680700379:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:23.269946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:23.270277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471746119721039:2106] [node 5] ICS09 handshake done sender: [6:7577471746119721030:2120] self: [6:7577471746119721032:0] peer: [5:7577471742321017719:0] socket: 502 qp: -1
2025-11-27T18:20:23.270434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471746119721039:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:23.271199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471742680700369:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.270627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:23.272033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:23.272081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471742680700369:2109] [node 6] ICP22 created new session: [4:7577471742680700407:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.272214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471742680700407:2109] [node 6] ICS09 handshake done sender: [4:7577471742680700379:2114] self: [4:7577471742680700378:0] peer: [6:7577471746119721037:0] socket: 512 qp: -1
2025-11-27T18:20:23.272261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471742680700407:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:23.272297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:23.273292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471746119721041:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57956
2025-11-27T18:20:23.273714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471746156468382:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:23.274570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471746156468364:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.275237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471746119721005:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2010) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471746119721035:2123] held: no
2025-11-27T18:20:23.275265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:23.275318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471746156468364:2108] [node 5] ICP22 created new session: [2:7577471746156468402:2108]
2025-11-27T18:20:23.275508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471742321017750:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60415
2025-11-27T18:20:23.275536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471746156468402:2108] [node 5] ICS09 handshake done sender: [2:7577471746156468382:2119] self: [2:7577471746156468381:0] peer: [5:7577471742321017749:0] socket: 500 qp: -1
2025-11-27T18:20:23.275760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471746156468402:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:23.275805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:23.277011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471738026050353:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15990) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471742321017733:2120] held: no
2025-11-27T18:20:23.277109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471746119721035:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:23.277342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471746119721005:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.277393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:23.277433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471746119721005:2107] [node 4] ICP22 created new session: [6:7577471746119721043:2107]
2025-11-27T18:20:23.277480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471746119721043:2107] [node 4] ICS09 handshake done sender: [6:7577471746119721035:2123] self: [6:7577471746119721037:0] peer: [4:7577471742680700378:0] socket: 513 qp: -1
2025-11-27T18:20:23.277522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471746119721043:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:23.277584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:23.277722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471742321017733:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:23.277883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471738026050353:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.277928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:23.277983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471738026050353:2049] [node 2] ICP22 created new session: [5:7577471742321017751:2049]
2025-11-27T18:20:23.278023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471742321017751:2049] [node 2] ICS09 handshake done sender: [5:7577471742321017733:2120] self: [5:7577471742321017749:0] peer: [2:7577471746156468381:0] socket: 475 qp: -1
2025-11-27T18:20:23.278078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471742321017751:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:23.278116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:23.320106Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:23.320366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471746156468399:2048] [node 1] ICS07 socket disconnect 455 reason# EndOfStream
2025-11-27T18:20:23.320407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471746156468399:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.320504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:23.320565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471746156468399:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:23.321405Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:23.321536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471745912754612:2048] [node 1] ICS07 socket disconnect 467 reason# EndOfStream
2025-11-27T18:20:23.321560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471745912754612:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.321613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:23.321640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471745912754612:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:23.322800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471741861501003:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Socket error# Connection reset by peer state# SendExRequestHeader processed# 0 remain# 8 incoming: [0:0:0] held: no
2025-11-27T18:20:23.322876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471746156468399:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:23.323874Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471741617787223:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:23.323913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471745912754612:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:23.324896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:23.324953Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471741861501003:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:23.325035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471746156468399:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:23.325069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471741861501003:2048] [node 1] ICP30 unregister session Session# [2:7577471746156468399:2048] VirtualId# [2:7577471746156468384:0]
2025-11-27T18:20:23.325484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471742680700402:2048] [node 1] ICS01 socket: 463 reason# EndOfStream
2025-11-27T18:20:23.325540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:23.325581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471741861501003:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.325542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471738385733009:2048] [node 1] ICP30 unregister session Session# [4:7577471742680700402:2048] VirtualId# [4:7577471742680700393:0]
2025-11-27T18:20:23.325595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.325577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471742680700402:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.325662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471741617787223:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:23.325640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:23.325756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471745912754612:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:23.325701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:23.325801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471741617787223:2048] [node 1] ICP30 unregister session Session# [3:7577471745912754612:2048] VirtualId# [3:7577471745912754606:0]
2025-11-27T18:20:23.325943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:23.325990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471741617787223:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.326015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.326621Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:23.326838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471742321017745:2048] [node 1] ICS07 socket disconnect 478 reason# EndOfStream
2025-11-27T18:20:23.326879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471742321017745:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.327066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:23.327129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471742321017745:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:23.327666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:23.327725Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471738026050352:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:23.327843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471742321017745:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:23.327878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471738026050352:2048] [node 1] ICP30 unregister session Session# [5:7577471742321017745:2048] VirtualId# [5:7577471742321017738:0]
2025-11-27T18:20:23.327806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471746119721027:2048] [node 1] ICS01 socket: 458 reason# EndOfStream
2025-11-27T18:20:23.327931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:23.327842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471741824753644:2048] [node 1] ICP30 unregister session Session# [6:7577471746119721027:2048] VirtualId# [6:7577471746119721021:0]
2025-11-27T18:20:23.327862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471746119721027:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.327980Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471738026050352:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.327994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.327904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:23.327930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:23.336256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471741617787223:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:23.338316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471738026050352:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:23.375867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:23.376106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471745912754616:2049] [node 2] ICS07 socket disconnect 483 reason# EndOfStream
2025-11-27T18:20:23.376141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471745912754616:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.376235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:23.376292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471745912754616:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:23.378692Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:23.378771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471741617787224:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15990) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:23.378818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471745912754616:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:23.378952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471742680700390:2049] [node 2] ICS01 socket: 474 reason# EndOfStream
2025-11-27T18:20:23.379010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471738385733010:2049] [node 2] ICP30 unregister session Session# [4:7577471742680700390:2049] VirtualId# [4:7577471742680700386:0]
2025-11-27T18:20:23.379049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471742680700390:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.379150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:23.379207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:23.379867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471741617787224:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15990) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:23.379971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471745912754616:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:23.380011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471741617787224:2049] [node 2] ICP30 unregister session Session# [3:7577471745912754616:2049] VirtualId# [3:7577471745912754608:0]
2025-11-27T18:20:23.380117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:23.380159Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471741617787224:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15990) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.380176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15990) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.380237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:23.380441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471742321017751:2049] [node 2] ICS01 socket: 475 reason# EndOfStream
2025-11-27T18:20:23.380505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471738026050353:2049] [node 2] ICP30 unregister session Session# [5:7577471742321017751:2049] VirtualId# [5:7577471742321017749:0]
2025-11-27T18:20:23.380535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471742321017751:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.380594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:23.380645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:23.381395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:23.381612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471746119721025:2049] [node 2] ICS07 socket disconnect 464 reason# EndOfStream
2025-11-27T18:20:23.381649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471746119721025:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.381713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:23.381769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471746119721025:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:23.382483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471741824753645:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15990) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:23.382551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471746119721025:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:23.382586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471741824753645:2049] [node 2] ICP30 unregister session Session# [6:7577471746119721025:2049] VirtualId# [6:7577471746119721022:0]
2025-11-27T18:20:23.382679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:23.382713Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471741824753645:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15990) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.382728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15990) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.390486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471741617787224:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:23.392560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471741824753645:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:23.455356Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:23.455531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471742680700404:2050] [node 3] ICS01 socket: 505 reason# EndOfStream
2025-11-27T18:20:23.455564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471738385733011:2050] [node 3] ICP30 unregister session Session# [4:7577471742680700404:2050] VirtualId# [4:7577471742680700394:0]
2025-11-27T18:20:23.455587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471742680700404:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.455641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:23.455673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:23.456384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:23.456581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471742321017734:2050] [node 3] ICS01 socket: 481 reason# EndOfStream
2025-11-27T18:20:23.456637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471738026050354:2050] [node 3] ICP30 unregister session Session# [5:7577471742321017734:2050] VirtualId# [5:7577471742321017727:0]
2025-11-27T18:20:23.456672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471742321017734:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.456736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:23.456793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:23.457435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:23.457617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471746119721038:2050] [node 3] ICS01 socket: 507 reason# EndOfStream
2025-11-27T18:20:23.457657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471741824753646:2050] [node 3] ICP30 unregister session Session# [6:7577471746119721038:2050] VirtualId# [6:7577471746119721033:0]
2025-11-27T18:20:23.457684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471746119721038:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.457741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:23.457788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:23.508070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:23.508334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471742321017746:2108] [node 4] ICS07 socket disconnect 484 reason# EndOfStream
2025-11-27T18:20:23.508370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471742321017746:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.508464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:23.508521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471742321017746:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:23.509299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471742321017712:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2010) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:23.509283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:23.509405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471742321017746:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:23.509476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471742321017712:2108] [node 4] ICP30 unregister session Session# [5:7577471742321017746:2108] VirtualId# [5:7577471742321017737:0]
2025-11-27T18:20:23.509512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471746119721043:2107] [node 4] ICS01 socket: 513 reason# EndOfStream
2025-11-27T18:20:23.509558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471746119721005:2107] [node 4] ICP30 unregister session Session# [6:7577471746119721043:2107] VirtualId# [6:7577471746119721037:0]
2025-11-27T18:20:23.509589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471746119721043:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.509585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:23.509624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471742321017712:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2010) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.509650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:23.509639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2010) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:23.509700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:23.519573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471742321017712:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:23.586908Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:23.587150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471746119721039:2106] [node 5] ICS01 socket: 502 reason# EndOfStream
2025-11-27T18:20:23.587199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471746119721000:2106] [node 5] ICP30 unregister session Session# [6:7577471746119721039:2106] VirtualId# [6:7577471746119721032:0]
2025-11-27T18:20:23.587231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471746119721039:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:23.587299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:23.587348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:23.679577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471742900927179:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:23.679686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471742900927178:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:23.699701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471743536739335:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:23.700022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471743536739336:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:23.720158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471745055850365:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:23.720522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471745055850366:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:23.740372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471745346385152:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:23.740457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471745346385151:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:23.740660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471745346385153:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:23.763408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471746059145133:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:23.763471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471746059145135:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:23.763861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471746059145134:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:23.784960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471745431196023:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:23.785026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471745431196025:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:23.785239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471745431196024:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:23.882843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471745431196084:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:23.891753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471742900927242:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:23.913545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471745055850440:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:23.913393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471743536739407:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:23.903043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471742900927243:2108] [node 6] ICP01 ready to work
2025-11-27T18:20:23.924003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471745055850439:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:23.913923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471742900927244:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:23.918159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471745055850438:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:23.924010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471743536739408:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:23.933149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471745055850365:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34698846ns) {Configure:104(34691576ns) {ProcessConfigured:135(62966ns) {StartInitialHandshake:157(59847ns) {DropHandshakes:470(2575ns) {DropIncomingHandshake:442(475ns) DropOutgoingHandshake:457(295ns)} GenerateSessionVirtualId:988(909ns) PrepareNewSessionHandshake:477(1292ns) {SwitchToState:179(529ns)}}}}}
2025-11-27T18:20:23.924113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471743536739409:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:23.933524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471746059145198:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:23.930505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471745431196097:2110] [node 4] ICP01 ready to work
2025-11-27T18:20:23.930901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471746059145197:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:23.930612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471745346385214:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:23.933794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471745346385215:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:23.939914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471742900927179:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(66302593ns) {Configure:104(66294783ns) {ProcessConfigured:135(581126ns) {StartInitialHandshake:157(577949ns) {DropHandshakes:470(2741ns) {DropIncomingHandshake:442(651ns) DropOutgoingHandshake:457(365ns)} GenerateSessionVirtualId:988(969ns) PrepareNewSessionHandshake:477(2185ns) {SwitchToState:179(911ns)}}}}}
2025-11-27T18:20:23.943135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471742900927178:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23896793ns) {Configure:104(23887321ns) {ProcessConfigured:135(932914ns) {StartInitialHandshake:157(930913ns) {DropHandshakes:470(2801ns) {DropIncomingHandshake:442(520ns) DropOutgoingHandshake:457(381ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:477(2634ns) {SwitchToState:179(1377ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.946089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471745346385151:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20650596ns) {Configure:104(20642546ns) {ProcessConfigured:135(423631ns) {StartInitialHandshake:157(421780ns) {DropHandshakes:470(1080ns) {DropIncomingHandshake:442(304ns) DropOutgoingHandshake:457(215ns)} GenerateSessionVirtualId:988(382ns) PrepareNewSessionHandshake:477(789ns) {SwitchToState:179(111ns)}}}}}
2025-11-27T18:20:23.947394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471742900927247:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27690
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.948773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742900927247:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.951259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471745346385230:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39197
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.949803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471746059145210:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25092
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:23.958633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471745346385229:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39197
2025-11-27T18:20:23.957486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471746059145210:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:23.964484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471745346385229:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.965488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471743536739425:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3133
2025-11-27T18:20:23.965490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471743536739422:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3133
2025-11-27T18:20:23.965613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471746059145135:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.965583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745055850455:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46917
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.966430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471742900927259:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27690
2025-11-27T18:20:23.966525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745055850448:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.968229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471745346385151:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.968191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471745431196108:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49737
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.966554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471745055850448:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.966034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471743536739424:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3133
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.969114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471745431196108:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.969680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471743536739422:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.969572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471745346385231:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39197
2025-11-27T18:20:23.969601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742900927254:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.969591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471746059145214:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25092
2025-11-27T18:20:23.968150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745055850454:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46917
2025-11-27T18:20:23.969660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471745346385230:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.972167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471745431196109:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49737
2025-11-27T18:20:23.969649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471742900927254:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.972126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471746059145213:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25092
2025-11-27T18:20:23.968372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745055850456:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46917
2025-11-27T18:20:23.972680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471745431196084:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.971576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471743536739425:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.968649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745055850455:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.969878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742900927259:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.976747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471745431196109:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.975279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471746059145205:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.972162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471743536739424:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.972101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471742900927261:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27690
2025-11-27T18:20:23.969174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471745055850448:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:23.975273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471743536739426:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3133
2025-11-27T18:20:23.979651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471745431196024:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.975319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471746059145205:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.980031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471745346385231:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.969299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471745055850365:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.975406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471743536739335:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.972694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742900927246:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.981718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471745346385218:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.977994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471743536739426:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.972735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471742900927246:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.977652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471746059145213:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.969865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745055850456:2127] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.981757Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471745346385218:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.979671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471746059145198:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.976313Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471742900927254:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:23.980251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471743536739415:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.984427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471745346385218:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:23.977590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745055850454:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.979774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471746059145214:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.976605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742900927261:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.980294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471743536739415:2120] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-27T18:20:23.986750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471745346385234:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39197
::ffff:127.1.0.02025-11-27T18:20:23.977625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471745055850439:2109] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.976682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471742900927246:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:23.980708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471746059145216:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25092
2025-11-27T18:20:23.982353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471743536739411:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.977976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:23.992639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471745431196084:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1122) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471745431196108:2125] held: no
2025-11-27T18:20:23.991950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471745346385214:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.979435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742900927250:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.979737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471745055850366:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:23.982383Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471743536739411:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.981876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471746059145205:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:23.979486Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471742900927250:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.979867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471745055850439:2109] [node 5] ICP22 created new session: [3:7577471745055850460:2109]
2025-11-27T18:20:23.992180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:23.987333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471743536739415:2120] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.986818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471746059145210:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:23.992193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471745346385229:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:23.982888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471742900927179:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.994614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471745431196115:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49737
2025-11-27T18:20:23.987341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471743536739411:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:23.980288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471745055850460:2109] [node 5] ICS09 handshake done sender: [3:7577471745055850448:2122] self: [3:7577471745055850445:0] peer: [5:7577471746059145215:0] socket: 453 qp: -1
2025-11-27T18:20:23.987505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471746059145197:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.980770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471745055850460:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.982897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471742900927242:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.992245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471745346385236:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39197
2025-11-27T18:20:23.987659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:23.995532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471743536739336:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.982947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.995089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471745346385214:2108] [node 5] ICP22 created new session: [4:7577471745346385238:2108]
2025-11-27T18:20:24.008795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471745431196116:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49737
No RDMA context found for GID: 2025-11-27T18:20:23.981788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
::ffff:127.1.0.0
2025-11-27T18:20:24.002889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:23.987724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471746059145135:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.982994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:24.009988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471745431196117:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49737
2025-11-27T18:20:24.003395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471743536739336:2049] [node 3] ICP22 created new session: [2:7577471747831706730:2049]
2025-11-27T18:20:23.987580Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471745055850365:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10860) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471745055850455:2126] held: no
2025-11-27T18:20:23.996130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471745346385238:2108] [node 5] ICS09 handshake done sender: [4:7577471745346385218:2110] self: [4:7577471745346385217:0] peer: [5:7577471746059145217:0] socket: 475 qp: -1
2025-11-27T18:20:23.988082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:23.983676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471742900927242:2107] [node 4] ICP22 created new session: [1:7577471742900927268:2107]
2025-11-27T18:20:23.996174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471745346385238:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:23.987822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745055850462:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46917
2025-11-27T18:20:24.004319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471743536739408:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.014369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471745431196024:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15628) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471745431196109:2126] held: no
2025-11-27T18:20:23.989666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471746059145197:2109] [node 6] ICP22 created new session: [5:7577471746059145221:2109]
2025-11-27T18:20:23.996236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.996174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471745055850465:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46917
2025-11-27T18:20:23.983891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471742900927179:2049] [node 3] ICP22 created new session: [1:7577471742900927267:2049]
2025-11-27T18:20:24.005140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471743536739433:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3133
2025-11-27T18:20:23.990139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471746059145135:2050] [node 3] ICP22 created new session: [5:7577471746059145222:2050]
2025-11-27T18:20:24.014796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471745431196117:2129] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.005174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471745055850454:2125] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:23.985764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471742900927267:2049] [node 3] ICS09 handshake done sender: [1:7577471742900927246:2111] self: [1:7577471742900927245:0] peer: [3:7577471745055850458:0] socket: 461 qp: -1
2025-11-27T18:20:24.005173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:23.996671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471745346385151:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10860) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471745346385229:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:23.992137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471746059145222:2050] [node 3] ICS09 handshake done sender: [5:7577471746059145210:2116] self: [5:7577471746059145215:0] peer: [3:7577471745055850445:0] socket: 454 qp: -1
2025-11-27T18:20:24.006673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471745055850366:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15628) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471745055850454:2125] held: no
2025-11-27T18:20:24.016379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471745431196109:2126] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.005486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471747831706730:2049] [node 3] ICS09 handshake done sender: [2:7577471743536739411:2119] self: [2:7577471743536739410:0] peer: [3:7577471745055850461:0] socket: 458 qp: -1
2025-11-27T18:20:23.986391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471742900927267:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:23.997544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471745346385236:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:23.992627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471746059145221:2109] [node 6] ICS09 handshake done sender: [5:7577471746059145205:2113] self: [5:7577471746059145204:0] peer: [6:7577471745431196113:0] socket: 456 qp: -1
2025-11-27T18:20:23.986422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471742900927268:2107] [node 4] ICS09 handshake done sender: [1:7577471742900927254:2115] self: [1:7577471742900927251:0] peer: [4:7577471745346385232:0] socket: 466 qp: -1
2025-11-27T18:20:24.004700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471745346385153:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.010255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745055850449:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.005526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471747831706730:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:24.020882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749726163414:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49737
2025-11-27T18:20:24.011002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471745346385151:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.010290Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471745055850449:2123] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:23.992660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471746059145222:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:23.986453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:24.022197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471745431196023:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.011044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:23.995035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:24.005561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471743536739408:2109] [node 6] ICP22 created new session: [2:7577471747831706732:2109]
2025-11-27T18:20:23.986804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471742900927268:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:24.014794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471745055850449:2123] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.005565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:24.022279Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471745431196084:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(17900430ns) {HandleHandshakeStatus:395(17898642ns) {LogHandshakeFail:472(9885ns) {UpdateErrorStateLog:545(3422ns)} DropOutgoingHandshake:457(1061ns) ProcessPendingSessionEvents:495(17281308ns) {unordered DropSessionEvent:512 num# 100 sum# 17213104ns max# 4922273ns}}}
2025-11-27T18:20:23.995083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471746059145221:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:23.986850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:24.013670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471745346385151:2048] [node 1] ICP22 created new session: [4:7577471749641352539:2048]
2025-11-27T18:20:24.022410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749726163415:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49737
2025-11-27T18:20:24.015868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471745055850366:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:23.996157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:23.987977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471742900927250:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:24.014618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471749641352539:2048] [node 1] ICS09 handshake done sender: [4:7577471745346385229:2115] self: [4:7577471745346385232:0] peer: [1:7577471742900927251:0] socket: 482 qp: -1
2025-11-27T18:20:24.006701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471743536739335:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10860) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471743536739425:2126] held: no
2025-11-27T18:20:23.995051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471742900927178:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.005137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471746059145198:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7577) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471746059145213:2117] held: no
2025-11-27T18:20:24.022519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471745431196024:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.009207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471747831706732:2109] [node 6] ICS09 handshake done sender: [2:7577471743536739415:2120] self: [2:7577471743536739412:0] peer: [6:7577471745431196114:0] socket: 465 qp: -1
2025-11-27T18:20:23.995161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:24.010596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471746059145224:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25092
2025-11-27T18:20:24.022552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:24.009252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471747831706732:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:24.024593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471745431196024:2049] [node 2] ICP22 created new session: [6:7577471749726163419:2049]
2025-11-27T18:20:24.015003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471749641352539:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:23.995444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471742900927178:2048] [node 2] ICP22 created new session: [1:7577471742900927273:2048]
2025-11-27T18:20:24.010814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471746059145220:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25092
2025-11-27T18:20:24.016946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471745055850438:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:23.996620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471742900927273:2048] [node 2] ICS09 handshake done sender: [1:7577471742900927250:2113] self: [1:7577471742900927248:0] peer: [2:7577471743536739429:0] socket: 471 qp: -1
2025-11-27T18:20:24.024831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471745431196108:2125] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:24.010010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:24.014643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471746059145213:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:24.016976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:24.015279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:24.017914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471747831706734:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3133
2025-11-27T18:20:24.002101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471742900927273:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:24.026408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471749726163419:2049] [node 2] ICS09 handshake done sender: [6:7577471745431196109:2126] self: [6:7577471745431196114:0] peer: [2:7577471743536739412:0] socket: 470 qp: -1
2025-11-27T18:20:24.020795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471749641352540:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39197
2025-11-27T18:20:24.020549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471743536739425:2126] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:24.015947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471746059145224:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.002928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:24.022480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471749641352542:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39197
2025-11-27T18:20:24.022186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471743536739335:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.026457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471749726163414:2130] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.024894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471745346385153:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17847) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471745346385236:2119] held: no
2025-11-27T18:20:24.016638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471746059145198:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.016340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471747195894571:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27690
2025-11-27T18:20:24.022212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471747831706734:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.017006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:24.028410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471749641352540:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.028279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471745431196084:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.017113Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471745055850365:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16805708ns) {HandleHandshakeStatus:395(16802368ns) {LogHandshakeFail:472(11612ns) {UpdateErrorStateLog:545(3282ns)} DropOutgoingHandshake:457(260ns) ProcessPendingSessionEvents:495(16752227ns) {unordered DropSessionEvent:512 num# 100 sum# 16693865ns max# 3958726ns}}}
2025-11-27T18:20:24.024559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742900927255:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.022483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:24.030883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471745346385152:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.016693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:24.028526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471749726163419:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.024592Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471742900927255:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.017550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471745055850438:2107] [node 4] ICP22 created new session: [3:7577471749350817766:2107]
2025-11-27T18:20:24.022539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471743536739335:2048] [node 1] ICP22 created new session: [2:7577471747831706737:2048]
2025-11-27T18:20:24.016741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471746059145198:2108] [node 4] ICP22 created new session: [5:7577471750354112523:2108]
2025-11-27T18:20:24.035597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471745346385225:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.018142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471745055850366:2049] [node 2] ICP22 created new session: [3:7577471749350817765:2049]
2025-11-27T18:20:24.025468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747195894571:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.028603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:24.025459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471747831706737:2048] [node 1] ICS09 handshake done sender: [2:7577471743536739425:2126] self: [2:7577471743536739429:0] peer: [1:7577471742900927248:0] socket: 474 qp: -1
2025-11-27T18:20:24.020440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471750354112523:2108] [node 4] ICS09 handshake done sender: [5:7577471746059145213:2117] self: [5:7577471746059145217:0] peer: [4:7577471745346385217:0] socket: 478 qp: -1
2025-11-27T18:20:24.035623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471745346385225:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.028812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471742900927253:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.020811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471749350817766:2107] [node 4] ICS09 handshake done sender: [3:7577471745055850449:2123] self: [3:7577471745055850447:0] peer: [4:7577471749641352537:0] socket: 498 qp: -1
2025-11-27T18:20:24.028676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471745431196084:2106] [node 5] ICP22 created new session: [6:7577471749726163420:2106]
2025-11-27T18:20:24.020850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471750354112523:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:24.025742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471747831706737:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:24.028845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471742900927253:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.036079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471745346385236:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:24.020989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471749350817765:2049] [node 2] ICS09 handshake done sender: [3:7577471745055850454:2125] self: [3:7577471745055850461:0] peer: [2:7577471743536739410:0] socket: 467 qp: -1
2025-11-27T18:20:24.028854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:24.021744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471746059145133:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.026430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:24.037900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471745346385153:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.020996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471749350817766:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:24.028929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471749726163415:2131] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.035190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471742900927255:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.037943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:24.021845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:24.032631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471743536739416:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.032653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471749726163420:2106] [node 5] ICS09 handshake done sender: [6:7577471745431196108:2125] self: [6:7577471745431196113:0] peer: [5:7577471746059145204:0] socket: 464 qp: -1
2025-11-27T18:20:24.035547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471742900927253:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:24.021023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471749350817765:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.030097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750354112524:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25092
2025-11-27T18:20:24.037874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471742900927242:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(42309977ns) {HandleHandshakeStatus:329(42307396ns) {DropIncomingHandshake:442(40726ns) DropOutgoingHandshake:457(500ns) SwitchToState:179(384ns) ProcessPendingSessionEvents:495(32866606ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32715517ns max# 5345867ns}}}
2025-11-27T18:20:24.039030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471745346385153:2050] [node 3] ICP22 created new session: [4:7577471749641352545:2050]
2025-11-27T18:20:24.032688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471743536739416:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.021056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:24.032682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471745431196025:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.033181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471750354112524:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.037928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471742900927244:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.041809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471749641352545:2050] [node 3] ICS09 handshake done sender: [4:7577471745346385236:2119] self: [4:7577471749641352537:0] peer: [3:7577471745055850447:0] socket: 501 qp: -1
2025-11-27T18:20:24.035615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471743536739417:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.021713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:24.033757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471746059145134:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.039684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471742900927243:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.032712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471749726163420:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:24.042025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471745346385225:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:24.035660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471743536739417:2122] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.040095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750354112528:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25092
2025-11-27T18:20:24.042119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471749641352545:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:24.025501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471745055850455:2126] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:24.032756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:24.039727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:24.045737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471746059145224:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:24.043644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:24.040115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:24.047474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471746059145133:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.041654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471742900927179:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(41634029ns) {HandleHandshakeStatus:329(41631590ns) {DropIncomingHandshake:442(11695ns) DropOutgoingHandshake:457(582ns) SwitchToState:179(388ns) ProcessPendingSessionEvents:495(34678063ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32251773ns max# 3961224ns}}}
2025-11-27T18:20:24.047191Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471745346385152:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15628) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471749641352540:2120] held: no
2025-11-27T18:20:24.048087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:24.036078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471743536739416:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:24.049596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471745346385215:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.048606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471746059145133:2048] [node 1] ICP22 created new session: [5:7577471750354112530:2048]
2025-11-27T18:20:24.041792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471742900927244:2109] [node 5] ICP22 created new session: [1:7577471747195894577:2109]
2025-11-27T18:20:24.049956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471745346385214:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(40774980ns) {HandleHandshakeStatus:329(40771881ns) {DropIncomingHandshake:442(12351ns) DropOutgoingHandshake:457(248ns) SwitchToState:179(412ns) ProcessPendingSessionEvents:495(29747623ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28075166ns max# 2667114ns}}}
2025-11-27T18:20:24.039146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471743536739409:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.051549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:24.049947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471750354112530:2048] [node 1] ICS09 handshake done sender: [5:7577471746059145224:2121] self: [5:7577471750354112525:0] peer: [1:7577471742900927252:0] socket: 450 qp: -1
2025-11-27T18:20:24.042026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471742900927243:2108] [node 6] ICP22 created new session: [1:7577471747195894576:2108]
2025-11-27T18:20:24.041441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:24.051819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471745346385215:2109] [node 6] ICP22 created new session: [4:7577471749641352548:2109]
2025-11-27T18:20:24.041678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471743536739409:2107] [node 4] ICP22 created new session: [2:7577471747831706741:2107]
2025-11-27T18:20:24.042271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471747195894577:2109] [node 5] ICS09 handshake done sender: [1:7577471742900927255:2116] self: [1:7577471742900927252:0] peer: [5:7577471750354112525:0] socket: 508 qp: -1
2025-11-27T18:20:24.045363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471747831706741:2107] [node 4] ICS09 handshake done sender: [2:7577471743536739416:2121] self: [2:7577471743536739413:0] peer: [4:7577471749641352543:0] socket: 487 qp: -1
2025-11-27T18:20:24.042310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471747195894577:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:24.053012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471749641352544:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39197
2025-11-27T18:20:24.054481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471749641352548:2109] [node 6] ICS09 handshake done sender: [4:7577471745346385225:2114] self: [4:7577471745346385223:0] peer: [6:7577471749726163424:0] socket: 472 qp: -1
2025-11-27T18:20:24.045709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471747831706741:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:24.042346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:24.050436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471750354112530:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:24.045744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:24.054528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471749641352548:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:24.033305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471745431196097:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.028810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471749350817764:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46917
2025-11-27T18:20:24.043643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471747195894576:2108] [node 6] ICS09 handshake done sender: [1:7577471742900927253:2114] self: [1:7577471742900927249:0] peer: [6:7577471749726163418:0] socket: 509 qp: -1
2025-11-27T18:20:24.045758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471743536739417:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:24.045349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749726163425:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49737
2025-11-27T18:20:24.050800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:24.058442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:24.047430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471743536739407:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.047484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:24.052329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471746059145135:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(44970318ns) {HandleHandshakeStatus:329(44967141ns) {DropIncomingHandshake:442(536ns) DropOutgoingHandshake:457(9315ns) SwitchToState:179(362ns) ProcessPendingSessionEvents:495(30595445ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29720118ns max# 2671772ns}}}
2025-11-27T18:20:24.059811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471749641352540:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:24.048587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471743536739407:2108] [node 5] ICP22 created new session: [2:7577471747831706744:2108]
2025-11-27T18:20:24.029925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471745055850365:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.048114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471745431196117:2129] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:24.055413Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471746059145197:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(45695761ns) {HandleHandshakeStatus:329(45692592ns) {DropIncomingHandshake:442(14882ns) DropOutgoingHandshake:457(443ns) SwitchToState:179(476ns) ProcessPendingSessionEvents:495(30204345ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28526244ns max# 2293947ns}}}
2025-11-27T18:20:24.045195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471747195894574:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27690
2025-11-27T18:20:24.062316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471745346385152:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.049973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471747831706744:2108] [node 5] ICS09 handshake done sender: [2:7577471743536739417:2122] self: [2:7577471743536739414:0] peer: [5:7577471750354112527:0] socket: 513 qp: -1
2025-11-27T18:20:24.055444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471746059145134:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15628) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471750354112524:2122] held: no
2025-11-27T18:20:24.030773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:24.050436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471747831706744:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:24.045381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471747195894576:2108] [node 6] ICS10 traffic start
2025-11-27T18:20:24.050321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471745431196023:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.063083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:24.032463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471745055850365:2048] [node 1] ICP22 created new session: [3:7577471749350817769:2048]
2025-11-27T18:20:24.063136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471745346385152:2049] [node 2] ICP22 created new session: [4:7577471749641352550:2049]
2025-11-27T18:20:24.032709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471749350817769:2048] [node 1] ICS09 handshake done sender: [3:7577471745055850455:2126] self: [3:7577471745055850458:0] peer: [1:7577471742900927245:0] socket: 477 qp: -1
2025-11-27T18:20:24.059563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750354112529:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25092
2025-11-27T18:20:24.064951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471749641352550:2049] [node 2] ICS09 handshake done sender: [4:7577471749641352540:2120] self: [4:7577471749641352543:0] peer: [2:7577471743536739413:0] socket: 490 qp: -1
2025-11-27T18:20:24.032753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471749350817769:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:24.069703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471749641352550:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:24.032808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:24.045731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:24.050419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471745431196025:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17847) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471749726163414:2130] held: no
2025-11-27T18:20:24.050612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471743536739336:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(40132606ns) {HandleHandshakeStatus:329(40127187ns) {DropIncomingHandshake:442(9428ns) DropOutgoingHandshake:457(170ns) SwitchToState:179(304ns) ProcessPendingSessionEvents:495(28527215ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28402826ns max# 2624234ns}}}
2025-11-27T18:20:24.069763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:24.047274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471742900927178:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(39043156ns) {HandleHandshakeStatus:329(39039649ns) {DropIncomingHandshake:442(30131ns) DropOutgoingHandshake:457(436ns) SwitchToState:179(509ns) ProcessPendingSessionEvents:495(27530214ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27377544ns max# 2610238ns}}}
2025-11-27T18:20:24.069695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471750354112524:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:24.035129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471749350817764:2130] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.072061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471746059145134:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.072751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:24.050772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747195894574:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.072897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471746059145134:2049] [node 2] ICP22 created new session: [5:7577471750354112532:2049]
2025-11-27T18:20:24.052081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:24.072944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471750354112532:2049] [node 2] ICS09 handshake done sender: [5:7577471750354112524:2122] self: [5:7577471750354112527:0] peer: [2:7577471743536739414:0] socket: 515 qp: -1
2025-11-27T18:20:24.055737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471743536739408:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(34021567ns) {HandleHandshakeStatus:329(34018479ns) {DropIncomingHandshake:442(16659ns) DropOutgoingHandshake:457(451ns) SwitchToState:179(2479ns) ProcessPendingSessionEvents:495(23975294ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23333075ns max# 2481320ns}}}
2025-11-27T18:20:24.035612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471745055850452:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.072973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471750354112532:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:24.035647Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471745055850452:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.073011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:24.037119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471745055850439:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(43501117ns) {HandleHandshakeStatus:329(43497965ns) {DropIncomingHandshake:442(485ns) DropOutgoingHandshake:457(436ns) SwitchToState:179(414ns) ProcessPendingSessionEvents:495(35200713ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 34082611ns max# 4888220ns}}}
2025-11-27T18:20:24.039004Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471745055850452:2124] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:24.043733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471745055850440:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.046798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:24.047458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471745055850440:2110] [node 6] ICP22 created new session: [3:7577471749350817773:2110]
2025-11-27T18:20:24.051559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471749350817773:2110] [node 6] ICS09 handshake done sender: [3:7577471745055850452:2124] self: [3:7577471745055850450:0] peer: [6:7577471749726163422:0] socket: 484 qp: -1
2025-11-27T18:20:24.051817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471749350817773:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:24.052095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:24.070897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471745055850438:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(35176237ns) {HandleHandshakeStatus:329(35172967ns) {DropIncomingHandshake:442(15188ns) DropOutgoingHandshake:457(289ns) SwitchToState:179(413ns) ProcessPendingSessionEvents:495(29158065ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28433084ns max# 2683495ns}}}
2025-11-27T18:20:24.050700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:24.051600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471745431196023:2048] [node 1] ICP22 created new session: [6:7577471749726163428:2048]
2025-11-27T18:20:24.052284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471749726163428:2048] [node 1] ICS09 handshake done sender: [6:7577471745431196117:2129] self: [6:7577471749726163418:0] peer: [1:7577471742900927249:0] socket: 510 qp: -1
2025-11-27T18:20:24.052317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471749726163428:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:24.052865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:24.054863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749726163426:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49737
2025-11-27T18:20:24.062768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471745431196097:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7577) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471749726163415:2131] held: no
2025-11-27T18:20:24.064088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749726163427:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49737
2025-11-27T18:20:24.069487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471749726163414:2130] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:24.071208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471745431196025:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.071431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471749726163415:2131] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:24.071435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:24.071503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471745431196025:2050] [node 3] ICP22 created new session: [6:7577471749726163430:2050]
2025-11-27T18:20:24.071795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471749726163430:2050] [node 3] ICS09 handshake done sender: [6:7577471749726163414:2130] self: [6:7577471749726163422:0] peer: [3:7577471745055850450:0] socket: 460 qp: -1
2025-11-27T18:20:24.071835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471749726163430:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:24.072376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:24.075313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471743536739407:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(27478293ns) {HandleHandshakeStatus:329(27475884ns) {DropIncomingHandshake:442(13386ns) DropOutgoingHandshake:457(190ns) SwitchToState:179(631ns) ProcessPendingSessionEvents:495(20550375ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18745200ns max# 4834351ns}}}
2025-11-27T18:20:24.074109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471745431196097:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.074202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:24.074483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471746059145133:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22251197ns) {HandleHandshakeStatus:329(22249363ns) {DropIncomingHandshake:442(864ns) DropOutgoingHandshake:457(3741ns) SwitchToState:179(388ns) ProcessPendingSessionEvents:495(17624409ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17524653ns max# 4681253ns}}}
2025-11-27T18:20:24.074250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471745431196097:2110] [node 4] ICP22 created new session: [6:7577471749726163432:2110]
2025-11-27T18:20:24.074605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471746059145133:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:24.074299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471749726163432:2110] [node 4] ICS09 handshake done sender: [6:7577471749726163415:2131] self: [6:7577471749726163424:0] peer: [4:7577471745346385223:0] socket: 485 qp: -1
2025-11-27T18:20:24.074339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471749726163432:2110] [node 4] ICS10 traffic start
2025-11-27T18:20:24.074372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:24.075611Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471745431196023:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22614774ns) {HandleHandshakeStatus:329(22612403ns) {DropIncomingHandshake:442(767ns) DropOutgoingHandshake:457(3950ns) SwitchToState:179(408ns) ProcessPendingSessionEvents:495(19029659ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18190988ns max# 5199472ns}}}
2025-11-27T18:20:24.077116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471745346385215:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(27667871ns) {HandleHandshakeStatus:329(27664256ns) {DropIncomingHandshake:442(14810ns) DropOutgoingHandshake:457(342ns) SwitchToState:179(429ns) ProcessPendingSessionEvents:495(15441609ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12844893ns max# 4875237ns}}}
2025-11-27T18:20:24.074725Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471742900927243:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(30615589ns) {HandleHandshakeStatus:329(30612810ns) {DropIncomingHandshake:442(889ns) DropOutgoingHandshake:457(711ns) SwitchToState:179(411ns) ProcessPendingSessionEvents:495(21357528ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21243692ns max# 4620323ns}}}
2025-11-27T18:20:24.075110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471742900927244:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(35378339ns) {HandleHandshakeStatus:329(35376377ns) {DropIncomingHandshake:442(1075815ns) DropOutgoingHandshake:457(2567ns) SwitchToState:179(288ns) ProcessPendingSessionEvents:495(25257567ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25132836ns max# 4698941ns}}}
2025-11-27T18:20:24.075220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471742900927243:2108] [node 6] ICP18 handshake (actor [1:7577471747195894574:2120]) is too old
2025-11-27T18:20:24.078118Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471743536739409:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(30305491ns) {HandleHandshakeStatus:329(30302560ns) {DropIncomingHandshake:442(11569ns) DropOutgoingHandshake:457(387ns) SwitchToState:179(3334ns) ProcessPendingSessionEvents:495(21979892ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21372913ns max# 4627109ns}}}
2025-11-27T18:20:24.078843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471745055850440:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(30357259ns) {HandleHandshakeStatus:329(30354536ns) {DropIncomingHandshake:442(1670128ns) DropOutgoingHandshake:457(920ns) SwitchToState:179(1907ns) ProcessPendingSessionEvents:495(19566608ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19465983ns max# 4546143ns}}}
2025-11-27T18:20:24.132649Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:24.132825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471747831706737:2048] [node 1] ICS01 socket: 474 reason# EndOfStream
2025-11-27T18:20:24.132863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471743536739335:2048] [node 1] ICP30 unregister session Session# [2:7577471747831706737:2048] VirtualId# [2:7577471743536739429:0]
2025-11-27T18:20:24.132889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471747831706737:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.132955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:24.133012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:24.134129Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:24.134317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471749350817769:2048] [node 1] ICS01 socket: 477 reason# EndOfStream
2025-11-27T18:20:24.134365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471745055850365:2048] [node 1] ICP30 unregister session Session# [3:7577471749350817769:2048] VirtualId# [3:7577471745055850458:0]
2025-11-27T18:20:24.134398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471749350817769:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.134499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:24.134554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:24.136703Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:24.136847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471749641352539:2048] [node 1] ICS01 socket: 482 reason# EndOfStream
2025-11-27T18:20:24.136882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471745346385151:2048] [node 1] ICP30 unregister session Session# [4:7577471749641352539:2048] VirtualId# [4:7577471745346385232:0]
2025-11-27T18:20:24.136912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471749641352539:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.136969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:24.137009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:24.137614Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:24.137882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471749726163428:2048] [node 1] ICS07 socket disconnect 510 reason# EndOfStream
2025-11-27T18:20:24.137944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471749726163428:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.138015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:24.138095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471749726163428:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:24.138359Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:24.138502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471750354112530:2048] [node 1] ICS01 socket: 450 reason# EndOfStream
2025-11-27T18:20:24.138532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471746059145133:2048] [node 1] ICP30 unregister session Session# [5:7577471750354112530:2048] VirtualId# [5:7577471750354112525:0]
2025-11-27T18:20:24.138552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471750354112530:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.138632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:24.138683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:24.138967Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471745431196023:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10860) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:24.139322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471749726163428:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:24.139387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471745431196023:2048] [node 1] ICP30 unregister session Session# [6:7577471749726163428:2048] VirtualId# [6:7577471749726163418:0]
2025-11-27T18:20:24.139485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:24.139524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471745431196023:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10860) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:24.139560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10860) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:24.149334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471745431196023:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:24.218586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:24.218777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471749350817765:2049] [node 2] ICS01 socket: 467 reason# EndOfStream
2025-11-27T18:20:24.218810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471745055850366:2049] [node 2] ICP30 unregister session Session# [3:7577471749350817765:2049] VirtualId# [3:7577471745055850461:0]
2025-11-27T18:20:24.218861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471749350817765:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.218937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:24.218988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:24.220966Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:24.221211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471749641352550:2049] [node 2] ICS01 socket: 490 reason# EndOfStream
2025-11-27T18:20:24.221281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471745346385152:2049] [node 2] ICP30 unregister session Session# [4:7577471749641352550:2049] VirtualId# [4:7577471749641352543:0]
2025-11-27T18:20:24.221317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471749641352550:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.221381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:24.221434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:24.222132Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:24.222348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471750354112532:2049] [node 2] ICS01 socket: 515 reason# EndOfStream
2025-11-27T18:20:24.222422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471746059145134:2049] [node 2] ICP30 unregister session Session# [5:7577471750354112532:2049] VirtualId# [5:7577471750354112527:0]
2025-11-27T18:20:24.222465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471750354112532:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.222543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:24.222596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:24.222909Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:24.223103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471749726163419:2049] [node 2] ICS01 socket: 470 reason# EndOfStream
2025-11-27T18:20:24.223143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471745431196024:2049] [node 2] ICP30 unregister session Session# [6:7577471749726163419:2049] VirtualId# [6:7577471745431196114:0]
2025-11-27T18:20:24.223197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471749726163419:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.223277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:24.223325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:24.282419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:24.282639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471749641352545:2050] [node 3] ICS01 socket: 501 reason# EndOfStream
2025-11-27T18:20:24.282683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471745346385153:2050] [node 3] ICP30 unregister session Session# [4:7577471749641352545:2050] VirtualId# [4:7577471749641352537:0]
2025-11-27T18:20:24.282721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471749641352545:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.282819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:24.282874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:24.283526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:24.283677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471746059145222:2050] [node 3] ICS07 socket disconnect 454 reason# EndOfStream
2025-11-27T18:20:24.283700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471746059145222:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.283749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:24.283782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471746059145222:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:24.284543Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471746059145135:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17847) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:24.284539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:24.284591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471746059145222:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:24.284661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471746059145135:2050] [node 3] ICP30 unregister session Session# [5:7577471746059145222:2050] VirtualId# [5:7577471746059145215:0]
2025-11-27T18:20:24.284696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471749726163430:2050] [node 3] ICS01 socket: 460 reason# EndOfStream
2025-11-27T18:20:24.284756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:24.284721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471745431196025:2050] [node 3] ICP30 unregister session Session# [6:7577471749726163430:2050] VirtualId# [6:7577471749726163422:0]
2025-11-27T18:20:24.284781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471746059145135:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17847) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:24.284750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471749726163430:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.284790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17847) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:24.284812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:24.284852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:24.294668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471746059145135:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:24.365409Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:24.365663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471750354112523:2108] [node 4] ICS01 socket: 478 reason# EndOfStream
2025-11-27T18:20:24.365722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471746059145198:2108] [node 4] ICP30 unregister session Session# [5:7577471750354112523:2108] VirtualId# [5:7577471746059145217:0]
2025-11-27T18:20:24.365769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471750354112523:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.365852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:24.365932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:24.366507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:24.366702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471749726163432:2110] [node 4] ICS01 socket: 485 reason# EndOfStream
2025-11-27T18:20:24.366739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471745431196097:2110] [node 4] ICP30 unregister session Session# [6:7577471749726163432:2110] VirtualId# [6:7577471749726163424:0]
2025-11-27T18:20:24.366767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471749726163432:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.366836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:24.366882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:24.448257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:24.448428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471749726163420:2106] [node 5] ICS01 socket: 464 reason# EndOfStream
2025-11-27T18:20:24.448466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471745431196084:2106] [node 5] ICP30 unregister session Session# [6:7577471749726163420:2106] VirtualId# [6:7577471745431196113:0]
2025-11-27T18:20:24.448499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471749726163420:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.448569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:24.448622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:24.582501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471747767107069:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:24.582552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471747767107070:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:24.603991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471749744197841:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:24.604059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471749744197842:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:24.622580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471748697513360:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:24.622649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471748697513361:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:24.644257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471748018774217:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:24.644336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471748018774218:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:24.644388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471748018774219:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:24.667242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471750103960518:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:24.667279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471750103960519:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:24.667352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471750103960520:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:24.690206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471749267703226:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:24.690279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471749267703227:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:24.690331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471749267703228:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:24.794605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471749267703290:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:24.827129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471748697513423:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:24.827423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471747767107132:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:24.827371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471748018774281:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:24.827578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471749744197904:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:24.827801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471748697513424:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:24.827803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471750103960582:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:24.827904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471747767107133:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:24.828015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471749744197905:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:24.828389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471748018774282:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:24.828482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471749267703291:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:24.828676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471750103960583:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:24.828745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471748697513425:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:24.828752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471747767107134:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:24.828840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471749744197906:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.842233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750103960597:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.842605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471748697513439:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13877
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.842716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471748697513440:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13877
2025-11-27T18:20:24.843038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471748697513441:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13877
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.843300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471749744197920:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18770
2025-11-27T18:20:24.843308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471748018774296:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38969
2025-11-27T18:20:24.843379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471749744197921:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18770
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.843398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471748018774297:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38969
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.844054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471749744197922:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18770
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:24.844064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471748018774298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38969
2025-11-27T18:20:24.844372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471749744197923:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18770
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.844918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471747767107148:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.845860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471747767107149:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.846050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471747767107150:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25681
2025-11-27T18:20:24.846962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471747767107151:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.849704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471749744197924:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18770
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.850033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471747767107152:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25681
2025-11-27T18:20:24.853482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471750103960597:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.855186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471748697513442:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13877
2025-11-27T18:20:24.855642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471748697513440:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.855671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471748018774296:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.856176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471748697513442:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.856401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471748018774219:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.856545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747767107152:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.856729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471748697513431:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.856733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747767107149:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.856759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471748697513431:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.856352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471749744197924:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.856874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747767107148:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.856617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471749744197920:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.857021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471748018774297:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.857037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471749744197921:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.857019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747767107150:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.857111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471748018774218:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.857208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471749744197923:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.857567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471748697513441:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.857635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471749744197912:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.857669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471749744197912:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.857693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471748697513439:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.857728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471748018774298:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.857720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747767107151:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.857861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471748018774217:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.858037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471748697513361:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.858222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471748697513360:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.858298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747767107140:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:24.858319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471747767107140:2112] [node 4] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-27T18:20:24.858353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471749744197922:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.858447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471748018774302:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38969
2025-11-27T18:20:24.858653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471748697513431:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:24.858736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471749744197910:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.858764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471749744197910:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.859687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471748697513423:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.859751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:24.859767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471748018774296:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:24.859768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747767107138:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.859799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471748697513423:2107] [node 4] ICP22 created new session: [3:7577471748697513448:2107]
2025-11-27T18:20:24.859799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471747767107138:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.859946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471748697513448:2107] [node 4] ICS09 handshake done sender: [3:7577471748697513431:2112] self: [3:7577471748697513430:0] peer: [4:7577471748018774299:0] socket: 469 qp: -1
2025-11-27T18:20:24.860224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471748697513448:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:24.860277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471749744197841:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.860270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:24.860439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471748018774219:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.860494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:24.860549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471748018774219:2050] [node 3] ICP22 created new session: [4:7577471748018774303:2050]
2025-11-27T18:20:24.860731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471748018774303:2050] [node 3] ICS09 handshake done sender: [4:7577471748018774296:2116] self: [4:7577471748018774299:0] peer: [3:7577471748697513430:0] socket: 479 qp: -1
2025-11-27T18:20:24.860768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471748018774303:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:24.860802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:24.860834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471749744197912:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:24.860860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747767107136:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.860891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471747767107136:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.861808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471749744197904:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.861864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:24.861925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471749744197904:2107] [node 4] ICP22 created new session: [2:7577471749744197932:2107]
2025-11-27T18:20:24.861979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471749744197932:2107] [node 4] ICS09 handshake done sender: [2:7577471749744197912:2112] self: [2:7577471749744197911:0] peer: [4:7577471748018774300:0] socket: 471 qp: -1
2025-11-27T18:20:24.862006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471749744197932:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:24.862030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:24.862171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471747767107140:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:24.862619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471747767107138:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:24.863023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471747767107132:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.863077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:24.863126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471747767107132:2107] [node 4] ICP22 created new session: [1:7577471747767107161:2107]
2025-11-27T18:20:24.863174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471747767107161:2107] [node 4] ICS09 handshake done sender: [1:7577471747767107140:2112] self: [1:7577471747767107139:0] peer: [4:7577471748018774301:0] socket: 470 qp: -1
2025-11-27T18:20:24.863217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471747767107161:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:24.863284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:24.863509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471749744197910:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:24.865201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471747767107070:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.865526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:24.865569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471747767107070:2049] [node 3] ICP22 created new session: [1:7577471747767107163:2049]
2025-11-27T18:20:24.865597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471747767107163:2049] [node 3] ICS09 handshake done sender: [1:7577471747767107138:2111] self: [1:7577471747767107137:0] peer: [3:7577471748697513447:0] socket: 465 qp: -1
2025-11-27T18:20:24.865623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471747767107163:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:24.865600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471749744197842:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.865653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:24.865660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:24.865711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471749744197842:2049] [node 3] ICP22 created new session: [2:7577471749744197935:2049]
2025-11-27T18:20:24.865757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471749744197935:2049] [node 3] ICS09 handshake done sender: [2:7577471749744197910:2111] self: [2:7577471749744197909:0] peer: [3:7577471748697513446:0] socket: 467 qp: -1
2025-11-27T18:20:24.865790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471749744197935:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:24.865824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.866013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471748697513450:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13877
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.866577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750103960599:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-27T18:20:24.866619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471748697513451:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13877
2025-11-27T18:20:24.867042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471748697513441:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:24.867172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471747767107136:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:24.867258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471748697513439:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:24.868941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471748697513361:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.868988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:24.869028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471748697513361:2049] [node 2] ICP22 created new session: [3:7577471748697513453:2049]
2025-11-27T18:20:24.869082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471748697513453:2049] [node 2] ICS09 handshake done sender: [3:7577471748697513441:2117] self: [3:7577471748697513446:0] peer: [2:7577471749744197909:0] socket: 476 qp: -1
2025-11-27T18:20:24.869112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471748697513453:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:24.869142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.870416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471749744197934:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18770
2025-11-27T18:20:24.870462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750103960600:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-27T18:20:24.870538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471748018774218:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21065) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471748018774297:2117] held: no
2025-11-27T18:20:24.871080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471747767107069:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.871170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471749744197922:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:24.871131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:24.871170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471747767107069:2048] [node 2] ICP22 created new session: [1:7577471747767107165:2048]
2025-11-27T18:20:24.871205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471747767107165:2048] [node 2] ICS09 handshake done sender: [1:7577471747767107136:2110] self: [1:7577471747767107135:0] peer: [2:7577471749744197931:0] socket: 460 qp: -1
2025-11-27T18:20:24.871224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471747767107165:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:24.871264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.872058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749267703305:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43301
2025-11-27T18:20:24.872179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471748018774305:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38969
2025-11-27T18:20:24.872606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471748697513360:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.872672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:24.872718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471748697513360:2048] [node 1] ICP22 created new session: [3:7577471748697513455:2048]
2025-11-27T18:20:24.872763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471748697513455:2048] [node 1] ICS09 handshake done sender: [3:7577471748697513439:2115] self: [3:7577471748697513447:0] peer: [1:7577471747767107137:0] socket: 472 qp: -1
2025-11-27T18:20:24.872757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471748018774306:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38969
2025-11-27T18:20:24.872797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471748697513455:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:24.872868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471749744197841:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.872834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:24.872912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:24.872962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471749744197841:2048] [node 1] ICP22 created new session: [2:7577471749744197939:2048]
2025-11-27T18:20:24.873006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471749744197939:2048] [node 1] ICS09 handshake done sender: [2:7577471749744197922:2117] self: [2:7577471749744197931:0] peer: [1:7577471747767107135:0] socket: 480 qp: -1
2025-11-27T18:20:24.873078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471749744197939:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:24.873116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:24.873662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471748018774217:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471748018774298:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:24.874339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750103960601:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
::ffff:127.1.0.0
2025-11-27T18:20:24.874460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749267703306:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43301
2025-11-27T18:20:24.874634Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471748018774298:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:24.875333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471748018774297:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.875765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471748018774217:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.875807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:24.875850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471748018774217:2048] [node 1] ICP22 created new session: [4:7577471748018774309:2048]
2025-11-27T18:20:24.875824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750103960602:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-27T18:20:24.875901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471748018774309:2048] [node 1] ICS09 handshake done sender: [4:7577471748018774298:2118] self: [4:7577471748018774301:0] peer: [1:7577471747767107139:0] socket: 486 qp: -1
2025-11-27T18:20:24.875877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749267703307:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43301
2025-11-27T18:20:24.875936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471748018774309:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:24.875962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:24.878652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471748018774218:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.878696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:24.878747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471748018774218:2049] [node 2] ICP22 created new session: [4:7577471748018774311:2049]
2025-11-27T18:20:24.878792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471748018774311:2049] [node 2] ICS09 handshake done sender: [4:7577471748018774297:2117] self: [4:7577471748018774300:0] peer: [2:7577471749744197911:0] socket: 482 qp: -1
2025-11-27T18:20:24.878804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471750103960600:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.878820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471748018774311:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:24.878854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:24.879008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471750103960519:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.880170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471748697513361:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:24.880740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471748697513360:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:24.880953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471749744197914:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.880991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471749744197914:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.881070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471750103960599:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.881199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471750103960520:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.881483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471748697513433:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.881517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471748697513433:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.882301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471749744197914:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:24.882633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471749267703305:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.883061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471749267703227:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.883080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471749744197905:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.883115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471750103960601:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.883251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:24.883281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750103960605:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-27T18:20:24.883302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471749744197905:2108] [node 5] ICP22 created new session: [2:7577471749744197942:2108]
2025-11-27T18:20:24.883369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471749744197942:2108] [node 5] ICS09 handshake done sender: [2:7577471749744197914:2113] self: [2:7577471749744197913:0] peer: [5:7577471750103960603:0] socket: 484 qp: -1
2025-11-27T18:20:24.883418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471749744197942:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:24.883376Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471748697513433:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:24.883462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.883559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749267703309:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43301
2025-11-27T18:20:24.884198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471750103960518:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.884249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471749267703306:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.884431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471750103960600:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:24.884530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471749267703226:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.884724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471749267703307:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.884792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747767107144:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.884823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471747767107144:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.884844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471750103960602:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.884830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471749267703291:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.884909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471750103960519:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.884942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:24.884983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471750103960519:2049] [node 2] ICP22 created new session: [5:7577471750103960608:2049]
2025-11-27T18:20:24.885020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471750103960608:2049] [node 2] ICS09 handshake done sender: [5:7577471750103960600:2118] self: [5:7577471750103960603:0] peer: [2:7577471749744197913:0] socket: 489 qp: -1
2025-11-27T18:20:24.885725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471748018774292:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.885669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471750103960608:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:24.885754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471748018774292:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.885696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:24.886389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471747767107142:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.886423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471747767107142:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.886512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471748697513424:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.886579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:24.886621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471748697513424:2108] [node 5] ICP22 created new session: [3:7577471748697513459:2108]
2025-11-27T18:20:24.886695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471748018774292:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:24.886671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471748697513459:2108] [node 5] ICS09 handshake done sender: [3:7577471748697513433:2113] self: [3:7577471748697513432:0] peer: [5:7577471750103960604:0] socket: 508 qp: -1
2025-11-27T18:20:24.887043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471748697513459:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:24.887073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:24.887265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471749744197916:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.887300Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471749744197916:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.887367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471748018774282:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.887427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:24.887460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471747767107144:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:24.887465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471748018774282:2110] [node 6] ICP22 created new session: [4:7577471748018774314:2110]
2025-11-27T18:20:24.887957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471748018774314:2110] [node 6] ICS09 handshake done sender: [4:7577471748018774292:2115] self: [4:7577471748018774291:0] peer: [6:7577471749267703311:0] socket: 512 qp: -1
2025-11-27T18:20:24.887989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471748018774314:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:24.888016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.888167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749267703312:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43301
2025-11-27T18:20:24.888379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471747767107142:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:24.888698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471747767107134:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.888860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:24.889007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471747767107134:2109] [node 6] ICP22 created new session: [1:7577471747767107171:2109]
2025-11-27T18:20:24.889106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471747767107171:2109] [node 6] ICS09 handshake done sender: [1:7577471747767107144:2114] self: [1:7577471747767107143:0] peer: [6:7577471749267703310:0] socket: 487 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.889136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471747767107171:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:24.889170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:24.889366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749267703313:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43301
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.889790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471749267703307:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:24.889907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471749744197916:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:24.889874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750103960606:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-27T18:20:24.890385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471747767107133:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.890435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:24.890403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471749267703309:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.890466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471749267703291:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.890480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471747767107133:2108] [node 5] ICP22 created new session: [1:7577471747767107173:2108]
2025-11-27T18:20:24.890506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:24.890515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471747767107173:2108] [node 5] ICS09 handshake done sender: [1:7577471747767107142:2113] self: [1:7577471747767107141:0] peer: [5:7577471750103960607:0] socket: 485 qp: -1
2025-11-27T18:20:24.890549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471749267703291:2109] [node 4] ICP22 created new session: [6:7577471749267703315:2109]
2025-11-27T18:20:24.890540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471747767107173:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:24.890564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:24.890612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471749267703315:2109] [node 4] ICS09 handshake done sender: [6:7577471749267703307:2118] self: [6:7577471749267703311:0] peer: [4:7577471748018774291:0] socket: 514 qp: -1
2025-11-27T18:20:24.890535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471749744197906:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.890620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:24.890646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471749267703315:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:24.890660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471749744197906:2109] [node 6] ICP22 created new session: [2:7577471749744197945:2109]
2025-11-27T18:20:24.890814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:24.890926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471749744197945:2109] [node 6] ICS09 handshake done sender: [2:7577471749744197916:2114] self: [2:7577471749744197915:0] peer: [6:7577471749267703308:0] socket: 481 qp: -1
2025-11-27T18:20:24.890967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471749744197945:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:24.891091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:24.892908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471750103960520:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13622) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471750103960599:2117] held: no
2025-11-27T18:20:24.893408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471749267703306:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:24.893864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471750103960582:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.893952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471750103960599:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:24.896175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471750103960520:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.896251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:24.896295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471750103960520:2050] [node 3] ICP22 created new session: [5:7577471750103960612:2050]
2025-11-27T18:20:24.896338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471750103960612:2050] [node 3] ICS09 handshake done sender: [5:7577471750103960599:2117] self: [5:7577471750103960604:0] peer: [3:7577471748697513432:0] socket: 509 qp: -1
2025-11-27T18:20:24.896366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471750103960612:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:24.896322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471748018774290:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.896404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:24.896359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471748018774290:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.897667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749267703314:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43301
2025-11-27T18:20:24.897860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471748018774290:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:24.898173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471749267703226:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.898218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:24.898236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471748018774281:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.898257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471749267703226:2048] [node 1] ICP22 created new session: [6:7577471749267703317:2048]
2025-11-27T18:20:24.898277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:24.898304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471749267703317:2048] [node 1] ICS09 handshake done sender: [6:7577471749267703306:2117] self: [6:7577471749267703310:0] peer: [1:7577471747767107143:0] socket: 510 qp: -1
2025-11-27T18:20:24.898315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471748018774281:2108] [node 5] ICP22 created new session: [4:7577471748018774317:2108]
2025-11-27T18:20:24.898338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471749267703317:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:24.898355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471748018774317:2108] [node 5] ICS09 handshake done sender: [4:7577471748018774290:2114] self: [4:7577471748018774289:0] peer: [5:7577471750103960611:0] socket: 511 qp: -1
2025-11-27T18:20:24.898366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:24.898384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471748018774317:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:24.898414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:24.898422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471750103960518:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471750103960601:2119] held: no
2025-11-27T18:20:24.898950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471749267703226:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.899130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750103960610:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-27T18:20:24.899897Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471749267703227:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21065) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471749267703305:2116] held: no
2025-11-27T18:20:24.900351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471749267703228:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.900755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471748018774319:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38969
2025-11-27T18:20:24.900796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471750103960601:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.900999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471749267703305:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:24.901083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471750103960616:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:261
2025-11-27T18:20:24.901498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471748697513435:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.901527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471749267703227:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.901525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471748697513435:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.901559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:24.901539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471750103960518:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.901583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:24.901585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471749267703227:2049] [node 2] ICP22 created new session: [6:7577471749267703322:2049]
2025-11-27T18:20:24.901628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471750103960518:2048] [node 1] ICP22 created new session: [5:7577471750103960617:2048]
2025-11-27T18:20:24.901649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471749267703322:2049] [node 2] ICS09 handshake done sender: [6:7577471749267703305:2116] self: [6:7577471749267703308:0] peer: [2:7577471749744197915:0] socket: 488 qp: -1
2025-11-27T18:20:24.901674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471750103960617:2048] [node 1] ICS09 handshake done sender: [5:7577471750103960601:2119] self: [5:7577471750103960607:0] peer: [1:7577471747767107141:0] socket: 493 qp: -1
2025-11-27T18:20:24.901691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471749267703322:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:24.901698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471750103960617:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:24.901724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:24.901727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:24.902493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471748697513435:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:24.902768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471748697513425:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.902796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:24.902831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471748697513425:2109] [node 6] ICP22 created new session: [3:7577471748697513462:2109]
2025-11-27T18:20:24.902871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471748697513462:2109] [node 6] ICS09 handshake done sender: [3:7577471748697513435:2114] self: [3:7577471748697513434:0] peer: [6:7577471749267703319:0] socket: 464 qp: -1
2025-11-27T18:20:24.902903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471748697513462:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:24.902929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.903319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749267703320:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43301
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.904343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471748697513464:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13877
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.904703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749267703324:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43301
2025-11-27T18:20:24.905332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471748697513464:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.905176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471749267703309:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:24.905609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471748697513425:2109] [node 6] ICP18 handshake (actor [3:7577471748697513464:2121]) is too old
2025-11-27T18:20:24.905669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471749267703228:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.905703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:24.905688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471750103960602:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:24.905737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471749267703228:2050] [node 3] ICP22 created new session: [6:7577471749267703325:2050]
2025-11-27T18:20:24.905788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471749267703325:2050] [node 3] ICS09 handshake done sender: [6:7577471749267703309:2119] self: [6:7577471749267703319:0] peer: [3:7577471748697513434:0] socket: 515 qp: -1
2025-11-27T18:20:24.905815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471749267703325:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:24.905847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:24.906182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471750103960582:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.906214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:24.906242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471750103960582:2108] [node 4] ICP22 created new session: [5:7577471750103960619:2108]
2025-11-27T18:20:24.906270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471750103960619:2108] [node 4] ICS09 handshake done sender: [5:7577471750103960602:2120] self: [5:7577471750103960611:0] peer: [4:7577471748018774289:0] socket: 513 qp: -1
2025-11-27T18:20:24.906292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471748018774319:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.906297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471750103960619:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:24.906319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:24.906397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471748018774281:2108] [node 5] ICP18 handshake (actor [4:7577471748018774319:2122]) is too old
2025-11-27T18:20:24.909465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471749267703320:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.909597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471749267703290:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:24.909842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471750103960593:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:24.909873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471750103960593:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:24.910829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471750103960593:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:24.911110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471749267703328:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43301
2025-11-27T18:20:24.911157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471750103960583:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:24.911203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:24.911248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471750103960583:2109] [node 6] ICP22 created new session: [5:7577471750103960622:2109]
2025-11-27T18:20:24.911286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471750103960622:2109] [node 6] ICS09 handshake done sender: [5:7577471750103960593:2115] self: [5:7577471750103960592:0] peer: [6:7577471749267703327:0] socket: 466 qp: -1
2025-11-27T18:20:24.911315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471750103960622:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:24.911340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:24.911497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471749267703320:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:24.911661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471749267703290:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:24.911702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:24.911776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471749267703290:2108] [node 5] ICP22 created new session: [6:7577471749267703329:2108]
2025-11-27T18:20:24.911825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471749267703329:2108] [node 5] ICS09 handshake done sender: [6:7577471749267703320:2123] self: [6:7577471749267703327:0] peer: [5:7577471750103960592:0] socket: 474 qp: -1
2025-11-27T18:20:24.911850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471749267703329:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:24.911873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:24.933998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:24.934168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471749744197939:2048] [node 1] ICS07 socket disconnect 480 reason# EndOfStream
2025-11-27T18:20:24.934205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471749744197939:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.934296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:24.934347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471749744197939:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:24.935427Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:24.935680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471748697513455:2048] [node 1] ICS07 socket disconnect 472 reason# EndOfStream
2025-11-27T18:20:24.935716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471748697513455:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.935823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:24.935876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471748697513455:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:24.937879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471749744197841:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:24.938566Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:24.938313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471749744197939:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:24.939826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471748018774309:2048] [node 1] ICS01 socket: 486 reason# EndOfStream
2025-11-27T18:20:24.939873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471748018774217:2048] [node 1] ICP30 unregister session Session# [4:7577471748018774309:2048] VirtualId# [4:7577471748018774301:0]
2025-11-27T18:20:24.939934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471748018774309:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.940005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:24.940068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:24.940532Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471749744197841:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:24.940583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471749744197939:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:24.940544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471748697513360:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:24.940622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471749744197841:2048] [node 1] ICP30 unregister session Session# [2:7577471749744197939:2048] VirtualId# [2:7577471749744197931:0]
2025-11-27T18:20:24.940581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471748697513455:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:24.940668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:24.940705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471749744197841:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:24.940720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:24.940995Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:24.941208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471750103960617:2048] [node 1] ICS01 socket: 493 reason# EndOfStream
2025-11-27T18:20:24.941307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471750103960518:2048] [node 1] ICP30 unregister session Session# [5:7577471750103960617:2048] VirtualId# [5:7577471750103960607:0]
2025-11-27T18:20:24.941347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471750103960617:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.941421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:24.941391Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471748697513360:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:24.941468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:24.941442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471748697513455:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:24.941473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471748697513360:2048] [node 1] ICP30 unregister session Session# [3:7577471748697513455:2048] VirtualId# [3:7577471748697513447:0]
2025-11-27T18:20:24.941585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:24.941623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471748697513360:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:24.941642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:24.942218Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:24.942406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471749267703317:2048] [node 1] ICS07 socket disconnect 510 reason# EndOfStream
2025-11-27T18:20:24.942443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471749267703317:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.942513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:24.942557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471749267703317:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:24.943331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471749267703226:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:24.943381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471749267703317:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:24.943416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471749267703226:2048] [node 1] ICP30 unregister session Session# [6:7577471749267703317:2048] VirtualId# [6:7577471749267703310:0]
2025-11-27T18:20:24.943501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:24.943533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471749267703226:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:24.943545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20836) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:24.950077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471749744197841:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:24.951124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471748697513360:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:24.953127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471749267703226:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:24.993506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:24.993742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471748697513453:2049] [node 2] ICS07 socket disconnect 476 reason# EndOfStream
2025-11-27T18:20:24.993780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471748697513453:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:24.993856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:24.993905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471748697513453:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:25.000227Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:25.000446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471748018774311:2049] [node 2] ICS01 socket: 482 reason# EndOfStream
2025-11-27T18:20:24.999978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471748697513361:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21065) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:25.000496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471748018774218:2049] [node 2] ICP30 unregister session Session# [4:7577471748018774311:2049] VirtualId# [4:7577471748018774300:0]
2025-11-27T18:20:25.000029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471748697513453:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:25.000533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471748018774311:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.000617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:25.000663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:25.000876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471748697513361:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21065) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.000928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471748697513453:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.000954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471748697513361:2049] [node 2] ICP30 unregister session Session# [3:7577471748697513453:2049] VirtualId# [3:7577471748697513446:0]
2025-11-27T18:20:25.001048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:25.001074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471748697513361:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.001083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.001506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:25.001731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471750103960608:2049] [node 2] ICS07 socket disconnect 489 reason# EndOfStream
2025-11-27T18:20:25.001768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471750103960608:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.001839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:25.001887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471750103960608:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:25.002585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:25.002676Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471750103960519:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21065) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.002730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471750103960608:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.002748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471749267703322:2049] [node 2] ICS01 socket: 488 reason# EndOfStream
2025-11-27T18:20:25.002763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471750103960519:2049] [node 2] ICP30 unregister session Session# [5:7577471750103960608:2049] VirtualId# [5:7577471750103960603:0]
2025-11-27T18:20:25.002798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471749267703227:2049] [node 2] ICP30 unregister session Session# [6:7577471749267703322:2049] VirtualId# [6:7577471749267703308:0]
2025-11-27T18:20:25.002824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471749267703322:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.002866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:25.002896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471750103960519:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.002907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21065) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.002902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:25.002955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:25.012555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471750103960519:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:25.029369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:25.029532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471748018774303:2050] [node 3] ICS07 socket disconnect 479 reason# EndOfStream
2025-11-27T18:20:25.029569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471748018774303:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.029646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:25.029695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471748018774303:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:25.030407Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471748018774219:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13622) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.030446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471748018774303:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.030469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471748018774219:2050] [node 3] ICP30 unregister session Session# [4:7577471748018774303:2050] VirtualId# [4:7577471748018774299:0]
2025-11-27T18:20:25.030562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:25.030589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471748018774219:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13622) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.030599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13622) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.030610Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:25.030839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471750103960612:2050] [node 3] ICS01 socket: 509 reason# EndOfStream
2025-11-27T18:20:25.030894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471750103960520:2050] [node 3] ICP30 unregister session Session# [5:7577471750103960612:2050] VirtualId# [5:7577471750103960604:0]
2025-11-27T18:20:25.030915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471750103960612:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.030977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:25.031037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:25.031496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:25.031649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471749267703325:2050] [node 3] ICS07 socket disconnect 515 reason# EndOfStream
2025-11-27T18:20:25.031684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471749267703325:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.031765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:25.031814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471749267703325:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:25.032495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471749267703228:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13622) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.032532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471749267703325:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.032554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471749267703228:2050] [node 3] ICP30 unregister session Session# [6:7577471749267703325:2050] VirtualId# [6:7577471749267703319:0]
2025-11-27T18:20:25.032672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:25.032700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471749267703228:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13622) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.032711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13622) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.042241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471749267703228:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:25.072261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:25.072494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471750103960619:2108] [node 4] ICS07 socket disconnect 513 reason# EndOfStream
2025-11-27T18:20:25.072529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471750103960619:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.072727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:25.072826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471750103960619:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:25.073287Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:25.073491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471749267703315:2109] [node 4] ICS07 socket disconnect 514 reason# EndOfStream
2025-11-27T18:20:25.073521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471749267703315:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.073630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:25.073675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471749267703315:2109] [node 4] ICS15 start handshake
2025-11-27T18:20:25.074066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471750103960582:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14744) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.074163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471750103960619:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.074211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471750103960582:2108] [node 4] ICP30 unregister session Session# [5:7577471750103960619:2108] VirtualId# [5:7577471750103960611:0]
2025-11-27T18:20:25.074401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:25.074444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471750103960582:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14744) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.074462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14744) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.074489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471749267703291:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14744) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.074530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471749267703315:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.074564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471749267703291:2109] [node 4] ICP30 unregister session Session# [6:7577471749267703315:2109] VirtualId# [6:7577471749267703311:0]
2025-11-27T18:20:25.074659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:25.074708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471749267703291:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14744) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.074722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14744) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.084238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471749267703291:2109] [node 4] ICP33 wake up from error state
2025-11-27T18:20:25.098036Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:25.098236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471749267703329:2108] [node 5] ICS07 socket disconnect 474 reason# EndOfStream
2025-11-27T18:20:25.098263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471749267703329:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.098333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:25.098380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471749267703329:2108] [node 5] ICS15 start handshake
2025-11-27T18:20:25.099220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471749267703290:2108] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1281) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.099264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471749267703329:2108] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.099299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471749267703290:2108] [node 5] ICP30 unregister session Session# [6:7577471749267703329:2108] VirtualId# [6:7577471749267703327:0]
2025-11-27T18:20:25.099424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:25.099488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471749267703290:2108] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1281) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.099546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1281) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.109837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471749267703290:2108] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:25.172162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471754935086585:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:25.172197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471754935086586:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:25.193115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471752758473252:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:25.193254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471752758473253:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:25.214378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471754881472663:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:25.214618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471754881472662:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:25.234668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471754196455937:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:25.234674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471754196455936:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:25.234710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471754196455938:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:25.253171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471752206269044:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:25.253218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471752206269045:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:25.253337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471752206269046:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:25.273233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471753890022368:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:25.273297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471753890022370:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:25.273327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471753890022369:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:25.321467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471753890022427:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:25.360692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471752758473333:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:25.364511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471754881472725:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:25.364816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471752206269109:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:25.364565Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471754935086586:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25167269ns) {Configure:104(25162529ns) {ProcessConfigured:135(20376550ns) {StartInitialHandshake:157(20373035ns) {DropHandshakes:470(1437ns) {DropIncomingHandshake:442(348ns) DropOutgoingHandshake:457(136ns)} GenerateSessionVirtualId:988(1064ns) PrepareNewSessionHandshake:477(4809ns) {SwitchToState:179(2562ns)}}}}}
2025-11-27T18:20:25.365086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471752758473334:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:25.364605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471754881472726:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:25.365142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471753890022436:2110] [node 4] ICP01 ready to work
2025-11-27T18:20:25.365330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471754196456001:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:25.365409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471754196456002:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:25.365453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471754881472727:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:25.365405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471752206269110:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:25.365541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471752758473335:2123] [node 6] ICP01 ready to work
2025-11-27T18:20:25.365692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471754935086652:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:25.365749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471754935086653:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:25.366364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471754935086654:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.371989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471754935086664:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45618
2025-11-27T18:20:25.375705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471754935086664:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.377118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471752206269108:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63797
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.377336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471754881472736:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4370
2025-11-27T18:20:25.377461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471754881472737:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4370
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.377963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471754881472736:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.377864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471752206269108:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.378158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471754881472737:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.378194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471752206269119:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63797
2025-11-27T18:20:25.378366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471754881472662:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.378540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471752758473344:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.378891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471752206269119:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.378910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471752758473350:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-11-27T18:20:25.378943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471754935086666:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45618
2025-11-27T18:20:25.379046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471752206269044:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.379000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471752758473351:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-11-27T18:20:25.379235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752758473344:2132] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.379239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471754935086651:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: 2025-11-27T18:20:25.379260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471754935086651:2111] [node 3] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:25.379259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471754935086667:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45618
2025-11-27T18:20:25.380569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471752758473352:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-11-27T18:20:25.380585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471754196456015:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12112
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.380999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471752758473252:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.380858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471754196456014:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12112
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.381237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471754935086668:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45618
2025-11-27T18:20:25.381359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471754881472745:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4370
2025-11-27T18:20:25.381312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471754935086656:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.382528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471754881472746:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4370
2025-11-27T18:20:25.381338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471754935086656:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.381515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471754935086646:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.381529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471754935086646:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.383424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471754935086669:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45618
2025-11-27T18:20:25.385080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471754935086651:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:25.385504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471754935086646:2107] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.387734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752758473351:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.386703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471754881472747:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4370
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.388106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752758473350:2134] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.386894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471752206269127:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63797
2025-11-27T18:20:25.388262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471754881472746:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.388120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471754935086656:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:25.388244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471752758473354:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.388434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471754196456015:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.388453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752758473352:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.388528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471754881472748:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4370
2025-11-27T18:20:25.388534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471754196455937:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.388548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471752206269119:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:25.388820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471754935086667:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.388855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471754196456014:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.388859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471752206269044:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.388971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471754935086585:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.389127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471754881472747:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.389239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471754196455938:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.388932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:25.389029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:25.389065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471754935086585:2048] [node 2] ICP22 created new session: [1:7577471754935086673:2048]
2025-11-27T18:20:25.389184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471752206269044:2048] [node 1] ICP22 created new session: [5:7577471752206269128:2048]
2025-11-27T18:20:25.389839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471754881472745:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.389737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752758473341:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.389314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471752206269128:2048] [node 1] ICS09 handshake done sender: [5:7577471752206269119:2116] self: [5:7577471752206269126:0] peer: [1:7577471754935086655:0] socket: 476 qp: -1
2025-11-27T18:20:25.389148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471754935086586:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.390059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471754881472736:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:25.389772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471752758473341:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.389397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471752206269128:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:25.389184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:25.390314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471754881472735:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.391025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471752758473344:2132] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:25.389189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471754935086673:2048] [node 2] ICS09 handshake done sender: [1:7577471754935086646:2107] self: [1:7577471754935086645:0] peer: [2:7577471752758473353:0] socket: 464 qp: -1
2025-11-27T18:20:25.389780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:25.390338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471754881472735:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.391269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471752758473252:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.389230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471754935086586:2049] [node 3] ICP22 created new session: [1:7577471754935086674:2049]
2025-11-27T18:20:25.391455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471754881472662:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.391305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:25.389314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471754935086673:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:25.391602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:25.391514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471752758473252:2048] [node 1] ICP22 created new session: [2:7577471752758473359:2048]
2025-11-27T18:20:25.389360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:25.391687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471752758473359:2048] [node 1] ICS09 handshake done sender: [2:7577471752758473344:2132] self: [2:7577471752758473353:0] peer: [1:7577471754935086645:0] socket: 474 qp: -1
2025-11-27T18:20:25.391668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471754881472663:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.391686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471754881472662:2048] [node 1] ICP22 created new session: [3:7577471754881472751:2048]
2025-11-27T18:20:25.389781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471754935086674:2049] [node 3] ICS09 handshake done sender: [1:7577471754935086651:2111] self: [1:7577471754935086650:0] peer: [3:7577471754881472743:0] socket: 465 qp: -1
2025-11-27T18:20:25.391761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471752758473359:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:25.391929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471754881472751:2048] [node 1] ICS09 handshake done sender: [3:7577471754881472736:2114] self: [3:7577471754881472743:0] peer: [1:7577471754935086650:0] socket: 472 qp: -1
2025-11-27T18:20:25.389905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471754935086674:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:25.391897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:25.391957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471754881472751:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:25.393871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471754196456020:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12112
2025-11-27T18:20:25.389995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:25.392047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:25.394119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471754196456021:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12112
2025-11-27T18:20:25.394246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471754881472735:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:25.394273Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471752758473341:2130] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.394544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471754196456022:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12112
2025-11-27T18:20:25.394763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471754881472726:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.394949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471754196456015:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:25.394979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471754196456021:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.394814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:25.395091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752758473339:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.394947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471754881472726:2107] [node 4] ICP22 created new session: [3:7577471754881472755:2107]
2025-11-27T18:20:25.395034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471754196456014:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:25.395163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471754935086653:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.395117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471752758473339:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.394990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471754881472755:2107] [node 4] ICS09 handshake done sender: [3:7577471754881472735:2113] self: [3:7577471754881472734:0] peer: [4:7577471754196456019:0] socket: 479 qp: -1
2025-11-27T18:20:25.395447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:25.395584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471754196455937:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.395018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471754881472755:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:25.395660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471754196455938:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.395572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471754935086653:2109] [node 5] ICP22 created new session: [1:7577471754935086677:2109]
2025-11-27T18:20:25.395046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:25.395686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:25.395695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471754935086669:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.395733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471754196455937:2049] [node 2] ICP22 created new session: [4:7577471754196456023:2049]
2025-11-27T18:20:25.395717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471754935086677:2109] [node 5] ICS09 handshake done sender: [1:7577471754935086656:2112] self: [1:7577471754935086655:0] peer: [5:7577471752206269126:0] socket: 466 qp: -1
2025-11-27T18:20:25.395762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:25.395746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471754935086677:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:25.395750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471754935086666:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.395764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471754196456023:2049] [node 2] ICS09 handshake done sender: [4:7577471754196456015:2117] self: [4:7577471754196456018:0] peer: [2:7577471752758473340:0] socket: 492 qp: -1
2025-11-27T18:20:25.395782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471754196456023:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:25.395779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:25.395789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471754196455938:2050] [node 3] ICP22 created new session: [4:7577471754196456024:2050]
2025-11-27T18:20:25.395889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471754935086668:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.395805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:25.395839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471754196456024:2050] [node 3] ICS09 handshake done sender: [4:7577471754196456014:2116] self: [4:7577471754196456019:0] peer: [3:7577471754881472734:0] socket: 487 qp: -1
2025-11-27T18:20:25.395874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471754196456024:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:25.396176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471752758473333:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.395909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:25.396298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:25.396341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471752758473333:2107] [node 4] ICP22 created new session: [2:7577471752758473361:2107]
2025-11-27T18:20:25.396387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471752758473361:2107] [node 4] ICS09 handshake done sender: [2:7577471752758473341:2130] self: [2:7577471752758473340:0] peer: [4:7577471754196456018:0] socket: 488 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.396416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471752758473361:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:25.396440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:25.398305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471752206269131:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63797
2025-11-27T18:20:25.398564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471752758473339:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:25.398554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471754935086653:2109] [node 5] ICP18 handshake (actor [1:7577471754935086667:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.399590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471752206269132:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63797
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.399630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471753890022448:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2885
2025-11-27T18:20:25.399633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471753890022447:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2885
2025-11-27T18:20:25.399730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471754881472759:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4370
2025-11-27T18:20:25.399727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471754935086586:2049] [node 3] ICP18 handshake (actor [1:7577471754935086666:2116]) is too old
2025-11-27T18:20:25.400132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471752758473253:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.400177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.400200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471752758473253:2049] [node 3] ICP22 created new session: [2:7577471752758473366:2049]
2025-11-27T18:20:25.400247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471752758473366:2049] [node 3] ICS09 handshake done sender: [2:7577471752758473339:2129] self: [2:7577471752758473338:0] peer: [3:7577471754881472753:0] socket: 483 qp: -1
2025-11-27T18:20:25.400354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471754935086585:2048] [node 2] ICP18 handshake (actor [1:7577471754935086668:2118]) is too old
2025-11-27T18:20:25.400379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471753890022449:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2885
2025-11-27T18:20:25.400275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471752758473366:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:25.400361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.402335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471754196456028:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12112
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.402919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471752206269133:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63797
2025-11-27T18:20:25.402989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471753890022450:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2885
2025-11-27T18:20:25.403250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471754881472745:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:25.403303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471752206269131:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.403891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471752206269046:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.404973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471753890022447:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.404952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471752206269132:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.405407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471753890022369:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.405490Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471754881472663:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18448) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471754881472745:2117] held: no
2025-11-27T18:20:25.405513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471752206269045:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.405549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471753890022449:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.405656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471752206269133:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.405660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471753890022368:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.405681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471754881472733:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.405668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752758473343:2131] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.405639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471754196456028:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.405772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471753890022450:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.405671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752758473346:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.405817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471752206269109:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.405704Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471754881472733:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.406447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471754196455936:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.405689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471752758473346:2133] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.405695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471752758473343:2131] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.406845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471754196456010:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.405788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471753890022448:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.406859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471754196456010:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.406860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471753890022370:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.407471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471752758473343:2131] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:25.406853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471754935086660:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.406869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471753890022436:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.406858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471754935086663:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.408081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471754881472663:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.406889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471754935086660:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.408120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:25.408179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471752758473346:2133] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:25.408092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471754196456017:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.406889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471754935086663:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.408656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471754881472663:2049] [node 2] ICP22 created new session: [3:7577471754881472761:2049]
2025-11-27T18:20:25.408124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471754196456017:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.408623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471752758473334:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.408645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471754935086660:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:25.408571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471754196456010:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:25.409006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471754881472761:2049] [node 2] ICS09 handshake done sender: [3:7577471754881472745:2117] self: [3:7577471754881472753:0] peer: [2:7577471752758473338:0] socket: 490 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.409500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471754935086652:2108] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.408902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.409408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471754881472739:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.408989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471752758473334:2108] [node 5] ICP22 created new session: [2:7577471752758473370:2108]
2025-11-27T18:20:25.410491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471752206269137:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63797
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.409434Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471754881472739:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.409745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:25.409491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471752758473370:2108] [node 5] ICS09 handshake done sender: [2:7577471752758473343:2131] self: [2:7577471752758473342:0] peer: [5:7577471752206269135:0] socket: 514 qp: -1
2025-11-27T18:20:25.409454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471754881472761:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:25.410777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471752206269138:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63797
2025-11-27T18:20:25.409591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471752758473370:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:25.410533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471754196456001:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.409770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:25.410532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471754935086652:2108] [node 4] ICP22 created new session: [1:7577471754935086684:2108]
2025-11-27T18:20:25.409614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:25.410822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471753890022455:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2885
2025-11-27T18:20:25.411020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471754935086684:2108] [node 4] ICS09 handshake done sender: [1:7577471754935086660:2113] self: [1:7577471754935086658:0] peer: [4:7577471754196456031:0] socket: 497 qp: -1
2025-11-27T18:20:25.410626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:25.411067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471753890022456:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2885
2025-11-27T18:20:25.411507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471754881472739:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:25.411688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471752758473335:2123] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.411056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471754196456001:2108] [node 5] ICP22 created new session: [4:7577471754196456035:2108]
2025-11-27T18:20:25.411061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471754935086684:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:25.411759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:25.411101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471754196456035:2108] [node 5] ICS09 handshake done sender: [4:7577471754196456010:2115] self: [4:7577471754196456009:0] peer: [5:7577471752206269136:0] socket: 478 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.411786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471752758473335:2123] [node 6] ICP22 created new session: [2:7577471752758473372:2123]
2025-11-27T18:20:25.411103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:25.411419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471754196456035:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:25.411818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471752758473372:2123] [node 6] ICS09 handshake done sender: [2:7577471752758473346:2133] self: [2:7577471752758473345:0] peer: [6:7577471753890022451:0] socket: 515 qp: -1
2025-11-27T18:20:25.411816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:25.412516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471754881472733:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:25.412430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471753890022457:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2885
2025-11-27T18:20:25.411819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471754935086663:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:25.411941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471752758473372:2123] [node 6] ICS10 traffic start
2025-11-27T18:20:25.414342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471752206269132:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:25.413867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471754935086654:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.411969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: 2025-11-27T18:20:25.414005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
2025-11-27T18:20:25.414044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471754935086654:2110] [node 6] ICP22 created new session: [1:7577471754935086686:2110]
2025-11-27T18:20:25.414864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471753890022447:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:25.414343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471754935086686:2110] [node 6] ICS09 handshake done sender: [1:7577471754935086663:2114] self: [1:7577471754935086662:0] peer: [6:7577471753890022452:0] socket: 520 qp: -1
2025-11-27T18:20:25.414550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471754935086686:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:25.415213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471754881472727:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.415141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471754196456033:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12112
2025-11-27T18:20:25.415203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471752206269133:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:25.415668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471752206269045:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18448) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471752206269132:2120] held: no
2025-11-27T18:20:25.415674Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471753890022449:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:25.414708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:25.415262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:25.415307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471754881472727:2109] [node 6] ICP22 created new session: [3:7577471754881472764:2109]
2025-11-27T18:20:25.416107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471753890022448:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:25.415596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471754881472764:2109] [node 6] ICS09 handshake done sender: [3:7577471754881472739:2116] self: [3:7577471754881472738:0] peer: [6:7577471753890022453:0] socket: 516 qp: -1
2025-11-27T18:20:25.415752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471754881472764:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:25.416532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471753890022369:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.415878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:25.416461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471754196456017:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:25.415996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471754881472725:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.416042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:25.416675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:25.416232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471754881472725:2108] [node 5] ICP22 created new session: [3:7577471754881472766:2108]
2025-11-27T18:20:25.416682Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471752206269109:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20527) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471752206269133:2121] held: no
2025-11-27T18:20:25.416739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471753890022369:2049] [node 2] ICP22 created new session: [6:7577471753890022459:2049]
2025-11-27T18:20:25.416271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471754881472766:2108] [node 5] ICS09 handshake done sender: [3:7577471754881472733:2112] self: [3:7577471754881472732:0] peer: [5:7577471752206269134:0] socket: 499 qp: -1
2025-11-27T18:20:25.416304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471754881472766:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:25.416336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:25.416902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471753890022459:2049] [node 2] ICS09 handshake done sender: [6:7577471753890022447:2116] self: [6:7577471753890022451:0] peer: [2:7577471752758473345:0] socket: 518 qp: -1
2025-11-27T18:20:25.416931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471752206269045:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.416909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471753890022368:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.416945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471753890022459:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:25.416977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:25.417104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:25.417168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471752206269045:2049] [node 2] ICP22 created new session: [5:7577471752206269140:2049]
2025-11-27T18:20:25.417059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:25.417122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471754196455936:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471754196456028:2122] held: no
2025-11-27T18:20:25.417112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471753890022368:2048] [node 1] ICP22 created new session: [6:7577471753890022461:2048]
2025-11-27T18:20:25.417164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471753890022461:2048] [node 1] ICS09 handshake done sender: [6:7577471753890022449:2118] self: [6:7577471753890022452:0] peer: [1:7577471754935086662:0] socket: 521 qp: -1
2025-11-27T18:20:25.417227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471752206269140:2049] [node 2] ICS09 handshake done sender: [5:7577471752206269132:2120] self: [5:7577471752206269135:0] peer: [2:7577471752758473342:0] socket: 494 qp: -1
2025-11-27T18:20:25.417200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471753890022461:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:25.417237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:25.417275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471752206269140:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:25.417296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:25.417378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471754196456002:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.417422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:25.417565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471752206269109:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.417558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471754196456002:2109] [node 6] ICP22 created new session: [4:7577471754196456037:2109]
2025-11-27T18:20:25.417645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471754196456037:2109] [node 6] ICS09 handshake done sender: [4:7577471754196456017:2118] self: [4:7577471754196456016:0] peer: [6:7577471753890022454:0] socket: 486 qp: -1
2025-11-27T18:20:25.417600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:25.417681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471754196456037:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:25.417635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471752206269109:2108] [node 4] ICP22 created new session: [5:7577471752206269142:2108]
2025-11-27T18:20:25.417711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:25.417681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471752206269142:2108] [node 4] ICS09 handshake done sender: [5:7577471752206269133:2121] self: [5:7577471752206269136:0] peer: [4:7577471754196456009:0] socket: 482 qp: -1
2025-11-27T18:20:25.417714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471752206269142:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:25.417868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:25.418355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471754196456028:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:25.418912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471754196455936:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.419000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:25.419053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471754196455936:2048] [node 1] ICP22 created new session: [4:7577471754196456039:2048]
2025-11-27T18:20:25.419089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471754196456039:2048] [node 1] ICS09 handshake done sender: [4:7577471754196456028:2122] self: [4:7577471754196456031:0] peer: [1:7577471754935086658:0] socket: 522 qp: -1
2025-11-27T18:20:25.419120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471754196456039:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:25.419154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.419410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471753890022370:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.419409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471752206269139:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63797
2025-11-27T18:20:25.419504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:25.419546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471753890022370:2050] [node 3] ICP22 created new session: [6:7577471753890022463:2050]
2025-11-27T18:20:25.419583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471753890022463:2050] [node 3] ICS09 handshake done sender: [6:7577471753890022448:2117] self: [6:7577471753890022453:0] peer: [3:7577471754881472738:0] socket: 519 qp: -1
2025-11-27T18:20:25.419652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471753890022463:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:25.420034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.420362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471752206269046:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4625) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471752206269131:2119] held: no
2025-11-27T18:20:25.420405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471753890022458:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2885
2025-11-27T18:20:25.420949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471752206269131:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:25.421379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471752206269046:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.421423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:25.421446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471752206269046:2050] [node 3] ICP22 created new session: [5:7577471752206269145:2050]
2025-11-27T18:20:25.421472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471752206269145:2050] [node 3] ICS09 handshake done sender: [5:7577471752206269131:2119] self: [5:7577471752206269134:0] peer: [3:7577471754881472732:0] socket: 502 qp: -1
2025-11-27T18:20:25.421494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471752206269145:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:25.421523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:25.422492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471753890022370:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:25.423192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471753890022450:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:25.423521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471753890022436:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.423573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:25.423596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471753890022436:2110] [node 4] ICP22 created new session: [6:7577471753890022467:2110]
2025-11-27T18:20:25.423622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471753890022467:2110] [node 4] ICS09 handshake done sender: [6:7577471753890022450:2119] self: [6:7577471753890022454:0] peer: [4:7577471754196456016:0] socket: 489 qp: -1
2025-11-27T18:20:25.423641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471753890022467:2110] [node 4] ICS10 traffic start
2025-11-27T18:20:25.423660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.424259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471754196456041:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12112
2025-11-27T18:20:25.424291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471753890022465:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2885
2025-11-27T18:20:25.424632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471753890022465:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.424983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471753890022427:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.425233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471752206269121:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.425276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471752206269121:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.425893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471752206269121:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.426066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471752206269110:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.426123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:25.426168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471752206269110:2109] [node 6] ICP22 created new session: [5:7577471752206269148:2109]
2025-11-27T18:20:25.426093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471753890022470:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2885
2025-11-27T18:20:25.426215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471752206269148:2109] [node 6] ICS09 handshake done sender: [5:7577471752206269121:2117] self: [5:7577471752206269120:0] peer: [6:7577471753890022469:0] socket: 471 qp: -1
2025-11-27T18:20:25.426249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471752206269148:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:25.426279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:25.426272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471753890022465:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:25.426584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471753890022427:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.426663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:25.426704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471753890022427:2105] [node 5] ICP22 created new session: [6:7577471753890022471:2105]
2025-11-27T18:20:25.426750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471753890022471:2105] [node 5] ICS09 handshake done sender: [6:7577471753890022465:2124] self: [6:7577471753890022469:0] peer: [5:7577471752206269120:0] socket: 469 qp: -1
2025-11-27T18:20:25.426783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471753890022471:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:25.426806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:25.427776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471753890022427:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:25.459577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:25.459741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471752758473359:2048] [node 1] ICS07 socket disconnect 474 reason# EndOfStream
2025-11-27T18:20:25.459765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471752758473359:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.459820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:25.459852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471752758473359:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:25.460588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:25.460710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471754881472751:2048] [node 1] ICS07 socket disconnect 472 reason# EndOfStream
2025-11-27T18:20:25.460728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471754881472751:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.460789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:25.460829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471754881472751:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:25.461770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471752758473252:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:25.461800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471752758473359:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:25.462768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471754881472662:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:25.462803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471754881472751:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:25.462859Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471752758473252:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.463994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471752758473359:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.464032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471752758473252:2048] [node 1] ICP30 unregister session Session# [2:7577471752758473359:2048] VirtualId# [2:7577471752758473353:0]
2025-11-27T18:20:25.464042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:25.464157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:25.464192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471752758473252:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.464202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.464297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471754196456039:2048] [node 1] ICS01 socket: 522 reason# EndOfStream
2025-11-27T18:20:25.464340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471754196455936:2048] [node 1] ICP30 unregister session Session# [4:7577471754196456039:2048] VirtualId# [4:7577471754196456031:0]
2025-11-27T18:20:25.464385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471754196456039:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.464463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:25.464522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:25.464521Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471754881472662:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.464555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471754881472751:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.464579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471754881472662:2048] [node 1] ICP30 unregister session Session# [3:7577471754881472751:2048] VirtualId# [3:7577471754881472743:0]
2025-11-27T18:20:25.464692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:25.464717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471754881472662:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.464732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.465124Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:25.465243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471752206269128:2048] [node 1] ICS01 socket: 476 reason# EndOfStream
2025-11-27T18:20:25.465292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471752206269044:2048] [node 1] ICP30 unregister session Session# [5:7577471752206269128:2048] VirtualId# [5:7577471752206269126:0]
2025-11-27T18:20:25.465316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471752206269128:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.465403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:25.465459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:25.465880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:25.466105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471753890022461:2048] [node 1] ICS07 socket disconnect 521 reason# EndOfStream
2025-11-27T18:20:25.466148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471753890022461:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.466226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:25.466269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471753890022461:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:25.466807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471753890022368:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.466844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471753890022461:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.466861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471753890022368:2048] [node 1] ICP30 unregister session Session# [6:7577471753890022461:2048] VirtualId# [6:7577471753890022452:0]
2025-11-27T18:20:25.466983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:25.467019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471753890022368:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.467030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12978) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.474346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471754881472662:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:25.474344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471752758473252:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:25.476400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471753890022368:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:25.511989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:25.512215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471754881472761:2049] [node 2] ICS01 socket: 490 reason# EndOfStream
2025-11-27T18:20:25.512263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471754881472663:2049] [node 2] ICP30 unregister session Session# [3:7577471754881472761:2049] VirtualId# [3:7577471754881472753:0]
2025-11-27T18:20:25.512297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471754881472761:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.512384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:25.512675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:25.514785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:25.514928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471754196456023:2049] [node 2] ICS01 socket: 492 reason# EndOfStream
2025-11-27T18:20:25.514958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471754196455937:2049] [node 2] ICP30 unregister session Session# [4:7577471754196456023:2049] VirtualId# [4:7577471754196456018:0]
2025-11-27T18:20:25.514980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471754196456023:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.515031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:25.515061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:25.515971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:25.516157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471752206269140:2049] [node 2] ICS01 socket: 494 reason# EndOfStream
2025-11-27T18:20:25.516189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471752206269045:2049] [node 2] ICP30 unregister session Session# [5:7577471752206269140:2049] VirtualId# [5:7577471752206269135:0]
2025-11-27T18:20:25.516225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471752206269140:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.516285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:25.516324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:25.516974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:25.517177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471753890022459:2049] [node 2] ICS07 socket disconnect 518 reason# EndOfStream
2025-11-27T18:20:25.517210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471753890022459:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.517323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:25.517381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471753890022459:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:25.518153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471753890022369:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18448) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.518261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471753890022459:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.518321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471753890022369:2049] [node 2] ICP30 unregister session Session# [6:7577471753890022459:2049] VirtualId# [6:7577471753890022451:0]
2025-11-27T18:20:25.518428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:25.518468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471753890022369:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18448) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.518490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18448) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.528666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471753890022369:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:25.556463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:25.556650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471754196456024:2050] [node 3] ICS07 socket disconnect 487 reason# EndOfStream
2025-11-27T18:20:25.556676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471754196456024:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.556737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:25.556767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471754196456024:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:25.557356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471754196455938:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4625) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.557418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471754196456024:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.557508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471754196455938:2050] [node 3] ICP30 unregister session Session# [4:7577471754196456024:2050] VirtualId# [4:7577471754196456019:0]
2025-11-27T18:20:25.557653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:25.557680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471754196455938:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.557689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.557659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:25.557828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471752206269145:2050] [node 3] ICS01 socket: 502 reason# EndOfStream
2025-11-27T18:20:25.557866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471752206269046:2050] [node 3] ICP30 unregister session Session# [5:7577471752206269145:2050] VirtualId# [5:7577471752206269134:0]
2025-11-27T18:20:25.557945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471752206269145:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.558038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:25.558100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:25.558511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:25.558724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471753890022463:2050] [node 3] ICS07 socket disconnect 519 reason# EndOfStream
2025-11-27T18:20:25.558753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471753890022463:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.558821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:25.558865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471753890022463:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:25.559467Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471753890022370:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4625) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.559551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471753890022463:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.559592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471753890022370:2050] [node 3] ICP30 unregister session Session# [6:7577471753890022463:2050] VirtualId# [6:7577471753890022453:0]
2025-11-27T18:20:25.559672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:25.559697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471753890022370:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.559707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.567547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471754196455938:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:25.569588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471753890022370:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:25.601932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:25.602143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471752206269142:2108] [node 4] ICS01 socket: 482 reason# EndOfStream
2025-11-27T18:20:25.602188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471752206269109:2108] [node 4] ICP30 unregister session Session# [5:7577471752206269142:2108] VirtualId# [5:7577471752206269136:0]
2025-11-27T18:20:25.602226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471752206269142:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.602302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:25.602347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:25.603027Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:25.603212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471753890022467:2110] [node 4] ICS07 socket disconnect 489 reason# EndOfStream
2025-11-27T18:20:25.603241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471753890022467:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.603304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:25.603347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471753890022467:2110] [node 4] ICS15 start handshake
2025-11-27T18:20:25.604027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471753890022436:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20527) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.604155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471753890022467:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.604214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471753890022436:2110] [node 4] ICP30 unregister session Session# [6:7577471753890022467:2110] VirtualId# [6:7577471753890022454:0]
2025-11-27T18:20:25.604306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:25.604345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471753890022436:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20527) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.604357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20527) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.614659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471753890022436:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:20:25.624292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:25.624563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471753890022471:2105] [node 5] ICS07 socket disconnect 469 reason# EndOfStream
2025-11-27T18:20:25.624598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471753890022471:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.624681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:25.624735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471753890022471:2105] [node 5] ICS15 start handshake
2025-11-27T18:20:25.625365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471753890022427:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13817) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:25.625426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471753890022471:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:25.625466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471753890022427:2105] [node 5] ICP30 unregister session Session# [6:7577471753890022471:2105] VirtualId# [6:7577471753890022469:0]
2025-11-27T18:20:25.625580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:25.625609Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471753890022427:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13817) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.625619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13817) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:25.635147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471753890022427:2105] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:25.685232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471752782723373:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:25.685280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471752782723374:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:25.703174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471752365654038:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:25.703476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471752365654039:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:25.721630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471751080856911:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:25.721699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471751080856912:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:25.739723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471752225611710:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:25.739723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471752225611709:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:25.739752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471752225611711:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:25.759515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471753858978094:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:25.759560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471753858978095:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:25.759598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471753858978096:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:25.777827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471752424283647:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:25.777879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471752424283648:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:25.778076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471752424283649:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:25.867827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471752225611772:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:25.867754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471752424283708:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:25.870308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471752782723438:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:25.871225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471751080856977:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:25.870382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471752782723437:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:25.871277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471752365654111:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:25.871492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471751080856976:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:25.871920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471752782723439:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:25.872283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471753858978157:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:25.872048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471752225611773:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:25.872667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471752365654113:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:25.872384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471752424283710:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:25.872126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471751080856978:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:25.873159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471753858978158:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:25.872992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471752365654112:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.883386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471751080856991:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:25.893506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471751080856994:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
::ffff:127.1.0.0
2025-11-27T18:20:25.893679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471752225611784:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29304
2025-11-27T18:20:25.894283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471753858978172:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30478
2025-11-27T18:20:25.893514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471751080856993:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-27T18:20:25.894683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471751080856995:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-27T18:20:25.894282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471753858978173:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30478
2025-11-27T18:20:25.894917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471751080856991:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.898627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471752225611784:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:25.899121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471752782723449:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65343
::ffff:127.1.0.0
2025-11-27T18:20:25.899284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471751080856994:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.899488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471753858978173:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.899648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471753858978157:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.899693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471752782723451:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65343
2025-11-27T18:20:25.899576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471752225611711:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.899746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471752782723450:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65343
2025-11-27T18:20:25.899812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471752365654120:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16673
2025-11-27T18:20:25.899818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471751080856984:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:25.899859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471751080856984:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.900146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471752782723449:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.899921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471751080856993:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.900652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471752225611779:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.900490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471752365654121:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16673
2025-11-27T18:20:25.900547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471752782723451:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.900683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471752225611779:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-27T18:20:25.901594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471752782723450:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.901772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471752225611779:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:25.901147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471752365654122:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16673
2025-11-27T18:20:25.901527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471752365654123:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16673
2025-11-27T18:20:25.901893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471753858978175:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30478
2025-11-27T18:20:25.901903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471751080856999:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-27T18:20:25.901636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471752365654125:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16673
2025-11-27T18:20:25.901781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752365654121:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.901977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471751080856984:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:25.902194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471752225611772:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.901823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752365654120:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.901857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752365654122:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.902397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.902605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471752225611772:2108] [node 5] ICP22 created new session: [4:7577471752225611791:2108]
2025-11-27T18:20:25.902063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752365654123:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.904135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471752782723456:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65343
2025-11-27T18:20:25.903807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471753858978176:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30478
2025-11-27T18:20:25.904057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471751080856976:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.902815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471752225611791:2108] [node 5] ICS09 handshake done sender: [4:7577471752225611779:2112] self: [4:7577471752225611777:0] peer: [5:7577471753858978174:0] socket: 482 qp: -1
2025-11-27T18:20:25.902865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471752225611791:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:25.904333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:25.904817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471753858978172:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.902973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:25.904472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471751080856976:2107] [node 4] ICP22 created new session: [3:7577471751080857001:2107]
2025-11-27T18:20:25.905678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471753858978173:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.904175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471752225611790:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29304
2025-11-27T18:20:25.904626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471751080857001:2107] [node 4] ICS09 handshake done sender: [3:7577471751080856984:2114] self: [3:7577471751080856981:0] peer: [4:7577471752225611788:0] socket: 476 qp: -1
2025-11-27T18:20:25.906702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471753858978157:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.907569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752365654125:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.907937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471752782723443:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.908507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471752225611784:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:25.905654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471751080857001:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:25.908466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471752365654038:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.907974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471752782723443:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.906556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:25.907522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471753858978094:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.908478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471752782723460:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65343
2025-11-27T18:20:25.907658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471751080856995:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.909404Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471752225611711:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471752225611784:2113] held: no
2025-11-27T18:20:25.908930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471752782723443:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:25.907912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:25.910184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471752424283726:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32820
2025-11-27T18:20:25.907890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471751080856911:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.909149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471752782723444:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.908574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471753858978157:2108] [node 4] ICP22 created new session: [5:7577471753858978178:2108]
2025-11-27T18:20:25.908887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471751080856999:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.909173Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471752782723444:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.910170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471751080856912:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.909002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471753858978178:2108] [node 4] ICS09 handshake done sender: [5:7577471753858978173:2116] self: [5:7577471753858978174:0] peer: [4:7577471752225611777:0] socket: 490 qp: -1
2025-11-27T18:20:25.909184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471752782723445:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.909224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471753858978178:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:25.909848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:25.910880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752365654119:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.910180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471753858978175:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.910914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471752365654119:2122] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.909208Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471752782723445:2113] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.910917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471752424283727:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32820
2025-11-27T18:20:25.910851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471753858978095:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.910307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471752782723456:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.911394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471752424283726:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.910520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471752782723438:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.913498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471751080857006:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.913637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471752424283649:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.913827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752365654117:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.913924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471752225611711:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.910575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:25.913877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471752365654117:2120] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.914399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:25.910619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471752782723438:2109] [node 5] ICP22 created new session: [1:7577471752782723464:2109]
2025-11-27T18:20:25.914391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471752365654119:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:25.915868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471751080856992:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.915195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471753858978181:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30478
2025-11-27T18:20:25.914917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471752225611711:2050] [node 3] ICP22 created new session: [4:7577471752225611795:2050]
2025-11-27T18:20:25.910660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471752782723464:2109] [node 5] ICS09 handshake done sender: [1:7577471752782723443:2111] self: [1:7577471752782723440:0] peer: [5:7577471753858978177:0] socket: 478 qp: -1
2025-11-27T18:20:25.915228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471753858978094:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16383) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471753858978172:2115] held: no
2025-11-27T18:20:25.915910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471751080856992:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.916050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471752365654137:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16673
2025-11-27T18:20:25.915235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471752225611795:2050] [node 3] ICS09 handshake done sender: [4:7577471752225611784:2113] self: [4:7577471752225611788:0] peer: [3:7577471751080856981:0] socket: 489 qp: -1
2025-11-27T18:20:25.910796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471752782723464:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:25.917008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471752424283727:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.916554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471752365654039:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.910823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:25.915810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471752225611795:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:25.916838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:25.916002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.911649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471752782723444:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.916993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471752365654039:2049] [node 3] ICP22 created new session: [2:7577471752365654140:2049]
2025-11-27T18:20:25.917783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471752424283708:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.918235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471751080856995:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:25.913175Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471752782723445:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:25.917175Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471752365654117:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:25.914718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471752782723373:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.918660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471752424283729:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32820
2025-11-27T18:20:25.917721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471752365654140:2049] [node 3] ICS09 handshake done sender: [2:7577471752365654119:2122] self: [2:7577471752365654116:0] peer: [3:7577471751080857005:0] socket: 506 qp: -1
2025-11-27T18:20:25.918786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471751080857007:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33069
2025-11-27T18:20:25.917753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471752365654140:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:25.919257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471753858978169:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.914963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471752782723374:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.919292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471753858978169:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.919560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471752424283729:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.920240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471751080856911:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.917790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:25.915131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:25.920493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471752424283710:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.918815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471752365654125:2127] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.920284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:25.920701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471753858978172:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:25.920932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471752225611787:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.915262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471752782723374:2049] [node 3] ICP22 created new session: [1:7577471752782723467:2049]
2025-11-27T18:20:25.920475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471751080856911:2048] [node 1] ICP22 created new session: [3:7577471751080857009:2048]
2025-11-27T18:20:25.920962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471752225611787:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.919954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471752365654038:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.915767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.920623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471751080857009:2048] [node 1] ICS09 handshake done sender: [3:7577471751080856995:2120] self: [3:7577471751080857004:0] peer: [1:7577471752782723442:0] socket: 500 qp: -1
2025-11-27T18:20:25.922638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471753858978183:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30478
2025-11-27T18:20:25.919969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471752365654111:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.915985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471752782723467:2049] [node 3] ICS09 handshake done sender: [1:7577471752782723444:2112] self: [1:7577471752782723442:0] peer: [3:7577471751080857004:0] socket: 495 qp: -1
2025-11-27T18:20:25.920822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471751080857009:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:25.920019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:25.916018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471752782723467:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:25.923605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471752424283732:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32820
2025-11-27T18:20:25.922661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471753858978094:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.923930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471752225611787:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:25.923692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:25.920111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:25.916073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471752782723373:2048] [node 2] ICP22 created new session: [1:7577471752782723468:2048]
2025-11-27T18:20:25.921044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:25.916519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.924028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471753858978094:2048] [node 1] ICP22 created new session: [5:7577471753858978185:2048]
2025-11-27T18:20:25.923614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471751080856912:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8513) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471751080856999:2121] held: no
2025-11-27T18:20:25.916670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471752782723468:2048] [node 2] ICS09 handshake done sender: [1:7577471752782723445:2113] self: [1:7577471752782723441:0] peer: [2:7577471752365654136:0] socket: 498 qp: -1
2025-11-27T18:20:25.920163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471752365654111:2108] [node 5] ICP22 created new session: [2:7577471752365654142:2108]
2025-11-27T18:20:25.924343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471753858978185:2048] [node 1] ICS09 handshake done sender: [5:7577471753858978172:2115] self: [5:7577471753858978177:0] peer: [1:7577471752782723440:0] socket: 487 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.925746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471752424283733:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32820
2025-11-27T18:20:25.916705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471752782723468:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:25.920460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471752365654038:2048] [node 1] ICP22 created new session: [2:7577471752365654143:2048]
2025-11-27T18:20:25.916749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:25.924376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471753858978185:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:25.920611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471752365654143:2048] [node 1] ICS09 handshake done sender: [2:7577471752365654125:2127] self: [2:7577471752365654136:0] peer: [1:7577471752782723441:0] socket: 499 qp: -1
2025-11-27T18:20:25.924419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:25.920621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471752365654142:2108] [node 5] ICS09 handshake done sender: [2:7577471752365654117:2120] self: [2:7577471752365654114:0] peer: [5:7577471753858978182:0] socket: 503 qp: -1
2025-11-27T18:20:25.924547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471753858978169:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:25.926300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471751080856999:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:25.926341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471752225611773:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.920770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471752365654143:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:20:25.920771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471752365654142:2108] [node 5] ICS10 traffic start
::ffff:127.1.0.02025-11-27T18:20:25.926317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471752424283734:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32820
2025-11-27T18:20:25.920874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:25.926594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471753858978175:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:25.926550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471752424283727:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:25.920908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:25.926382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:25.926815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471753858978158:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.926569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471752782723460:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.926506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471752225611773:2109] [node 6] ICP22 created new session: [4:7577471752225611799:2109]
2025-11-27T18:20:25.926577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471751080856912:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.926958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471752782723373:2048] [node 2] ICP18 handshake (actor [1:7577471752782723456:2119]) is too old
2025-11-27T18:20:25.926638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471752225611799:2109] [node 6] ICS09 handshake done sender: [4:7577471752225611787:2115] self: [4:7577471752225611786:0] peer: [6:7577471752424283731:0] socket: 524 qp: -1
2025-11-27T18:20:25.927369Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471752424283729:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:25.926785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471752225611799:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:25.927369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471752424283708:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.927397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:25.926914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471752225611798:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29304
2025-11-27T18:20:25.927445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:25.926623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:25.927536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471752424283708:2105] [node 5] ICP22 created new session: [6:7577471752424283735:2105]
2025-11-27T18:20:25.926842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471751080856912:2049] [node 2] ICP22 created new session: [3:7577471751080857011:2049]
2025-11-27T18:20:25.926965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:25.927451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471753858978158:2109] [node 6] ICP22 created new session: [5:7577471753858978187:2109]
2025-11-27T18:20:25.927855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471752424283735:2105] [node 5] ICS09 handshake done sender: [6:7577471752424283727:2117] self: [6:7577471752424283730:0] peer: [5:7577471753858978168:0] socket: 519 qp: -1
2025-11-27T18:20:25.926971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471751080857011:2049] [node 2] ICS09 handshake done sender: [3:7577471751080856999:2121] self: [3:7577471751080857005:0] peer: [2:7577471752365654116:0] socket: 508 qp: -1
2025-11-27T18:20:25.927668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471753858978187:2109] [node 6] ICS09 handshake done sender: [5:7577471753858978169:2113] self: [5:7577471753858978168:0] peer: [6:7577471752424283730:0] socket: 518 qp: -1
2025-11-27T18:20:25.928005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471752424283735:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:25.927912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471753858978187:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:25.928430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:25.927147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471751080857011:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:25.927955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:25.928671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471752424283710:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.928601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471753858978095:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.927335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471751080856992:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:25.927452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:25.928947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471751080856977:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.929110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:25.929149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471751080856977:2109] [node 6] ICP22 created new session: [3:7577471751080857014:2109]
2025-11-27T18:20:25.928969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:25.928846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:25.929194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471751080857014:2109] [node 6] ICS09 handshake done sender: [3:7577471751080856992:2117] self: [3:7577471751080856989:0] peer: [6:7577471752424283728:0] socket: 515 qp: -1
2025-11-27T18:20:25.929330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471751080857014:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:25.929022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471753858978095:2049] [node 2] ICP22 created new session: [5:7577471753858978189:2049]
2025-11-27T18:20:25.929158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471753858978189:2049] [node 2] ICS09 handshake done sender: [5:7577471753858978175:2117] self: [5:7577471753858978182:0] peer: [2:7577471752365654114:0] socket: 505 qp: -1
2025-11-27T18:20:25.928884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471752424283710:2107] [node 4] ICP22 created new session: [6:7577471752424283737:2107]
2025-11-27T18:20:25.929582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:25.929196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471753858978189:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:25.928956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471752424283737:2107] [node 4] ICS09 handshake done sender: [6:7577471752424283729:2118] self: [6:7577471752424283731:0] peer: [4:7577471752225611786:0] socket: 525 qp: -1
2025-11-27T18:20:25.929321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:25.929052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471752424283737:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:25.929096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:25.929151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471752424283726:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:25.931850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471752424283649:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.931926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:25.931960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471752424283649:2050] [node 3] ICP22 created new session: [6:7577471752424283739:2050]
2025-11-27T18:20:25.932124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471752424283739:2050] [node 3] ICS09 handshake done sender: [6:7577471752424283726:2116] self: [6:7577471752424283728:0] peer: [3:7577471751080856989:0] socket: 516 qp: -1
2025-11-27T18:20:25.932235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471752424283739:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:25.932260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.932908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471752225611801:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29304
2025-11-27T18:20:25.937805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471752225611801:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.937902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471752225611798:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.938474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471752225611710:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.939727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471752225611709:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.940376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752365654128:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.940413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471752365654128:2128] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.940632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471752782723453:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.941169Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471753858978095:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:25.940673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471752782723453:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.941653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471753858978191:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30478
2025-11-27T18:20:25.941841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471752365654128:2128] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:25.942555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471753858978191:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.942739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471752424283649:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:25.942711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471753858978096:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.943005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471752365654113:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.942998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471752225611804:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29304
2025-11-27T18:20:25.943069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471752782723453:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:25.943187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471751080856990:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.943180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:25.943240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471752365654113:2107] [node 4] ICP22 created new session: [2:7577471752365654149:2107]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:25.943216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471751080856990:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.943252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471752782723439:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.943664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471752225611801:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:25.943506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471752365654149:2107] [node 4] ICS09 handshake done sender: [2:7577471752365654128:2128] self: [2:7577471752365654127:0] peer: [4:7577471752225611802:0] socket: 472 qp: -1
2025-11-27T18:20:25.943380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.943424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471752782723439:2107] [node 4] ICP22 created new session: [1:7577471752782723474:2107]
2025-11-27T18:20:25.943553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471752365654149:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:25.944751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471752225611805:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29304
2025-11-27T18:20:25.944816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471751080856990:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:25.944382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471752782723474:2107] [node 4] ICS09 handshake done sender: [1:7577471752782723453:2117] self: [1:7577471752782723452:0] peer: [4:7577471752225611803:0] socket: 532 qp: -1
2025-11-27T18:20:25.945023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471752424283741:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32820
2025-11-27T18:20:25.943679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:25.945449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471751080856978:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.945456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471752424283742:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32820
2025-11-27T18:20:25.944419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471752782723474:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:25.945721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471752424283741:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.944469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:25.945561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:25.945446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471752225611710:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.946238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471752424283647:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.945734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471751080856978:2108] [node 5] ICP22 created new session: [3:7577471751080857017:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.946278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471751080857017:2108] [node 5] ICS09 handshake done sender: [3:7577471751080856990:2115] self: [3:7577471751080856988:0] peer: [5:7577471753858978192:0] socket: 470 qp: -1
2025-11-27T18:20:25.946274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:25.946301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471752424283742:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.946326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471751080857017:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:25.946456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471753858978096:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471753858978191:2121] held: no
2025-11-27T18:20:25.946338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471752225611710:2049] [node 2] ICP22 created new session: [4:7577471752225611806:2049]
2025-11-27T18:20:25.946528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471752424283648:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:25.948070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471752225611806:2049] [node 2] ICS09 handshake done sender: [4:7577471752225611801:2118] self: [4:7577471752225611802:0] peer: [2:7577471752365654127:0] socket: 484 qp: -1
2025-11-27T18:20:25.948308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471752782723455:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.946542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:25.948876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471752365654129:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:25.948275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471753858978193:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30478
2025-11-27T18:20:25.948345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471752225611709:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16383) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471752225611798:2117] held: no
2025-11-27T18:20:25.948907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471752365654129:2129] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.948344Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471752782723455:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:25.948354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471752225611806:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:25.950515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471752365654129:2129] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:25.948390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:25.950639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471752225611798:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:25.949930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471752782723455:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.950842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471752782723437:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.955332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:25.955307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471752365654112:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:25.956560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471752424283745:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32820
2025-11-27T18:20:25.955493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471752782723437:2110] [node 6] ICP22 created new session: [1:7577471752782723477:2110]
2025-11-27T18:20:25.956954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471753858978191:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:25.955370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:25.956965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471752782723477:2110] [node 6] ICS09 handshake done sender: [1:7577471752782723455:2118] self: [1:7577471752782723454:0] peer: [6:7577471752424283743:0] socket: 488 qp: -1
2025-11-27T18:20:25.955495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471752365654112:2110] [node 6] ICP22 created new session: [2:7577471752365654152:2110]
2025-11-27T18:20:25.957098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471752782723477:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:25.957234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471753858978096:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.956934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471752365654152:2110] [node 6] ICS09 handshake done sender: [2:7577471752365654129:2129] self: [2:7577471752365654124:0] peer: [6:7577471752424283744:0] socket: 504 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:25.957158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471752424283647:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16383) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471752424283741:2122] held: no
2025-11-27T18:20:25.957160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:25.956975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471752365654152:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:25.959281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471752225611709:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.957022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:25.959351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:25.959796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:25.959782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471752424283746:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32820
2025-11-27T18:20:25.959534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471752225611709:2048] [node 1] ICP22 created new session: [4:7577471752225611808:2048]
2025-11-27T18:20:25.959664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471752225611808:2048] [node 1] ICS09 handshake done sender: [4:7577471752225611798:2117] self: [4:7577471752225611803:0] peer: [1:7577471752782723452:0] socket: 533 qp: -1
2025-11-27T18:20:25.959908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471753858978096:2050] [node 3] ICP22 created new session: [5:7577471753858978194:2050]
2025-11-27T18:20:25.959951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471753858978194:2050] [node 3] ICS09 handshake done sender: [5:7577471753858978191:2121] self: [5:7577471753858978192:0] peer: [3:7577471751080856988:0] socket: 468 qp: -1
2025-11-27T18:20:25.959827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471752225611808:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:25.959981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471753858978194:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:25.959875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:25.960094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:25.961440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471752225611710:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:25.961679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471752424283648:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8513) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471752424283742:2123] held: no
2025-11-27T18:20:25.961970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471752424283741:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:25.962201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471752365654113:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17069084ns) {HandleHandshakeStatus:329(17054364ns) {DropIncomingHandshake:442(31134ns) DropOutgoingHandshake:457(688ns) SwitchToState:179(313ns) ProcessPendingSessionEvents:495(14649505ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14555431ns max# 2049756ns}}}
2025-11-27T18:20:25.962237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471752424283647:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.962368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:25.962393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471752424283742:2123] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:25.962422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471752424283647:2048] [node 1] ICP22 created new session: [6:7577471752424283747:2048]
2025-11-27T18:20:25.962458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471752424283747:2048] [node 1] ICS09 handshake done sender: [6:7577471752424283741:2122] self: [6:7577471752424283743:0] peer: [1:7577471752782723454:0] socket: 534 qp: -1
2025-11-27T18:20:25.962487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471752424283747:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:25.962486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471752424283648:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:25.962515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:25.962684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:25.962801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471752424283648:2049] [node 2] ICP22 created new session: [6:7577471752424283749:2049]
2025-11-27T18:20:25.962847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471752424283749:2049] [node 2] ICS09 handshake done sender: [6:7577471752424283742:2123] self: [6:7577471752424283744:0] peer: [2:7577471752365654124:0] socket: 535 qp: -1
2025-11-27T18:20:25.962874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471752424283749:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:25.962907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:25.996426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:25.996648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471752365654143:2048] [node 1] ICS07 socket disconnect 499 reason# EndOfStream
2025-11-27T18:20:25.996686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471752365654143:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.996764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:25.996807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471752365654143:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:25.997587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:25.997779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471751080857009:2048] [node 1] ICS07 socket disconnect 500 reason# EndOfStream
2025-11-27T18:20:25.997803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471751080857009:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:25.997854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:25.997885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471751080857009:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:25.999180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471752365654038:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16383) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:25.999220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471752365654143:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:25.999787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:26.000377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471751080856911:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16383) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:26.000374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471752225611808:2048] [node 1] ICS01 socket: 533 reason# EndOfStream
2025-11-27T18:20:26.000424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471752225611709:2048] [node 1] ICP30 unregister session Session# [4:7577471752225611808:2048] VirtualId# [4:7577471752225611803:0]
2025-11-27T18:20:26.000411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471751080857009:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:26.000503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471752225611808:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.000570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:26.000616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:26.000978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471752365654038:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16383) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.001034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471752365654143:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.001152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471752365654038:2048] [node 1] ICP30 unregister session Session# [2:7577471752365654143:2048] VirtualId# [2:7577471752365654136:0]
2025-11-27T18:20:26.001313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:26.001364Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471752365654038:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.001382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.001418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471751080856911:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16383) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.001532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471751080857009:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.001563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471751080856911:2048] [node 1] ICP30 unregister session Session# [3:7577471751080857009:2048] VirtualId# [3:7577471751080857004:0]
2025-11-27T18:20:26.001675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:26.001712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471751080856911:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.001727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16383) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.001844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:26.002008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471753858978185:2048] [node 1] ICS01 socket: 487 reason# EndOfStream
2025-11-27T18:20:26.002038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471753858978094:2048] [node 1] ICP30 unregister session Session# [5:7577471753858978185:2048] VirtualId# [5:7577471753858978177:0]
2025-11-27T18:20:26.002063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471753858978185:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.002106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:26.002132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:26.002746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:26.002916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471752424283747:2048] [node 1] ICS01 socket: 534 reason# EndOfStream
2025-11-27T18:20:26.002956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471752424283647:2048] [node 1] ICP30 unregister session Session# [6:7577471752424283747:2048] VirtualId# [6:7577471752424283743:0]
2025-11-27T18:20:26.002980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471752424283747:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.003053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:26.003092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:26.011978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471751080856911:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:26.049826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:26.050059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471751080857011:2049] [node 2] ICS01 socket: 508 reason# EndOfStream
2025-11-27T18:20:26.050109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471751080856912:2049] [node 2] ICP30 unregister session Session# [3:7577471751080857011:2049] VirtualId# [3:7577471751080857005:0]
2025-11-27T18:20:26.050151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471751080857011:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.050238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:26.050298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:26.052754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:26.052946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471752225611806:2049] [node 2] ICS01 socket: 484 reason# EndOfStream
2025-11-27T18:20:26.052991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471752225611710:2049] [node 2] ICP30 unregister session Session# [4:7577471752225611806:2049] VirtualId# [4:7577471752225611802:0]
2025-11-27T18:20:26.053014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471752225611806:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.053065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:26.053107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:26.054026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:26.054204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471753858978189:2049] [node 2] ICS07 socket disconnect 505 reason# EndOfStream
2025-11-27T18:20:26.054233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471753858978189:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.054288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:26.054339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471753858978189:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:26.054958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471753858978095:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8513) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.055043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471753858978189:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.055075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471753858978095:2049] [node 2] ICP30 unregister session Session# [5:7577471753858978189:2049] VirtualId# [5:7577471753858978182:0]
2025-11-27T18:20:26.055117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:26.055196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:26.055228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471753858978095:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8513) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.055238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8513) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.055306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471752424283749:2049] [node 2] ICS01 socket: 535 reason# EndOfStream
2025-11-27T18:20:26.055347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471752424283648:2049] [node 2] ICP30 unregister session Session# [6:7577471752424283749:2049] VirtualId# [6:7577471752424283744:0]
2025-11-27T18:20:26.055373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471752424283749:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.055474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:26.055549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:26.065208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471753858978095:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:26.121856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:26.122058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471752225611795:2050] [node 3] ICS01 socket: 489 reason# EndOfStream
2025-11-27T18:20:26.122093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471752225611711:2050] [node 3] ICP30 unregister session Session# [4:7577471752225611795:2050] VirtualId# [4:7577471752225611788:0]
2025-11-27T18:20:26.122116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471752225611795:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.122178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:26.122232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:26.122944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:26.123071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471753858978194:2050] [node 3] ICS01 socket: 468 reason# EndOfStream
2025-11-27T18:20:26.123098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471753858978096:2050] [node 3] ICP30 unregister session Session# [5:7577471753858978194:2050] VirtualId# [5:7577471753858978192:0]
2025-11-27T18:20:26.123126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471753858978194:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.123187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:26.123233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:26.123975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:26.124104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471752424283739:2050] [node 3] ICS07 socket disconnect 516 reason# EndOfStream
2025-11-27T18:20:26.124126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471752424283739:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.124178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:26.124220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471752424283739:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:26.124813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471752424283649:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11649) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.124864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471752424283739:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.124915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471752424283649:2050] [node 3] ICP30 unregister session Session# [6:7577471752424283739:2050] VirtualId# [6:7577471752424283728:0]
2025-11-27T18:20:26.124996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:26.125017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471752424283649:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11649) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.125026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11649) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.134839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471752424283649:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:26.147385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:26.147554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471753858978178:2108] [node 4] ICS07 socket disconnect 490 reason# EndOfStream
2025-11-27T18:20:26.147583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471753858978178:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.147651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:26.147714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471753858978178:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:26.148250Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:26.148339Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471753858978157:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30834) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.148413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471753858978178:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.148516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471752424283737:2107] [node 4] ICS07 socket disconnect 525 reason# EndOfStream
2025-11-27T18:20:26.148447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471753858978157:2108] [node 4] ICP30 unregister session Session# [5:7577471753858978178:2108] VirtualId# [5:7577471753858978174:0]
2025-11-27T18:20:26.148555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471752424283737:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.148574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:26.148607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471753858978157:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30834) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.148636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:26.148618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30834) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.148690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471752424283737:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:26.149381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471752424283710:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30834) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.149638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471752424283737:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.149676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471752424283710:2107] [node 4] ICP30 unregister session Session# [6:7577471752424283737:2107] VirtualId# [6:7577471752424283731:0]
2025-11-27T18:20:26.149716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:26.149751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471752424283710:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30834) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.149763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30834) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.158424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471753858978157:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:26.159408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471752424283710:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:26.206146Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:26.206355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471752424283735:2105] [node 5] ICS07 socket disconnect 519 reason# EndOfStream
2025-11-27T18:20:26.206389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471752424283735:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.206460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:26.206510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471752424283735:2105] [node 5] ICS15 start handshake
2025-11-27T18:20:26.207052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471752424283708:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3703) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.207095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471752424283735:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.207154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471752424283708:2105] [node 5] ICP30 unregister session Session# [6:7577471752424283735:2105] VirtualId# [6:7577471752424283730:0]
2025-11-27T18:20:26.207191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:26.207212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471752424283708:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3703) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.207221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3703) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:26.266242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471759153012810:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:26.266242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471759153012809:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:26.283724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471758005147141:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:26.283790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471758005147140:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:26.305744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471756563384381:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:26.306120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471756563384382:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:26.326154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471756162790310:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:26.326197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471756162790312:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:26.326553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471756162790311:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:26.343787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471756540899201:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:26.343828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471756540899203:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:26.344324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471756540899202:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:26.361780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471755424995234:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:26.361978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471755424995235:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:26.362019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471755424995236:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:26.454049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471756540899264:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:26.455098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471759153012873:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:26.454797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471755424995295:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:26.455098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471756563384446:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:26.455149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471759153012874:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:26.455566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471756563384447:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:26.455602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471756162790373:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:26.455652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471755424995298:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:26.456215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471756162790374:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:26.455821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471759153012875:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:26.456068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471758005147230:2117] [node 5] ICP01 ready to work
2025-11-27T18:20:26.456039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471756563384448:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:26.456255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471758005147231:2126] [node 6] ICP01 ready to work
2025-11-27T18:20:26.456506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471756540899265:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:26.456595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471758005147232:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:26.460254Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471759153012810:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(55052991ns) {Configure:104(55045403ns) {ProcessConfigured:135(81183ns) {StartInitialHandshake:157(78380ns) {DropHandshakes:470(2516ns) {DropIncomingHandshake:442(629ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(1292ns) PrepareNewSessionHandshake:477(1702ns) {SwitchToState:179(836ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.467482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471756162790379:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16418
2025-11-27T18:20:26.473549Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471756563384382:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16334231ns) {Configure:104(16326062ns) {ProcessConfigured:135(1069389ns) {StartInitialHandshake:157(1067108ns) {DropHandshakes:470(1539ns) {DropIncomingHandshake:442(712ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(364ns) PrepareNewSessionHandshake:477(2316ns) {SwitchToState:179(1045ns)}}}}}
2025-11-27T18:20:26.474760Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471756540899201:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17093884ns) {Configure:104(17089421ns) {ProcessConfigured:135(1957770ns) {StartInitialHandshake:157(1956362ns) {DropHandshakes:470(1102ns) {DropIncomingHandshake:442(341ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:477(1539ns) {SwitchToState:179(622ns)}}}}}
2025-11-27T18:20:26.475123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471756162790310:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18279413ns) {Configure:104(18274255ns) {ProcessConfigured:135(2554392ns) {StartInitialHandshake:157(2552276ns) {DropHandshakes:470(1168ns) {DropIncomingHandshake:442(302ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(566ns) PrepareNewSessionHandshake:477(1061ns) {SwitchToState:179(807ns)}}}}}
2025-11-27T18:20:26.474976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471756563384381:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17024923ns) {Configure:104(17020860ns) {ProcessConfigured:135(2743345ns) {StartInitialHandshake:157(2741754ns) {DropHandshakes:470(594ns) {DropIncomingHandshake:442(226ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(2395ns) {SwitchToState:179(1355ns)}}}}}
2025-11-27T18:20:26.475470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471756162790312:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18051087ns) {Configure:104(18044893ns) {ProcessConfigured:135(2683896ns) {StartInitialHandshake:157(2682267ns) {DropHandshakes:470(706ns) {DropIncomingHandshake:442(203ns) DropOutgoingHandshake:457(140ns)} GenerateSessionVirtualId:988(392ns) PrepareNewSessionHandshake:477(1905ns) {SwitchToState:179(856ns)}}}}}
2025-11-27T18:20:26.474846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471756540899202:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17689325ns) {Configure:104(17685345ns) {ProcessConfigured:135(1727751ns) {StartInitialHandshake:157(1725923ns) {DropHandshakes:470(422774ns) {DropIncomingHandshake:442(329ns) DropOutgoingHandshake:457(660ns)} GenerateSessionVirtualId:988(740ns) PrepareNewSessionHandshake:477(1105ns) {SwitchToState:179(531ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.475824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471758005147241:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31295
2025-11-27T18:20:26.476058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471756162790379:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.477191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471756162790310:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.477033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471758005147241:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.477976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471758005147242:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31295
2025-11-27T18:20:26.478142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471759153012879:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.479742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471756563384456:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56913
2025-11-27T18:20:26.478177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471759153012879:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.480469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471756540899272:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4439
2025-11-27T18:20:26.480946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471759153012879:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.481816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471756162790389:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16418
2025-11-27T18:20:26.481821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471756563384456:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.481829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471756540899273:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4439
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.481843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471756540899272:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.482322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471756563384458:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56913
2025-11-27T18:20:26.482801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471755424995310:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27948
2025-11-27T18:20:26.483019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471759153012889:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59773
2025-11-27T18:20:26.483288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471756540899273:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.483811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471756563384381:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.484161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471755424995310:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.484254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471759153012874:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.484969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471758005147242:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.483751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471756540899275:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4439
2025-11-27T18:20:26.483994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471756563384459:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56913
2025-11-27T18:20:26.485249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471755424995298:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.484286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:26.485550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471756162790391:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16418
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.487949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471758005147247:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31295
2025-11-27T18:20:26.484295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471756563384458:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.483827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471756540899201:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.488209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471756162790385:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.487341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471756540899265:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.484614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471759153012889:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.488124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471755424995313:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27948
2025-11-27T18:20:26.488258Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471756162790385:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-27T18:20:26.485030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471759153012874:2107] [node 4] ICP22 created new session: [1:7577471759153012892:2107]
::ffff:127.1.0.0
2025-11-27T18:20:26.489079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471756563384459:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.485499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471759153012883:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.488246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471756540899275:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.490988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471756563384465:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56913
2025-11-27T18:20:26.485516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471759153012883:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.489035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471756162790392:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16418
2025-11-27T18:20:26.492907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471755424995314:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27948
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.491043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471756563384382:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.485571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471759153012892:2107] [node 4] ICS09 handshake done sender: [1:7577471759153012879:2112] self: [1:7577471759153012878:0] peer: [4:7577471756162790390:0] socket: 475 qp: -1
2025-11-27T18:20:26.491008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471756162790379:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:26.493361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471758005147236:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.495086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471755424995314:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.495444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471756563384465:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.491238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471756162790389:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.486918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471759153012892:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:26.493399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471758005147236:2138] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.496585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471755424995295:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.492007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471756162790378:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.486962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:26.492020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471756162790310:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.492032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471756162790378:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.497947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471756540899270:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.495566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471758005147251:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31295
2025-11-27T18:20:26.496598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471755424995313:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.493027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:26.487348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471759153012877:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-27T18:20:26.493074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471756162790311:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.495802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471758005147247:2143] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:26.497979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471756540899270:2111] [node 6] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-27T18:20:26.497718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471755424995236:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.498030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471756563384453:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.487374Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471759153012877:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.498064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471756563384453:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.499370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471755424995316:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27948
2025-11-27T18:20:26.496280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471758005147239:2139] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.498414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471756540899285:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4439
2025-11-27T18:20:26.491288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471759153012891:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59773
2025-11-27T18:20:26.493366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471756162790385:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:26.499320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471756563384469:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56913
2025-11-27T18:20:26.496305Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471758005147239:2139] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.492150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471759153012877:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:26.499583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471756540899286:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4439
2025-11-27T18:20:26.499580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471756563384471:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56913
2025-11-27T18:20:26.493419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471756162790310:2048] [node 1] ICP22 created new session: [4:7577471756162790394:2048]
2025-11-27T18:20:26.497638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471758005147236:2138] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:26.492936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471759153012883:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:26.495080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471756162790394:2048] [node 1] ICS09 handshake done sender: [4:7577471756162790379:2112] self: [4:7577471756162790390:0] peer: [1:7577471759153012878:0] socket: 478 qp: -1
2025-11-27T18:20:26.498074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471758005147239:2139] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:26.495429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471759153012810:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.500045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471756540899272:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:26.499312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471758005147141:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.495388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471756162790394:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:26.496037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471759153012873:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.500559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471755424995310:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:26.500331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471756540899270:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:26.499479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:26.496249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:26.496163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:26.502103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471756563384453:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:26.501231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471756540899273:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:26.499514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471758005147141:2049] [node 3] ICP22 created new session: [2:7577471758005147256:2049]
2025-11-27T18:20:26.496336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471756162790373:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.497635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471759153012873:2109] [node 5] ICP22 created new session: [1:7577471759153012898:2109]
2025-11-27T18:20:26.503498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471756563384382:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16250) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471756563384459:2118] held: no
2025-11-27T18:20:26.503476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471755424995298:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.499547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471758005147256:2049] [node 3] ICS09 handshake done sender: [2:7577471758005147236:2138] self: [2:7577471758005147233:0] peer: [3:7577471756563384468:0] socket: 497 qp: -1
2025-11-27T18:20:26.497860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471756162790378:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:26.503802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471756563384456:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:26.497660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:26.501415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471756540899201:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.504058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:26.497888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:26.501582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:26.497713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471759153012810:2049] [node 3] ICP22 created new session: [1:7577471759153012899:2049]
2025-11-27T18:20:26.499553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471758005147251:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.497996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471756162790392:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.504105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471755424995298:2107] [node 4] ICP22 created new session: [6:7577471755424995322:2107]
2025-11-27T18:20:26.506296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471756563384459:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:26.502069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471756540899201:2048] [node 1] ICP22 created new session: [5:7577471756540899289:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.506255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471755424995322:2107] [node 4] ICS09 handshake done sender: [6:7577471755424995310:2114] self: [6:7577471755424995315:0] peer: [4:7577471756162790382:0] socket: 498 qp: -1
2025-11-27T18:20:26.498036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471756162790373:2109] [node 6] ICP22 created new session: [4:7577471756162790398:2109]
2025-11-27T18:20:26.499568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471758005147256:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:26.497880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471759153012898:2109] [node 5] ICS09 handshake done sender: [1:7577471759153012883:2114] self: [1:7577471759153012882:0] peer: [5:7577471756540899280:0] socket: 476 qp: -1
2025-11-27T18:20:26.507020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471756563384446:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.502103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471756540899264:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.498382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471756162790398:2109] [node 6] ICS09 handshake done sender: [4:7577471756162790385:2113] self: [4:7577471756162790382:0] peer: [6:7577471755424995315:0] socket: 495 qp: -1
2025-11-27T18:20:26.499619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:26.507072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471755424995322:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:26.497906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471759153012898:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.507076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471755424995319:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27948
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.507031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471756563384381:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.502112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471756540899289:2048] [node 1] ICS09 handshake done sender: [5:7577471756540899272:2112] self: [5:7577471756540899280:0] peer: [1:7577471759153012882:0] socket: 484 qp: -1
2025-11-27T18:20:26.500134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471758005147232:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.507093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:26.498995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471756162790374:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.497940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:26.507150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:26.507348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471755424995321:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27948
2025-11-27T18:20:26.500182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:26.502267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471756540899289:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:26.498914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471756162790398:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:26.507185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471756563384446:2110] [node 6] ICP22 created new session: [3:7577471756563384474:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.500234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471758005147232:2107] [node 4] ICP22 created new session: [2:7577471758005147258:2107]
2025-11-27T18:20:26.508591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471755424995320:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27948
2025-11-27T18:20:26.497985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471759153012899:2049] [node 3] ICS09 handshake done sender: [1:7577471759153012877:2111] self: [1:7577471759153012876:0] peer: [3:7577471756563384464:0] socket: 474 qp: -1
2025-11-27T18:20:26.503327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:26.503538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471756540899265:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.498020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471759153012899:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:26.508626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471755424995323:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27948
2025-11-27T18:20:26.507453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471756563384474:2110] [node 6] ICS09 handshake done sender: [3:7577471756563384453:2113] self: [3:7577471756563384451:0] peer: [6:7577471755424995318:0] socket: 503 qp: -1
2025-11-27T18:20:26.498050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:26.503783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:26.499339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:26.508921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471755424995313:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:26.503863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:26.499605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:26.503909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471756540899265:2108] [node 4] ICP22 created new session: [5:7577471756540899291:2108]
2025-11-27T18:20:26.500757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471758005147258:2107] [node 4] ICS09 handshake done sender: [2:7577471758005147239:2139] self: [2:7577471758005147238:0] peer: [4:7577471756162790395:0] socket: 493 qp: -1
2025-11-27T18:20:26.507484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471756563384474:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:26.498063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471759153012891:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.499648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471756162790374:2108] [node 5] ICP22 created new session: [4:7577471756162790402:2108]
2025-11-27T18:20:26.500921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471758005147258:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:26.503793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471759153012881:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.504310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471756540899264:2109] [node 6] ICP22 created new session: [5:7577471756540899292:2109]
2025-11-27T18:20:26.507515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:26.510505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471755424995295:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471755424995314:2117] held: no
2025-11-27T18:20:26.500162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471756162790402:2108] [node 5] ICS09 handshake done sender: [4:7577471756162790378:2111] self: [4:7577471756162790377:0] peer: [5:7577471756540899283:0] socket: 480 qp: -1
2025-11-27T18:20:26.503820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471759153012881:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.507704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:26.505212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471756540899291:2108] [node 4] ICS09 handshake done sender: [5:7577471756540899273:2113] self: [5:7577471756540899283:0] peer: [4:7577471756162790377:0] socket: 487 qp: -1
2025-11-27T18:20:26.501174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:26.500309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471756162790402:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:26.507887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471756563384381:2048] [node 1] ICP22 created new session: [3:7577471756563384476:2048]
2025-11-27T18:20:26.510535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471755424995321:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.505910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471756540899292:2109] [node 6] ICS09 handshake done sender: [5:7577471756540899270:2111] self: [5:7577471756540899266:0] peer: [6:7577471755424995317:0] socket: 512 qp: -1
2025-11-27T18:20:26.502563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471758005147140:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.506700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471759153012881:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:26.511150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471755424995323:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.508152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471756563384382:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.500556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:26.506302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471756540899292:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:26.507852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471759153012874:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16460587ns) {HandleHandshakeStatus:329(16457944ns) {DropIncomingHandshake:442(1024ns) DropOutgoingHandshake:457(452ns) SwitchToState:179(74ns) ProcessPendingSessionEvents:495(12789866ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11815666ns max# 1281666ns}}}
2025-11-27T18:20:26.511195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471755424995236:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.503506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471756162790312:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.512041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:26.507435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471756162790400:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16418
2025-11-27T18:20:26.509409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471758005147262:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31295
2025-11-27T18:20:26.512086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471755424995236:2050] [node 3] ICP22 created new session: [6:7577471755424995326:2050]
2025-11-27T18:20:26.509106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471756162790311:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16250) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471756162790389:2116] held: no
2025-11-27T18:20:26.506331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:26.511178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471758005147140:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471758005147251:2145] held: no
2025-11-27T18:20:26.507914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471759153012809:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.512134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471755424995326:2050] [node 3] ICS09 handshake done sender: [6:7577471755424995313:2116] self: [6:7577471755424995318:0] peer: [3:7577471756563384451:0] socket: 507 qp: -1
2025-11-27T18:20:26.508217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:26.511989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471758005147251:2145] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:26.507959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:26.512396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471755424995326:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:26.506678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471756540899291:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:26.508275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471759153012809:2048] [node 2] ICP22 created new session: [1:7577471759153012904:2048]
2025-11-27T18:20:26.512437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:26.506722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:26.508501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471759153012904:2048] [node 2] ICS09 handshake done sender: [1:7577471759153012881:2113] self: [1:7577471759153012880:0] peer: [2:7577471758005147261:0] socket: 518 qp: -1
2025-11-27T18:20:26.514361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471756162790310:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16472894ns) {HandleHandshakeStatus:329(16469649ns) {DropIncomingHandshake:442(404ns) DropOutgoingHandshake:457(4826ns) SwitchToState:179(166ns) ProcessPendingSessionEvents:495(10424808ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10346103ns max# 1089967ns}}}
2025-11-27T18:20:26.508257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471756563384382:2049] [node 2] ICP22 created new session: [3:7577471756563384477:2049]
2025-11-27T18:20:26.511185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471756540899288:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4439
2025-11-27T18:20:26.508547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471759153012904:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:26.515018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471756162790389:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:26.508578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:26.515156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471756540899288:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.515385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471755424995234:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.508366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471756563384476:2048] [node 1] ICS09 handshake done sender: [3:7577471756563384456:2115] self: [3:7577471756563384464:0] peer: [1:7577471759153012876:0] socket: 486 qp: -1
2025-11-27T18:20:26.515101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471759153012873:2109] [node 5] ICP18 handshake (actor [1:7577471759153012891:2117]) is too old
2025-11-27T18:20:26.508523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471756563384476:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:26.515565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471755424995235:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.515541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471758005147140:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.508558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:26.515579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:26.508659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471756563384473:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56913
2025-11-27T18:20:26.515629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471758005147140:2048] [node 1] ICP22 created new session: [2:7577471758005147264:2048]
2025-11-27T18:20:26.515592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471756162790311:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.516249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:26.508764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471756563384477:2049] [node 2] ICS09 handshake done sender: [3:7577471756563384459:2118] self: [3:7577471756563384468:0] peer: [2:7577471758005147233:0] socket: 500 qp: -1
2025-11-27T18:20:26.516285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471758005147264:2048] [node 1] ICS09 handshake done sender: [2:7577471758005147251:2145] self: [2:7577471758005147261:0] peer: [1:7577471759153012880:0] socket: 519 qp: -1
2025-11-27T18:20:26.516298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471756162790311:2049] [node 2] ICP22 created new session: [4:7577471756162790406:2049]
2025-11-27T18:20:26.516312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471755424995314:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:26.516422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471758005147264:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.508935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471756563384477:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:26.516441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471756162790406:2049] [node 2] ICS09 handshake done sender: [4:7577471756162790389:2116] self: [4:7577471756162790395:0] peer: [2:7577471758005147238:0] socket: 496 qp: -1
2025-11-27T18:20:26.509327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:26.516563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:26.516482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471756162790406:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:26.516638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471756540899202:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.517552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471758005147240:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.516521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:26.512380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471756563384455:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.517582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471758005147240:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.517459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471756540899296:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4439
2025-11-27T18:20:26.517334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471759153012906:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59773
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.512407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471756563384455:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.515609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471756563384455:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:26.517808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471759153012888:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.517615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471756563384447:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.517989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:26.517998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471758005147263:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31295
2025-11-27T18:20:26.518504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471756563384447:2107] [node 4] ICP22 created new session: [3:7577471756563384483:2107]
2025-11-27T18:20:26.517828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471759153012888:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.519194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471755424995295:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.519248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:26.519256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471756563384483:2107] [node 4] ICS09 handshake done sender: [3:7577471756563384455:2114] self: [3:7577471756563384449:0] peer: [4:7577471756162790404:0] socket: 508 qp: -1
2025-11-27T18:20:26.518486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471758005147249:2144] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.519302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471756563384483:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:26.518506Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471758005147249:2144] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.519524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471755424995295:2106] [node 5] ICP22 created new session: [6:7577471755424995330:2106]
2025-11-27T18:20:26.519582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.520128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471759153012906:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.518946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471758005147240:2140] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:26.520142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471755424995330:2106] [node 5] ICS09 handshake done sender: [6:7577471755424995314:2117] self: [6:7577471755424995317:0] peer: [5:7577471756540899266:0] socket: 515 qp: -1
2025-11-27T18:20:26.519863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471758005147263:2147] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.520419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471755424995330:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:26.520151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471758005147230:2117] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.520528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471756162790405:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16418
2025-11-27T18:20:26.521183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471756540899298:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4439
2025-11-27T18:20:26.520832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471759153012888:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:26.520170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471758005147249:2144] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:26.521195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471756563384473:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.520474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:26.520179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:26.520334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471758005147230:2117] [node 5] ICP22 created new session: [2:7577471758005147268:2117]
2025-11-27T18:20:26.521662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471759153012875:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:26.520553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471758005147268:2117] [node 5] ICS09 handshake done sender: [2:7577471758005147240:2140] self: [2:7577471758005147237:0] peer: [5:7577471756540899297:0] socket: 534 qp: -1
2025-11-27T18:20:26.520672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471758005147268:2117] [node 5] ICS10 traffic start
2025-11-27T18:20:26.520937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:26.521266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471758005147231:2126] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.521307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:26.521349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471758005147231:2126] [node 6] ICP22 created new session: [2:7577471758005147270:2126]
2025-11-27T18:20:26.521982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:26.521837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471758005147270:2126] [node 6] ICS09 handshake done sender: [2:7577471758005147249:2144] self: [2:7577471758005147248:0] peer: [6:7577471755424995329:0] socket: 482 qp: -1
2025-11-27T18:20:26.522445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471756162790392:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:26.522026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471759153012875:2110] [node 6] ICP22 created new session: [1:7577471759153012909:2110]
2025-11-27T18:20:26.522267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471755424995331:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27948
2025-11-27T18:20:26.522407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471759153012909:2110] [node 6] ICS09 handshake done sender: [1:7577471759153012888:2115] self: [1:7577471759153012887:0] peer: [6:7577471755424995328:0] socket: 538 qp: -1
2025-11-27T18:20:26.521872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471758005147270:2126] [node 6] ICS10 traffic start
2025-11-27T18:20:26.522690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471756540899288:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:26.522692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471759153012909:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:26.521907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:26.522888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471756540899296:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.522729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:26.523306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471756540899202:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.523144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471756162790312:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.523274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:26.523404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:26.523561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471756162790312:2050] [node 3] ICP22 created new session: [4:7577471756162790408:2050]
2025-11-27T18:20:26.523415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471756540899203:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:26.525424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471756162790408:2050] [node 3] ICS09 handshake done sender: [4:7577471756162790392:2118] self: [4:7577471756162790404:0] peer: [3:7577471756563384449:0] socket: 509 qp: -1
2025-11-27T18:20:26.523558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471756540899202:2049] [node 2] ICP22 created new session: [5:7577471756540899299:2049]
2025-11-27T18:20:26.525458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471756162790408:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:26.523978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471756540899299:2049] [node 2] ICS09 handshake done sender: [5:7577471756540899288:2120] self: [5:7577471756540899297:0] peer: [2:7577471758005147237:0] socket: 537 qp: -1
2025-11-27T18:20:26.525497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:26.525463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471756540899299:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:26.525501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:26.525521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471758005147230:2117] [node 5] ICP18 handshake (actor [2:7577471758005147263:2147]) is too old
2025-11-27T18:20:26.525519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471755424995235:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16250) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471755424995323:2122] held: no
2025-11-27T18:20:26.526215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471756563384466:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:26.526257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471756563384466:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:26.527106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471755424995234:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471755424995321:2121] held: no
2025-11-27T18:20:26.527163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471755424995323:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:26.527240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471756563384447:2107] [node 4] ICP18 handshake (actor [3:7577471756563384473:2123]) is too old
2025-11-27T18:20:26.527568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471755424995235:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.527883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:26.527898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471756563384466:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:26.527947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471755424995235:2049] [node 2] ICP22 created new session: [6:7577471755424995334:2049]
2025-11-27T18:20:26.528003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471755424995334:2049] [node 2] ICS09 handshake done sender: [6:7577471755424995323:2122] self: [6:7577471755424995329:0] peer: [2:7577471758005147248:0] socket: 502 qp: -1
2025-11-27T18:20:26.528034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471755424995334:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:26.528067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.528125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471756563384448:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:26.528185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:26.528266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471756563384448:2108] [node 5] ICP22 created new session: [3:7577471756563384486:2108]
2025-11-27T18:20:26.528308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471756563384486:2108] [node 5] ICS09 handshake done sender: [3:7577471756563384466:2120] self: [3:7577471756563384463:0] peer: [5:7577471756540899300:0] socket: 494 qp: -1
2025-11-27T18:20:26.528303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471756540899302:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4439
2025-11-27T18:20:26.528332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471755424995333:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27948
2025-11-27T18:20:26.528357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471756563384486:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:26.528389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:26.528529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471756540899296:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:26.528958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471755424995321:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:26.529440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471756540899203:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20958) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471756540899296:2121] held: no
2025-11-27T18:20:26.529483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471755424995234:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.529921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:26.530098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471755424995234:2048] [node 1] ICP22 created new session: [6:7577471755424995336:2048]
2025-11-27T18:20:26.530139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471755424995336:2048] [node 1] ICS09 handshake done sender: [6:7577471755424995321:2121] self: [6:7577471755424995328:0] peer: [1:7577471759153012887:0] socket: 540 qp: -1
2025-11-27T18:20:26.530187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471755424995336:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:26.530219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:26.530538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471756540899203:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:26.530576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:26.530608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471756540899203:2050] [node 3] ICP22 created new session: [5:7577471756540899303:2050]
2025-11-27T18:20:26.530654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471756540899303:2050] [node 3] ICS09 handshake done sender: [5:7577471756540899296:2121] self: [5:7577471756540899300:0] peer: [3:7577471756563384463:0] socket: 542 qp: -1
2025-11-27T18:20:26.530685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471756540899303:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:26.530716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:26.563722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:26.563890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471758005147264:2048] [node 1] ICS01 socket: 519 reason# EndOfStream
2025-11-27T18:20:26.563928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471758005147140:2048] [node 1] ICP30 unregister session Session# [2:7577471758005147264:2048] VirtualId# [2:7577471758005147261:0]
2025-11-27T18:20:26.563950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471758005147264:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.564021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:26.564064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:26.564959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:26.565100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471756563384476:2048] [node 1] ICS07 socket disconnect 486 reason# EndOfStream
2025-11-27T18:20:26.565120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471756563384476:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.565170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:26.565210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471756563384476:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:26.567267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471756563384381:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:26.567292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471756563384476:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:26.567632Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:26.567884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471756162790394:2048] [node 1] ICS07 socket disconnect 478 reason# EndOfStream
2025-11-27T18:20:26.567919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471756162790394:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.567985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:26.568014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471756563384381:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.568051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471756563384476:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.568027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471756162790394:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:26.568082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471756563384381:2048] [node 1] ICP30 unregister session Session# [3:7577471756563384476:2048] VirtualId# [3:7577471756563384464:0]
2025-11-27T18:20:26.568185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:26.568215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471756563384381:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.568231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.568685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471756162790310:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.568722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471756162790394:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.568741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471756162790310:2048] [node 1] ICP30 unregister session Session# [4:7577471756162790394:2048] VirtualId# [4:7577471756162790390:0]
2025-11-27T18:20:26.568882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:26.568830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:26.568892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471756162790310:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.568902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.569012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471756540899289:2048] [node 1] ICS07 socket disconnect 484 reason# EndOfStream
2025-11-27T18:20:26.569032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471756540899289:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.569089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:26.569125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471756540899289:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:26.569619Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471756540899201:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.569688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471756540899289:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.569722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471756540899201:2048] [node 1] ICP30 unregister session Session# [5:7577471756540899289:2048] VirtualId# [5:7577471756540899280:0]
2025-11-27T18:20:26.569806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:26.569838Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471756540899201:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.569852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.570009Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:26.570154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471755424995336:2048] [node 1] ICS01 socket: 540 reason# EndOfStream
2025-11-27T18:20:26.570183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471755424995234:2048] [node 1] ICP30 unregister session Session# [6:7577471755424995336:2048] VirtualId# [6:7577471755424995328:0]
2025-11-27T18:20:26.570204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471755424995336:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.570259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:26.570306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:26.578226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471756162790310:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:26.578252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471756563384381:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:26.579237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471756540899201:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:26.626770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:26.626941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471756563384477:2049] [node 2] ICS01 socket: 500 reason# EndOfStream
2025-11-27T18:20:26.626976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471756563384382:2049] [node 2] ICP30 unregister session Session# [3:7577471756563384477:2049] VirtualId# [3:7577471756563384468:0]
2025-11-27T18:20:26.627001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471756563384477:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.627069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:26.627108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:26.629189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:26.629421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471756162790406:2049] [node 2] ICS01 socket: 496 reason# EndOfStream
2025-11-27T18:20:26.629462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471756162790311:2049] [node 2] ICP30 unregister session Session# [4:7577471756162790406:2049] VirtualId# [4:7577471756162790395:0]
2025-11-27T18:20:26.629485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471756162790406:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.629538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:26.629584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:26.630257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:26.630471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471756540899299:2049] [node 2] ICS07 socket disconnect 537 reason# EndOfStream
2025-11-27T18:20:26.630505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471756540899299:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.630571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:26.630618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471756540899299:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:26.631193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471756540899202:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16250) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.631267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471756540899299:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.631303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471756540899202:2049] [node 2] ICP30 unregister session Session# [5:7577471756540899299:2049] VirtualId# [5:7577471756540899297:0]
2025-11-27T18:20:26.631322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:26.631406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:26.631464Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471756540899202:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.631485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.631528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471755424995334:2049] [node 2] ICS01 socket: 502 reason# EndOfStream
2025-11-27T18:20:26.631568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471755424995235:2049] [node 2] ICP30 unregister session Session# [6:7577471755424995334:2049] VirtualId# [6:7577471755424995329:0]
2025-11-27T18:20:26.631593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471755424995334:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.631667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:26.631725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:26.641709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471756540899202:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:26.667802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:26.667960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471756162790408:2050] [node 3] ICS07 socket disconnect 509 reason# EndOfStream
2025-11-27T18:20:26.667984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471756162790408:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.668039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:26.668071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471756162790408:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:26.668626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471756162790312:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20958) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.668663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471756162790408:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.668723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471756162790312:2050] [node 3] ICP30 unregister session Session# [4:7577471756162790408:2050] VirtualId# [4:7577471756162790404:0]
2025-11-27T18:20:26.668829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:26.668795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:26.668855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471756162790312:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20958) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.668868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20958) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.669028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471756540899303:2050] [node 3] ICS01 socket: 542 reason# EndOfStream
2025-11-27T18:20:26.669076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471756540899203:2050] [node 3] ICP30 unregister session Session# [5:7577471756540899303:2050] VirtualId# [5:7577471756540899300:0]
2025-11-27T18:20:26.669110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471756540899303:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.669186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:26.669249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:26.669622Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:26.669798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471755424995326:2050] [node 3] ICS07 socket disconnect 507 reason# EndOfStream
2025-11-27T18:20:26.669832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471755424995326:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.669935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:26.669984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471755424995326:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:26.670651Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471755424995236:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20958) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.670714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471755424995326:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.670742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471755424995236:2050] [node 3] ICP30 unregister session Session# [6:7577471755424995326:2050] VirtualId# [6:7577471755424995318:0]
2025-11-27T18:20:26.670829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:26.670861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471755424995236:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20958) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.670876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20958) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.678578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471756162790312:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:26.680658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471755424995236:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:26.727384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:26.727627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471756540899291:2108] [node 4] ICS01 socket: 487 reason# EndOfStream
2025-11-27T18:20:26.727678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471756540899265:2108] [node 4] ICP30 unregister session Session# [5:7577471756540899291:2108] VirtualId# [5:7577471756540899283:0]
2025-11-27T18:20:26.727709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471756540899291:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.727782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:26.727843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:26.728557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:26.728779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471755424995322:2107] [node 4] ICS07 socket disconnect 498 reason# EndOfStream
2025-11-27T18:20:26.728812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471755424995322:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.728881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:26.728920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471755424995322:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:26.729598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471755424995298:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8768) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:26.729691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471755424995322:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:26.729732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471755424995298:2107] [node 4] ICP30 unregister session Session# [6:7577471755424995322:2107] VirtualId# [6:7577471755424995315:0]
2025-11-27T18:20:26.729833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:26.729871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471755424995298:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8768) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.729885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8768) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:26.740034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471755424995298:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:26.798499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:26.798655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471755424995330:2106] [node 5] ICS01 socket: 515 reason# EndOfStream
2025-11-27T18:20:26.798689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471755424995295:2106] [node 5] ICP30 unregister session Session# [6:7577471755424995330:2106] VirtualId# [6:7577471755424995317:0]
2025-11-27T18:20:26.798715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471755424995330:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:26.798774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:26.798814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:26.875606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471758403762577:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:26.875624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471758403762578:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:26.895932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471756968388432:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:26.895935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471756968388431:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:26.915776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471758694826896:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:26.915848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471758694826895:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:26.936415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471755537645330:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:26.936572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471755537645332:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:26.936630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471755537645331:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:26.956634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471755586061574:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:26.957104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471755586061576:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:26.957147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471755586061575:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:26.978535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471757128280745:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:26.978581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471757128280747:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:26.978684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471757128280746:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:27.020770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471761423248102:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:27.043839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471762698729936:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:27.061614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471761263355790:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:27.064097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471762698729937:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:27.063787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471762989794254:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:27.064600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471759832612689:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:27.064392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471759881028934:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:27.063789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471762989794255:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:27.064595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471761263355791:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:27.064613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471759832612690:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:27.064610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471761263355792:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:27.065105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471762698729938:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:27.064776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471762989794256:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:27.064614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471759881028933:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:27.065468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471761423248105:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:27.069280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471758403762577:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48520822ns) {Configure:104(48515795ns) {ProcessConfigured:135(122166ns) {StartInitialHandshake:157(119195ns) {DropHandshakes:470(3855ns) {DropIncomingHandshake:442(896ns) DropOutgoingHandshake:457(506ns)} GenerateSessionVirtualId:988(991ns) PrepareNewSessionHandshake:477(2381ns) {SwitchToState:179(981ns)}}}}}
2025-11-27T18:20:27.078723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471755586061576:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17270410ns) {Configure:104(17260138ns) {ProcessConfigured:135(4694413ns) {StartInitialHandshake:157(4692714ns) {DropHandshakes:470(1270ns) {DropIncomingHandshake:442(458ns) DropOutgoingHandshake:457(283ns)} GenerateSessionVirtualId:988(694ns) PrepareNewSessionHandshake:477(2102ns) {SwitchToState:179(707ns)}}}}}
2025-11-27T18:20:27.079405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471755537645331:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16378349ns) {Configure:104(16372884ns) {ProcessConfigured:135(228394ns) {StartInitialHandshake:157(225818ns) {DropHandshakes:470(1760ns) {DropIncomingHandshake:442(763ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(387ns) PrepareNewSessionHandshake:477(1353ns) {SwitchToState:179(612ns)}}}}}
2025-11-27T18:20:27.080918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471756968388431:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16480932ns) {Configure:104(16472007ns) {ProcessConfigured:135(5395201ns) {StartInitialHandshake:157(5393043ns) {DropHandshakes:470(1325ns) {DropIncomingHandshake:442(606ns) DropOutgoingHandshake:457(166ns)} GenerateSessionVirtualId:988(405ns) PrepareNewSessionHandshake:477(338950ns) {SwitchToState:179(2511ns)}}}}}
2025-11-27T18:20:27.082260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471761423248102:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(21372631ns) {Configure:104(21365425ns) {ProcessConfigured:135(727173ns) {StartInitialHandshake:157(724104ns) {DropHandshakes:470(1240ns) {DropIncomingHandshake:442(394ns) DropOutgoingHandshake:457(226ns)} GenerateSessionVirtualId:988(2365ns) PrepareNewSessionHandshake:477(1061ns) {SwitchToState:179(492ns)}}}}}
2025-11-27T18:20:27.082788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471758694826895:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20450526ns) {Configure:104(20443992ns) {ProcessConfigured:135(404504ns) {StartInitialHandshake:157(402624ns) {DropHandshakes:470(1278ns) {DropIncomingHandshake:442(313ns) DropOutgoingHandshake:457(247ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:477(1679ns) {SwitchToState:179(802ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.082974Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471755537645332:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(41806854ns) {Configure:104(41793826ns) {ProcessConfigured:135(1402170ns) {StartInitialHandshake:157(1400539ns) {DropHandshakes:470(1148ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(593ns) PrepareNewSessionHandshake:477(2637ns) {SwitchToState:179(1002ns)}}}}}
2025-11-27T18:20:27.083258Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471758694826896:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18407019ns) {Configure:104(18391064ns) {ProcessConfigured:135(541198ns) {StartInitialHandshake:157(539273ns) {DropHandshakes:470(1331ns) {DropIncomingHandshake:442(348ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(807ns) PrepareNewSessionHandshake:477(968ns) {SwitchToState:179(384ns)}}}}}
2025-11-27T18:20:27.084109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471759881028945:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34669
2025-11-27T18:20:27.084470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471762989794254:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17425878ns) {Configure:104(17421664ns) {ProcessConfigured:135(856595ns) {StartInitialHandshake:157(854111ns) {DropHandshakes:470(207838ns) {DropIncomingHandshake:442(436ns) DropOutgoingHandshake:457(681ns)} GenerateSessionVirtualId:988(1127ns) PrepareNewSessionHandshake:477(1687ns) {SwitchToState:179(917ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.085429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471759881028945:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.086490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471755586061574:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.086913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762698729945:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.086897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761263355801:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44820
2025-11-27T18:20:27.086960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471762698729945:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.090977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761263355801:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.093231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471762698729945:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.091702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471756968388431:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.094041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471759881028949:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34669
2025-11-27T18:20:27.094686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759832612704:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
2025-11-27T18:20:27.093774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471761423248119:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32262
2025-11-27T18:20:27.093593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471762698729952:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19518
2025-11-27T18:20:27.093753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761263355807:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44820
2025-11-27T18:20:27.097225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471759881028950:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34669
2025-11-27T18:20:27.094636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471762698729953:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19518
2025-11-27T18:20:27.098853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471762989794265:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62218
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.099310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471761423248120:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32262
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.094665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762698729940:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.099906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471762989794268:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62218
2025-11-27T18:20:27.100999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471761423248119:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.094701Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471762698729940:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.100291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471759832612704:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.100758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471759881028949:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.101079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471761423248120:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.098805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471762698729954:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19518
2025-11-27T18:20:27.100966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471762989794265:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:27.101082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471762989794269:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62218
::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.100293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471762698729940:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:27.101231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471761423248105:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.100969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471759881028950:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.101715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471759832612697:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.101972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761263355807:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.101094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471758694826895:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.100683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471762698729936:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.102343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471761423248102:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.101058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471755586061575:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.101100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471762989794268:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.101763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471759832612697:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.102331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761263355800:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.101079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.102211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471759881028953:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34669
2025-11-27T18:20:27.105226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471759832612697:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:27.102353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471761263355800:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.101141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471762698729936:2108] [node 5] ICP22 created new session: [1:7577471762698729958:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.102352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471759881028938:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.101704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471762989794269:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.102359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761263355810:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44820
2025-11-27T18:20:27.101987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471762989794270:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62218
2025-11-27T18:20:27.102375Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471759881028938:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.101170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762698729952:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.110520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471761423248123:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32262
2025-11-27T18:20:27.101468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471762698729958:2108] [node 5] ICS09 handshake done sender: [1:7577471762698729945:2112] self: [1:7577471762698729943:0] peer: [5:7577471759881028951:0] socket: 481 qp: -1
2025-11-27T18:20:27.102682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761263355809:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44820
2025-11-27T18:20:27.102100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471758694826896:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.105172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471759881028952:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34669
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.111850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471759832612689:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.104785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761263355797:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.101573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471758403762577:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.104893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471762989794270:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.113674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:27.111557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471759881028938:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:27.104813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471761263355797:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.101612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:27.114098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471761423248124:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32262
2025-11-27T18:20:27.112551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471759881028945:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:27.113985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471759832612689:2109] [node 6] ICP22 created new session: [4:7577471759832612709:2109]
2025-11-27T18:20:27.106086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471761263355800:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.101633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471762698729958:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.114115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471761423248123:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.113927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471759881028934:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.114167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471759832612709:2109] [node 6] ICS09 handshake done sender: [4:7577471759832612697:2111] self: [4:7577471759832612692:0] peer: [6:7577471761423248122:0] socket: 502 qp: -1
2025-11-27T18:20:27.106375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471761263355801:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:27.102011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471758403762577:2048] [node 2] ICP22 created new session: [1:7577471762698729960:2048]
2025-11-27T18:20:27.115814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471758694826896:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471762989794269:2117] held: no
2025-11-27T18:20:27.114020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471759881028953:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.114331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471761423248125:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32262
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.106609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471756968388431:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.116808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471762989794279:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62218
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.102015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:27.114210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471759832612709:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:27.115431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471757128280745:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.114101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471755586061574:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.115322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:27.118319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471762989794278:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62218
2025-11-27T18:20:27.109658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471761263355791:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.102231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762698729942:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.114600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:27.116284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471761423248120:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:27.120865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759832612708:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.120484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471762989794265:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:27.110570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:27.102259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471762698729942:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.122455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471759832612708:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.115089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:27.116284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471761423248119:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:27.122073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471758694826895:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.103473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471762698729960:2048] [node 2] ICS09 handshake done sender: [1:7577471762698729940:2110] self: [1:7577471762698729939:0] peer: [2:7577471761263355808:0] socket: 480 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.111435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471756968388431:2048] [node 1] ICP22 created new session: [2:7577471761263355814:2048]
2025-11-27T18:20:27.115313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471755586061574:2048] [node 1] ICP22 created new session: [5:7577471759881028958:2048]
2025-11-27T18:20:27.119743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471761423248105:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.123411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:27.123159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471755537645330:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.111817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:27.103528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762698729954:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.115374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471759881028934:2109] [node 6] ICP22 created new session: [5:7577471759881028960:2109]
2025-11-27T18:20:27.123503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759832612710:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
2025-11-27T18:20:27.104476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471762698729960:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:27.112193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471761263355814:2048] [node 1] ICS09 handshake done sender: [2:7577471761263355801:2113] self: [2:7577471761263355808:0] peer: [1:7577471762698729939:0] socket: 485 qp: -1
2025-11-27T18:20:27.123645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471758694826895:2048] [node 1] ICP22 created new session: [3:7577471762989794281:2048]
2025-11-27T18:20:27.119791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.115900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471759881028958:2048] [node 1] ICS09 handshake done sender: [5:7577471759881028945:2115] self: [5:7577471759881028951:0] peer: [1:7577471762698729943:0] socket: 484 qp: -1
2025-11-27T18:20:27.125756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471759832612710:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.119922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471761423248102:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28039) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471761423248119:2115] held: no
2025-11-27T18:20:27.120522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471761423248105:2108] [node 4] ICP22 created new session: [6:7577471761423248128:2108]
2025-11-27T18:20:27.123726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471762989794269:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:27.112241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761263355809:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.104778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762698729953:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.126954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471755537645331:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.121654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471761423248128:2108] [node 4] ICS09 handshake done sender: [6:7577471761423248120:2116] self: [6:7577471761423248122:0] peer: [4:7577471759832612692:0] socket: 505 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.116764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471759881028958:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:27.124270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471762989794281:2048] [node 1] ICS09 handshake done sender: [3:7577471762989794265:2114] self: [3:7577471762989794274:0] peer: [1:7577471762698729941:0] socket: 486 qp: -1
2025-11-27T18:20:27.104914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:27.121692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471761423248128:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:27.112542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471761263355791:2108] [node 5] ICP22 created new session: [2:7577471761263355815:2108]
2025-11-27T18:20:27.125703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471762989794281:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.116836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.107944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471762698729957:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19518
2025-11-27T18:20:27.112565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471761263355814:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:27.122851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:27.126232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:27.118338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471759881028960:2109] [node 6] ICS09 handshake done sender: [5:7577471759881028938:2110] self: [5:7577471759881028935:0] peer: [6:7577471761423248121:0] socket: 487 qp: -1
2025-11-27T18:20:27.109614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471762698729942:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.112603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:27.126788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471761423248127:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32262
2025-11-27T18:20:27.126847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471762989794260:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.118392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471759881028960:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:27.112614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471761263355797:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:27.127365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471761423248102:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.112985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762698729957:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.126870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471762989794260:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.119055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:27.133337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471759832612698:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.127678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471758694826896:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.127997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471761423248127:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.122044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471759881028956:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34669
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.133368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471759832612698:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.127720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:27.128118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471758694826896:2049] [node 2] ICP22 created new session: [3:7577471762989794283:2049]
2025-11-27T18:20:27.135864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471759832612698:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:27.122838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471755586061576:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.128544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471762989794283:2049] [node 2] ICS09 handshake done sender: [3:7577471762989794269:2117] self: [3:7577471762989794276:0] peer: [2:7577471761263355793:0] socket: 511 qp: -1
2025-11-27T18:20:27.128576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:27.113880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471758403762578:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.114126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471761263355815:2108] [node 5] ICS09 handshake done sender: [2:7577471761263355800:2112] self: [2:7577471761263355795:0] peer: [5:7577471759881028954:0] socket: 492 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.124292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471759881028949:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:27.129164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471762989794283:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:27.127008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471759881028959:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34669
2025-11-27T18:20:27.129165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471761423248102:2106] [node 5] ICP22 created new session: [6:7577471761423248133:2106]
2025-11-27T18:20:27.130370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:27.131170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471762989794260:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:27.113913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:27.136313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759832612714:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
2025-11-27T18:20:27.129678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471761423248130:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32262
2025-11-27T18:20:27.132690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471762989794255:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.113943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471758403762578:2049] [node 3] ICP22 created new session: [1:7577471762698729966:2049]
2025-11-27T18:20:27.127690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471755586061575:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.114405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471761263355815:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:27.129747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471757128280746:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.132948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:27.130006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471761423248133:2106] [node 5] ICS09 handshake done sender: [6:7577471761423248119:2115] self: [6:7577471761423248121:0] peer: [5:7577471759881028935:0] socket: 500 qp: -1
2025-11-27T18:20:27.136364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759832612715:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
2025-11-27T18:20:27.133391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471762989794255:2108] [node 5] ICP22 created new session: [3:7577471762989794288:2108]
2025-11-27T18:20:27.114632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471762698729966:2049] [node 3] ICS09 handshake done sender: [1:7577471762698729942:2111] self: [1:7577471762698729941:0] peer: [3:7577471762989794274:0] socket: 482 qp: -1
2025-11-27T18:20:27.133430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471762989794288:2108] [node 5] ICS09 handshake done sender: [3:7577471762989794260:2110] self: [3:7577471762989794257:0] peer: [5:7577471759881028964:0] socket: 520 qp: -1
2025-11-27T18:20:27.130033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471761423248133:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:27.136781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759832612716:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
2025-11-27T18:20:27.115302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471762698729966:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:27.127727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:27.133453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471762989794288:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.114654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471756968388432:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.131056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:27.136656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471759832612690:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.127977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471755586061575:2049] [node 2] ICP22 created new session: [5:7577471759881028966:2049]
2025-11-27T18:20:27.134479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471762989794284:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62218
2025-11-27T18:20:27.115093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:27.115666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:27.131202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471761423248131:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32262
2025-11-27T18:20:27.137052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:27.128173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471759881028966:2049] [node 2] ICS09 handshake done sender: [5:7577471759881028949:2116] self: [5:7577471759881028954:0] peer: [2:7577471761263355795:0] socket: 494 qp: -1
2025-11-27T18:20:27.135464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:27.132460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471761423248123:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:27.136280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471762989794284:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.129760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471759881028966:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:27.115141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471756968388432:2049] [node 3] ICP22 created new session: [2:7577471761263355820:2049]
2025-11-27T18:20:27.137105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471759832612690:2108] [node 5] ICP22 created new session: [4:7577471759832612718:2108]
2025-11-27T18:20:27.137065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471762989794264:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.133367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471757128280745:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.129985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:27.115294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:27.137735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471759832612718:2108] [node 5] ICS09 handshake done sender: [4:7577471759832612698:2112] self: [4:7577471759832612695:0] peer: [5:7577471759881028969:0] socket: 539 qp: -1
2025-11-27T18:20:27.131214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471759881028959:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.133418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:27.137086Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471762989794264:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.137755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471759832612718:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:27.138780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471759832612715:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.132300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471759881028933:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.115650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471761263355820:2049] [node 3] ICS09 handshake done sender: [2:7577471761263355797:2111] self: [2:7577471761263355793:0] peer: [3:7577471762989794276:0] socket: 495 qp: -1
2025-11-27T18:20:27.133925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471761423248131:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.138661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:27.115879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762698729947:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.136796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471759881028968:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34669
2025-11-27T18:20:27.139497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471759832612710:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:27.141077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471762989794264:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:27.115917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471761263355820:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:27.134516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471757128280745:2048] [node 1] ICP22 created new session: [6:7577471761423248136:2048]
2025-11-27T18:20:27.115912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471762698729947:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.137774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471759881028970:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34669
2025-11-27T18:20:27.139956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471759832612708:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:27.134575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471761423248136:2048] [node 1] ICS09 handshake done sender: [6:7577471761423248123:2117] self: [6:7577471761423248126:0] peer: [1:7577471762698729944:0] socket: 527 qp: -1
2025-11-27T18:20:27.116799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:27.141588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471759881028953:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:27.120489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471762698729947:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:27.135234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471761423248136:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:27.141703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471759881028933:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20292) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471759881028959:2121] held: no
2025-11-27T18:20:27.128098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761263355802:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.135684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:27.141539Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471755537645330:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15948) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471759832612708:2116] held: no
2025-11-27T18:20:27.128137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471761263355802:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.122414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471762698729938:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.135694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471757128280747:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.142349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471762989794256:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.131366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761263355803:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.139974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471761423248139:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32262
2025-11-27T18:20:27.122451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:27.141755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471755537645332:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.142402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:27.131388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471761263355803:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.142162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471761423248127:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:27.143000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471762989794256:2109] [node 6] ICP22 created new session: [3:7577471762989794291:2109]
2025-11-27T18:20:27.142127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471755537645331:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.142649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471757128280746:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.143358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471762989794291:2109] [node 6] ICS09 handshake done sender: [3:7577471762989794264:2113] self: [3:7577471762989794258:0] peer: [6:7577471761423248137:0] socket: 498 qp: -1
2025-11-27T18:20:27.143216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:27.143524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471762989794267:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.143322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471755586061574:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(22996693ns) {HandleHandshakeStatus:329(22993702ns) {DropIncomingHandshake:442(960ns) DropOutgoingHandshake:457(257759ns) SwitchToState:179(618ns) ProcessPendingSessionEvents:495(15490571ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14972881ns max# 1234423ns}}}
2025-11-27T18:20:27.142545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:27.132646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761263355823:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44820
2025-11-27T18:20:27.123040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471762698729938:2109] [node 6] ICP22 created new session: [1:7577471762698729969:2109]
2025-11-27T18:20:27.143374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471755586061576:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.143549Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471762989794267:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.143135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471755537645331:2049] [node 2] ICP22 created new session: [4:7577471759832612722:2049]
2025-11-27T18:20:27.143267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471757128280746:2049] [node 2] ICP22 created new session: [6:7577471761423248141:2049]
2025-11-27T18:20:27.123585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471762698729969:2109] [node 6] ICS09 handshake done sender: [1:7577471762698729947:2113] self: [1:7577471762698729944:0] peer: [6:7577471761423248126:0] socket: 526 qp: -1
2025-11-27T18:20:27.135204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471761263355802:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.143521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471761423248141:2049] [node 2] ICS09 handshake done sender: [6:7577471761423248127:2120] self: [6:7577471761423248134:0] peer: [2:7577471761263355799:0] socket: 518 qp: -1
2025-11-27T18:20:27.143923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471762989794291:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:27.136014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471761263355803:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:27.123716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471762698729969:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:27.143320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471759832612722:2049] [node 2] ICS09 handshake done sender: [4:7577471759832612710:2117] self: [4:7577471759832612713:0] peer: [2:7577471761263355798:0] socket: 541 qp: -1
2025-11-27T18:20:27.144788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:27.143976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:27.143558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471761423248141:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:27.143547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471759832612722:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:27.123756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:27.136732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471761263355790:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.143594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:27.145297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471762989794255:2108] [node 5] ICP18 handshake (actor [3:7577471762989794284:2121]) is too old
2025-11-27T18:20:27.143589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:27.144837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471759881028934:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(22792856ns) {HandleHandshakeStatus:329(22790304ns) {DropIncomingHandshake:442(16246ns) DropOutgoingHandshake:457(387ns) SwitchToState:179(659ns) ProcessPendingSessionEvents:495(16117301ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14924319ns max# 1414684ns}}}
2025-11-27T18:20:27.125676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762698729948:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.136764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:27.145679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471755537645330:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.145282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471755586061576:2050] [node 3] ICP22 created new session: [5:7577471759881028971:2050]
2025-11-27T18:20:27.144778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471761423248140:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32262
2025-11-27T18:20:27.136800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471761263355790:2107] [node 4] ICP22 created new session: [2:7577471761263355826:2107]
2025-11-27T18:20:27.125708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471762698729948:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.146086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471759881028971:2050] [node 3] ICS09 handshake done sender: [5:7577471759881028953:2119] self: [5:7577471759881028964:0] peer: [3:7577471762989794257:0] socket: 521 qp: -1
2025-11-27T18:20:27.146814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:27.147075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471762989794267:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:27.146808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471761423248105:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(17628639ns) {HandleHandshakeStatus:329(17625949ns) {DropIncomingHandshake:442(2400ns) DropOutgoingHandshake:457(5930ns) SwitchToState:179(545ns) ProcessPendingSessionEvents:495(13134035ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13031096ns max# 1198833ns}}}
2025-11-27T18:20:27.136838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471761263355826:2107] [node 4] ICS09 handshake done sender: [2:7577471761263355802:2114] self: [2:7577471761263355798:0] peer: [4:7577471759832612713:0] socket: 540 qp: -1
2025-11-27T18:20:27.146279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471759881028971:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:27.132266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471762698729971:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19518
2025-11-27T18:20:27.148620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471762989794254:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.147090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471755537645330:2048] [node 1] ICP22 created new session: [4:7577471759832612724:2048]
2025-11-27T18:20:27.148173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471759832612724:2048] [node 1] ICS09 handshake done sender: [4:7577471759832612708:2116] self: [4:7577471759832612712:0] peer: [1:7577471762698729946:0] socket: 538 qp: -1
2025-11-27T18:20:27.136858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471761263355826:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:27.147895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471757128280747:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2803) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471761423248131:2122] held: no
2025-11-27T18:20:27.146310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:27.149131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471761423248131:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:27.148403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471759832612689:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(25954985ns) {HandleHandshakeStatus:329(25952560ns) {DropIncomingHandshake:442(525ns) DropOutgoingHandshake:457(671ns) SwitchToState:179(237ns) ProcessPendingSessionEvents:495(18470294ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18357281ns max# 1424725ns}}}
2025-11-27T18:20:27.133082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471762698729948:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:27.137051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:27.148858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:27.135841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471762698729936:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(27285341ns) {HandleHandshakeStatus:329(27282031ns) {DropIncomingHandshake:442(736ns) DropOutgoingHandshake:457(630ns) SwitchToState:179(1427ns) ProcessPendingSessionEvents:495(25202354ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25081009ns max# 2185796ns}}}
2025-11-27T18:20:27.137560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471761263355792:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.148420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471759832612724:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:27.149064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471762989794254:2107] [node 4] ICP22 created new session: [3:7577471762989794294:2107]
2025-11-27T18:20:27.139933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:27.135937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762698729971:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.149401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471762989794294:2107] [node 4] ICS09 handshake done sender: [3:7577471762989794267:2115] self: [3:7577471762989794266:0] peer: [4:7577471759832612721:0] socket: 550 qp: -1
2025-11-27T18:20:27.148471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.149447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471762989794294:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:27.135984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471762698729937:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.139998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471761263355792:2109] [node 6] ICP22 created new session: [2:7577471761263355828:2109]
2025-11-27T18:20:27.136045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:27.141491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471761263355828:2109] [node 6] ICS09 handshake done sender: [2:7577471761263355803:2115] self: [2:7577471761263355799:0] peer: [6:7577471761423248134:0] socket: 543 qp: -1
2025-11-27T18:20:27.136087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471762698729937:2107] [node 4] ICP22 created new session: [1:7577471762698729973:2107]
2025-11-27T18:20:27.141541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471761263355828:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:27.136551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471762698729973:2107] [node 4] ICS09 handshake done sender: [1:7577471762698729948:2114] self: [1:7577471762698729946:0] peer: [4:7577471759832612712:0] socket: 532 qp: -1
2025-11-27T18:20:27.149787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759832612725:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17487
2025-11-27T18:20:27.149494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:27.141580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:27.136829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471762698729973:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:27.142471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471756968388431:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(29069053ns) {HandleHandshakeStatus:329(29066800ns) {DropIncomingHandshake:442(312ns) DropOutgoingHandshake:457(5946ns) SwitchToState:179(351ns) ProcessPendingSessionEvents:495(21277496ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21166875ns max# 1595491ns}}}
2025-11-27T18:20:27.136851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471762698729936:2108] [node 5] ICP18 handshake (actor [1:7577471762698729952:2115]) is too old
2025-11-27T18:20:27.142981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471756968388432:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(21849470ns) {HandleHandshakeStatus:329(21848283ns) {DropIncomingHandshake:442(13405ns) DropOutgoingHandshake:457(263ns) SwitchToState:179(164ns) ProcessPendingSessionEvents:495(16305270ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15420645ns max# 1447965ns}}}
2025-11-27T18:20:27.136880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:27.143201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471761263355791:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(26380340ns) {HandleHandshakeStatus:329(26377450ns) {DropIncomingHandshake:442(239ns) DropOutgoingHandshake:457(334ns) SwitchToState:179(130ns) ProcessPendingSessionEvents:495(19729695ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18849903ns max# 1681743ns}}}
2025-11-27T18:20:27.138609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471758403762577:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(29220474ns) {HandleHandshakeStatus:329(29217630ns) {DropIncomingHandshake:442(712ns) DropOutgoingHandshake:457(365ns) SwitchToState:179(303ns) ProcessPendingSessionEvents:495(24308408ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22920441ns max# 2770786ns}}}
2025-11-27T18:20:27.150136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471759832612715:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.143537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471758403762577:2048] [node 2] ICP18 handshake (actor [1:7577471762698729953:2116]) is too old
2025-11-27T18:20:27.150335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471757128280747:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.144474Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471758403762578:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(23773487ns) {HandleHandshakeStatus:329(23770948ns) {DropIncomingHandshake:442(1668ns) DropOutgoingHandshake:457(665ns) SwitchToState:179(155ns) ProcessPendingSessionEvents:495(17110747ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16964970ns max# 1241721ns}}}
2025-11-27T18:20:27.151192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761263355830:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44820
2025-11-27T18:20:27.150718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:27.144536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471762698729938:2109] [node 6] ICP18 handshake (actor [1:7577471762698729971:2119]) is too old
2025-11-27T18:20:27.151212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471755537645332:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.150713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471759881028959:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:27.144874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471758403762578:2049] [node 3] ICP18 handshake (actor [1:7577471762698729957:2118]) is too old
2025-11-27T18:20:27.151293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471757128280747:2050] [node 3] ICP22 created new session: [6:7577471761423248143:2050]
2025-11-27T18:20:27.152094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471761423248143:2050] [node 3] ICS09 handshake done sender: [6:7577471761423248131:2122] self: [6:7577471761423248137:0] peer: [3:7577471762989794258:0] socket: 517 qp: -1
2025-11-27T18:20:27.151318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:27.151366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471755537645332:2050] [node 3] ICP22 created new session: [4:7577471759832612727:2050]
2025-11-27T18:20:27.152411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471759881028933:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.151406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471759832612727:2050] [node 3] ICS09 handshake done sender: [4:7577471759832612715:2119] self: [4:7577471759832612721:0] peer: [3:7577471762989794266:0] socket: 554 qp: -1
2025-11-27T18:20:27.152323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471761423248143:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:27.152502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:27.151432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471759832612727:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:27.152467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:27.152308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:27.152757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471759881028933:2108] [node 4] ICP22 created new session: [5:7577471759881028973:2108]
2025-11-27T18:20:27.152889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471759881028973:2108] [node 4] ICS09 handshake done sender: [5:7577471759881028959:2121] self: [5:7577471759881028969:0] peer: [4:7577471759832612695:0] socket: 542 qp: -1
2025-11-27T18:20:27.152918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471759881028973:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:27.153197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471757128280745:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16019497ns) {HandleHandshakeStatus:329(16017476ns) {DropIncomingHandshake:442(545ns) DropOutgoingHandshake:457(5254ns) SwitchToState:179(182ns) ProcessPendingSessionEvents:495(12278226ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11457245ns max# 1583619ns}}}
2025-11-27T18:20:27.152957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:27.154173Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471755537645332:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:27.195882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:27.196045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471761263355814:2048] [node 1] ICS07 socket disconnect 485 reason# EndOfStream
2025-11-27T18:20:27.196071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471761263355814:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.196134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:27.196173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471761263355814:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:27.197001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:27.197176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471762989794281:2048] [node 1] ICS01 socket: 486 reason# EndOfStream
2025-11-27T18:20:27.197239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471758694826895:2048] [node 1] ICP30 unregister session Session# [3:7577471762989794281:2048] VirtualId# [3:7577471762989794274:0]
2025-11-27T18:20:27.197292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471762989794281:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.197363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:27.197403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:27.198264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471756968388431:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15948) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:27.198302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471761263355814:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:27.199203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471756968388431:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15948) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.199868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471761263355814:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.199917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471756968388431:2048] [node 1] ICP30 unregister session Session# [2:7577471761263355814:2048] VirtualId# [2:7577471761263355808:0]
2025-11-27T18:20:27.200012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:27.200075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471756968388431:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15948) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.200036Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:27.200097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15948) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.200191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471759832612724:2048] [node 1] ICS01 socket: 538 reason# EndOfStream
2025-11-27T18:20:27.200229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471755537645330:2048] [node 1] ICP30 unregister session Session# [4:7577471759832612724:2048] VirtualId# [4:7577471759832612712:0]
2025-11-27T18:20:27.200251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471759832612724:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.200326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:27.200397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:27.201063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:27.201200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471759881028958:2048] [node 1] ICS07 socket disconnect 484 reason# EndOfStream
2025-11-27T18:20:27.201223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471759881028958:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.201289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:27.201316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471759881028958:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:27.201874Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471755586061574:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15948) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.202006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471759881028958:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.202043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471755586061574:2048] [node 1] ICP30 unregister session Session# [5:7577471759881028958:2048] VirtualId# [5:7577471759881028951:0]
2025-11-27T18:20:27.202035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:27.202155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:27.202189Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471755586061574:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15948) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.202202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15948) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.202182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471761423248136:2048] [node 1] ICS07 socket disconnect 527 reason# EndOfStream
2025-11-27T18:20:27.202211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471761423248136:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.202257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:27.202282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471761423248136:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:27.202867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471757128280745:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15948) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.202902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471761423248136:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.203009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471757128280745:2048] [node 1] ICP30 unregister session Session# [6:7577471761423248136:2048] VirtualId# [6:7577471761423248126:0]
2025-11-27T18:20:27.203110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:27.203146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471757128280745:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15948) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.203160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15948) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.212066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471755586061574:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:27.213094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471757128280745:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:27.253370Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:27.253614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471762989794283:2049] [node 2] ICS01 socket: 511 reason# EndOfStream
2025-11-27T18:20:27.253665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471758694826896:2049] [node 2] ICP30 unregister session Session# [3:7577471762989794283:2049] VirtualId# [3:7577471762989794276:0]
2025-11-27T18:20:27.253699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471762989794283:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.253789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:27.253860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:27.255699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:27.255918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471759832612722:2049] [node 2] ICS07 socket disconnect 541 reason# EndOfStream
2025-11-27T18:20:27.255960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471759832612722:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.256040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:27.256106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471759832612722:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:27.256538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:27.256710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471759881028966:2049] [node 2] ICS01 socket: 494 reason# EndOfStream
2025-11-27T18:20:27.256753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471755586061575:2049] [node 2] ICP30 unregister session Session# [5:7577471759881028966:2049] VirtualId# [5:7577471759881028954:0]
2025-11-27T18:20:27.256810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471755537645331:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5295) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.256783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471759881028966:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.256943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471759832612722:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.256856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:27.257041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471755537645331:2049] [node 2] ICP30 unregister session Session# [4:7577471759832612722:2049] VirtualId# [4:7577471759832612713:0]
2025-11-27T18:20:27.256909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:27.257159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:27.257201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471755537645331:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.257220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.257618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:27.257825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471761423248141:2049] [node 2] ICS07 socket disconnect 518 reason# EndOfStream
2025-11-27T18:20:27.257880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471761423248141:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.257968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:27.258017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471761423248141:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:27.258700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471757128280746:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5295) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.258752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471761423248141:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.258818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471757128280746:2049] [node 2] ICP30 unregister session Session# [6:7577471761423248141:2049] VirtualId# [6:7577471761423248134:0]
2025-11-27T18:20:27.258853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:27.258875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471757128280746:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.258887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.267412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471755537645331:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:27.268326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471757128280746:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:27.317517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:27.317714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471759832612727:2050] [node 3] ICS07 socket disconnect 554 reason# EndOfStream
2025-11-27T18:20:27.317755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471759832612727:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.317814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:27.317847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471759832612727:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:27.318439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:27.318382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471755537645332:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2803) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.318419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471759832612727:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.318441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471755537645332:2050] [node 3] ICP30 unregister session Session# [4:7577471759832612727:2050] VirtualId# [4:7577471759832612721:0]
2025-11-27T18:20:27.318589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471759881028971:2050] [node 3] ICS07 socket disconnect 521 reason# EndOfStream
2025-11-27T18:20:27.318548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:27.318612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471759881028971:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.318575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471755537645332:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2803) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.318660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:27.318588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2803) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.318696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471759881028971:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:27.319275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471755586061576:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2803) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.319336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:27.319307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471759881028971:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.319329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471755586061576:2050] [node 3] ICP30 unregister session Session# [5:7577471759881028971:2050] VirtualId# [5:7577471759881028964:0]
2025-11-27T18:20:27.319505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:27.319518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471761423248143:2050] [node 3] ICS01 socket: 517 reason# EndOfStream
2025-11-27T18:20:27.319544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471755586061576:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2803) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.319550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471757128280747:2050] [node 3] ICP30 unregister session Session# [6:7577471761423248143:2050] VirtualId# [6:7577471761423248137:0]
2025-11-27T18:20:27.319563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2803) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.319571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471761423248143:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.319617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:27.319646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:27.328802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471755537645332:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:27.329808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471755586061576:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:27.373048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:27.373209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471759881028973:2108] [node 4] ICS01 socket: 542 reason# EndOfStream
2025-11-27T18:20:27.373244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471759881028933:2108] [node 4] ICP30 unregister session Session# [5:7577471759881028973:2108] VirtualId# [5:7577471759881028969:0]
2025-11-27T18:20:27.373287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471759881028973:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.373342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:27.373380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:27.374132Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:27.374316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471761423248128:2108] [node 4] ICS07 socket disconnect 505 reason# EndOfStream
2025-11-27T18:20:27.374351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471761423248128:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.374427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:27.374473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471761423248128:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:27.375186Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471761423248105:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20292) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.375228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471761423248128:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.375318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471761423248105:2108] [node 4] ICP30 unregister session Session# [6:7577471761423248128:2108] VirtualId# [6:7577471761423248122:0]
2025-11-27T18:20:27.375405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:27.375443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471761423248105:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20292) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.375453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20292) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.385146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471761423248105:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:27.434057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:27.434237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471761423248133:2106] [node 5] ICS01 socket: 500 reason# EndOfStream
2025-11-27T18:20:27.434284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471761423248102:2106] [node 5] ICP30 unregister session Session# [6:7577471761423248133:2106] VirtualId# [6:7577471761423248121:0]
2025-11-27T18:20:27.434323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471761423248133:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.434410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:27.434466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:27.541545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471762270407177:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:27.541605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471762270407178:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:27.558767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471761074504741:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:27.558817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471761074504742:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:27.576203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471760723387532:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:27.576247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471760723387533:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:27.593953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471759572856421:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:27.594014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471759572856422:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:27.594048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471759572856423:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:27.610908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471762203520601:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:27.610948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471762203520602:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:27.610981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471762203520603:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:27.626988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471763012934888:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:27.627032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471763012934889:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:27.627061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471763012934890:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:27.695026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471763012934949:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:27.713969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471759572856484:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:27.713809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471760723387595:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:27.714062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471761074504804:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:27.714142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471762270407241:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:27.714230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471760723387596:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:27.714721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471762203520664:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:27.715006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471759572856485:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:27.715091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471760723387597:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:27.715128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471761074504805:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:27.715140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471762203520665:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:27.715162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471762270407242:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:27.715201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471763012934952:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:27.715743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471761074504806:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:27.715809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471762270407243:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.725623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471760723387611:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44542
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.727810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759572856496:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42797
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.728766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761074504817:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6409
2025-11-27T18:20:27.729776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761074504818:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6409
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.729886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761074504819:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6409
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.730521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761074504820:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6409
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.732330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471762203520679:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50023
2025-11-27T18:20:27.732535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761074504817:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.733271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471762270407254:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55847
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.733523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471762270407255:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55847
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.733571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471762203520679:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.733868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471762270407256:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55847
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.733981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471762270407257:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55847
2025-11-27T18:20:27.734418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762270407254:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:27.734524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762270407256:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.734592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471762270407258:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55847
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.734582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471760723387611:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.734648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471760723387612:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44542
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.734898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762270407255:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.735062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471760723387613:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44542
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.735134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762270407257:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.735147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471760723387614:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44542
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.735225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471760723387612:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.735180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471759572856496:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.735231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759572856500:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42797
2025-11-27T18:20:27.735268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759572856501:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42797
2025-11-27T18:20:27.735339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471759572856423:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.735500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471760723387603:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.735520Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471760723387603:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.736048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471760723387603:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.736559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759572856503:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42797
2025-11-27T18:20:27.736545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761074504818:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.736950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471760723387595:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.737018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:27.737059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471760723387595:2107] [node 4] ICP22 created new session: [3:7577471760723387618:2107]
2025-11-27T18:20:27.737095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471760723387618:2107] [node 4] ICS09 handshake done sender: [3:7577471760723387603:2112] self: [3:7577471760723387602:0] peer: [4:7577471759572856502:0] socket: 491 qp: -1
2025-11-27T18:20:27.737149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471760723387618:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:27.737182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:27.737188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762270407258:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.737245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761074504819:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.737405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761074504824:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6409
2025-11-27T18:20:27.737487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761074504820:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.737442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471759572856500:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.737483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471759572856496:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:27.737725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471759572856423:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.737771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:27.737804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471759572856423:2050] [node 3] ICP22 created new session: [4:7577471759572856504:2050]
2025-11-27T18:20:27.737829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471759572856504:2050] [node 3] ICS09 handshake done sender: [4:7577471759572856496:2115] self: [4:7577471759572856502:0] peer: [3:7577471760723387602:0] socket: 499 qp: -1
2025-11-27T18:20:27.737853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471759572856504:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:27.737959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:27.738758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471760723387613:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.739328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471760723387533:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.739673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471762203520681:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50023
2025-11-27T18:20:27.740456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761074504810:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.740478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471761074504810:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.741381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471761074504810:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:27.741498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471759572856422:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.741696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471761074504742:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.741834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:27.741940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471761074504742:2049] [node 3] ICP22 created new session: [2:7577471761074504830:2049]
2025-11-27T18:20:27.741971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471761074504830:2049] [node 3] ICS09 handshake done sender: [2:7577471761074504810:2111] self: [2:7577471761074504809:0] peer: [3:7577471760723387621:0] socket: 515 qp: -1
2025-11-27T18:20:27.741992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471761074504830:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:27.742030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.742139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471760723387614:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.742205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471762203520682:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50023
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.742404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471760723387622:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44542
2025-11-27T18:20:27.742422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471762203520681:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.742548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471762203520603:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.742900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471760723387532:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.742995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471760723387605:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.743011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471760723387605:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.742966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471759572856501:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.743065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761074504824:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.743108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471759572856421:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.743127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471762203520682:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.743225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471762203520664:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.743373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471759572856493:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.743381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762270407249:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.743449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471760723387613:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:27.743395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471759572856493:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.743427Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471762270407249:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.743795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761074504812:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.743816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471761074504812:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.744480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471759572856493:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:27.744957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762270407247:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.744995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471762270407247:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.746345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471762203520685:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50023
2025-11-27T18:20:27.746590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471759572856484:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.746625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:27.746713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471761074504741:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.746662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471759572856484:2108] [node 5] ICP22 created new session: [4:7577471759572856510:2108]
2025-11-27T18:20:27.746755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471760723387533:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.746801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:27.746695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471759572856510:2108] [node 5] ICS09 handshake done sender: [4:7577471759572856493:2113] self: [4:7577471759572856492:0] peer: [5:7577471762203520684:0] socket: 531 qp: -1
2025-11-27T18:20:27.746847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471762270407249:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:27.746965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471761074504812:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:27.746856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471760723387533:2049] [node 2] ICP22 created new session: [3:7577471760723387625:2049]
2025-11-27T18:20:27.746725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471759572856510:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:27.746902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471760723387625:2049] [node 2] ICS09 handshake done sender: [3:7577471760723387613:2117] self: [3:7577471760723387621:0] peer: [2:7577471761074504809:0] socket: 521 qp: -1
2025-11-27T18:20:27.746750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:27.746931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471760723387625:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:27.746953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:27.747466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471762203520682:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:27.747496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471761074504804:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.747572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:27.747647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471761074504804:2107] [node 4] ICP22 created new session: [2:7577471761074504835:2107]
2025-11-27T18:20:27.747673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471762270407247:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:27.747731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471762203520664:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.747708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471761074504835:2107] [node 4] ICS09 handshake done sender: [2:7577471761074504812:2112] self: [2:7577471761074504811:0] peer: [4:7577471759572856506:0] socket: 516 qp: -1
2025-11-27T18:20:27.747763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:27.747733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471761074504835:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:27.747883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471762203520664:2108] [node 4] ICP22 created new session: [5:7577471762203520687:2108]
2025-11-27T18:20:27.747751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:27.747885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471760723387533:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:27.747978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471762203520687:2108] [node 4] ICS09 handshake done sender: [5:7577471762203520682:2117] self: [5:7577471762203520684:0] peer: [4:7577471759572856492:0] socket: 532 qp: -1
2025-11-27T18:20:27.748006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471762203520687:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:27.748054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:27.748188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762270407245:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.748224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471762270407245:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.749026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471762270407241:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.749091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:27.749142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471762270407241:2107] [node 4] ICP22 created new session: [1:7577471762270407269:2107]
2025-11-27T18:20:27.749204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471762270407269:2107] [node 4] ICS09 handshake done sender: [1:7577471762270407249:2113] self: [1:7577471762270407248:0] peer: [4:7577471759572856508:0] socket: 523 qp: -1
2025-11-27T18:20:27.749240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471762270407269:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:27.749290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.750072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471760723387605:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:27.750132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471762203520686:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50023
2025-11-27T18:20:27.750523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471762270407178:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.750471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471760723387596:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.750511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:27.750623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:27.750534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471760723387596:2108] [node 5] ICP22 created new session: [3:7577471760723387629:2108]
2025-11-27T18:20:27.750671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471762270407178:2049] [node 3] ICP22 created new session: [1:7577471762270407271:2049]
2025-11-27T18:20:27.750556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471760723387629:2108] [node 5] ICS09 handshake done sender: [3:7577471760723387605:2113] self: [3:7577471760723387604:0] peer: [5:7577471762203520683:0] socket: 510 qp: -1
2025-11-27T18:20:27.750575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471760723387629:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:27.750598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:27.750736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471762270407271:2049] [node 3] ICS09 handshake done sender: [1:7577471762270407247:2112] self: [1:7577471762270407246:0] peer: [3:7577471760723387623:0] socket: 520 qp: -1
2025-11-27T18:20:27.750844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471762270407271:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:27.750869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.751681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471762203520689:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50023
2025-11-27T18:20:27.751737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759572856512:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42797
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.752396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471759572856513:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42797
2025-11-27T18:20:27.752431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471762203520681:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.752900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471760723387627:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44542
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.752977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471762203520686:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.752963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471759572856422:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2329) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471759572856500:2116] held: no
2025-11-27T18:20:27.753008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471762203520603:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.752987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471763012934966:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26976
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.753071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:27.753117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471762203520603:2050] [node 3] ICP22 created new session: [5:7577471762203520691:2050]
2025-11-27T18:20:27.753113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471763012934967:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26976
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.753144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471762203520691:2050] [node 3] ICS09 handshake done sender: [5:7577471762203520681:2116] self: [5:7577471762203520683:0] peer: [3:7577471760723387604:0] socket: 530 qp: -1
2025-11-27T18:20:27.753232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471762203520691:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:27.753304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471763012934968:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26976
2025-11-27T18:20:27.753282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.754083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471763012934969:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26976
2025-11-27T18:20:27.754191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471759572856500:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:27.754239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471763012934967:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.754389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471763012934889:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.754371Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471760723387614:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:27.754592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471759572856501:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:27.754677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761074504816:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.754706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471761074504816:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.756596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471760723387532:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.756658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:27.756744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471760723387532:2048] [node 1] ICP22 created new session: [3:7577471760723387631:2048]
2025-11-27T18:20:27.756799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471762270407245:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:27.756825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471760723387631:2048] [node 1] ICS09 handshake done sender: [3:7577471760723387614:2118] self: [3:7577471760723387623:0] peer: [1:7577471762270407246:0] socket: 522 qp: -1
2025-11-27T18:20:27.756865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471760723387631:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:27.756905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.756877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471759572856422:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.756907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:27.757109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471761074504838:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6409
2025-11-27T18:20:27.756943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471759572856422:2049] [node 2] ICP22 created new session: [4:7577471759572856515:2049]
2025-11-27T18:20:27.757040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471759572856515:2049] [node 2] ICS09 handshake done sender: [4:7577471759572856500:2116] self: [4:7577471759572856506:0] peer: [2:7577471761074504811:0] socket: 524 qp: -1
2025-11-27T18:20:27.757061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471759572856515:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:27.757081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:27.757281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471762270407177:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.757339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:27.757388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471762270407177:2048] [node 2] ICP22 created new session: [1:7577471762270407276:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.757447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471762270407276:2048] [node 2] ICS09 handshake done sender: [1:7577471762270407245:2111] self: [1:7577471762270407244:0] peer: [2:7577471761074504834:0] socket: 519 qp: -1
2025-11-27T18:20:27.757502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471762270407276:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:27.757538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.757530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471763012934971:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26976
2025-11-27T18:20:27.757604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471760723387532:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:27.757640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471761074504816:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:27.757664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471763012934972:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26976
2025-11-27T18:20:27.759323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471759572856421:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.759441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471763012934967:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:27.759413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:27.759441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471759572856421:2048] [node 1] ICP22 created new session: [4:7577471759572856517:2048]
2025-11-27T18:20:27.759473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471759572856517:2048] [node 1] ICS09 handshake done sender: [4:7577471759572856501:2117] self: [4:7577471759572856508:0] peer: [1:7577471762270407248:0] socket: 525 qp: -1
2025-11-27T18:20:27.760143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471761074504806:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.759679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471759572856517:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:27.760276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:27.759706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:27.760316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471763012934889:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.760325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471761074504806:2109] [node 6] ICP22 created new session: [2:7577471761074504840:2109]
2025-11-27T18:20:27.760379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:27.760379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471761074504840:2109] [node 6] ICS09 handshake done sender: [2:7577471761074504816:2114] self: [2:7577471761074504815:0] peer: [6:7577471763012934970:0] socket: 544 qp: -1
2025-11-27T18:20:27.760427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471763012934889:2049] [node 2] ICP22 created new session: [6:7577471763012934973:2049]
2025-11-27T18:20:27.760414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471761074504840:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:27.760448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:27.760478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471763012934973:2049] [node 2] ICS09 handshake done sender: [6:7577471763012934967:2116] self: [6:7577471763012934970:0] peer: [2:7577471761074504815:0] socket: 546 qp: -1
2025-11-27T18:20:27.760588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471759572856421:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:27.760638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471763012934973:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:27.760714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:27.762289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471762203520602:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.762550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471762203520693:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50023
2025-11-27T18:20:27.762872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471761074504824:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:27.764370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471763012934969:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.764447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471763012934968:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.764495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471763012934966:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.764745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471763012934890:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.764843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471763012934949:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.764926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471763012934952:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.765065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471760723387633:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44542
2025-11-27T18:20:27.765203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471760723387607:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.765225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471761074504814:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.765225Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471760723387607:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.767162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471759572856495:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.765270Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471761074504814:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.767200Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471759572856495:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.765962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471761074504741:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.767812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471762203520675:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.767935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471760723387607:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:27.767850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471762203520675:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.767207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:27.768030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471759572856495:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:27.767349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471761074504741:2048] [node 1] ICP22 created new session: [2:7577471761074504842:2048]
2025-11-27T18:20:27.767862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471761074504842:2048] [node 1] ICS09 handshake done sender: [2:7577471761074504824:2119] self: [2:7577471761074504834:0] peer: [1:7577471762270407244:0] socket: 529 qp: -1
2025-11-27T18:20:27.768637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471760723387597:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.767906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471761074504842:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:27.768001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:27.768707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:27.768818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471760723387597:2109] [node 6] ICP22 created new session: [3:7577471760723387635:2109]
2025-11-27T18:20:27.768983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471760723387635:2109] [node 6] ICS09 handshake done sender: [3:7577471760723387607:2114] self: [3:7577471760723387606:0] peer: [6:7577471763012934976:0] socket: 548 qp: -1
2025-11-27T18:20:27.768887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471762203520675:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:27.769007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471760723387635:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:27.768843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471759572856485:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.769033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:27.769021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:27.769183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471759572856485:2109] [node 6] ICP22 created new session: [4:7577471759572856520:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.769398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471759572856520:2109] [node 6] ICS09 handshake done sender: [4:7577471759572856495:2114] self: [4:7577471759572856494:0] peer: [6:7577471763012934978:0] socket: 505 qp: -1
2025-11-27T18:20:27.769420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471759572856520:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:27.769563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471762203520665:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.769586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471763012934979:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26976
2025-11-27T18:20:27.769489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:27.769641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:27.769659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471761074504741:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:27.769695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471762203520665:2109] [node 6] ICP22 created new session: [5:7577471762203520696:2109]
2025-11-27T18:20:27.769738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471762203520696:2109] [node 6] ICS09 handshake done sender: [5:7577471762203520675:2114] self: [5:7577471762203520674:0] peer: [6:7577471763012934977:0] socket: 545 qp: -1
2025-11-27T18:20:27.769763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471762203520696:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:27.769792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.769975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471763012934980:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26976
2025-11-27T18:20:27.770411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471760723387633:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.770432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471763012934969:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:27.770544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471760723387597:2109] [node 6] ICP18 handshake (actor [3:7577471760723387633:2121]) is too old
2025-11-27T18:20:27.770639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471763012934966:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.771090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471763012934981:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26976
2025-11-27T18:20:27.771250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471762203520693:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.771434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471762203520601:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.771459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471763012934890:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.771495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:27.771521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471763012934890:2050] [node 3] ICP22 created new session: [6:7577471763012934982:2050]
2025-11-27T18:20:27.771546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471763012934982:2050] [node 3] ICS09 handshake done sender: [6:7577471763012934969:2118] self: [6:7577471763012934976:0] peer: [3:7577471760723387606:0] socket: 549 qp: -1
2025-11-27T18:20:27.771564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471763012934982:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:27.771581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:27.771777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762270407251:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.771809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471761074504814:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:27.771813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471762270407251:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:27.771956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471762203520699:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50023
2025-11-27T18:20:27.772650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471761074504805:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.772710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:27.772769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471761074504805:2108] [node 5] ICP22 created new session: [2:7577471761074504845:2108]
2025-11-27T18:20:27.772818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471761074504845:2108] [node 5] ICS09 handshake done sender: [2:7577471761074504814:2113] self: [2:7577471761074504813:0] peer: [5:7577471762203520694:0] socket: 536 qp: -1
2025-11-27T18:20:27.772821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471762270407251:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:27.772845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471761074504845:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:27.772861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471762203520686:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:27.772917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:27.773353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471762270407242:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.773415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:27.773394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471762203520602:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.773476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471762270407242:2109] [node 5] ICP22 created new session: [1:7577471762270407279:2109]
2025-11-27T18:20:27.773452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:27.773482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471762203520602:2049] [node 2] ICP22 created new session: [5:7577471762203520701:2049]
2025-11-27T18:20:27.773535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471762270407279:2109] [node 5] ICS09 handshake done sender: [1:7577471762270407251:2114] self: [1:7577471762270407250:0] peer: [5:7577471762203520698:0] socket: 489 qp: -1
2025-11-27T18:20:27.773521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471762203520701:2049] [node 2] ICS09 handshake done sender: [5:7577471762203520686:2119] self: [5:7577471762203520694:0] peer: [2:7577471761074504813:0] socket: 538 qp: -1
2025-11-27T18:20:27.773574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471762270407279:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:27.773537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471762203520701:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:27.773556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:27.773612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:27.773999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471763012934952:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.774033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:27.774058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471763012934952:2108] [node 4] ICP22 created new session: [6:7577471763012934984:2108]
2025-11-27T18:20:27.774081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471763012934984:2108] [node 4] ICS09 handshake done sender: [6:7577471763012934966:2115] self: [6:7577471763012934978:0] peer: [4:7577471759572856494:0] socket: 543 qp: -1
2025-11-27T18:20:27.774099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471763012934984:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:27.774122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.776865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471762203520700:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50023
2025-11-27T18:20:27.777547Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471763012934949:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26563) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471763012934968:2117] held: no
2025-11-27T18:20:27.778230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471763012934971:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.778332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471762203520601:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471762203520693:2121] held: no
2025-11-27T18:20:27.778685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471763012934890:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:27.778764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471762203520693:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:27.778776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471763012934968:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:27.779234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471762203520601:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.779285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:27.779317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471762203520601:2048] [node 1] ICP22 created new session: [5:7577471762203520703:2048]
2025-11-27T18:20:27.779354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471762203520703:2048] [node 1] ICS09 handshake done sender: [5:7577471762203520693:2121] self: [5:7577471762203520698:0] peer: [1:7577471762270407250:0] socket: 488 qp: -1
2025-11-27T18:20:27.779374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471762203520703:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:27.779396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:27.779546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471763012934949:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.779576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:27.779615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471763012934949:2106] [node 5] ICP22 created new session: [6:7577471763012934986:2106]
2025-11-27T18:20:27.779655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471763012934986:2106] [node 5] ICS09 handshake done sender: [6:7577471763012934968:2117] self: [6:7577471763012934977:0] peer: [5:7577471762203520674:0] socket: 547 qp: -1
2025-11-27T18:20:27.779680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471763012934986:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:27.779699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:27.780599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471763012934888:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:27.780858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471762270407253:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:27.780891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471762270407253:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:27.781959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471762270407253:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:27.782058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471763012934989:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26976
2025-11-27T18:20:27.782256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471762270407243:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:27.782333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:27.782382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471762270407243:2110] [node 6] ICP22 created new session: [1:7577471762270407282:2110]
2025-11-27T18:20:27.782364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471763012934971:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:27.782425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471762270407282:2110] [node 6] ICS09 handshake done sender: [1:7577471762270407253:2115] self: [1:7577471762270407252:0] peer: [6:7577471763012934988:0] socket: 513 qp: -1
2025-11-27T18:20:27.782473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471762270407282:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:27.782523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:27.782533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471763012934888:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:27.782566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:27.782595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471763012934888:2048] [node 1] ICP22 created new session: [6:7577471763012934990:2048]
2025-11-27T18:20:27.782623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471763012934990:2048] [node 1] ICS09 handshake done sender: [6:7577471763012934971:2119] self: [6:7577471763012934988:0] peer: [1:7577471762270407252:0] socket: 504 qp: -1
2025-11-27T18:20:27.782676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471763012934990:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:27.782701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:27.803875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:27.804066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471761074504842:2048] [node 1] ICS07 socket disconnect 529 reason# EndOfStream
2025-11-27T18:20:27.804093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471761074504842:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.804154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:27.804186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471761074504842:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:27.805011Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:27.805144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471760723387631:2048] [node 1] ICS07 socket disconnect 522 reason# EndOfStream
2025-11-27T18:20:27.805178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471760723387631:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.805229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:27.805273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471760723387631:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:27.806803Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471761074504741:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:27.806829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471761074504842:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:27.808555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471760723387532:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:27.808588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471760723387631:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:27.808711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:27.808869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471759572856517:2048] [node 1] ICS07 socket disconnect 525 reason# EndOfStream
2025-11-27T18:20:27.808894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471759572856517:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.808886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471761074504741:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.808936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471761074504842:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.808941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:27.808968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471761074504741:2048] [node 1] ICP30 unregister session Session# [2:7577471761074504842:2048] VirtualId# [2:7577471761074504834:0]
2025-11-27T18:20:27.808966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471759572856517:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:27.809086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:27.809126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471761074504741:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.809148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.809232Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471760723387532:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.809302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471760723387631:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.809334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471760723387532:2048] [node 1] ICP30 unregister session Session# [3:7577471760723387631:2048] VirtualId# [3:7577471760723387623:0]
2025-11-27T18:20:27.809374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:27.809410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471760723387532:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.809425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.809672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471759572856421:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.809717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471759572856517:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.809748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471759572856421:2048] [node 1] ICP30 unregister session Session# [4:7577471759572856517:2048] VirtualId# [4:7577471759572856508:0]
2025-11-27T18:20:27.809783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:27.809801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471759572856421:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.809809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.809874Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:27.810050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471762203520703:2048] [node 1] ICS01 socket: 488 reason# EndOfStream
2025-11-27T18:20:27.810113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471762203520601:2048] [node 1] ICP30 unregister session Session# [5:7577471762203520703:2048] VirtualId# [5:7577471762203520698:0]
2025-11-27T18:20:27.810136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471762203520703:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.810208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:27.810264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:27.810665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:27.810806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471763012934990:2048] [node 1] ICS07 socket disconnect 504 reason# EndOfStream
2025-11-27T18:20:27.810827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471763012934990:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.810874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:27.810911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471763012934990:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:27.811408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471763012934888:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.811438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471763012934990:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.811456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471763012934888:2048] [node 1] ICP30 unregister session Session# [6:7577471763012934990:2048] VirtualId# [6:7577471763012934988:0]
2025-11-27T18:20:27.811478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:27.811497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471763012934888:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.811505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10202) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.819303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471761074504741:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:27.819294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471760723387532:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:27.819301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471759572856421:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:27.821374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471763012934888:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:27.844290Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:27.844459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471760723387625:2049] [node 2] ICS07 socket disconnect 521 reason# EndOfStream
2025-11-27T18:20:27.844484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471760723387625:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.844550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:27.844588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471760723387625:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:27.846753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471760723387533:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2329) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:27.846797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471760723387625:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:27.847627Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:27.847760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471759572856515:2049] [node 2] ICS01 socket: 524 reason# EndOfStream
2025-11-27T18:20:27.847791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471759572856422:2049] [node 2] ICP30 unregister session Session# [4:7577471759572856515:2049] VirtualId# [4:7577471759572856506:0]
2025-11-27T18:20:27.847820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471759572856515:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.847887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:27.847925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:27.847911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471760723387533:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2329) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.847945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471760723387625:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.847967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471760723387533:2049] [node 2] ICP30 unregister session Session# [3:7577471760723387625:2049] VirtualId# [3:7577471760723387621:0]
2025-11-27T18:20:27.848066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:27.848095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471760723387533:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.848109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.848473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:27.848655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471762203520701:2049] [node 2] ICS07 socket disconnect 538 reason# EndOfStream
2025-11-27T18:20:27.848679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471762203520701:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.848747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:27.848804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471762203520701:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:27.849306Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:27.849512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471762203520602:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2329) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.849517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471763012934973:2049] [node 2] ICS07 socket disconnect 546 reason# EndOfStream
2025-11-27T18:20:27.849554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471763012934973:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.849567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471762203520701:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.849598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471762203520602:2049] [node 2] ICP30 unregister session Session# [5:7577471762203520701:2049] VirtualId# [5:7577471762203520694:0]
2025-11-27T18:20:27.849624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:27.849681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:27.849675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471763012934973:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:27.849727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471762203520602:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.849742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.850293Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471763012934889:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2329) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.850339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471763012934973:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.850369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471763012934889:2049] [node 2] ICP30 unregister session Session# [6:7577471763012934973:2049] VirtualId# [6:7577471763012934970:0]
2025-11-27T18:20:27.850399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:27.850421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471763012934889:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.850429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2329) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.859209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471762203520602:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:27.860258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471763012934889:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:27.871193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:27.871377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471759572856504:2050] [node 3] ICS07 socket disconnect 499 reason# EndOfStream
2025-11-27T18:20:27.871408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471759572856504:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.871464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:27.871510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471759572856504:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:27.872066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:27.872106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471759572856423:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65197) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.872157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471759572856504:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.872200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471759572856423:2050] [node 3] ICP30 unregister session Session# [4:7577471759572856504:2050] VirtualId# [4:7577471759572856502:0]
2025-11-27T18:20:27.872231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471762203520691:2050] [node 3] ICS07 socket disconnect 530 reason# EndOfStream
2025-11-27T18:20:27.872263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471762203520691:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.872280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:27.872321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:27.872304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471759572856423:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65197) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.872314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65197) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.872354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471762203520691:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:27.872940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:27.873013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471762203520603:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65197) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.873046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471762203520691:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.873091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471763012934982:2050] [node 3] ICS07 socket disconnect 549 reason# EndOfStream
2025-11-27T18:20:27.873072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471762203520603:2050] [node 3] ICP30 unregister session Session# [5:7577471762203520691:2050] VirtualId# [5:7577471762203520683:0]
2025-11-27T18:20:27.873112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471763012934982:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.873151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:27.873165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:27.873185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471762203520603:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65197) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.873208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471763012934982:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:27.873204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65197) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.873863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471763012934890:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65197) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.873904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471763012934982:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.873927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471763012934890:2050] [node 3] ICP30 unregister session Session# [6:7577471763012934982:2050] VirtualId# [6:7577471763012934976:0]
2025-11-27T18:20:27.873987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:27.874007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471763012934890:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65197) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.874016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65197) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.881773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471759572856423:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:27.882774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471762203520603:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:27.883799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471763012934890:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:27.902997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:27.903159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471762203520687:2108] [node 4] ICS07 socket disconnect 532 reason# EndOfStream
2025-11-27T18:20:27.903203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471762203520687:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.903300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:27.903336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471762203520687:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:27.903822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:27.903958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471763012934984:2108] [node 4] ICS07 socket disconnect 543 reason# EndOfStream
2025-11-27T18:20:27.903989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471763012934984:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.904047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:27.904026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471762203520664:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11687) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.904079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471763012934984:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:27.904070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471762203520687:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.904100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471762203520664:2108] [node 4] ICP30 unregister session Session# [5:7577471762203520687:2108] VirtualId# [5:7577471762203520684:0]
2025-11-27T18:20:27.904197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:27.904233Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471762203520664:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11687) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.904248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11687) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.904651Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471763012934952:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11687) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:27.904685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471763012934984:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:27.904713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471763012934952:2108] [node 4] ICP30 unregister session Session# [6:7577471763012934984:2108] VirtualId# [6:7577471763012934978:0]
2025-11-27T18:20:27.904794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:27.904817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471763012934952:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11687) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.904826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11687) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:27.914520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471763012934952:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:27.926903Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:27.927061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471763012934986:2106] [node 5] ICS01 socket: 547 reason# EndOfStream
2025-11-27T18:20:27.927107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471763012934949:2106] [node 5] ICP30 unregister session Session# [6:7577471763012934986:2106] VirtualId# [6:7577471763012934977:0]
2025-11-27T18:20:27.927137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471763012934986:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:27.927199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:27.927234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:28.005671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471765089412277:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:28.005801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471765089412278:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:28.031408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471764116190877:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:28.031439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471764116190878:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:28.055129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471765647532207:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:28.055164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471765647532208:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:28.074754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471767911196326:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:28.074782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471767911196327:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:28.074801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471767911196328:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:28.092098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471766865405440:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:28.092103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471766865405441:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:28.092135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471766865405442:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:28.109607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471765559214213:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:28.109608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471765559214212:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:28.109640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471765559214214:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:28.180527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471765559214274:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:28.180518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471765089412340:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:28.180530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471765647532272:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:28.183141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471764116190941:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:28.183195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471766865405503:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:28.183203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471767911196389:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:28.183350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471767911196390:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:28.183382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471764116190942:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:28.183342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471765089412341:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:28.183363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471765647532273:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:28.183243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471766865405504:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:28.183418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471765559214275:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:28.183363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471765089412342:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:28.183574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471765647532274:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:28.184212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471764116190943:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.197218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766865405518:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9737
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.197236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471764116190955:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16732
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.197337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471765089412354:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26719
2025-11-27T18:20:28.197335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765647532286:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11280
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.197395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765647532285:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11280
2025-11-27T18:20:28.197393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471764116190958:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16732
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.197674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471765089412353:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26719
2025-11-27T18:20:28.197813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765647532287:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11280
2025-11-27T18:20:28.197927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471765089412355:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26719
2025-11-27T18:20:28.199307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471765089412359:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26719
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.200770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471767911196404:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56098
2025-11-27T18:20:28.200905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471764116190959:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16732
2025-11-27T18:20:28.200815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765647532291:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11280
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.201098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766865405519:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9737
2025-11-27T18:20:28.201468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765647532286:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.202886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766865405520:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9737
2025-11-27T18:20:28.202809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471765089412360:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26719
2025-11-27T18:20:28.203075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471766865405519:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.203080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471765089412353:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.203451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471766865405518:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.203653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766865405521:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9737
2025-11-27T18:20:28.203722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471766865405503:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.203862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471767911196396:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.203897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471767911196396:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.206063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471764116190958:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.206317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471767911196396:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:28.206748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471765089412355:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.206775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765647532285:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.206767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471764116190960:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16732
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.206933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471764116190955:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.206958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471767911196389:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.207231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766865405524:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9737
2025-11-27T18:20:28.207045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:28.207360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765647532287:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.207940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471764116190960:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.207576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471765089412359:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.207168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471767911196389:2108] [node 5] ICP22 created new session: [4:7577471767911196406:2108]
2025-11-27T18:20:28.207834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765647532291:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.207398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471767911196406:2108] [node 5] ICS09 handshake done sender: [4:7577471767911196396:2112] self: [4:7577471767911196395:0] peer: [5:7577471766865405523:0] socket: 496 qp: -1
2025-11-27T18:20:28.207438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471767911196406:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:28.208308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471765647532207:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.207480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:28.208322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471765647532208:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.208370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471765089412360:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.208461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471766865405520:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.208454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471765089412354:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.208500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471764116190959:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.209098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471766865405518:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:28.208673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471764116190945:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.209157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471765089412346:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.208702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471764116190945:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.209574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471766865405521:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.209962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471764116190964:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16732
2025-11-27T18:20:28.209196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471765089412346:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.209966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471764116190877:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.210059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471766865405503:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.210064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471766865405441:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.210098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:28.210307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471764116190945:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:28.210124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471766865405503:2108] [node 4] ICP22 created new session: [5:7577471766865405526:2108]
2025-11-27T18:20:28.210516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471764116190949:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.210468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471765089412344:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.210549Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471764116190949:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.210293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471766865405526:2108] [node 4] ICS09 handshake done sender: [5:7577471766865405518:2115] self: [5:7577471766865405523:0] peer: [4:7577471767911196395:0] socket: 504 qp: -1
2025-11-27T18:20:28.210818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765647532296:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11280
2025-11-27T18:20:28.210492Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471765089412344:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.210849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471767911196404:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.210473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471766865405526:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:28.210503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471765089412346:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:28.211158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471767911196326:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.210505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:28.210526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471766865405442:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.211788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471764116190878:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.211935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471765089412278:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.211970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765647532280:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.211836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:28.211976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:28.211997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471765647532280:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-27T18:20:28.211981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471764116190878:2049] [node 3] ICP22 created new session: [2:7577471764116190967:2049]
2025-11-27T18:20:28.212041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471765089412349:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:28.212064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471765089412349:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.212061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471764116190967:2049] [node 3] ICS09 handshake done sender: [2:7577471764116190945:2111] self: [2:7577471764116190944:0] peer: [3:7577471765647532295:0] socket: 497 qp: -1
2025-11-27T18:20:28.212757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471765647532287:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.213251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765647532297:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11280
2025-11-27T18:20:28.212087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471765089412278:2049] [node 3] ICP22 created new session: [1:7577471765089412368:2049]
2025-11-27T18:20:28.213748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471765559214290:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11609
2025-11-27T18:20:28.212507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471764116190967:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:28.213825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471765647532208:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.212625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471765089412368:2049] [node 3] ICS09 handshake done sender: [1:7577471765089412346:2111] self: [1:7577471765089412345:0] peer: [3:7577471765647532294:0] socket: 520 qp: -1
2025-11-27T18:20:28.212622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:28.213884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:28.212670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471765089412368:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:28.213913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471765647532208:2049] [node 2] ICP22 created new session: [3:7577471765647532299:2049]
2025-11-27T18:20:28.212767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:28.213956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471765647532299:2049] [node 2] ICS09 handshake done sender: [3:7577471765647532287:2119] self: [3:7577471765647532295:0] peer: [2:7577471764116190944:0] socket: 511 qp: -1
2025-11-27T18:20:28.213914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471765089412344:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:28.213980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471765647532299:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:28.214000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:28.214196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471764116190964:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.214193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471765559214290:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.214379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471765559214274:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.214407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766865405530:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9737
2025-11-27T18:20:28.214539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471766865405517:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.214567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471766865405517:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.215626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471764116190968:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16732
2025-11-27T18:20:28.215708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471765089412277:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.215850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:28.215881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471765089412277:2048] [node 2] ICP22 created new session: [1:7577471765089412371:2048]
2025-11-27T18:20:28.215913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471765089412371:2048] [node 2] ICS09 handshake done sender: [1:7577471765089412344:2110] self: [1:7577471765089412343:0] peer: [2:7577471764116190966:0] socket: 519 qp: -1
2025-11-27T18:20:28.216045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471765089412371:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:28.216069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:28.216306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471764116190949:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.216345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471765647532208:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:28.216429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766865405531:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9737
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.217541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471765559214292:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11609
2025-11-27T18:20:28.217701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471764116190941:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.217802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:28.217803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471765089412349:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:28.217826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471765647532280:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:28.217842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471764116190941:2108] [node 5] ICP22 created new session: [2:7577471764116190972:2108]
2025-11-27T18:20:28.217892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471766865405517:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:28.217895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471764116190972:2108] [node 5] ICS09 handshake done sender: [2:7577471764116190949:2113] self: [2:7577471764116190948:0] peer: [5:7577471766865405525:0] socket: 508 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.218262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471766865405520:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:28.217924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471764116190972:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:28.217952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:28.218687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471766865405504:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.218744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471765647532291:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:28.218499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471765089412340:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.219076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471765559214290:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:28.218774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:28.218856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471767911196410:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56098
2025-11-27T18:20:28.218551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:28.218961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471766865405504:2109] [node 6] ICP22 created new session: [5:7577471766865405534:2109]
2025-11-27T18:20:28.218753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471765089412340:2107] [node 4] ICP22 created new session: [1:7577471765089412375:2107]
2025-11-27T18:20:28.219269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471766865405534:2109] [node 6] ICS09 handshake done sender: [5:7577471766865405517:2114] self: [5:7577471766865405513:0] peer: [6:7577471765559214291:0] socket: 543 qp: -1
2025-11-27T18:20:28.218882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471765089412375:2107] [node 4] ICS09 handshake done sender: [1:7577471765089412349:2112] self: [1:7577471765089412348:0] peer: [4:7577471767911196409:0] socket: 521 qp: -1
2025-11-27T18:20:28.219452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471765647532207:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24424) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471765647532291:2120] held: no
2025-11-27T18:20:28.219299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471766865405534:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:28.219004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471765089412375:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:28.219337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:28.219040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:28.219633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471765559214274:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.219808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:28.219586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471766865405441:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.219623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:28.219849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471765559214274:2107] [node 5] ICP22 created new session: [6:7577471765559214294:2107]
2025-11-27T18:20:28.219670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471766865405441:2049] [node 2] ICP22 created new session: [5:7577471766865405536:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.219904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471765559214294:2107] [node 5] ICS09 handshake done sender: [6:7577471765559214290:2115] self: [6:7577471765559214291:0] peer: [5:7577471766865405513:0] socket: 544 qp: -1
2025-11-27T18:20:28.219807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471766865405536:2049] [node 2] ICS09 handshake done sender: [5:7577471766865405520:2117] self: [5:7577471766865405525:0] peer: [2:7577471764116190948:0] socket: 515 qp: -1
2025-11-27T18:20:28.219945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471765559214294:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:28.219835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471766865405536:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:28.219975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471767911196411:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56098
2025-11-27T18:20:28.219977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:28.219869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:28.220035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471767911196404:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:28.220169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471767911196326:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.221115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471765647532207:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.220647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:28.221444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471765647532273:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.220809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471767911196326:2048] [node 1] ICP22 created new session: [4:7577471767911196412:2048]
2025-11-27T18:20:28.221126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471767911196412:2048] [node 1] ICS09 handshake done sender: [4:7577471767911196404:2115] self: [4:7577471767911196409:0] peer: [1:7577471765089412348:0] socket: 523 qp: -1
2025-11-27T18:20:28.221482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:28.221143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471767911196412:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:28.221519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471765647532273:2109] [node 5] ICP22 created new session: [3:7577471765647532301:2109]
2025-11-27T18:20:28.221521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:28.221583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471764116190959:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:28.221531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:28.221555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471765647532301:2109] [node 5] ICS09 handshake done sender: [3:7577471765647532280:2114] self: [3:7577471765647532279:0] peer: [5:7577471766865405528:0] socket: 529 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.221575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471765647532301:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:28.221575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471765647532207:2048] [node 1] ICP22 created new session: [3:7577471765647532302:2048]
2025-11-27T18:20:28.221601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:28.221663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471765647532302:2048] [node 1] ICS09 handshake done sender: [3:7577471765647532291:2120] self: [3:7577471765647532294:0] peer: [1:7577471765089412345:0] socket: 524 qp: -1
2025-11-27T18:20:28.221694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471765647532302:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:28.222372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471765559214293:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11609
2025-11-27T18:20:28.221753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:28.223345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471764116190877:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24424) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471764116190959:2118] held: no
2025-11-27T18:20:28.223544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471766865405521:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:28.223842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471764116190877:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.223875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:28.223930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471764116190877:2048] [node 1] ICP22 created new session: [2:7577471764116190976:2048]
2025-11-27T18:20:28.223981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471764116190976:2048] [node 1] ICS09 handshake done sender: [2:7577471764116190959:2118] self: [2:7577471764116190966:0] peer: [1:7577471765089412343:0] socket: 522 qp: -1
2025-11-27T18:20:28.224009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471764116190976:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:28.224029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:28.226173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471766865405442:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.226222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:28.226325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471766865405442:2050] [node 3] ICP22 created new session: [5:7577471766865405538:2050]
2025-11-27T18:20:28.226370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471766865405538:2050] [node 3] ICS09 handshake done sender: [5:7577471766865405521:2118] self: [5:7577471766865405528:0] peer: [3:7577471765647532279:0] socket: 533 qp: -1
2025-11-27T18:20:28.226403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471766865405538:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:28.226432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.226784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471767911196415:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56098
No RDMA context found for GID: 2025-11-27T18:20:28.226840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765647532307:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11280
::ffff:127.1.0.0
2025-11-27T18:20:28.226933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471765559214297:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11609
2025-11-27T18:20:28.226963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471765559214293:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.227121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471765559214298:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11609
2025-11-27T18:20:28.227278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471765559214297:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:28.227304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471765559214212:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.227392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471765559214299:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11609
2025-11-27T18:20:28.227510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471765559214275:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.227526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471765089412352:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.227560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471765089412352:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.228833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471767911196400:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.228830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765647532307:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.228857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471767911196400:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.229944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471765089412352:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:28.229812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471765559214299:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.230084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471767911196415:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:28.230014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471765559214298:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:28.230142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471765559214213:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.230174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471767911196411:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.230176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766865405533:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9737
2025-11-27T18:20:28.230274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471765559214214:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.230318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471767911196328:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.230192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471765089412342:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.230371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.230446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471767911196400:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:28.230427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471765089412342:2109] [node 6] ICP22 created new session: [1:7577471765089412378:2109]
2025-11-27T18:20:28.230443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471764116190957:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.230464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471764116190957:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.230477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471767911196327:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.230511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471765089412378:2109] [node 6] ICS09 handshake done sender: [1:7577471765089412352:2114] self: [1:7577471765089412351:0] peer: [6:7577471765559214300:0] socket: 552 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.230544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471765089412378:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:28.230670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471767911196390:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.230986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765647532282:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.230528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471765559214302:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11609
2025-11-27T18:20:28.231131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471766865405442:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:28.231045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:28.230663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:28.231306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471764116190951:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.231013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471765647532282:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.231840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471765559214305:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11609
2025-11-27T18:20:28.231331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471767911196390:2109] [node 6] ICP22 created new session: [4:7577471767911196419:2109]
2025-11-27T18:20:28.231336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471764116190951:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.231614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471767911196419:2109] [node 6] ICS09 handshake done sender: [4:7577471767911196400:2114] self: [4:7577471767911196399:0] peer: [6:7577471765559214301:0] socket: 500 qp: -1
2025-11-27T18:20:28.232067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471764116190957:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:28.232201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471767911196419:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:28.232289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:28.232512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471764116190943:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.232910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471766865405533:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.232323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765647532284:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.232716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:28.233348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471766865405440:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.232348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471765647532284:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.232926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471764116190943:2110] [node 6] ICP22 created new session: [2:7577471764116190979:2110]
2025-11-27T18:20:28.233604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471764116190979:2110] [node 6] ICS09 handshake done sender: [2:7577471764116190957:2116] self: [2:7577471764116190956:0] peer: [6:7577471765559214303:0] socket: 555 qp: -1
2025-11-27T18:20:28.233635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471764116190979:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:28.233659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:28.233723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471765089412350:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.233919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471765647532284:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:28.233742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471765089412350:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.233742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471765559214293:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:28.234476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471765647532282:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:28.234668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471765559214212:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24424) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471765559214293:2117] held: no
2025-11-27T18:20:28.234881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471765089412350:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:28.234940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471764116190951:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:28.235034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471765647532274:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.235092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.235130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471765647532274:2110] [node 6] ICP22 created new session: [3:7577471765647532311:2110]
2025-11-27T18:20:28.235500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471765089412341:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.235176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471765647532311:2110] [node 6] ICS09 handshake done sender: [3:7577471765647532284:2116] self: [3:7577471765647532283:0] peer: [6:7577471765559214304:0] socket: 502 qp: -1
No RDMA context found for GID: 2025-11-27T18:20:28.235291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471765647532311:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:28.235657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
::ffff:127.1.0.0
2025-11-27T18:20:28.235633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766865405541:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9737
2025-11-27T18:20:28.235456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:28.235708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471765089412341:2108] [node 5] ICP22 created new session: [1:7577471765089412381:2108]
2025-11-27T18:20:28.235748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471764116190942:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.235735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471765559214212:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.235497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471765647532272:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.235792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471765089412381:2108] [node 5] ICS09 handshake done sender: [1:7577471765089412350:2113] self: [1:7577471765089412347:0] peer: [5:7577471766865405540:0] socket: 551 qp: -1
2025-11-27T18:20:28.235806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:28.235787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:28.235638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:28.235854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471765089412381:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:28.235820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471764116190942:2107] [node 4] ICP22 created new session: [2:7577471764116190982:2107]
2025-11-27T18:20:28.235832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471765559214212:2048] [node 1] ICP22 created new session: [6:7577471765559214308:2048]
2025-11-27T18:20:28.235685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471765647532272:2108] [node 4] ICP22 created new session: [3:7577471765647532313:2108]
2025-11-27T18:20:28.235858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471765559214308:2048] [node 1] ICS09 handshake done sender: [6:7577471765559214293:2117] self: [6:7577471765559214300:0] peer: [1:7577471765089412351:0] socket: 553 qp: -1
2025-11-27T18:20:28.235866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471766865405533:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:28.235845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471764116190982:2107] [node 4] ICS09 handshake done sender: [2:7577471764116190951:2114] self: [2:7577471764116190950:0] peer: [4:7577471767911196418:0] socket: 503 qp: -1
2025-11-27T18:20:28.235845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471767911196421:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56098
2025-11-27T18:20:28.235742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471765647532313:2108] [node 4] ICS09 handshake done sender: [3:7577471765647532282:2115] self: [3:7577471765647532281:0] peer: [4:7577471767911196417:0] socket: 493 qp: -1
2025-11-27T18:20:28.235892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:28.235884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471764116190982:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:28.235889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471765559214308:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:28.235773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471765647532313:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:28.236942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471766865405440:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.235807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:28.235954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:28.235904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:28.237074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:28.237320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471766865405440:2048] [node 1] ICP22 created new session: [5:7577471766865405542:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.237422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471766865405542:2048] [node 1] ICS09 handshake done sender: [5:7577471766865405533:2122] self: [5:7577471766865405540:0] peer: [1:7577471765089412347:0] socket: 554 qp: -1
2025-11-27T18:20:28.237567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471766865405542:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:28.237688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:28.237721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471767911196422:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56098
2025-11-27T18:20:28.237905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471765559214297:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:28.237939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471767911196415:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:28.238776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471767911196328:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.238849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:28.239029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471767911196328:2050] [node 3] ICP22 created new session: [4:7577471767911196423:2050]
2025-11-27T18:20:28.239138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471767911196423:2050] [node 3] ICS09 handshake done sender: [4:7577471767911196415:2118] self: [4:7577471767911196417:0] peer: [3:7577471765647532281:0] socket: 507 qp: -1
2025-11-27T18:20:28.239175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471767911196423:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:28.239211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:28.240301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471765559214213:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23617) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471765559214299:2120] held: no
2025-11-27T18:20:28.240425Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471767911196411:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:28.240704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471765559214275:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.240797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:28.240826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471765559214275:2108] [node 4] ICP22 created new session: [6:7577471765559214310:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.240854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471765559214310:2108] [node 4] ICS09 handshake done sender: [6:7577471765559214297:2118] self: [6:7577471765559214301:0] peer: [4:7577471767911196399:0] socket: 505 qp: -1
2025-11-27T18:20:28.240875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471765559214310:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:28.240930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:28.240930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471765559214306:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11609
2025-11-27T18:20:28.242023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471767911196327:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23617) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471767911196411:2117] held: no
2025-11-27T18:20:28.242494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471767911196327:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.242519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:28.242543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471767911196327:2049] [node 2] ICP22 created new session: [4:7577471767911196425:2049]
2025-11-27T18:20:28.242570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471767911196425:2049] [node 2] ICS09 handshake done sender: [4:7577471767911196411:2117] self: [4:7577471767911196418:0] peer: [2:7577471764116190950:0] socket: 526 qp: -1
2025-11-27T18:20:28.242589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471767911196425:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:28.242605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.243165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471765559214307:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11609
2025-11-27T18:20:28.243416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471765559214214:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4140) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471765559214298:2119] held: no
2025-11-27T18:20:28.243635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471765559214299:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:28.243941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471765559214213:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.243988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:28.244010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471765559214298:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:28.244019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471765559214213:2049] [node 2] ICP22 created new session: [6:7577471765559214313:2049]
2025-11-27T18:20:28.244043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471765559214313:2049] [node 2] ICS09 handshake done sender: [6:7577471765559214299:2120] self: [6:7577471765559214303:0] peer: [2:7577471764116190956:0] socket: 556 qp: -1
2025-11-27T18:20:28.244060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471765559214313:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:28.244078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:28.244237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471765559214214:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.244263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:28.244287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471765559214214:2050] [node 3] ICP22 created new session: [6:7577471765559214315:2050]
2025-11-27T18:20:28.244321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471765559214315:2050] [node 3] ICS09 handshake done sender: [6:7577471765559214298:2119] self: [6:7577471765559214304:0] peer: [3:7577471765647532283:0] socket: 513 qp: -1
2025-11-27T18:20:28.244347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471765559214315:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:28.244366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:28.272758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:28.272926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471764116190976:2048] [node 1] ICS01 socket: 522 reason# EndOfStream
2025-11-27T18:20:28.272961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471764116190877:2048] [node 1] ICP30 unregister session Session# [2:7577471764116190976:2048] VirtualId# [2:7577471764116190966:0]
2025-11-27T18:20:28.272983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471764116190976:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.273042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:28.273084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:28.273686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:28.273848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471765647532302:2048] [node 1] ICS01 socket: 524 reason# EndOfStream
2025-11-27T18:20:28.273878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471765647532207:2048] [node 1] ICP30 unregister session Session# [3:7577471765647532302:2048] VirtualId# [3:7577471765647532294:0]
2025-11-27T18:20:28.273895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471765647532302:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.273968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:28.274026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:28.275568Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:28.275785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471767911196412:2048] [node 1] ICS07 socket disconnect 523 reason# EndOfStream
2025-11-27T18:20:28.275817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471767911196412:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.275898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:28.275947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471767911196412:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:28.276282Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:28.276459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471766865405542:2048] [node 1] ICS07 socket disconnect 554 reason# EndOfStream
2025-11-27T18:20:28.276487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471766865405542:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.276564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471767911196326:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24424) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.276574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:28.276646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471766865405542:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:28.276688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471767911196412:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.276750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471767911196326:2048] [node 1] ICP30 unregister session Session# [4:7577471767911196412:2048] VirtualId# [4:7577471767911196409:0]
2025-11-27T18:20:28.276860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:28.276898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471767911196326:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24424) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.276918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24424) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.277016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:28.277208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471765559214308:2048] [node 1] ICS01 socket: 553 reason# EndOfStream
2025-11-27T18:20:28.277276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471765559214212:2048] [node 1] ICP30 unregister session Session# [6:7577471765559214308:2048] VirtualId# [6:7577471765559214300:0]
2025-11-27T18:20:28.277247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471766865405440:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24424) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.277307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471765559214308:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.277315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471766865405542:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.277383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:28.277382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471766865405440:2048] [node 1] ICP30 unregister session Session# [5:7577471766865405542:2048] VirtualId# [5:7577471766865405540:0]
2025-11-27T18:20:28.277495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:28.277572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:28.277617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471766865405440:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24424) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.277627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24424) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.286279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471767911196326:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:28.287285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471766865405440:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:28.335019Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:28.335253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471765647532299:2049] [node 2] ICS07 socket disconnect 511 reason# EndOfStream
2025-11-27T18:20:28.335285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471765647532299:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.335371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:28.335437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471765647532299:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:28.337064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:28.337164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471765647532208:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23617) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:28.337196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471765647532299:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:28.337536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471767911196425:2049] [node 2] ICS01 socket: 526 reason# EndOfStream
2025-11-27T18:20:28.337603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471767911196327:2049] [node 2] ICP30 unregister session Session# [4:7577471767911196425:2049] VirtualId# [4:7577471767911196418:0]
2025-11-27T18:20:28.337639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471767911196425:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.337722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:28.337779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:28.338015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471765647532208:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23617) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.338069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471765647532299:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.338145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471765647532208:2049] [node 2] ICP30 unregister session Session# [3:7577471765647532299:2049] VirtualId# [3:7577471765647532295:0]
2025-11-27T18:20:28.338224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:28.338248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471765647532208:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23617) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.338257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23617) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.338247Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:28.338442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471766865405536:2049] [node 2] ICS07 socket disconnect 515 reason# EndOfStream
2025-11-27T18:20:28.338495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471766865405536:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.338551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:28.338587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471766865405536:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:28.339107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:28.339154Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471766865405441:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23617) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.339248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471766865405536:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.339304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471766865405441:2049] [node 2] ICP30 unregister session Session# [5:7577471766865405536:2049] VirtualId# [5:7577471766865405525:0]
2025-11-27T18:20:28.339341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471765559214313:2049] [node 2] ICS01 socket: 556 reason# EndOfStream
2025-11-27T18:20:28.339423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:28.339394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471765559214213:2049] [node 2] ICP30 unregister session Session# [6:7577471765559214313:2049] VirtualId# [6:7577471765559214303:0]
2025-11-27T18:20:28.339473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471766865405441:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23617) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.339435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471765559214313:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.339493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23617) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.339513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:28.339563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:28.349771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471766865405441:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:28.373578Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:28.373803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471767911196423:2050] [node 3] ICS07 socket disconnect 507 reason# EndOfStream
2025-11-27T18:20:28.373841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471767911196423:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.373932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:28.373990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471767911196423:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:28.374364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:28.374500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471766865405538:2050] [node 3] ICS07 socket disconnect 533 reason# EndOfStream
2025-11-27T18:20:28.374523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471766865405538:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.374572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:28.374597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471766865405538:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:28.374673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471767911196328:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4140) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.374726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471767911196423:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.374836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471767911196328:2050] [node 3] ICP30 unregister session Session# [4:7577471767911196423:2050] VirtualId# [4:7577471767911196417:0]
2025-11-27T18:20:28.374977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:28.375013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471767911196328:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.375030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.375175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:28.375230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471766865405442:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4140) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.375274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471766865405538:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.375420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471765559214315:2050] [node 3] ICS01 socket: 513 reason# EndOfStream
2025-11-27T18:20:28.375377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471766865405442:2050] [node 3] ICP30 unregister session Session# [5:7577471766865405538:2050] VirtualId# [5:7577471766865405528:0]
2025-11-27T18:20:28.375481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471765559214214:2050] [node 3] ICP30 unregister session Session# [6:7577471765559214315:2050] VirtualId# [6:7577471765559214304:0]
2025-11-27T18:20:28.375472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:28.375510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471765559214315:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.375493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471766865405442:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.375502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.375577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:28.375637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:28.384569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471767911196328:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:28.385566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471766865405442:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:28.412758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:28.412999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471766865405526:2108] [node 4] ICS07 socket disconnect 504 reason# EndOfStream
2025-11-27T18:20:28.413041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471766865405526:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.413144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:28.413204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471766865405526:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:28.413635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:28.413858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471765559214310:2108] [node 4] ICS07 socket disconnect 505 reason# EndOfStream
2025-11-27T18:20:28.413870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471766865405503:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8923) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.413888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471765559214310:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.413914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471766865405526:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.413990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471766865405503:2108] [node 4] ICP30 unregister session Session# [5:7577471766865405526:2108] VirtualId# [5:7577471766865405523:0]
2025-11-27T18:20:28.413966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:28.414011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471765559214310:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:28.414119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:28.414149Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471766865405503:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8923) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.414163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8923) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.414667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471765559214275:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8923) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.414704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471765559214310:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.414762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471765559214275:2108] [node 4] ICP30 unregister session Session# [6:7577471765559214310:2108] VirtualId# [6:7577471765559214301:0]
2025-11-27T18:20:28.414825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:28.414856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471765559214275:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8923) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.414872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8923) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.424522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471765559214275:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:28.424521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471766865405503:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:28.480066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:28.480319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471765559214294:2107] [node 5] ICS07 socket disconnect 544 reason# EndOfStream
2025-11-27T18:20:28.480359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471765559214294:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.480459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:28.480508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471765559214294:2107] [node 5] ICS15 start handshake
2025-11-27T18:20:28.481140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471765559214274:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2342) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.481184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471765559214294:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.481273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471765559214274:2107] [node 5] ICP30 unregister session Session# [6:7577471765559214294:2107] VirtualId# [6:7577471765559214291:0]
2025-11-27T18:20:28.481390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:28.481417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471765559214274:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2342) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.481432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2342) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:28.572153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471767438637079:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:28.572235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471767438637080:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:28.590946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471767234681849:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:28.590975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471767234681850:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:28.608581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471765147955779:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:28.608675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471765147955780:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:28.626384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471766015103993:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:28.626440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471766015103994:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:28.626634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471766015103995:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:28.644674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471766766255892:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:28.644784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471766766255893:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:28.644790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471766766255894:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:28.661920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471764579702682:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:28.661984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471764579702684:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:28.662083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471764579702683:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:28.741819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471766766255956:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:28.744408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471766015104074:2125] [node 6] ICP01 ready to work
2025-11-27T18:20:28.744286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471767438637169:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:28.745728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471764579702743:2104] [node 5] ICP01 ready to work
2025-11-27T18:20:28.745663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471767234681930:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:28.745835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471765147955862:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:28.744342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471767438637170:2114] [node 5] ICP01 ready to work
2025-11-27T18:20:28.745769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471766766255957:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:28.745656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471767234681931:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:28.745955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471765147955860:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:28.746652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471766015104075:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:28.745860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471764579702746:2103] [node 4] ICP01 ready to work
2025-11-27T18:20:28.746836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471767438637171:2123] [node 6] ICP01 ready to work
2025-11-27T18:20:28.746177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471767234681932:2126] [node 6] ICP01 ready to work
2025-11-27T18:20:28.747077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471765147955861:2125] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.761974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766766255967:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63771
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.762193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471767234681939:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23356
2025-11-27T18:20:28.763084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471767438637185:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47990
2025-11-27T18:20:28.763126Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471766766255967:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.763252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471767438637186:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47990
2025-11-27T18:20:28.764217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471766766255892:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.764615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471767234681939:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.764902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471766015104082:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62046
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.765249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471767234681946:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23356
2025-11-27T18:20:28.765557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471767438637177:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.765445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471767234681849:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.765595Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471767438637177:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.768080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471767234681948:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23356
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.768143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471766015104082:2131] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.768305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471767438637186:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.768285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765147955869:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14085
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.769181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471767438637175:2138] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.769578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765147955875:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14085
2025-11-27T18:20:28.769594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766766255973:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63771
2025-11-27T18:20:28.772014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471767234681948:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.770146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765147955874:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14085
2025-11-27T18:20:28.769229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471767438637175:2138] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.772153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471767234681949:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23356
2025-11-27T18:20:28.772498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765147955869:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.772848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765147955874:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:28.769238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471767438637187:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47990
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: 2025-11-27T18:20:28.769242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471767438637188:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47990
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:28.772990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471767234681946:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:28.772004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471767438637185:2142] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.773059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471765147955779:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.772508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471767438637177:2139] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:28.773098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766766255975:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63771
2025-11-27T18:20:28.773069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765147955879:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14085
2025-11-27T18:20:28.773098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766766255974:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63771
2025-11-27T18:20:28.773034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471767438637188:2145] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.774067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471767234681949:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.773160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471767438637175:2138] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:28.773579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765147955875:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.774856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471766766255973:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.773170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471767438637190:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47990
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.774925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471767234681951:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23356
2025-11-27T18:20:28.773717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765147955879:2136] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.774882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471766766255967:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:28.773552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471767438637187:2144] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.775804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471767234681937:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.775515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471766766255892:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.774862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471765147955780:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.775830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471767234681937:2129] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.773626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471767438637170:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.775581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471766766255893:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.776051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471767234681952:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23356
2025-11-27T18:20:28.775640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:28.773796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:28.774373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471767438637170:2114] [node 5] ICP22 created new session: [1:7577471767438637194:2114]
2025-11-27T18:20:28.774871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471767438637194:2114] [node 5] ICS09 handshake done sender: [1:7577471767438637177:2139] self: [1:7577471767438637176:0] peer: [5:7577471766766255972:0] socket: 500 qp: -1
2025-11-27T18:20:28.775938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471766766255892:2048] [node 1] ICP22 created new session: [5:7577471766766255977:2048]
2025-11-27T18:20:28.780473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471767234681937:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:28.774925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471767438637194:2114] [node 5] ICS10 traffic start
2025-11-27T18:20:28.777652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471766766255977:2048] [node 1] ICS09 handshake done sender: [5:7577471766766255967:2114] self: [5:7577471766766255972:0] peer: [1:7577471767438637176:0] socket: 504 qp: -1
2025-11-27T18:20:28.780588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471767234681938:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.774925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471767438637173:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.778664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471766766255977:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:28.774953Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471767438637173:2137] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.778701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.774968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471767438637079:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.780620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471767234681938:2130] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.775209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:28.782249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471766766255975:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.775511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:28.782082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471767234681939:2131] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:28.782779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471766766255894:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.783192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765147955884:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14085
2025-11-27T18:20:28.775571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471767438637079:2048] [node 2] ICP22 created new session: [1:7577471767438637196:2048]
2025-11-27T18:20:28.775782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471767438637196:2048] [node 2] ICS09 handshake done sender: [1:7577471767438637175:2138] self: [1:7577471767438637174:0] peer: [2:7577471767234681950:0] socket: 499 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.783378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471766015104091:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62046
2025-11-27T18:20:28.782229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471767234681951:2137] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.782276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471767234681850:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.783963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765147955866:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.775935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471767438637196:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:28.784546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471764579702763:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5938
2025-11-27T18:20:28.777633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:28.782317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:28.784009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471765147955866:2129] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.784595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471766015104091:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.782747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471767234681850:2049] [node 3] ICP22 created new session: [2:7577471767234681957:2049]
2025-11-27T18:20:28.783701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471767438637190:2146] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.784907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765147955885:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14085
2025-11-27T18:20:28.785329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766766255981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63771
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.783017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471767234681957:2049] [node 3] ICS09 handshake done sender: [2:7577471767234681937:2129] self: [2:7577471767234681933:0] peer: [3:7577471765147955882:0] socket: 519 qp: -1
2025-11-27T18:20:28.786002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471765147955875:2135] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:28.785851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471766015104092:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62046
2025-11-27T18:20:28.784064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471767438637173:2137] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:28.783045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471767234681957:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:28.787360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471764579702764:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5938
2025-11-27T18:20:28.785702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471767438637080:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.787786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471766766255973:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:28.787079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471765147955866:2129] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:28.783392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:28.785736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.787567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471765147955780:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.783456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471767234681849:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.786025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471767438637080:2049] [node 3] ICP22 created new session: [1:7577471767438637202:2049]
2025-11-27T18:20:28.783600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:28.787050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471767438637202:2049] [node 3] ICS09 handshake done sender: [1:7577471767438637173:2137] self: [1:7577471767438637172:0] peer: [3:7577471765147955880:0] socket: 498 qp: -1
2025-11-27T18:20:28.787693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:28.783644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471767234681849:2048] [node 1] ICP22 created new session: [2:7577471767234681961:2048]
2025-11-27T18:20:28.788738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766766255982:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63771
2025-11-27T18:20:28.784435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471767234681961:2048] [node 1] ICS09 handshake done sender: [2:7577471767234681939:2131] self: [2:7577471767234681950:0] peer: [1:7577471767438637174:0] socket: 503 qp: -1
2025-11-27T18:20:28.787254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471767438637202:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:28.787300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:28.787750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471765147955869:2131] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:28.788794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471766766255893:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.784607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471767234681961:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:28.788013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471765147955780:2049] [node 2] ICP22 created new session: [3:7577471765147955887:2049]
2025-11-27T18:20:28.784693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:28.788838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:28.788474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471765147955862:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.788730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471765147955887:2049] [node 2] ICS09 handshake done sender: [3:7577471765147955875:2135] self: [3:7577471765147955882:0] peer: [2:7577471767234681933:0] socket: 525 qp: -1
2025-11-27T18:20:28.785055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471767234681938:2130] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:28.788807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:28.789414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471766766255893:2049] [node 2] ICP22 created new session: [5:7577471766766255983:2049]
2025-11-27T18:20:28.788354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471767234681930:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.788820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:28.789925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471766766255983:2049] [node 2] ICS09 handshake done sender: [5:7577471766766255973:2117] self: [5:7577471766766255976:0] peer: [2:7577471767234681936:0] socket: 524 qp: -1
2025-11-27T18:20:28.788826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471765147955887:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:28.788869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471767234681930:2108] [node 5] ICP22 created new session: [2:7577471767234681963:2108]
2025-11-27T18:20:28.789964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471766766255983:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:28.788862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:28.789899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471767234681963:2108] [node 5] ICS09 handshake done sender: [2:7577471767234681938:2130] self: [2:7577471767234681936:0] peer: [5:7577471766766255976:0] socket: 521 qp: -1
2025-11-27T18:20:28.788866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471765147955862:2108] [node 5] ICP22 created new session: [3:7577471765147955888:2108]
2025-11-27T18:20:28.789934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471767234681963:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:28.790203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:28.790005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:28.789912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471765147955888:2108] [node 5] ICS09 handshake done sender: [3:7577471765147955866:2129] self: [3:7577471765147955864:0] peer: [5:7577471766766255980:0] socket: 536 qp: -1
2025-11-27T18:20:28.789953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471765147955888:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:28.789993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:28.790932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471767234681850:2049] [node 3] ICP18 handshake (actor [2:7577471767234681951:2137]) is too old
2025-11-27T18:20:28.790167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471765147955779:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.790217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:28.791263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471767438637079:2048] [node 2] ICP18 handshake (actor [1:7577471767438637187:2144]) is too old
2025-11-27T18:20:28.791319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471767438637080:2049] [node 3] ICP18 handshake (actor [1:7577471767438637190:2146]) is too old
2025-11-27T18:20:28.790269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471765147955779:2048] [node 1] ICP22 created new session: [3:7577471765147955891:2048]
2025-11-27T18:20:28.790322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471765147955891:2048] [node 1] ICS09 handshake done sender: [3:7577471765147955869:2131] self: [3:7577471765147955880:0] peer: [1:7577471767438637172:0] socket: 502 qp: -1
2025-11-27T18:20:28.791487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471766766255975:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:28.790616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471765147955891:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:28.790914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:28.792219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471764579702763:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.792452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471764579702682:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.792515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471766766255894:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.792578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:28.792616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471766766255894:2050] [node 3] ICP22 created new session: [5:7577471766766255986:2050]
2025-11-27T18:20:28.792772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471766766255986:2050] [node 3] ICS09 handshake done sender: [5:7577471766766255975:2119] self: [5:7577471766766255980:0] peer: [3:7577471765147955864:0] socket: 539 qp: -1
2025-11-27T18:20:28.792808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471766766255986:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:28.792837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.794715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471764579702766:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5938
2025-11-27T18:20:28.795615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471766015104092:2135] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.796487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471766015103993:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.796795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471767438637182:2140] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.796831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471767438637182:2140] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.797163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471767438637184:2141] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.797199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471767438637184:2141] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.800557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471764579702766:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.800151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471767438637184:2141] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:28.800176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471767438637182:2140] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.800717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471764579702764:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.801217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471766015104095:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62046
2025-11-27T18:20:28.801511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471767438637169:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.801379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471766015104096:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62046
2025-11-27T18:20:28.801758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471767438637171:2123] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.801721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471766015104097:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62046
2025-11-27T18:20:28.801741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471764579702767:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5938
2025-11-27T18:20:28.801795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-11-27T18:20:28.801746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471764579702683:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-27T18:20:28.801805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:28.801805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471764579702743:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.802005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471764579702768:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5938
2025-11-27T18:20:28.801859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471767438637171:2123] [node 6] ICP22 created new session: [1:7577471767438637207:2123]
2025-11-27T18:20:28.802027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471766015104095:2136] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.802055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471766766255970:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.801887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471767438637169:2107] [node 4] ICP22 created new session: [1:7577471767438637206:2107]
2025-11-27T18:20:28.802091Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471766766255970:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.801912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471767438637207:2123] [node 6] ICS09 handshake done sender: [1:7577471767438637182:2140] self: [1:7577471767438637181:0] peer: [6:7577471764579702765:0] socket: 548 qp: -1
2025-11-27T18:20:28.802111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471767234681942:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.801938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471767438637206:2107] [node 4] ICS09 handshake done sender: [1:7577471767438637184:2141] self: [1:7577471767438637183:0] peer: [4:7577471766015104094:0] socket: 550 qp: -1
2025-11-27T18:20:28.803123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471766015104092:2135] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:28.801946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471767438637207:2123] [node 6] ICS10 traffic start
2025-11-27T18:20:28.803407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471764579702767:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.802148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471767234681942:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.803215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471766015104096:2137] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.801975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471767438637206:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:28.804443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471767234681942:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:28.801981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:28.804248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471766015103995:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.802010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:28.804098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471764579702763:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:28.804602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471766766255970:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:28.804319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471766015103993:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.804516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471764579702684:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.804784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471764579702682:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.805689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471767234681932:2126] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.805651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:28.804665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:28.806053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471766766255956:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.805747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:28.806066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471764579702682:2048] [node 1] ICP22 created new session: [6:7577471764579702773:2048]
2025-11-27T18:20:28.806122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471764579702773:2048] [node 1] ICS09 handshake done sender: [6:7577471764579702763:2118] self: [6:7577471764579702765:0] peer: [1:7577471767438637181:0] socket: 549 qp: -1
2025-11-27T18:20:28.805627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471766015103994:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.805780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471767234681932:2126] [node 6] ICP22 created new session: [2:7577471767234681967:2126]
2025-11-27T18:20:28.806524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765147955872:2132] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.806200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:28.806510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471764579702773:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:28.807102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:28.806425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471767234681967:2126] [node 6] ICS09 handshake done sender: [2:7577471767234681942:2132] self: [2:7577471767234681940:0] peer: [6:7577471764579702769:0] socket: 554 qp: -1
2025-11-27T18:20:28.806563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471765147955872:2132] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.806249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471766766255956:2110] [node 6] ICP22 created new session: [5:7577471766766255989:2110]
2025-11-27T18:20:28.805630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471766015103993:2048] [node 1] ICP22 created new session: [4:7577471766015104099:2048]
2025-11-27T18:20:28.807142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471767234681967:2126] [node 6] ICS10 traffic start
2025-11-27T18:20:28.806057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471766015104099:2048] [node 1] ICS09 handshake done sender: [4:7577471766015104092:2135] self: [4:7577471766015104094:0] peer: [1:7577471767438637183:0] socket: 551 qp: -1
2025-11-27T18:20:28.807237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471765147955873:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.807142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471766766255989:2110] [node 6] ICS09 handshake done sender: [5:7577471766766255970:2115] self: [5:7577471766766255968:0] peer: [6:7577471764579702770:0] socket: 505 qp: -1
2025-11-27T18:20:28.807228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:28.806187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471766015104099:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:28.807264Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471765147955873:2133] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.807180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471766766255989:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:28.806599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:28.807251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471767234681943:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.808232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:28.808258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471765147955872:2132] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.807281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471767234681943:2133] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.810027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471765147955860:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.811398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471764579702683:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15451) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471764579702764:2119] held: no
2025-11-27T18:20:28.810350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471765147955873:2133] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:28.811829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471766015104103:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62046
2025-11-27T18:20:28.811203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:28.810072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471767234681943:2133] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:28.811231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471767234681931:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.811242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471765147955860:2107] [node 4] ICP22 created new session: [3:7577471765147955897:2107]
2025-11-27T18:20:28.812049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471764579702772:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5938
2025-11-27T18:20:28.811948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471765147955897:2107] [node 4] ICS09 handshake done sender: [3:7577471765147955872:2132] self: [3:7577471765147955871:0] peer: [4:7577471766015104098:0] socket: 510 qp: -1
2025-11-27T18:20:28.811831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:28.812097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766766255991:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63771
2025-11-27T18:20:28.812181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471766015104095:2136] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:28.811997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471767234681931:2107] [node 4] ICP22 created new session: [2:7577471767234681970:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.811991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471765147955897:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:28.812025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:28.812100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471767234681970:2107] [node 4] ICS09 handshake done sender: [2:7577471767234681943:2133] self: [2:7577471767234681941:0] peer: [4:7577471766015104100:0] socket: 522 qp: -1
2025-11-27T18:20:28.812136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471767234681970:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.812535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471765147955861:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.813188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471766015104104:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62046
2025-11-27T18:20:28.812176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:28.812734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:28.812857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471765147955861:2125] [node 6] ICP22 created new session: [3:7577471765147955899:2125]
2025-11-27T18:20:28.813170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471765147955899:2125] [node 6] ICS09 handshake done sender: [3:7577471765147955873:2133] self: [3:7577471765147955870:0] peer: [6:7577471764579702771:0] socket: 518 qp: -1
2025-11-27T18:20:28.813370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471764579702774:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5938
2025-11-27T18:20:28.813225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471765147955899:2125] [node 6] ICS10 traffic start
2025-11-27T18:20:28.813282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:28.813849Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471766015103993:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:28.814090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471766766255991:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.814915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471766766255957:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.815330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471764579702766:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:28.816306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471764579702764:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.816552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471766015103995:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.817240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:28.817353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471766015103995:2050] [node 3] ICP22 created new session: [4:7577471766015104105:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.817563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471766015104105:2050] [node 3] ICS09 handshake done sender: [4:7577471766015104095:2136] self: [4:7577471766015104098:0] peer: [3:7577471765147955871:0] socket: 520 qp: -1
2025-11-27T18:20:28.817516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471764579702776:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5938
2025-11-27T18:20:28.817587Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471766015103994:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15451) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471766015104096:2137] held: no
2025-11-27T18:20:28.817677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471766015104105:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:28.817665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471765147955901:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14085
2025-11-27T18:20:28.817720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:28.817786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471764579702743:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.817835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:28.817961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471764579702743:2104] [node 5] ICP22 created new session: [6:7577471764579702779:2104]
2025-11-27T18:20:28.818091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471764579702779:2104] [node 5] ICS09 handshake done sender: [6:7577471764579702766:2120] self: [6:7577471764579702770:0] peer: [5:7577471766766255968:0] socket: 513 qp: -1
2025-11-27T18:20:28.818169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471764579702779:2104] [node 5] ICS10 traffic start
2025-11-27T18:20:28.818210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:28.818276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471766015104096:2137] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:28.818303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471766015104084:2132] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.818335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471766015104084:2132] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.818393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471764579702683:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.818491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:28.818815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471764579702683:2049] [node 2] ICP22 created new session: [6:7577471764579702781:2049]
2025-11-27T18:20:28.819197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471764579702781:2049] [node 2] ICS09 handshake done sender: [6:7577471764579702764:2119] self: [6:7577471764579702769:0] peer: [2:7577471767234681940:0] socket: 555 qp: -1
2025-11-27T18:20:28.819239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471764579702781:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:28.819268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:28.819362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471766015103995:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.819629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471766015103994:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.819651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766766255993:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63771
2025-11-27T18:20:28.819736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:28.819790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471766015103994:2049] [node 2] ICP22 created new session: [4:7577471766015104107:2049]
2025-11-27T18:20:28.819868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471766015104107:2049] [node 2] ICS09 handshake done sender: [4:7577471766015104096:2137] self: [4:7577471766015104100:0] peer: [2:7577471767234681941:0] socket: 535 qp: -1
2025-11-27T18:20:28.819891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471764579702767:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:28.819903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471766015104107:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:28.819948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:28.820685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471764579702684:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.820724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:28.820771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471764579702684:2050] [node 3] ICP22 created new session: [6:7577471764579702783:2050]
2025-11-27T18:20:28.820803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471764579702783:2050] [node 3] ICS09 handshake done sender: [6:7577471764579702767:2121] self: [6:7577471764579702771:0] peer: [3:7577471765147955870:0] socket: 533 qp: -1
2025-11-27T18:20:28.820832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471764579702783:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:28.820880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.821245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471766766255994:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63771
2025-11-27T18:20:28.821375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471766015104084:2132] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:28.821510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471766766255991:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:28.821667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471766766255957:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.821867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471766015104075:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.821715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:28.821845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471766766255957:2108] [node 4] ICP22 created new session: [5:7577471766766255995:2108]
2025-11-27T18:20:28.823045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:28.822478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471766766255995:2108] [node 4] ICS09 handshake done sender: [5:7577471766766255991:2122] self: [5:7577471766766255992:0] peer: [4:7577471766015104083:0] socket: 516 qp: -1
2025-11-27T18:20:28.823339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471766015104075:2108] [node 5] ICP22 created new session: [4:7577471766015104111:2108]
2025-11-27T18:20:28.822515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471766766255995:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:28.823528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471766015104111:2108] [node 5] ICS09 handshake done sender: [4:7577471766015104084:2132] self: [4:7577471766015104083:0] peer: [5:7577471766766255992:0] socket: 562 qp: -1
2025-11-27T18:20:28.822723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:28.823572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471766015104111:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:28.823619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.824925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471764579702785:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5938
2025-11-27T18:20:28.825153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471765147955861:2125] [node 6] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:28.825427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471764579702785:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.825576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471764579702746:2103] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:28.825757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471766015104089:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:28.825785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471766015104089:2133] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:28.826664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471766015104089:2133] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:28.826898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471766015104074:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:28.826963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:28.826959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471764579702787:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5938
2025-11-27T18:20:28.827209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471766015104074:2125] [node 6] ICP22 created new session: [4:7577471766015104114:2125]
2025-11-27T18:20:28.827284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471766015104114:2125] [node 6] ICS09 handshake done sender: [4:7577471766015104089:2133] self: [4:7577471766015104088:0] peer: [6:7577471764579702786:0] socket: 532 qp: -1
2025-11-27T18:20:28.827311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471766015104114:2125] [node 6] ICS10 traffic start
2025-11-27T18:20:28.827423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:28.827437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471764579702785:2126] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:28.828751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471764579702746:2103] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471764579702785:2126] held: no
2025-11-27T18:20:28.829411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471764579702746:2103] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:28.829452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:28.829498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471764579702746:2103] [node 4] ICP22 created new session: [6:7577471764579702788:2103]
2025-11-27T18:20:28.829560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471764579702788:2103] [node 4] ICS09 handshake done sender: [6:7577471764579702785:2126] self: [6:7577471764579702786:0] peer: [4:7577471766015104088:0] socket: 567 qp: -1
2025-11-27T18:20:28.829604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471764579702788:2103] [node 4] ICS10 traffic start
2025-11-27T18:20:28.829650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:28.868849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:28.869106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471767234681961:2048] [node 1] ICS01 socket: 503 reason# EndOfStream
2025-11-27T18:20:28.869165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471767234681849:2048] [node 1] ICP30 unregister session Session# [2:7577471767234681961:2048] VirtualId# [2:7577471767234681950:0]
2025-11-27T18:20:28.869211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471767234681961:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.869325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:28.869384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:28.870250Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:28.870425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471765147955891:2048] [node 1] ICS07 socket disconnect 502 reason# EndOfStream
2025-11-27T18:20:28.870452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471765147955891:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.870512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:28.870554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471765147955891:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:28.871209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471765147955779:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.871254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471765147955891:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.871280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471765147955779:2048] [node 1] ICP30 unregister session Session# [3:7577471765147955891:2048] VirtualId# [3:7577471765147955880:0]
2025-11-27T18:20:28.871449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:28.871490Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471765147955779:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.871512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.872656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:28.872862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471766015104099:2048] [node 1] ICS07 socket disconnect 551 reason# EndOfStream
2025-11-27T18:20:28.872900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471766015104099:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.873009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:28.873058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471766015104099:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:28.873524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:28.873734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471766766255977:2048] [node 1] ICS07 socket disconnect 504 reason# EndOfStream
2025-11-27T18:20:28.873768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471766766255977:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.873866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471766015103993:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.873884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:28.873955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471766766255977:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:28.873916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471766015104099:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.873973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471766015103993:2048] [node 1] ICP30 unregister session Session# [4:7577471766015104099:2048] VirtualId# [4:7577471766015104094:0]
2025-11-27T18:20:28.874111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:28.874156Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471766015103993:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.874199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.874487Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:28.874712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471764579702773:2048] [node 1] ICS07 socket disconnect 549 reason# EndOfStream
2025-11-27T18:20:28.874778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471764579702773:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.874747Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471766766255892:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.874871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:28.874922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471766766255977:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.874939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471764579702773:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:28.874972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471766766255892:2048] [node 1] ICP30 unregister session Session# [5:7577471766766255977:2048] VirtualId# [5:7577471766766255972:0]
2025-11-27T18:20:28.875093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:28.875131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471766766255892:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.875151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.875635Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471764579702682:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.875718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471764579702773:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.875781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471764579702682:2048] [node 1] ICP30 unregister session Session# [6:7577471764579702773:2048] VirtualId# [6:7577471764579702765:0]
2025-11-27T18:20:28.875904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:28.875948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471764579702682:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.875989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30395) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.881199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471765147955779:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:28.884267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471766015103993:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:28.885303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471766766255892:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:28.886324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471764579702682:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:28.920690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:28.920929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471765147955887:2049] [node 2] ICS07 socket disconnect 525 reason# EndOfStream
2025-11-27T18:20:28.920962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471765147955887:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.921042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:28.921092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471765147955887:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:28.923100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:28.923228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471765147955780:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15451) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:28.923257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471765147955887:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:28.923323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471766015104107:2049] [node 2] ICS01 socket: 535 reason# EndOfStream
2025-11-27T18:20:28.923380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471766015103994:2049] [node 2] ICP30 unregister session Session# [4:7577471766015104107:2049] VirtualId# [4:7577471766015104100:0]
2025-11-27T18:20:28.923412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471766015104107:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.923958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:28.924072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:28.924183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471765147955780:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15451) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.924240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471765147955887:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.924319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471765147955780:2049] [node 2] ICP30 unregister session Session# [3:7577471765147955887:2049] VirtualId# [3:7577471765147955882:0]
2025-11-27T18:20:28.924435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:28.924495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471765147955780:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15451) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.924493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:28.924531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15451) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.924727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471766766255983:2049] [node 2] ICS01 socket: 524 reason# EndOfStream
2025-11-27T18:20:28.924782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471766766255893:2049] [node 2] ICP30 unregister session Session# [5:7577471766766255983:2049] VirtualId# [5:7577471766766255976:0]
2025-11-27T18:20:28.924806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471766766255983:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.924874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:28.924911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:28.925320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:28.925449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471764579702781:2049] [node 2] ICS01 socket: 555 reason# EndOfStream
2025-11-27T18:20:28.925483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471764579702683:2049] [node 2] ICP30 unregister session Session# [6:7577471764579702781:2049] VirtualId# [6:7577471764579702769:0]
2025-11-27T18:20:28.925502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471764579702781:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.925567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:28.925620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:28.934447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471765147955780:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:28.984130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:28.984351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471766015104105:2050] [node 3] ICS07 socket disconnect 520 reason# EndOfStream
2025-11-27T18:20:28.984378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471766015104105:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.984454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:28.984492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471766015104105:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:28.985179Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:28.985272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471766015103995:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1335) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.985321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471766015104105:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.985345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471766015103995:2050] [node 3] ICP30 unregister session Session# [4:7577471766015104105:2050] VirtualId# [4:7577471766015104098:0]
2025-11-27T18:20:28.985370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471766766255986:2050] [node 3] ICS07 socket disconnect 539 reason# EndOfStream
2025-11-27T18:20:28.985480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:28.985407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471766766255986:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.985530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471766015103995:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1335) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.985478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:28.985543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1335) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.985533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471766766255986:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:28.986312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471766766255894:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1335) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.986378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471766766255986:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.986382Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:28.986404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471766766255894:2050] [node 3] ICP30 unregister session Session# [5:7577471766766255986:2050] VirtualId# [5:7577471766766255980:0]
2025-11-27T18:20:28.986489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:28.986513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471766766255894:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1335) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.986530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1335) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.986579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471764579702783:2050] [node 3] ICS07 socket disconnect 533 reason# EndOfStream
2025-11-27T18:20:28.986608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471764579702783:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:28.986687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:28.986762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471764579702783:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:28.987383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471764579702684:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1335) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:28.987452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471764579702783:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:28.987494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471764579702684:2050] [node 3] ICP30 unregister session Session# [6:7577471764579702783:2050] VirtualId# [6:7577471764579702771:0]
2025-11-27T18:20:28.987566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:28.987587Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471764579702684:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1335) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.987609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1335) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:28.994858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471766015103995:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:28.995863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471766766255894:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:28.997965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471764579702684:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:29.042682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:29.042890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471766766255995:2108] [node 4] ICS01 socket: 516 reason# EndOfStream
2025-11-27T18:20:29.042943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471766766255957:2108] [node 4] ICP30 unregister session Session# [5:7577471766766255995:2108] VirtualId# [5:7577471766766255992:0]
2025-11-27T18:20:29.042972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471766766255995:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.043087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:29.043151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:29.043581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:29.043785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471764579702788:2103] [node 4] ICS01 socket: 567 reason# EndOfStream
2025-11-27T18:20:29.043817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471764579702746:2103] [node 4] ICP30 unregister session Session# [6:7577471764579702788:2103] VirtualId# [6:7577471764579702786:0]
2025-11-27T18:20:29.043848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471764579702788:2103] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.043928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:29.043981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:29.089505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:29.089738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471764579702779:2104] [node 5] ICS07 socket disconnect 513 reason# EndOfStream
2025-11-27T18:20:29.089782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471764579702779:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.089879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:29.089930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471764579702779:2104] [node 5] ICS15 start handshake
2025-11-27T18:20:29.091057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471764579702743:2104] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7161) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:29.091177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471764579702779:2104] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:29.091233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471764579702743:2104] [node 5] ICP30 unregister session Session# [6:7577471764579702779:2104] VirtualId# [6:7577471764579702770:0]
2025-11-27T18:20:29.091348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:29.091408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471764579702743:2104] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7161) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.091426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7161) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.101417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471764579702743:2104] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:29.200350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471768699741703:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:29.200425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471768699741704:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:29.219296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471768685907149:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:29.219350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471768685907150:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:29.238031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471771230767761:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:29.238068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471771230767762:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:29.260231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471772118961136:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:29.260274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471772118961138:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:29.260710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471772118961137:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:29.280642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471768378660857:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:29.280672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471768378660858:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:29.280874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471768378660859:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:29.301100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471768020718366:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:29.301139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471768020718368:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:29.301280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471768020718367:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:29.386414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471768020718434:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:29.382754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471768699741766:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:29.389079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471768699741767:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:29.390312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471768685907240:2132] [node 5] ICP01 ready to work
2025-11-27T18:20:29.389838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471771230767842:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:29.390769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471768378660929:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:29.390801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471772118961218:2117] [node 6] ICP01 ready to work
2025-11-27T18:20:29.390557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471768699741768:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:29.390571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471768020718436:2106] [node 4] ICP01 ready to work
2025-11-27T18:20:29.390760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471771230767843:2115] [node 6] ICP01 ready to work
2025-11-27T18:20:29.390543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471768685907239:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:29.390779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471768378660930:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:29.390980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471772118961217:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:29.391038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471771230767844:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:29.390964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471768685907241:2133] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.403806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471768020718446:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53058
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.407027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471771230767853:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3858
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:29.411077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471768685907250:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53578
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.410860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471771230767856:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3858
2025-11-27T18:20:29.411036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471768020718450:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53058
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.412288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471768699741780:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1849
2025-11-27T18:20:29.411381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471768685907251:2142] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53578
2025-11-27T18:20:29.412539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471768020718446:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.412314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471771230767860:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3858
2025-11-27T18:20:29.415777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471768699741785:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1849
2025-11-27T18:20:29.411770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471768685907252:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53578
2025-11-27T18:20:29.415787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471768378660942:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.416206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471768699741784:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1849
2025-11-27T18:20:29.415875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471768020718366:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.414743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471771230767853:2132] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.412565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471768685907250:2141] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.416209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471768699741782:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1849
2025-11-27T18:20:29.417677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772118961230:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18990
2025-11-27T18:20:29.416208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471768378660943:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22139
No RDMA context found for GID: 2025-11-27T18:20:29.416266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471771230767761:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-27T18:20:29.414738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471768685907251:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.417452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471768378660942:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.417485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471771230767860:2135] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.419624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471768020718450:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.416561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471768699741780:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.420744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471771230767856:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.419641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471768378660943:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.415131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471768685907149:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.420548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772118961233:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18990
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.420960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471771230767861:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3858
2025-11-27T18:20:29.420630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471768378660859:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.421011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471768020718434:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.421482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471768685907252:2143] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.422606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772118961233:2134] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.422988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471772118961137:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.417457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471768699741773:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.422869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471771230767861:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.421234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471768378660932:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.423122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772118961230:2132] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.417486Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471768699741773:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.421281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471768378660932:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.423282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471772118961138:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.422885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471771230767849:2129] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.423317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471768378660932:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.417584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471768699741775:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.422920Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471771230767849:2129] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.417611Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471768699741775:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:29.423420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471768685907246:2137] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.419600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471768699741774:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.423453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471768685907248:2139] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.423456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471768685907246:2137] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.423014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471771230767762:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.419641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471768699741774:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.421215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471768699741785:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.423481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471768685907248:2139] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.427029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471768020718455:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53058
2025-11-27T18:20:29.424825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471771230767850:2130] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.422600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471768699741784:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.427095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471768685907260:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53578
2025-11-27T18:20:29.427084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471768020718456:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53058
2025-11-27T18:20:29.422611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471768699741773:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:29.422723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471768699741786:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1849
2025-11-27T18:20:29.424848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471771230767850:2130] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.427703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471768378660929:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.427394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471768685907246:2137] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:29.422774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471768699741775:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:29.427343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471771230767866:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3858
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.423284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471768699741782:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.423651Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471768699741774:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.427864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471768020718446:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:29.427131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471768699741766:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.427385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471771230767849:2129] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:29.428014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471768685907239:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.427775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.429201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772118961236:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18990
2025-11-27T18:20:29.428411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471771230767844:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.428645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471768020718450:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:29.428134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471768685907248:2139] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:29.427813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471768378660929:2109] [node 6] ICP22 created new session: [5:7577471768378660950:2109]
2025-11-27T18:20:29.427346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471768699741703:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.427870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471768378660950:2109] [node 6] ICS09 handshake done sender: [5:7577471768378660932:2119] self: [5:7577471768378660931:0] peer: [6:7577471768020718454:0] socket: 514 qp: -1
2025-11-27T18:20:29.429712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471768020718366:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.431266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772118961237:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18990
2025-11-27T18:20:29.428322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:29.428456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:29.427404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:29.429885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471768685907239:2107] [node 4] ICP22 created new session: [2:7577471768685907264:2107]
2025-11-27T18:20:29.428150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471768378660950:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:29.435306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471772118961137:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19153) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471772118961233:2134] held: no
2025-11-27T18:20:29.429391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471771230767850:2130] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:29.430317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471768685907150:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.430010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:29.428200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:29.427405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:29.429407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471771230767844:2108] [node 5] ICP22 created new session: [3:7577471771230767869:2108]
2025-11-27T18:20:29.429892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471768378660949:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22139
2025-11-27T18:20:29.430544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471768020718366:2048] [node 1] ICP22 created new session: [6:7577471768020718457:2048]
2025-11-27T18:20:29.430067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471771230767869:2108] [node 5] ICS09 handshake done sender: [3:7577471771230767849:2129] self: [3:7577471771230767846:0] peer: [5:7577471768378660947:0] socket: 526 qp: -1
2025-11-27T18:20:29.427476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471768699741786:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.430376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:29.436504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471772118961233:2134] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:29.431581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471768378660943:2124] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:29.430542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471768685907250:2141] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:29.430478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471771230767869:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:29.436711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471772118961230:2132] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:29.427503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471768699741766:2109] [node 6] ICP22 created new session: [1:7577471768699741795:2109]
2025-11-27T18:20:29.431383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471768020718434:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.435879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471768378660859:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4623) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471768378660943:2124] held: no
2025-11-27T18:20:29.431348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471768685907264:2107] [node 4] ICS09 handshake done sender: [2:7577471768685907246:2137] self: [2:7577471768685907242:0] peer: [4:7577471772118961234:0] socket: 539 qp: -1
2025-11-27T18:20:29.439232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471772118961137:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.431377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471768685907264:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:29.431712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:29.439790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:29.431681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:29.427573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471768699741704:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.431402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:29.440003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471772118961137:2049] [node 2] ICP22 created new session: [4:7577471772118961239:2049]
2025-11-27T18:20:29.431763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471771230767842:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.427633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471768699741703:2048] [node 2] ICP22 created new session: [1:7577471768699741794:2048]
2025-11-27T18:20:29.440046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471772118961239:2049] [node 2] ICS09 handshake done sender: [4:7577471772118961233:2134] self: [4:7577471772118961234:0] peer: [2:7577471768685907242:0] socket: 541 qp: -1
2025-11-27T18:20:29.431700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471768685907150:2049] [node 3] ICP22 created new session: [2:7577471768685907265:2049]
2025-11-27T18:20:29.431713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471768020718457:2048] [node 1] ICS09 handshake done sender: [6:7577471768020718446:2121] self: [6:7577471768020718453:0] peer: [1:7577471768699741770:0] socket: 508 qp: -1
2025-11-27T18:20:29.431799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:29.435260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471768685907265:2049] [node 3] ICS09 handshake done sender: [2:7577471768685907248:2139] self: [2:7577471768685907245:0] peer: [3:7577471771230767865:0] socket: 540 qp: -1
2025-11-27T18:20:29.427845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:29.440089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471772118961138:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.442980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471768378660859:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.427865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471768699741795:2109] [node 6] ICS09 handshake done sender: [1:7577471768699741775:2112] self: [1:7577471768699741770:0] peer: [6:7577471768020718453:0] socket: 506 qp: -1
2025-11-27T18:20:29.431929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471768020718457:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:29.440092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471772118961239:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:29.435534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471768685907265:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:29.431909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471771230767853:2132] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:29.443682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:29.427880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471768699741794:2048] [node 2] ICS09 handshake done sender: [1:7577471768699741773:2110] self: [1:7577471768699741772:0] peer: [2:7577471768685907258:0] socket: 505 qp: -1
2025-11-27T18:20:29.431946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471768020718434:2107] [node 5] ICP22 created new session: [6:7577471768020718458:2107]
2025-11-27T18:20:29.444029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471768378660859:2050] [node 3] ICP22 created new session: [5:7577471768378660955:2050]
2025-11-27T18:20:29.440131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:29.435957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:29.431986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471771230767842:2107] [node 4] ICP22 created new session: [3:7577471771230767872:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.428016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471768699741795:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:29.431956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:29.444222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471768378660954:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22139
2025-11-27T18:20:29.437370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471768685907149:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.428055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471768699741794:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:29.435881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471771230767872:2107] [node 4] ICS09 handshake done sender: [3:7577471771230767850:2130] self: [3:7577471771230767845:0] peer: [4:7577471772118961235:0] socket: 531 qp: -1
2025-11-27T18:20:29.444275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471768378660955:2050] [node 3] ICS09 handshake done sender: [5:7577471768378660943:2124] self: [5:7577471768378660947:0] peer: [3:7577471771230767846:0] socket: 530 qp: -1
2025-11-27T18:20:29.437414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:29.428060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:29.435509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471768020718458:2107] [node 5] ICS09 handshake done sender: [6:7577471768020718450:2122] self: [6:7577471768020718454:0] peer: [5:7577471768378660931:0] socket: 515 qp: -1
2025-11-27T18:20:29.428279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471768699741704:2049] [node 3] ICP22 created new session: [1:7577471768699741796:2049]
2025-11-27T18:20:29.444445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471768378660955:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:29.437782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471768685907149:2048] [node 1] ICP22 created new session: [2:7577471768685907270:2048]
2025-11-27T18:20:29.442968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:29.436303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471771230767872:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:20:29.444592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
::ffff:127.1.0.02025-11-27T18:20:29.428656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471768699741796:2049] [node 3] ICS09 handshake done sender: [1:7577471768699741774:2111] self: [1:7577471768699741771:0] peer: [3:7577471771230767862:0] socket: 504 qp: -1
2025-11-27T18:20:29.439970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471768685907270:2048] [node 1] ICS09 handshake done sender: [2:7577471768685907250:2141] self: [2:7577471768685907258:0] peer: [1:7577471768699741772:0] socket: 510 qp: -1
2025-11-27T18:20:29.435909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471768020718458:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:29.435943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:29.436512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:29.443004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471772118961138:2050] [node 3] ICP22 created new session: [4:7577471772118961242:2050]
2025-11-27T18:20:29.428690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:29.439997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471768685907270:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:20:29.429292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471768699741796:2049] [node 3] ICS10 traffic start
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:29.440025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
::ffff:127.1.0.0
2025-11-27T18:20:29.440061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471771230767761:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.445014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471768685907267:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53578
2025-11-27T18:20:29.443489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471772118961242:2050] [node 3] ICS09 handshake done sender: [4:7577471772118961230:2132] self: [4:7577471772118961235:0] peer: [3:7577471771230767845:0] socket: 535 qp: -1
2025-11-27T18:20:29.440102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.445616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471768378660956:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22139
2025-11-27T18:20:29.429359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:29.442938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471771230767761:2048] [node 1] ICP22 created new session: [3:7577471771230767875:2048]
2025-11-27T18:20:29.444013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471772118961242:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.445738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471768020718434:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:29.443427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471771230767875:2048] [node 1] ICS09 handshake done sender: [3:7577471771230767853:2132] self: [3:7577471771230767862:0] peer: [1:7577471768699741771:0] socket: 509 qp: -1
2025-11-27T18:20:29.444139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:29.443448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471771230767875:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:29.443554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:29.445932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471768685907267:2146] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.445567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772118961243:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18990
2025-11-27T18:20:29.445928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471771230767870:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3858
2025-11-27T18:20:29.445624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772118961238:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18990
2025-11-27T18:20:29.445923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772118961241:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18990
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.446147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471768378660956:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.446119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772118961238:2137] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.446174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471768020718463:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53058
2025-11-27T18:20:29.446139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772118961243:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.446367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471768378660958:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22139
2025-11-27T18:20:29.446334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772118961241:2138] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.446379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471768378660857:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.446445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471772118961138:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.446632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471768699741783:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.446816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471772118961136:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.446688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471771230767761:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.446858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471771230767877:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3858
2025-11-27T18:20:29.446822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471768020718464:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53058
2025-11-27T18:20:29.448102Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471768685907239:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17360331ns) {HandleHandshakeStatus:329(17358544ns) {DropIncomingHandshake:442(10001ns) DropOutgoingHandshake:457(280ns) SwitchToState:179(1025ns) ProcessPendingSessionEvents:495(9449246ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9385350ns max# 3085406ns}}}
2025-11-27T18:20:29.446661Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471768699741783:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.446864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471771230767844:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(16862980ns) {HandleHandshakeStatus:329(16860812ns) {DropIncomingHandshake:442(10484ns) DropOutgoingHandshake:457(254ns) SwitchToState:179(85ns) ProcessPendingSessionEvents:495(9727617ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9646100ns max# 1979564ns}}}
2025-11-27T18:20:29.448095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471768020718465:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53058
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.448092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471771230767861:2136] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:29.449090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471768378660929:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(17882759ns) {HandleHandshakeStatus:329(17880143ns) {DropIncomingHandshake:442(12365ns) DropOutgoingHandshake:457(279ns) SwitchToState:179(315ns) ProcessPendingSessionEvents:495(15466735ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15365985ns max# 2711260ns}}}
2025-11-27T18:20:29.448965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471768699741776:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.449208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471768020718366:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16111713ns) {HandleHandshakeStatus:329(16109837ns) {DropIncomingHandshake:442(500ns) DropOutgoingHandshake:457(4117ns) SwitchToState:179(284ns) ProcessPendingSessionEvents:495(9478034ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9141485ns max# 2740442ns}}}
2025-11-27T18:20:29.449193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471768378660954:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.448997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471768699741776:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.450663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471768685907273:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53578
2025-11-27T18:20:29.449448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471768699741703:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(18182904ns) {HandleHandshakeStatus:329(18179860ns) {DropIncomingHandshake:442(215ns) DropOutgoingHandshake:457(618ns) SwitchToState:179(660ns) ProcessPendingSessionEvents:495(15467038ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15372776ns max# 2734329ns}}}
2025-11-27T18:20:29.449295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471768020718366:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:29.449487Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471768699741766:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18212384ns) {HandleHandshakeStatus:329(18210739ns) {DropIncomingHandshake:442(10700ns) DropOutgoingHandshake:457(1121ns) SwitchToState:179(226ns) ProcessPendingSessionEvents:495(15425709ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15328887ns max# 1902883ns}}}
2025-11-27T18:20:29.449531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471768699741783:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:29.450729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471768685907249:2140] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.449458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471768378660858:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.450744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471768685907249:2140] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.450741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471771230767762:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.450698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471768378660958:2129] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.451419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471768378660930:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.451400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471768699741768:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.450789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:29.451473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:29.451518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471768378660960:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22139
2025-11-27T18:20:29.450964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471771230767877:2139] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.451734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471768020718465:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.451545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471768699741768:2108] [node 5] ICP22 created new session: [1:7577471768699741803:2108]
2025-11-27T18:20:29.451367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471771230767762:2049] [node 2] ICP22 created new session: [3:7577471771230767879:2049]
2025-11-27T18:20:29.451949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471768020718463:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.451691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471768699741803:2108] [node 5] ICS09 handshake done sender: [1:7577471768699741783:2116] self: [1:7577471768699741781:0] peer: [5:7577471768378660959:0] socket: 534 qp: -1
2025-11-27T18:20:29.451763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471768685907249:2140] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:29.451449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471771230767879:2049] [node 2] ICS09 handshake done sender: [3:7577471771230767861:2136] self: [3:7577471771230767865:0] peer: [2:7577471768685907245:0] socket: 542 qp: -1
2025-11-27T18:20:29.451476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471771230767879:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:29.451717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471768699741803:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:29.452233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471768020718367:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.451931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772118961225:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.452290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471768378660956:2128] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:29.451738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:29.451959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471772118961225:2130] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.451526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:29.453864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471768685907240:2132] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.454248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772118961232:2133] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.452323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471768020718436:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.454270Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471772118961232:2133] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.455297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471768378660857:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.454378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:29.455341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:29.454408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471768685907257:2144] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.454429Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471768685907257:2144] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.454984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471768685907240:2132] [node 5] ICP22 created new session: [2:7577471768685907276:2132]
2025-11-27T18:20:29.455332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471772118961225:2130] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:29.455462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471768378660857:2048] [node 1] ICP22 created new session: [5:7577471768378660964:2048]
2025-11-27T18:20:29.455379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471768685907276:2132] [node 5] ICS09 handshake done sender: [2:7577471768685907249:2140] self: [2:7577471768685907244:0] peer: [5:7577471768378660961:0] socket: 513 qp: -1
2025-11-27T18:20:29.455416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471768685907276:2132] [node 5] ICS10 traffic start
2025-11-27T18:20:29.455574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471768378660964:2048] [node 1] ICS09 handshake done sender: [5:7577471768378660956:2128] self: [5:7577471768378660959:0] peer: [1:7577471768699741781:0] socket: 563 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.455554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471768699741776:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:29.455444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:29.456455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471772118961217:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.456079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471768378660964:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:29.456789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:29.458588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772118961251:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18990
2025-11-27T18:20:29.458595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:29.458650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471772118961217:2108] [node 5] ICP22 created new session: [4:7577471772118961253:2108]
2025-11-27T18:20:29.458851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471768685907257:2144] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:29.459174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471772118961253:2108] [node 5] ICS09 handshake done sender: [4:7577471772118961225:2130] self: [4:7577471772118961223:0] peer: [5:7577471768378660962:0] socket: 564 qp: -1
2025-11-27T18:20:29.459469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471772118961232:2133] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:29.459495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471768699741767:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.459494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471772118961253:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:29.459533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:29.459533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471768020718464:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.459555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.459683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471768685907241:2133] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.459695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471768699741767:2107] [node 4] ICP22 created new session: [1:7577471768699741806:2107]
2025-11-27T18:20:29.459709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471772118961241:2138] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:29.459796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471768020718368:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:29.459726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:29.461168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471772118961136:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.460297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471768699741806:2107] [node 4] ICS09 handshake done sender: [1:7577471768699741776:2113] self: [1:7577471768699741769:0] peer: [4:7577471772118961249:0] socket: 518 qp: -1
2025-11-27T18:20:29.460366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471768685907150:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:29.461526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471772118961218:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.460587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471768020718468:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53058
2025-11-27T18:20:29.460995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471768020718469:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53058
2025-11-27T18:20:29.461529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:29.460698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471768685907241:2133] [node 6] ICP22 created new session: [2:7577471768685907279:2133]
2025-11-27T18:20:29.460332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471768699741806:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:29.461575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:29.460365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:29.462218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471771230767858:2134] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:29.461751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471772118961136:2048] [node 1] ICP22 created new session: [4:7577471772118961255:2048]
2025-11-27T18:20:29.460762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471768685907279:2133] [node 6] ICS09 handshake done sender: [2:7577471768685907257:2144] self: [2:7577471768685907256:0] peer: [6:7577471768020718466:0] socket: 566 qp: -1
2025-11-27T18:20:29.462266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471768020718465:2128] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:29.461801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471772118961218:2117] [node 6] ICP22 created new session: [4:7577471772118961256:2117]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.462249Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471771230767858:2134] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:29.461013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471768685907279:2133] [node 6] ICS10 traffic start
2025-11-27T18:20:29.462055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471772118961256:2117] [node 6] ICS09 handshake done sender: [4:7577471772118961232:2133] self: [4:7577471772118961231:0] peer: [6:7577471768020718467:0] socket: 570 qp: -1
2025-11-27T18:20:29.463593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471772118961256:2117] [node 6] ICS10 traffic start
2025-11-27T18:20:29.465565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471768699741703:2048] [node 2] ICP18 handshake (actor [1:7577471768699741786:2119]) is too old
2025-11-27T18:20:29.465071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471771230767858:2134] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:29.464946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471768020718463:2126] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:29.463619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:29.462010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:29.464781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471772118961255:2048] [node 1] ICS09 handshake done sender: [4:7577471772118961241:2138] self: [4:7577471772118961249:0] peer: [1:7577471768699741769:0] socket: 562 qp: -1
2025-11-27T18:20:29.465619Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471768020718367:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19153) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471768020718463:2126] held: no
2025-11-27T18:20:29.464968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471772118961255:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:29.464997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:29.466449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471768020718436:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.466171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471768378660963:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22139
2025-11-27T18:20:29.466544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:29.466555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471768378660858:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19153) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471768378660954:2127] held: no
2025-11-27T18:20:29.467404Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471768378660857:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:29.466388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471771230767843:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:29.466585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471768020718436:2106] [node 4] ICP22 created new session: [6:7577471768020718472:2106]
2025-11-27T18:20:29.468158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:29.466840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471768020718472:2106] [node 4] ICS09 handshake done sender: [6:7577471768020718465:2128] self: [6:7577471768020718467:0] peer: [4:7577471772118961231:0] socket: 572 qp: -1
2025-11-27T18:20:29.468203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471771230767843:2115] [node 6] ICP22 created new session: [3:7577471771230767883:2115]
2025-11-27T18:20:29.466868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471768020718472:2106] [node 4] ICS10 traffic start
2025-11-27T18:20:29.468445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471771230767883:2115] [node 6] ICS09 handshake done sender: [3:7577471771230767858:2134] self: [3:7577471771230767857:0] peer: [6:7577471768020718470:0] socket: 569 qp: -1
2025-11-27T18:20:29.467334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:29.468593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471771230767883:2115] [node 6] ICS10 traffic start
2025-11-27T18:20:29.468473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471768378660930:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11850) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471768378660958:2129] held: no
2025-11-27T18:20:29.468645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:29.468551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471768020718367:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.468579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:29.468702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471768020718367:2049] [node 2] ICP22 created new session: [6:7577471768020718474:2049]
2025-11-27T18:20:29.468933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471768020718474:2049] [node 2] ICS09 handshake done sender: [6:7577471768020718463:2126] self: [6:7577471768020718466:0] peer: [2:7577471768685907256:0] socket: 567 qp: -1
2025-11-27T18:20:29.468968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471768020718474:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.469049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:29.469450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471768378660966:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22139
2025-11-27T18:20:29.469698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471772118961136:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:29.469734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471768378660954:2127] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:29.471206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471768378660858:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.471284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471768378660958:2129] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:29.471323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:29.471455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471768378660858:2049] [node 2] ICP22 created new session: [5:7577471768378660967:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:29.472521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471768378660930:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.472544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471768378660967:2049] [node 2] ICS09 handshake done sender: [5:7577471768378660954:2127] self: [5:7577471768378660961:0] peer: [2:7577471768685907244:0] socket: 529 qp: -1
2025-11-27T18:20:29.472576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471768020718368:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4623) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471768020718464:2127] held: no
2025-11-27T18:20:29.472566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:29.472575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471768378660967:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:29.472619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471768020718471:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53058
2025-11-27T18:20:29.472604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:29.472613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471768378660930:2108] [node 4] ICP22 created new session: [5:7577471768378660968:2108]
2025-11-27T18:20:29.472758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471768378660968:2108] [node 4] ICS09 handshake done sender: [5:7577471768378660958:2129] self: [5:7577471768378660962:0] peer: [4:7577471772118961223:0] socket: 565 qp: -1
2025-11-27T18:20:29.472932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471768378660968:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:29.472958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:29.473816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471768020718464:2127] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:29.473937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471768020718368:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:29.473974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:29.474012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471768020718368:2050] [node 3] ICP22 created new session: [6:7577471768020718476:2050]
2025-11-27T18:20:29.474083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471768020718476:2050] [node 3] ICS09 handshake done sender: [6:7577471768020718464:2127] self: [6:7577471768020718470:0] peer: [3:7577471771230767857:0] socket: 571 qp: -1
2025-11-27T18:20:29.474141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471768020718476:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:29.474170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:29.535196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:29.535528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471768685907270:2048] [node 1] ICS01 socket: 510 reason# EndOfStream
2025-11-27T18:20:29.535618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471768685907149:2048] [node 1] ICP30 unregister session Session# [2:7577471768685907270:2048] VirtualId# [2:7577471768685907258:0]
2025-11-27T18:20:29.535652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471768685907270:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.535738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:29.535802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:29.536362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:29.536583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471771230767875:2048] [node 1] ICS07 socket disconnect 509 reason# EndOfStream
2025-11-27T18:20:29.536633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471771230767875:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.536712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:29.536799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471771230767875:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:29.537554Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471771230767761:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:29.537677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471771230767875:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:29.537789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471771230767761:2048] [node 1] ICP30 unregister session Session# [3:7577471771230767875:2048] VirtualId# [3:7577471771230767862:0]
2025-11-27T18:20:29.537895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:29.537919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471771230767761:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.537929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.538606Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:29.538791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471772118961255:2048] [node 1] ICS07 socket disconnect 562 reason# EndOfStream
2025-11-27T18:20:29.538820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471772118961255:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.538889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:29.538960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471772118961255:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:29.539361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:29.539483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471768378660964:2048] [node 1] ICS07 socket disconnect 563 reason# EndOfStream
2025-11-27T18:20:29.539514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471768378660964:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.539574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:29.539570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471772118961136:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:29.539620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471768378660964:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:29.539623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471772118961255:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:29.539741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471772118961136:2048] [node 1] ICP30 unregister session Session# [4:7577471772118961255:2048] VirtualId# [4:7577471772118961249:0]
2025-11-27T18:20:29.539878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:29.539940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471772118961136:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.539960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.540352Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471768378660857:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:29.540322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:29.540386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471768378660964:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:29.540409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471768378660857:2048] [node 1] ICP30 unregister session Session# [5:7577471768378660964:2048] VirtualId# [5:7577471768378660959:0]
2025-11-27T18:20:29.540482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471768020718457:2048] [node 1] ICS07 socket disconnect 508 reason# EndOfStream
2025-11-27T18:20:29.540483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:29.540515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471768020718457:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.540529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471768378660857:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.540539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.540581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:29.540622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471768020718457:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:29.541311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471768020718366:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:29.541383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471768020718457:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:29.541421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471768020718366:2048] [node 1] ICP30 unregister session Session# [6:7577471768020718457:2048] VirtualId# [6:7577471768020718453:0]
2025-11-27T18:20:29.541459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:29.541480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471768020718366:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.541489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.547843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471771230767761:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:29.549824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471768378660857:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:29.550017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471772118961136:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:29.550881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471768020718366:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:29.561716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:29.561943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471771230767879:2049] [node 2] ICS07 socket disconnect 542 reason# EndOfStream
2025-11-27T18:20:29.561982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471771230767879:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.562078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:29.562139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471771230767879:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:29.562926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471771230767762:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19153) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:29.563012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471771230767879:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:29.563056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471771230767762:2049] [node 2] ICP30 unregister session Session# [3:7577471771230767879:2049] VirtualId# [3:7577471771230767865:0]
2025-11-27T18:20:29.563155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:29.563211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471771230767762:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19153) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.563233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19153) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.564223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:29.564400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471772118961239:2049] [node 2] ICS01 socket: 541 reason# EndOfStream
2025-11-27T18:20:29.564440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471772118961137:2049] [node 2] ICP30 unregister session Session# [4:7577471772118961239:2049] VirtualId# [4:7577471772118961234:0]
2025-11-27T18:20:29.564480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471772118961239:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.564546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:29.564597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:29.565092Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:29.565250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471768378660967:2049] [node 2] ICS01 socket: 529 reason# EndOfStream
2025-11-27T18:20:29.565311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471768378660858:2049] [node 2] ICP30 unregister session Session# [5:7577471768378660967:2049] VirtualId# [5:7577471768378660961:0]
2025-11-27T18:20:29.565342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471768378660967:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.565418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:29.565466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:29.565887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:29.566047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471768020718474:2049] [node 2] ICS01 socket: 567 reason# EndOfStream
2025-11-27T18:20:29.566079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471768020718367:2049] [node 2] ICP30 unregister session Session# [6:7577471768020718474:2049] VirtualId# [6:7577471768020718466:0]
2025-11-27T18:20:29.566102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471768020718474:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.566181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:29.566230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:29.573419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471771230767762:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:29.630265Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:29.630422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471772118961242:2050] [node 3] ICS07 socket disconnect 535 reason# EndOfStream
2025-11-27T18:20:29.630451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471772118961242:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.630509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:29.630542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471772118961242:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:29.631225Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471772118961138:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4623) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:29.631270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471772118961242:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:29.631219Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:29.631340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471772118961138:2050] [node 3] ICP30 unregister session Session# [4:7577471772118961242:2050] VirtualId# [4:7577471772118961235:0]
2025-11-27T18:20:29.631437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:29.631424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471768378660955:2050] [node 3] ICS01 socket: 530 reason# EndOfStream
2025-11-27T18:20:29.631460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471768378660859:2050] [node 3] ICP30 unregister session Session# [5:7577471768378660955:2050] VirtualId# [5:7577471768378660947:0]
2025-11-27T18:20:29.631497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471772118961138:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4623) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.631491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471768378660955:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.631522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4623) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.631570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:29.631623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:29.632280Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:29.632475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471768020718476:2050] [node 3] ICS01 socket: 571 reason# EndOfStream
2025-11-27T18:20:29.632528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471768020718368:2050] [node 3] ICP30 unregister session Session# [6:7577471768020718476:2050] VirtualId# [6:7577471768020718470:0]
2025-11-27T18:20:29.632565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471768020718476:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.632627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:29.632663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:29.676248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:29.676418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471768378660968:2108] [node 4] ICS01 socket: 565 reason# EndOfStream
2025-11-27T18:20:29.676451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471768378660930:2108] [node 4] ICP30 unregister session Session# [5:7577471768378660968:2108] VirtualId# [5:7577471768378660962:0]
2025-11-27T18:20:29.676479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471768378660968:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.676542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:29.676577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:29.677124Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:29.677325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471768020718472:2106] [node 4] ICS07 socket disconnect 572 reason# EndOfStream
2025-11-27T18:20:29.677357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471768020718472:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.677443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:29.677480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471768020718472:2106] [node 4] ICS15 start handshake
2025-11-27T18:20:29.678086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471768020718436:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11850) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:29.678125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471768020718472:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:29.678184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471768020718436:2106] [node 4] ICP30 unregister session Session# [6:7577471768020718472:2106] VirtualId# [6:7577471768020718467:0]
2025-11-27T18:20:29.678289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:29.678315Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471768020718436:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11850) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.678343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11850) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:29.688121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471768020718436:2106] [node 4] ICP33 wake up from error state
2025-11-27T18:20:29.757467Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:29.757659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471768020718458:2107] [node 5] ICS01 socket: 515 reason# EndOfStream
2025-11-27T18:20:29.757693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471768020718434:2107] [node 5] ICP30 unregister session Session# [6:7577471768020718458:2107] VirtualId# [6:7577471768020718454:0]
2025-11-27T18:20:29.757716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471768020718458:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:29.757783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:29.757857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:29.849767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471771277346447:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:29.849829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471771277346448:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:29.867327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471770280835713:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:29.867515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471770280835714:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:29.885242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471768442554735:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:29.885543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471768442554734:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:29.903724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471769291186584:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:29.903772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471769291186586:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:29.903790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471769291186585:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:29.923355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471772087649211:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:29.923355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471772087649209:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:29.923359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471772087649210:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:29.943100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471769650015562:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:29.943379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471769650015564:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:29.943438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471769650015563:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:30.053014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471775572313825:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:30.053775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471772737522095:2110] [node 5] ICP01 ready to work
2025-11-27T18:20:30.053780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471775572313826:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:30.053525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471774575803073:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:30.054176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471772737522096:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:30.054620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471774575803074:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:30.054699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471773586153943:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:30.054729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471772737522097:2111] [node 6] ICP01 ready to work
2025-11-27T18:20:30.054658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471774575803072:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:30.053856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471776382616586:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:30.054732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471775572313827:2117] [node 6] ICP01 ready to work
2025-11-27T18:20:30.053882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471776382616587:2117] [node 6] ICP01 ready to work
2025-11-27T18:20:30.054801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471773944982921:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:30.055065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471773586153944:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:30.055197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471773944982922:2106] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.071553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471773944982933:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5667
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.073969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471774575803086:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63315
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:30.075399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471775572313841:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.076169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471774575803088:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63315
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:30.077934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471776382616602:2134] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4638
2025-11-27T18:20:30.077585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471773586153957:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20593
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.079642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471773586153959:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20593
2025-11-27T18:20:30.078017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471776382616601:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4638
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:30.083695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471774575803088:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.084002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471773944982937:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5667
2025-11-27T18:20:30.084044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471773586153957:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.083992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471774575803089:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63315
2025-11-27T18:20:30.081371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471776382616601:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.078070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471775572313842:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.084353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471772087649209:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.084390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471774575803091:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63315
2025-11-27T18:20:30.084382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471776382616603:2135] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4638
2025-11-27T18:20:30.084484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471773944982933:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.084537Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471773586153959:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.084375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471772737522111:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36973
2025-11-27T18:20:30.084630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471773944982938:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5667
2025-11-27T18:20:30.084713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471769650015562:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.081411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471775572313842:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.085070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471774575803086:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.084726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471769291186586:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.084580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471772737522112:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36973
2025-11-27T18:20:30.084729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471776382616602:2134] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.085453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471773944982937:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.084016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471775572313843:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
2025-11-27T18:20:30.085193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471774575803091:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.085114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471776382616603:2135] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.085247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471772737522111:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.084376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471775572313844:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
2025-11-27T18:20:30.085543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471773944982938:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.086397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471773586153952:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.085458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471772737522112:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.085576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471776382616586:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.085404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471774575803089:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.085699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471772737522105:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.086800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471769650015564:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.084668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471775572313834:2130] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.086447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471773586153952:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.085823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471770280835713:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.086818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471769650015563:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.084705Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471775572313834:2130] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.085726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471772737522105:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.089808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471774575803081:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.090620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471773586153952:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:30.084901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471775572313841:2134] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.089842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471774575803081:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.086532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471768442554735:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.085290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471775572313844:2137] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.091780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471774575803080:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.093122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471776382616607:2136] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4638
2025-11-27T18:20:30.094588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471773586153943:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.085449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471775572313843:2136] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.089449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471772737522103:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.091816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471774575803080:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.094662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:30.095449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471776382616608:2137] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4638
2025-11-27T18:20:30.085664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471775572313833:2129] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.089477Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471772737522103:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.092613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471774575803081:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:30.094703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471773586153943:2108] [node 5] ICP22 created new session: [4:7577471773586153964:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.097653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471772087649209:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471776382616601:2133] held: no
2025-11-27T18:20:30.095161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471774575803096:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63315
2025-11-27T18:20:30.090612Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471772737522105:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:30.085690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471775572313833:2129] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.101175Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471776382616601:2133] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:30.095618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471773586153963:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20593
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.094489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471772737522103:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:30.103629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471773944982942:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5667
2025-11-27T18:20:30.088774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471775572313835:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.096202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471774575803086:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:30.095654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471773586153964:2108] [node 5] ICS09 handshake done sender: [4:7577471773586153952:2112] self: [4:7577471773586153949:0] peer: [5:7577471776382616605:0] socket: 525 qp: -1
2025-11-27T18:20:30.104230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471769650015562:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471773944982933:2115] held: no
2025-11-27T18:20:30.094527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471772737522096:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.096157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471773586153964:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:30.105273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471773944982943:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5667
2025-11-27T18:20:30.103611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471776382616586:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29008) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471776382616602:2134] held: no
2025-11-27T18:20:30.094592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:30.096221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:30.088800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471775572313835:2131] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.095370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471772737522096:2107] [node 4] ICP22 created new session: [3:7577471772737522118:2107]
2025-11-27T18:20:30.106385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471776382616602:2134] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:30.096516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471769291186586:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28048) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471773586153959:2116] held: no
2025-11-27T18:20:30.096542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471774575803080:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.096121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471772737522118:2107] [node 4] ICS09 handshake done sender: [3:7577471772737522105:2114] self: [3:7577471772737522100:0] peer: [4:7577471773586153960:0] socket: 521 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.088855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471775572313834:2130] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:30.099807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471770280835714:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.096454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471772737522118:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.111050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471773586153967:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20593
2025-11-27T18:20:30.099889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:30.096455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471772737522097:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.111199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471772087649209:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.111132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471773944982937:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:30.092459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471775572313833:2129] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:30.096497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:30.104127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471770280835714:2049] [node 3] ICP22 created new session: [2:7577471774575803100:2049]
2025-11-27T18:20:30.111831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471773944982944:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5667
2025-11-27T18:20:30.111574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471773586153959:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:30.092482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471775572313835:2131] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:30.111251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:30.112344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471769291186586:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.096743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:30.104228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471770280835713:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.114371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471769650015563:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21495) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471773944982938:2117] held: no
2025-11-27T18:20:30.104270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:30.104300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471770280835713:2048] [node 1] ICP22 created new session: [2:7577471774575803101:2048]
2025-11-27T18:20:30.114094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:30.105710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471774575803101:2048] [node 1] ICS09 handshake done sender: [2:7577471774575803086:2115] self: [2:7577471774575803093:0] peer: [1:7577471775572313830:0] socket: 523 qp: -1
2025-11-27T18:20:30.111897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471772087649209:2048] [node 1] ICP22 created new session: [5:7577471776382616609:2048]
2025-11-27T18:20:30.092729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471775572313826:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.106327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471774575803100:2049] [node 3] ICS09 handshake done sender: [2:7577471774575803081:2112] self: [2:7577471774575803076:0] peer: [3:7577471772737522113:0] socket: 549 qp: -1
2025-11-27T18:20:30.114455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471769291186586:2050] [node 3] ICP22 created new session: [4:7577471773586153969:2050]
2025-11-27T18:20:30.093184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: 2025-11-27T18:20:30.106334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471774575803101:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:30.099825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471772737522097:2111] [node 6] ICP22 created new session: [3:7577471772737522121:2111]
::ffff:127.1.0.02025-11-27T18:20:30.114779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471773586153969:2050] [node 3] ICS09 handshake done sender: [4:7577471773586153959:2116] self: [4:7577471773586153960:0] peer: [3:7577471772737522100:0] socket: 536 qp: -1
2025-11-27T18:20:30.106665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471774575803073:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.106744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471774575803100:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:30.115054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471773586153969:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:30.094406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471771277346447:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.104160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471772737522121:2111] [node 6] ICS09 handshake done sender: [3:7577471772737522103:2113] self: [3:7577471772737522099:0] peer: [6:7577471773944982941:0] socket: 524 qp: -1
2025-11-27T18:20:30.115105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471773586153967:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.106786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:30.106981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:30.094449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471775572313827:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.104218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471772737522121:2111] [node 6] ICS10 traffic start
2025-11-27T18:20:30.107741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471774575803073:2109] [node 6] ICP22 created new session: [2:7577471774575803103:2109]
2025-11-27T18:20:30.115274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:30.094466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471775572313826:2108] [node 5] ICP22 created new session: [1:7577471775572313852:2108]
2025-11-27T18:20:30.104873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:30.107840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:30.094588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:30.109056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471774575803103:2109] [node 6] ICS09 handshake done sender: [2:7577471774575803080:2111] self: [2:7577471774575803077:0] peer: [6:7577471773944982940:0] socket: 540 qp: -1
2025-11-27T18:20:30.095250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471771277346447:2048] [node 2] ICP22 created new session: [1:7577471775572313854:2048]
2025-11-27T18:20:30.110015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471774575803103:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.107807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471772737522117:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36973
2025-11-27T18:20:30.115070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471769650015564:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.112253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471776382616609:2048] [node 1] ICS09 handshake done sender: [5:7577471776382616601:2133] self: [5:7577471776382616604:0] peer: [1:7577471775572313829:0] socket: 520 qp: -1
2025-11-27T18:20:30.110593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:30.095337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.114341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471774575803099:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63315
2025-11-27T18:20:30.110639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471768442554735:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21495) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471772737522112:2118] held: no
2025-11-27T18:20:30.095350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471775572313852:2108] [node 5] ICS09 handshake done sender: [1:7577471775572313834:2130] self: [1:7577471775572313829:0] peer: [5:7577471776382616604:0] socket: 513 qp: -1
2025-11-27T18:20:30.115804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471774575803099:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.117684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471769291186584:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.114396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471776382616609:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:30.095399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471775572313852:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:30.118396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471773586153972:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20593
2025-11-27T18:20:30.114442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:30.115282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:30.095618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: 2025-11-27T18:20:30.115065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471776382616586:2108] [node 4] ICP19 incoming handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:30.095662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471775572313854:2048] [node 2] ICS09 handshake done sender: [1:7577471775572313835:2131] self: [1:7577471775572313830:0] peer: [2:7577471774575803093:0] socket: 514 qp: -1
2025-11-27T18:20:30.118241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471772737522112:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:30.115740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471773944982938:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.115749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:30.095710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471775572313827:2117] [node 6] ICP22 created new session: [1:7577471775572313855:2117]
2025-11-27T18:20:30.116148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471769650015564:2050] [node 3] ICP22 created new session: [6:7577471773944982947:2050]
2025-11-27T18:20:30.119437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471772737522124:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36973
2025-11-27T18:20:30.118751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471770280835713:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:30.115812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471776382616586:2108] [node 4] ICP22 created new session: [5:7577471776382616615:2108]
2025-11-27T18:20:30.120187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471774575803084:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.117618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471773944982947:2050] [node 3] ICS09 handshake done sender: [6:7577471773944982937:2116] self: [6:7577471773944982941:0] peer: [3:7577471772737522099:0] socket: 533 qp: -1
2025-11-27T18:20:30.117526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471776382616615:2108] [node 4] ICS09 handshake done sender: [5:7577471776382616602:2134] self: [5:7577471776382616605:0] peer: [4:7577471773586153949:0] socket: 535 qp: -1
2025-11-27T18:20:30.095839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471775572313854:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.120062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471768442554735:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.117574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471776382616615:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:30.117670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471773944982947:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:30.120224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471774575803084:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.120110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:30.095879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:30.117632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:30.096747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471775572313855:2117] [node 6] ICS09 handshake done sender: [1:7577471775572313833:2129] self: [1:7577471775572313828:0] peer: [6:7577471773944982939:0] socket: 510 qp: -1
2025-11-27T18:20:30.120147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471768442554735:2049] [node 2] ICP22 created new session: [3:7577471772737522127:2049]
2025-11-27T18:20:30.117636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471776382616611:2139] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4638
2025-11-27T18:20:30.097686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471775572313855:2117] [node 6] ICS10 traffic start
2025-11-27T18:20:30.120895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471772737522127:2049] [node 2] ICS09 handshake done sender: [3:7577471772737522112:2118] self: [3:7577471772737522113:0] peer: [2:7577471774575803076:0] socket: 551 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.117886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:30.117968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471773944982933:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:30.097724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:30.118104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471776382616611:2139] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.122444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471772737522127:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:30.111974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471775572313860:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
2025-11-27T18:20:30.117694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471775572313860:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.123503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471772737522124:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:30.117969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471769650015563:2049] [node 2] ICP19 incoming handshake succeeded
::ffff:127.1.0.02025-11-27T18:20:30.118183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471776382616610:2138] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4638
2025-11-27T18:20:30.117849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471775572313837:2133] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.118196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:30.117872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471775572313837:2133] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.123516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:30.118515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471772087649210:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.118471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471769650015563:2049] [node 2] ICP22 created new session: [6:7577471773944982950:2049]
2025-11-27T18:20:30.119832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471775572313837:2133] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.124205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471774575803084:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:30.122444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471776382616610:2138] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.124098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471772737522125:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36973
2025-11-27T18:20:30.120231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471775572313825:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.124107Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471769291186584:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471773586153967:2118] held: no
2025-11-27T18:20:30.124112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471772087649211:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.119680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471773944982950:2049] [node 2] ICS09 handshake done sender: [6:7577471773944982938:2117] self: [6:7577471773944982940:0] peer: [2:7577471774575803077:0] socket: 541 qp: -1
2025-11-27T18:20:30.120807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:30.119884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471773944982950:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:30.124315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471773586153973:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20593
2025-11-27T18:20:30.121003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471775572313825:2107] [node 4] ICP22 created new session: [1:7577471775572313863:2107]
2025-11-27T18:20:30.119889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471773944982945:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5667
2025-11-27T18:20:30.124417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471774575803074:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.123873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471775572313863:2107] [node 4] ICS09 handshake done sender: [1:7577471775572313837:2133] self: [1:7577471775572313832:0] peer: [4:7577471773586153971:0] socket: 543 qp: -1
2025-11-27T18:20:30.119910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471769650015562:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.124640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:30.123932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471775572313863:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:30.124322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471768442554734:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.119920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:30.125180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471773586153953:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.125209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471774575803074:2108] [node 5] ICP22 created new session: [2:7577471774575803110:2108]
2025-11-27T18:20:30.124126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:30.119956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:30.125213Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471773586153953:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.124337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471772737522126:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36973
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.125242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471775572313836:2132] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.120287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471769650015562:2048] [node 1] ICP22 created new session: [6:7577471773944982953:2048]
2025-11-27T18:20:30.125281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471774575803110:2108] [node 5] ICS09 handshake done sender: [2:7577471774575803084:2114] self: [2:7577471774575803082:0] peer: [5:7577471776382616617:0] socket: 519 qp: -1
2025-11-27T18:20:30.125435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471773586153943:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(24196845ns) {HandleHandshakeStatus:329(24193330ns) {DropIncomingHandshake:442(20627ns) DropOutgoingHandshake:457(351ns) SwitchToState:179(258ns) ProcessPendingSessionEvents:495(17776709ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17670273ns max# 2646783ns}}}
2025-11-27T18:20:30.125179Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471772737522096:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(23626640ns) {HandleHandshakeStatus:329(23623504ns) {DropIncomingHandshake:442(20867ns) DropOutgoingHandshake:457(323ns) SwitchToState:179(322ns) ProcessPendingSessionEvents:495(12741080ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12063679ns max# 2083472ns}}}
2025-11-27T18:20:30.125766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471772087649210:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21495) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471776382616611:2139] held: no
2025-11-27T18:20:30.125289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471775572313836:2132] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.125713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471774575803110:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:30.125786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471772737522125:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.127082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471775572313826:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(24860728ns) {HandleHandshakeStatus:329(24857911ns) {DropIncomingHandshake:442(13124ns) DropOutgoingHandshake:457(363ns) SwitchToState:179(1277ns) ProcessPendingSessionEvents:495(20781569ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20678485ns max# 4240640ns}}}
2025-11-27T18:20:30.128593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471773586153974:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20593
2025-11-27T18:20:30.128464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:30.120867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471773944982945:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.128306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471775572313827:2117] [node 6] ICP35 event processing took too much time PendingConnection:279(24033405ns) {HandleHandshakeStatus:329(24031215ns) {DropIncomingHandshake:442(14915ns) DropOutgoingHandshake:457(268ns) SwitchToState:179(292ns) ProcessPendingSessionEvents:495(13286877ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13174129ns max# 1781177ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.128191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471772737522107:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.120900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471773944982953:2048] [node 1] ICS09 handshake done sender: [6:7577471773944982933:2115] self: [6:7577471773944982939:0] peer: [1:7577471775572313828:0] socket: 511 qp: -1
2025-11-27T18:20:30.130544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471775572313836:2132] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:30.128606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471776382616597:2132] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.130440Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471770280835714:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(22909858ns) {HandleHandshakeStatus:329(22908309ns) {DropIncomingHandshake:442(20793ns) DropOutgoingHandshake:457(414ns) SwitchToState:179(302ns) ProcessPendingSessionEvents:495(8586385ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8510135ns max# 1758796ns}}}
2025-11-27T18:20:30.130940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471773586153953:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:30.120974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471773944982953:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:30.128641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471776382616597:2132] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.128218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471772737522107:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.131161Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471771277346447:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(24527103ns) {HandleHandshakeStatus:329(24069866ns) {DropIncomingHandshake:442(13699ns) DropOutgoingHandshake:457(1336ns) SwitchToState:179(484ns) ProcessPendingSessionEvents:495(14339107ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14237204ns max# 1834177ns}}}
2025-11-27T18:20:30.130974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471773586153972:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.130820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471776382616619:2140] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4638
2025-11-27T18:20:30.131980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471773586153974:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.123470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:30.130423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471772737522097:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(21724827ns) {HandleHandshakeStatus:329(21721948ns) {DropIncomingHandshake:442(468ns) DropOutgoingHandshake:457(349ns) SwitchToState:179(332ns) ProcessPendingSessionEvents:495(10378246ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10285987ns max# 1782167ns}}}
2025-11-27T18:20:30.132463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471773586153944:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.131978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471771277346448:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.133110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471774575803083:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.132633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:30.123484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471773944982948:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5667
2025-11-27T18:20:30.133232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471776382616597:2132] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:30.131022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471772737522107:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:30.132138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:30.124319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471773944982922:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.132796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471773586153967:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:30.133149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471774575803083:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.132189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471771277346448:2049] [node 3] ICP22 created new session: [1:7577471775572313866:2049]
2025-11-27T18:20:30.124629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471773944982948:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.133032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471772737522095:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.132801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471769291186585:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.132821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471773586153944:2109] [node 6] ICP22 created new session: [4:7577471773586153976:2109]
2025-11-27T18:20:30.133119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:30.132663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471775572313866:2049] [node 3] ICS09 handshake done sender: [1:7577471775572313836:2132] self: [1:7577471775572313831:0] peer: [3:7577471772737522129:0] socket: 517 qp: -1
2025-11-27T18:20:30.128606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471773944982921:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.133312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471769291186584:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.133617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471772737522095:2110] [node 5] ICP22 created new session: [3:7577471772737522133:2110]
2025-11-27T18:20:30.132705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471775572313866:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:30.135172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471776382616620:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4638
2025-11-27T18:20:30.133456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471773586153976:2109] [node 6] ICS09 handshake done sender: [4:7577471773586153953:2113] self: [4:7577471773586153946:0] peer: [6:7577471773944982955:0] socket: 526 qp: -1
2025-11-27T18:20:30.135238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471772737522133:2110] [node 5] ICS09 handshake done sender: [3:7577471772737522107:2116] self: [3:7577471772737522104:0] peer: [5:7577471776382616618:0] socket: 568 qp: -1
2025-11-27T18:20:30.135973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471774575803083:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:30.135329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471776382616611:2139] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:30.135266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471773944982957:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5667
2025-11-27T18:20:30.133044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:30.133486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471773586153976:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:30.135271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471772737522132:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36973
2025-11-27T18:20:30.136741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471774575803072:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.136719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471776382616587:2117] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.136061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471773944982945:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:30.133604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: 2025-11-27T18:20:30.136777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
::ffff:127.1.0.0
2025-11-27T18:20:30.135434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471772737522133:2110] [node 5] ICS10 traffic start
2025-11-27T18:20:30.138385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471773944982922:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.133860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:30.138408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471774575803072:2107] [node 4] ICP22 created new session: [2:7577471774575803113:2107]
2025-11-27T18:20:30.135348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471769291186584:2048] [node 1] ICP22 created new session: [4:7577471773586153978:2048]
2025-11-27T18:20:30.138426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:30.138435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:30.135678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:30.138777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471774575803113:2107] [node 4] ICS09 handshake done sender: [2:7577471774575803083:2113] self: [2:7577471774575803079:0] peer: [4:7577471773586153977:0] socket: 571 qp: -1
2025-11-27T18:20:30.135604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471773586153978:2048] [node 1] ICS09 handshake done sender: [4:7577471773586153967:2118] self: [4:7577471773586153971:0] peer: [1:7577471775572313832:0] socket: 512 qp: -1
2025-11-27T18:20:30.138470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471773944982922:2106] [node 4] ICP22 created new session: [6:7577471773944982959:2106]
2025-11-27T18:20:30.138492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471772737522124:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:30.138583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471772087649210:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.135640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471773586153978:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:30.139097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471774575803113:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:30.139055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471773944982958:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5667
2025-11-27T18:20:30.141054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471768442554734:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471772737522124:2120] held: no
2025-11-27T18:20:30.139144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.138626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:30.135680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:30.138799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471772087649210:2049] [node 2] ICP22 created new session: [5:7577471776382616623:2049]
2025-11-27T18:20:30.141018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471773944982959:2106] [node 4] ICS09 handshake done sender: [6:7577471773944982945:2121] self: [6:7577471773944982955:0] peer: [4:7577471773586153946:0] socket: 539 qp: -1
2025-11-27T18:20:30.138801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471776382616587:2117] [node 6] ICP22 created new session: [5:7577471776382616622:2117]
2025-11-27T18:20:30.142603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471769291186585:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21495) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471773586153972:2119] held: no
2025-11-27T18:20:30.139130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471776382616622:2117] [node 6] ICS09 handshake done sender: [5:7577471776382616597:2132] self: [5:7577471776382616595:0] peer: [6:7577471773944982956:0] socket: 570 qp: -1
2025-11-27T18:20:30.141586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471773944982959:2106] [node 4] ICS10 traffic start
2025-11-27T18:20:30.139263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471776382616623:2049] [node 2] ICS09 handshake done sender: [5:7577471776382616611:2139] self: [5:7577471776382616617:0] peer: [2:7577471774575803082:0] socket: 544 qp: -1
2025-11-27T18:20:30.142214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:30.145027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471773586153981:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20593
2025-11-27T18:20:30.139831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471776382616622:2117] [node 6] ICS10 traffic start
2025-11-27T18:20:30.140555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:30.145420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471773944982948:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:30.140594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471776382616623:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:30.146649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471775572313825:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17510759ns) {HandleHandshakeStatus:329(17507184ns) {DropIncomingHandshake:442(10478ns) DropOutgoingHandshake:457(200ns) SwitchToState:179(189ns) ProcessPendingSessionEvents:495(13330352ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12958732ns max# 1385568ns}}}
2025-11-27T18:20:30.146058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471768442554734:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.140638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:30.146561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471773586153944:2109] [node 6] ICP18 handshake (actor [4:7577471773586153974:2121]) is too old
2025-11-27T18:20:30.146121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:30.140732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471772087649211:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28048) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471776382616610:2138] held: no
2025-11-27T18:20:30.146506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471768442554734:2048] [node 1] ICP22 created new session: [3:7577471772737522135:2048]
2025-11-27T18:20:30.146553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471772737522135:2048] [node 1] ICS09 handshake done sender: [3:7577471772737522124:2120] self: [3:7577471772737522129:0] peer: [1:7577471775572313831:0] socket: 538 qp: -1
2025-11-27T18:20:30.146924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471776382616610:2138] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:30.148465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471772087649211:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.148403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471773586153972:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:30.146881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471772737522135:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:30.148702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471773944982921:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.148509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:30.148554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471772087649211:2050] [node 3] ICP22 created new session: [5:7577471776382616626:2050]
2025-11-27T18:20:30.149135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471769291186585:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.149442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471776382616626:2050] [node 3] ICS09 handshake done sender: [5:7577471776382616610:2138] self: [5:7577471776382616618:0] peer: [3:7577471772737522104:0] socket: 516 qp: -1
2025-11-27T18:20:30.149456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:30.147401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:30.149894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471776382616626:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:30.149931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:30.149459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:30.150043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471773944982921:2107] [node 5] ICP22 created new session: [6:7577471773944982961:2107]
2025-11-27T18:20:30.150110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471773944982961:2107] [node 5] ICS09 handshake done sender: [6:7577471773944982948:2122] self: [6:7577471773944982956:0] peer: [5:7577471776382616595:0] socket: 574 qp: -1
2025-11-27T18:20:30.150142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471773944982961:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:30.150497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:30.150919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471774575803074:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(16659080ns) {HandleHandshakeStatus:329(16656168ns) {DropIncomingHandshake:442(11849ns) DropOutgoingHandshake:457(271ns) SwitchToState:179(219ns) ProcessPendingSessionEvents:495(10618653ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10514493ns max# 1163123ns}}}
2025-11-27T18:20:30.149503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471769291186585:2049] [node 2] ICP22 created new session: [4:7577471773586153982:2049]
2025-11-27T18:20:30.150648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471773586153982:2049] [node 2] ICS09 handshake done sender: [4:7577471773586153972:2119] self: [4:7577471773586153977:0] peer: [2:7577471774575803079:0] socket: 572 qp: -1
2025-11-27T18:20:30.151323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471771277346448:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(16576873ns) {HandleHandshakeStatus:329(15815775ns) {DropIncomingHandshake:442(11739ns) DropOutgoingHandshake:457(366ns) SwitchToState:179(380ns) ProcessPendingSessionEvents:495(14614883ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14535165ns max# 1769989ns}}}
2025-11-27T18:20:30.150687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471773586153982:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:30.150725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:30.152169Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471773944982921:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:30.213691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:30.213881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471774575803101:2048] [node 1] ICS01 socket: 523 reason# EndOfStream
2025-11-27T18:20:30.213920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471770280835713:2048] [node 1] ICP30 unregister session Session# [2:7577471774575803101:2048] VirtualId# [2:7577471774575803093:0]
2025-11-27T18:20:30.213947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471774575803101:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.214033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:30.214086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:30.215083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:30.215267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471772737522135:2048] [node 1] ICS01 socket: 538 reason# EndOfStream
2025-11-27T18:20:30.215301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471768442554734:2048] [node 1] ICP30 unregister session Session# [3:7577471772737522135:2048] VirtualId# [3:7577471772737522129:0]
2025-11-27T18:20:30.215334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471772737522135:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.215418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:30.215463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:30.217573Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:30.217721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471773586153978:2048] [node 1] ICS01 socket: 512 reason# EndOfStream
2025-11-27T18:20:30.217753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471769291186584:2048] [node 1] ICP30 unregister session Session# [4:7577471773586153978:2048] VirtualId# [4:7577471773586153971:0]
2025-11-27T18:20:30.217776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471773586153978:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.217836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:30.217872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:30.218619Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:30.218835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471776382616609:2048] [node 1] ICS01 socket: 520 reason# EndOfStream
2025-11-27T18:20:30.218891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471772087649209:2048] [node 1] ICP30 unregister session Session# [5:7577471776382616609:2048] VirtualId# [5:7577471776382616604:0]
2025-11-27T18:20:30.218924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471776382616609:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.219004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:30.219073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:30.219584Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:30.219720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471773944982953:2048] [node 1] ICS01 socket: 511 reason# EndOfStream
2025-11-27T18:20:30.219749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471769650015562:2048] [node 1] ICP30 unregister session Session# [6:7577471773944982953:2048] VirtualId# [6:7577471773944982939:0]
2025-11-27T18:20:30.219771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471773944982953:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.219832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:30.219877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:30.316257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:30.316476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471772737522127:2049] [node 2] ICS01 socket: 551 reason# EndOfStream
2025-11-27T18:20:30.316526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471768442554735:2049] [node 2] ICP30 unregister session Session# [3:7577471772737522127:2049] VirtualId# [3:7577471772737522113:0]
2025-11-27T18:20:30.316561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471772737522127:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.316657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:30.316715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:30.320230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:30.320401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471773586153982:2049] [node 2] ICS01 socket: 572 reason# EndOfStream
2025-11-27T18:20:30.320432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471769291186585:2049] [node 2] ICP30 unregister session Session# [4:7577471773586153982:2049] VirtualId# [4:7577471773586153977:0]
2025-11-27T18:20:30.320463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471773586153982:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.320560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:30.320613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:30.321395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:30.321615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471776382616623:2049] [node 2] ICS01 socket: 544 reason# EndOfStream
2025-11-27T18:20:30.321663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471772087649210:2049] [node 2] ICP30 unregister session Session# [5:7577471776382616623:2049] VirtualId# [5:7577471776382616617:0]
2025-11-27T18:20:30.321700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471776382616623:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.321791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:30.321857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:30.322419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:30.322642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471773944982950:2049] [node 2] ICS01 socket: 541 reason# EndOfStream
2025-11-27T18:20:30.322690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471769650015563:2049] [node 2] ICP30 unregister session Session# [6:7577471773944982950:2049] VirtualId# [6:7577471773944982940:0]
2025-11-27T18:20:30.322719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471773944982950:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.322790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:30.322848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:30.383906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:30.384070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471773586153969:2050] [node 3] ICS01 socket: 536 reason# EndOfStream
2025-11-27T18:20:30.384105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471769291186586:2050] [node 3] ICP30 unregister session Session# [4:7577471773586153969:2050] VirtualId# [4:7577471773586153960:0]
2025-11-27T18:20:30.384133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471773586153969:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.384211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:30.384248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:30.385140Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:30.385373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471776382616626:2050] [node 3] ICS01 socket: 516 reason# EndOfStream
2025-11-27T18:20:30.385417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471772087649211:2050] [node 3] ICP30 unregister session Session# [5:7577471776382616626:2050] VirtualId# [5:7577471776382616618:0]
2025-11-27T18:20:30.385460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471776382616626:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.385548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:30.385609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:30.386174Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:30.386392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471773944982947:2050] [node 3] ICS07 socket disconnect 533 reason# EndOfStream
2025-11-27T18:20:30.386430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471773944982947:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.386500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:30.386549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471773944982947:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:30.387249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471769650015564:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28048) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:30.387386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471773944982947:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:30.387435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471769650015564:2050] [node 3] ICP30 unregister session Session# [6:7577471773944982947:2050] VirtualId# [6:7577471773944982941:0]
2025-11-27T18:20:30.387533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:30.387569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471769650015564:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28048) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:30.387583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28048) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:30.397777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471769650015564:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:30.463145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:30.463389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471776382616615:2108] [node 4] ICS01 socket: 535 reason# EndOfStream
2025-11-27T18:20:30.463454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471776382616586:2108] [node 4] ICP30 unregister session Session# [5:7577471776382616615:2108] VirtualId# [5:7577471776382616605:0]
2025-11-27T18:20:30.463499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471776382616615:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.463609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:30.463673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:30.464469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:30.464695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471773944982959:2106] [node 4] ICS07 socket disconnect 539 reason# EndOfStream
2025-11-27T18:20:30.464732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471773944982959:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.464804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:30.464849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471773944982959:2106] [node 4] ICS15 start handshake
2025-11-27T18:20:30.465640Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471773944982922:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29008) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:30.465798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471773944982959:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:30.465854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471773944982922:2106] [node 4] ICP30 unregister session Session# [6:7577471773944982959:2106] VirtualId# [6:7577471773944982955:0]
2025-11-27T18:20:30.465974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:30.466013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471773944982922:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29008) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:30.466052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29008) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:30.475601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471773944982922:2106] [node 4] ICP33 wake up from error state
2025-11-27T18:20:30.558829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:30.559077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471773944982961:2107] [node 5] ICS07 socket disconnect 574 reason# EndOfStream
2025-11-27T18:20:30.559126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471773944982961:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.559205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:30.559244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471773944982961:2107] [node 5] ICS15 start handshake
2025-11-27T18:20:30.560001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471773944982921:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7698) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:30.560067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471773944982961:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:30.560098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471773944982921:2107] [node 5] ICP30 unregister session Session# [6:7577471773944982961:2107] VirtualId# [6:7577471773944982956:0]
2025-11-27T18:20:30.560203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:30.560277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471773944982921:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7698) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:30.560292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7698) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:30.669483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471772835868475:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:30.669541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471772835868476:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:30.693029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471772575368434:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:30.693107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471772575368435:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:30.716155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471774602630452:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:30.716217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471774602630453:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:30.739382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471772801009298:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:30.739441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471772801009299:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:30.739476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471772801009300:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:30.763156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471774058889435:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:30.763238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471774058889436:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:30.763289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471774058889437:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:30.786194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471775953976486:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:30.786235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471775953976487:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:30.786265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471775953976488:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:30.879459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471774602630525:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:30.881839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471775953976550:2106] [node 4] ICP01 ready to work
2025-11-27T18:20:30.881847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471772575368506:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:30.881897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471774602630526:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:30.881919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471774058889508:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:30.881929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471772835868538:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:30.882293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471772835868539:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:30.882342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471772575368507:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:30.882495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471772801009363:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:30.882600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471775953976551:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:30.882846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471774602630527:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:30.883062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471772835868540:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:30.883179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471772801009364:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:30.885343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471774058889511:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:30.885592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471772575368510:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.894497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772801009375:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28783
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.898499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471774602630536:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54260
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:30.899460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471774602630539:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54260
::ffff:127.1.0.0
2025-11-27T18:20:30.899800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471774602630540:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54260
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.899762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471772835868551:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21107
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.899869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471774602630541:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54260
2025-11-27T18:20:30.899927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471772835868552:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21107
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.899914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471772575368522:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53830
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.900033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471772575368523:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53830
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.900066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471772835868553:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21107
2025-11-27T18:20:30.900116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471774602630536:2124] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.900149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471772575368524:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53830
2025-11-27T18:20:30.900226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471772835868554:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21107
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.900635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471774602630545:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54260
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.900949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471772575368525:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53830
2025-11-27T18:20:30.901344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471772575368526:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53830
2025-11-27T18:20:30.901557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471772575368523:2125] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.901659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471772835868551:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.901790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471772835868553:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.901868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471772575368522:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.902167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471772835868558:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21107
2025-11-27T18:20:30.902369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471772835868552:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.902375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471774602630453:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.903484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471772575368524:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.904423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772801009375:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.904603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772801009379:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28783
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.904704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772801009380:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28783
2025-11-27T18:20:30.904701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471772835868554:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.904796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471772801009299:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.905007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471772575368514:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.905028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471772575368514:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.905972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471772575368514:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.906365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772801009382:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28783
2025-11-27T18:20:30.906702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471772575368506:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.906758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:30.906796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471772575368506:2107] [node 4] ICP22 created new session: [2:7577471772575368531:2107]
2025-11-27T18:20:30.906971Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471772801009375:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:30.906950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471772575368531:2107] [node 4] ICS09 handshake done sender: [2:7577471772575368514:2121] self: [2:7577471772575368513:0] peer: [4:7577471772801009381:0] socket: 518 qp: -1
2025-11-27T18:20:30.907088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471772575368531:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:30.907139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:30.907312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471772801009299:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.907487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:30.907533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471772801009299:2049] [node 2] ICP22 created new session: [4:7577471772801009383:2049]
2025-11-27T18:20:30.907680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471774602630540:2127] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.907772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471772801009383:2049] [node 2] ICS09 handshake done sender: [4:7577471772801009375:2117] self: [4:7577471772801009381:0] peer: [2:7577471772575368513:0] socket: 525 qp: -1
2025-11-27T18:20:30.907788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471774602630539:2126] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.907820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471772801009383:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:30.907859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:30.909108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471772575368512:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.909128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471772575368512:2120] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.909424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471772835868558:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.909702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471774602630545:2129] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.909817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471772575368525:2127] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.909867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471774602630452:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.910086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471772835868544:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.910113Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471772835868544:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.910908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471772835868544:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.911223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471772575368526:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.911331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471774602630550:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54260
2025-11-27T18:20:30.911361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471772835868476:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.911410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:30.911470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471772835868476:2049] [node 3] ICP22 created new session: [1:7577471772835868565:2049]
2025-11-27T18:20:30.911544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471772835868565:2049] [node 3] ICS09 handshake done sender: [1:7577471772835868544:2111] self: [1:7577471772835868543:0] peer: [3:7577471774602630549:0] socket: 548 qp: -1
2025-11-27T18:20:30.911584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471772835868565:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:30.911647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:30.911845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471774602630541:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.911909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471774602630545:2129] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:30.911957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772801009379:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.912100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471772575368434:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.912345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471772801009300:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.912349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471774602630452:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.912401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471772575368512:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:30.912398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:30.912351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471774058889523:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29205
2025-11-27T18:20:30.912435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471774602630452:2048] [node 1] ICP22 created new session: [3:7577471774602630552:2048]
2025-11-27T18:20:30.912460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471774602630552:2048] [node 1] ICS09 handshake done sender: [3:7577471774602630545:2129] self: [3:7577471774602630549:0] peer: [1:7577471772835868543:0] socket: 552 qp: -1
2025-11-27T18:20:30.912488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471774602630552:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:30.912620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:30.912734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471772575368435:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.912772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:30.912824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471772575368435:2049] [node 3] ICP22 created new session: [2:7577471772575368537:2049]
2025-11-27T18:20:30.912866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471772575368537:2049] [node 3] ICS09 handshake done sender: [2:7577471772575368512:2120] self: [2:7577471772575368511:0] peer: [3:7577471774602630546:0] socket: 517 qp: -1
2025-11-27T18:20:30.912852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471774058889524:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29205
2025-11-27T18:20:30.912902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471772575368537:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:30.912930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:30.913874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471772835868542:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.913903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471772835868542:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.914083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772801009380:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.914287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471772801009298:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.914301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471774058889524:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.914811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471774058889508:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.915051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772801009372:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.915081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471772801009372:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.915736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471772801009372:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:30.916950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471772835868546:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.916987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471772835868546:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.917036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471772801009363:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.917110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:30.917107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471774602630533:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.917164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471772801009363:2108] [node 5] ICP22 created new session: [4:7577471772801009389:2108]
2025-11-27T18:20:30.917166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471774058889526:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29205
2025-11-27T18:20:30.917146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471774602630533:2122] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.917214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471772801009389:2108] [node 5] ICS09 handshake done sender: [4:7577471772801009372:2115] self: [4:7577471772801009371:0] peer: [5:7577471774058889525:0] socket: 564 qp: -1
2025-11-27T18:20:30.917248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471772801009389:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:30.917953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.919512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471775953976565:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57193
2025-11-27T18:20:30.919609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471774058889523:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.919735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471774058889524:2126] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:30.920051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471775953976565:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.920070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471772835868542:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:30.920112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471774058889508:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.920161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471775953976487:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.920159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:30.920198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471774058889508:2108] [node 4] ICP22 created new session: [5:7577471774058889528:2108]
2025-11-27T18:20:30.920204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471772575368540:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53830
2025-11-27T18:20:30.920234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471774058889528:2108] [node 4] ICS09 handshake done sender: [5:7577471774058889524:2126] self: [5:7577471774058889525:0] peer: [4:7577471772801009371:0] socket: 565 qp: -1
2025-11-27T18:20:30.921017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471774058889528:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:30.921050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:30.921284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471772835868546:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.921656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471772575368518:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.921688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471772575368518:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.921675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471774602630551:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54260
2025-11-27T18:20:30.921747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471772835868475:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.921808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:30.921853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471772835868475:2048] [node 2] ICP22 created new session: [1:7577471772835868570:2048]
2025-11-27T18:20:30.921922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471772835868570:2048] [node 2] ICS09 handshake done sender: [1:7577471772835868542:2110] self: [1:7577471772835868541:0] peer: [2:7577471772575368536:0] socket: 545 qp: -1
2025-11-27T18:20:30.921950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471772835868570:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:30.922127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:30.922356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471772575368526:2128] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:30.922615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471774602630453:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18130) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471774602630536:2124] held: no
2025-11-27T18:20:30.922717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471772575368434:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.922750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:30.922787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471772575368434:2048] [node 1] ICP22 created new session: [2:7577471772575368542:2048]
2025-11-27T18:20:30.922811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471772575368542:2048] [node 1] ICS09 handshake done sender: [2:7577471772575368526:2128] self: [2:7577471772575368536:0] peer: [1:7577471772835868541:0] socket: 551 qp: -1
2025-11-27T18:20:30.922830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471772575368542:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:30.922858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.922984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772801009391:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28783
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.923186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471775953976567:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57193
2025-11-27T18:20:30.923273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471775953976568:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57193
2025-11-27T18:20:30.923359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471772835868538:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.923404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:30.923434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471772835868538:2107] [node 4] ICP22 created new session: [1:7577471772835868572:2107]
2025-11-27T18:20:30.923464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471772835868572:2107] [node 4] ICS09 handshake done sender: [1:7577471772835868546:2112] self: [1:7577471772835868545:0] peer: [4:7577471772801009387:0] socket: 549 qp: -1
2025-11-27T18:20:30.923506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471772835868572:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:30.923533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:30.923888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471774058889436:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.923942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471774602630536:2124] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:30.924405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471774602630453:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.924447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:30.924484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471774602630453:2049] [node 2] ICP22 created new session: [3:7577471774602630558:2049]
2025-11-27T18:20:30.924512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471774602630558:2049] [node 2] ICS09 handshake done sender: [3:7577471774602630536:2124] self: [3:7577471774602630546:0] peer: [2:7577471772575368511:0] socket: 520 qp: -1
2025-11-27T18:20:30.924537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471774602630558:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:30.924567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.925072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471774058889527:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29205
2025-11-27T18:20:30.925827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471772801009380:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.926164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471775953976569:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57193
2025-11-27T18:20:30.926327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471772801009298:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.926408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:30.926551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471774602630533:2122] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:30.926462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471772801009298:2048] [node 1] ICP22 created new session: [4:7577471772801009394:2048]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:30.926653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471772801009394:2048] [node 1] ICS09 handshake done sender: [4:7577471772801009380:2119] self: [4:7577471772801009387:0] peer: [1:7577471772835868545:0] socket: 557 qp: -1
2025-11-27T18:20:30.926690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471772801009394:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:30.927283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471775953976570:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57193
2025-11-27T18:20:30.926729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:30.927364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471772575368518:2123] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.927980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471774058889532:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29205
2025-11-27T18:20:30.928079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471775953976565:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:30.928127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471774602630525:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.928197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:30.928139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471772575368516:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.928280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471774602630525:2107] [node 4] ICP22 created new session: [3:7577471774602630560:2107]
2025-11-27T18:20:30.928171Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471772575368516:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.928463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471775953976487:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.928495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471774602630560:2107] [node 4] ICS09 handshake done sender: [3:7577471774602630533:2122] self: [3:7577471774602630532:0] peer: [4:7577471772801009386:0] socket: 547 qp: -1
2025-11-27T18:20:30.928548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:30.928549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471774602630560:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:30.928589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:30.928636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471775953976487:2049] [node 2] ICP22 created new session: [6:7577471775953976571:2049]
2025-11-27T18:20:30.928686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471775953976571:2049] [node 2] ICS09 handshake done sender: [6:7577471775953976565:2116] self: [6:7577471775953976566:0] peer: [2:7577471772575368517:0] socket: 566 qp: -1
2025-11-27T18:20:30.928714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471775953976571:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:30.928796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:30.929229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471772575368510:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.929298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:30.929357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471772575368510:2109] [node 6] ICP22 created new session: [2:7577471772575368544:2109]
2025-11-27T18:20:30.929403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471772575368544:2109] [node 6] ICS09 handshake done sender: [2:7577471772575368518:2123] self: [2:7577471772575368517:0] peer: [6:7577471775953976566:0] socket: 544 qp: -1
2025-11-27T18:20:30.929439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471772575368544:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:30.929466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:30.930008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471775953976569:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.931161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471774058889527:2128] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.931819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471774058889435:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.932270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471772835868548:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.932306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471772835868548:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.934051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471774058889534:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29205
2025-11-27T18:20:30.934196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471775953976551:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.935810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471772575368516:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:30.935698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772801009393:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28783
2025-11-27T18:20:30.935750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471772835868548:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:30.936230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471772575368507:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.936428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:30.936577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471772575368507:2108] [node 5] ICP22 created new session: [2:7577471772575368547:2108]
No RDMA context found for GID: 2025-11-27T18:20:30.936631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471772835868539:2108] [node 5] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:30.936683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:30.936648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471772575368547:2108] [node 5] ICS09 handshake done sender: [2:7577471772575368516:2122] self: [2:7577471772575368515:0] peer: [5:7577471774058889530:0] socket: 561 qp: -1
2025-11-27T18:20:30.936691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471772575368547:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:30.936730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:30.936732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471775953976573:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57193
2025-11-27T18:20:30.936734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471772835868539:2108] [node 5] ICP22 created new session: [1:7577471772835868576:2108]
2025-11-27T18:20:30.936788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471772835868576:2108] [node 5] ICS09 handshake done sender: [1:7577471772835868548:2113] self: [1:7577471772835868547:0] peer: [5:7577471774058889533:0] socket: 568 qp: -1
2025-11-27T18:20:30.936737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471774058889519:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.936773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471774058889519:2124] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.936815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471772835868576:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:30.937478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471772801009300:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62675) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471772801009379:2118] held: no
2025-11-27T18:20:30.936847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:30.937511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471775953976570:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.937604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471775953976568:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.937592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471774058889532:2129] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.937710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471774058889535:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29205
2025-11-27T18:20:30.938191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471774058889527:2128] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:30.938499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471772801009379:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:30.939062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471774058889435:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.939116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:30.939197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471772801009300:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.939204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471774058889435:2048] [node 1] ICP22 created new session: [5:7577471774058889537:2048]
2025-11-27T18:20:30.939329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471774058889537:2048] [node 1] ICS09 handshake done sender: [5:7577471774058889527:2128] self: [5:7577471774058889533:0] peer: [1:7577471772835868547:0] socket: 571 qp: -1
2025-11-27T18:20:30.939365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471774058889537:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:30.939343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:30.939401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:30.939394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471772801009300:2050] [node 3] ICP22 created new session: [4:7577471772801009397:2050]
2025-11-27T18:20:30.939430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471772801009397:2050] [node 3] ICS09 handshake done sender: [4:7577471772801009379:2118] self: [4:7577471772801009386:0] peer: [3:7577471774602630532:0] socket: 556 qp: -1
2025-11-27T18:20:30.939434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471775953976573:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.939457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471772801009397:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:30.939510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471775953976488:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.939482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:30.939576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471775953976550:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.939922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471774602630538:2125] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.939965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471775953976576:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57193
2025-11-27T18:20:30.939955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471774602630538:2125] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.940050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471775953976486:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.940279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471772835868550:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.940310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471772835868550:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.941239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772801009374:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.941284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471772801009374:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.941723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471774602630538:2125] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:30.941782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471774058889523:2125] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.942447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471772801009396:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28783
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.942470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471774058889437:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:30.942540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471774602630527:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.942604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:30.942558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471775953976580:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57193
2025-11-27T18:20:30.942652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471774602630527:2110] [node 6] ICP22 created new session: [3:7577471774602630563:2110]
2025-11-27T18:20:30.942712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471774602630563:2110] [node 6] ICS09 handshake done sender: [3:7577471774602630538:2125] self: [3:7577471774602630537:0] peer: [6:7577471775953976577:0] socket: 579 qp: -1
2025-11-27T18:20:30.942742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471774602630563:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:30.942777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:30.942907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471772801009374:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:30.942993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471772835868550:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:30.944045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471772801009364:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.943996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471772835868540:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.944091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:30.944056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:30.944142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471772801009364:2109] [node 6] ICP22 created new session: [4:7577471772801009400:2109]
2025-11-27T18:20:30.944113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471772835868540:2109] [node 6] ICP22 created new session: [1:7577471772835868579:2109]
2025-11-27T18:20:30.944197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471772801009400:2109] [node 6] ICS09 handshake done sender: [4:7577471772801009374:2116] self: [4:7577471772801009373:0] peer: [6:7577471775953976578:0] socket: 574 qp: -1
2025-11-27T18:20:30.944169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471772835868579:2109] [node 6] ICS09 handshake done sender: [1:7577471772835868550:2114] self: [1:7577471772835868549:0] peer: [6:7577471775953976579:0] socket: 522 qp: -1
2025-11-27T18:20:30.944227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471772801009400:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:30.944206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471772835868579:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:30.944218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471775953976570:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:30.944255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:30.944232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:30.944931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471775953976488:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.945023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:30.945077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471775953976488:2050] [node 3] ICP22 created new session: [6:7577471775953976583:2050]
2025-11-27T18:20:30.945583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471775953976583:2050] [node 3] ICS09 handshake done sender: [6:7577471775953976570:2120] self: [6:7577471775953976577:0] peer: [3:7577471774602630537:0] socket: 580 qp: -1
2025-11-27T18:20:30.945626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471775953976583:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:30.945665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:30.945663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471772801009396:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.946308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471772801009364:2109] [node 6] ICP18 handshake (actor [4:7577471772801009396:2123]) is too old
2025-11-27T18:20:30.946634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471774058889436:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.946696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:30.946767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471774058889436:2049] [node 2] ICP22 created new session: [5:7577471774058889540:2049]
2025-11-27T18:20:30.946809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471774058889540:2049] [node 2] ICS09 handshake done sender: [5:7577471774058889523:2125] self: [5:7577471774058889530:0] peer: [2:7577471772575368515:0] socket: 562 qp: -1
2025-11-27T18:20:30.946847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471774058889540:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:30.946875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:30.947476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471774058889436:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:30.948814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471774058889435:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:30.948861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471774602630535:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:30.948882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471774602630535:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:30.949038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471774058889519:2124] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:30.950010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471774602630535:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:30.950175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471774058889511:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.950392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:30.950445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471774058889511:2110] [node 6] ICP22 created new session: [5:7577471774058889542:2110]
2025-11-27T18:20:30.950489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471774058889542:2110] [node 6] ICS09 handshake done sender: [5:7577471774058889519:2124] self: [5:7577471774058889518:0] peer: [6:7577471775953976574:0] socket: 577 qp: -1
2025-11-27T18:20:30.950505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471774602630526:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:30.950559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:30.950526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471774058889542:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:30.950612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471774602630526:2108] [node 5] ICP22 created new session: [3:7577471774602630566:2108]
2025-11-27T18:20:30.950580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:30.950663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471774602630566:2108] [node 5] ICS09 handshake done sender: [3:7577471774602630535:2123] self: [3:7577471774602630534:0] peer: [5:7577471774058889539:0] socket: 516 qp: -1
2025-11-27T18:20:30.950699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471774602630566:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:30.950731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.950957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471775953976581:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57193
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.951422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471775953976582:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57193
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.952068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471774058889544:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29205
2025-11-27T18:20:30.952121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471775953976486:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471775953976573:2121] held: no
2025-11-27T18:20:30.953382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471775953976550:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28528) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471775953976568:2118] held: no
2025-11-27T18:20:30.953517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471774058889437:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62675) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471774058889532:2129] held: no
2025-11-27T18:20:30.954046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471774058889532:2129] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:30.954078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471775953976573:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:30.954563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471774058889437:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.954560Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471775953976568:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:30.954605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:30.954691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471774058889437:2050] [node 3] ICP22 created new session: [5:7577471774058889545:2050]
2025-11-27T18:20:30.954741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471774058889545:2050] [node 3] ICS09 handshake done sender: [5:7577471774058889532:2129] self: [5:7577471774058889539:0] peer: [3:7577471774602630534:0] socket: 581 qp: -1
2025-11-27T18:20:30.954776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471774058889545:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:30.954782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471775953976486:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.954814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:30.954812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:30.954841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471775953976486:2048] [node 1] ICP22 created new session: [6:7577471775953976586:2048]
2025-11-27T18:20:30.954871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471775953976586:2048] [node 1] ICS09 handshake done sender: [6:7577471775953976573:2121] self: [6:7577471775953976579:0] peer: [1:7577471772835868549:0] socket: 540 qp: -1
2025-11-27T18:20:30.954887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471775953976586:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:30.954916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:30.955571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471775953976569:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:30.955779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471775953976550:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.955806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:30.955834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471775953976550:2106] [node 4] ICP22 created new session: [6:7577471775953976588:2106]
2025-11-27T18:20:30.955864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471775953976588:2106] [node 4] ICS09 handshake done sender: [6:7577471775953976568:2118] self: [6:7577471775953976578:0] peer: [4:7577471772801009373:0] socket: 575 qp: -1
2025-11-27T18:20:30.955884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471775953976588:2106] [node 4] ICS10 traffic start
2025-11-27T18:20:30.955902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:30.956947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471774058889547:2133] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29205
2025-11-27T18:20:30.956976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471775953976551:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:30.957013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:30.957042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471775953976551:2107] [node 5] ICP22 created new session: [6:7577471775953976590:2107]
2025-11-27T18:20:30.957077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471775953976590:2107] [node 5] ICS09 handshake done sender: [6:7577471775953976569:2119] self: [6:7577471775953976574:0] peer: [5:7577471774058889518:0] socket: 578 qp: -1
2025-11-27T18:20:30.957101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471775953976590:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:30.957130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:30.975883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:30.976153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471772575368542:2048] [node 1] ICS07 socket disconnect 551 reason# EndOfStream
2025-11-27T18:20:30.976209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471772575368542:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.976322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:30.976380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471772575368542:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:30.977369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:30.977583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471774602630552:2048] [node 1] ICS07 socket disconnect 552 reason# EndOfStream
2025-11-27T18:20:30.977620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471774602630552:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.977695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:30.977742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471774602630552:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:30.979882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471772575368434:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:30.979923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471772575368542:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:30.980891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:30.981111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471772801009394:2048] [node 1] ICS01 socket: 557 reason# EndOfStream
2025-11-27T18:20:30.981161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471772801009298:2048] [node 1] ICP30 unregister session Session# [4:7577471772801009394:2048] VirtualId# [4:7577471772801009387:0]
2025-11-27T18:20:30.981198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471772801009394:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.981324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:30.981325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471774602630452:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:30.981379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:30.981365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471774602630552:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:30.981675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471772575368434:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:30.981768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471772575368542:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:30.981829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471772575368434:2048] [node 1] ICP30 unregister session Session# [2:7577471772575368542:2048] VirtualId# [2:7577471772575368536:0]
2025-11-27T18:20:30.982009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:30.982051Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471772575368434:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:30.982071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:30.982237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471774602630452:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:30.982289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471774602630552:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:30.982344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471774602630452:2048] [node 1] ICP30 unregister session Session# [3:7577471774602630552:2048] VirtualId# [3:7577471774602630549:0]
2025-11-27T18:20:30.982454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:30.982501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471774602630452:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:30.982522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:30.982517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:30.982723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471774058889537:2048] [node 1] ICS07 socket disconnect 571 reason# EndOfStream
2025-11-27T18:20:30.982768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471774058889537:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.982849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:30.982895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471774058889537:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:30.983606Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:30.983663Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471774058889435:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:30.983718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471774058889537:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:30.983757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471774058889435:2048] [node 1] ICP30 unregister session Session# [5:7577471774058889537:2048] VirtualId# [5:7577471774058889533:0]
2025-11-27T18:20:30.983759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471775953976586:2048] [node 1] ICS01 socket: 540 reason# EndOfStream
2025-11-27T18:20:30.983795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471775953976486:2048] [node 1] ICP30 unregister session Session# [6:7577471775953976586:2048] VirtualId# [6:7577471775953976579:0]
2025-11-27T18:20:30.983814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471775953976586:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:30.983853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:30.983868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:30.983889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471774058889435:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:30.983899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:30.983902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29522) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:30.991680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471772575368434:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:30.992684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471774602630452:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:30.993715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471774058889435:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:31.045381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:31.045620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471774602630558:2049] [node 2] ICS01 socket: 520 reason# EndOfStream
2025-11-27T18:20:31.045670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471774602630453:2049] [node 2] ICP30 unregister session Session# [3:7577471774602630558:2049] VirtualId# [3:7577471774602630546:0]
2025-11-27T18:20:31.045708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471774602630558:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.045794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:31.045863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:31.048088Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:31.048252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471772801009383:2049] [node 2] ICS07 socket disconnect 525 reason# EndOfStream
2025-11-27T18:20:31.048276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471772801009383:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.048336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:31.048381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471772801009383:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:31.049241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471772801009299:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18130) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.049329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471772801009383:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.049366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471772801009299:2049] [node 2] ICP30 unregister session Session# [4:7577471772801009383:2049] VirtualId# [4:7577471772801009381:0]
2025-11-27T18:20:31.049370Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:31.049468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:31.049506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471772801009299:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.049526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471774058889540:2049] [node 2] ICS07 socket disconnect 562 reason# EndOfStream
2025-11-27T18:20:31.049531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.049558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471774058889540:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.049618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:31.049653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471774058889540:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:31.050333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471774058889436:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18130) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.050386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471774058889540:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.050421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471774058889436:2049] [node 2] ICP30 unregister session Session# [5:7577471774058889540:2049] VirtualId# [5:7577471774058889530:0]
2025-11-27T18:20:31.050501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:31.050526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471774058889436:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.050537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.050540Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:31.050751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471775953976571:2049] [node 2] ICS07 socket disconnect 566 reason# EndOfStream
2025-11-27T18:20:31.050799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471775953976571:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.050851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:31.050880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471775953976571:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:31.051539Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471775953976487:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18130) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.051585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471775953976571:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.051618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471775953976487:2049] [node 2] ICP30 unregister session Session# [6:7577471775953976571:2049] VirtualId# [6:7577471775953976566:0]
2025-11-27T18:20:31.051651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:31.051679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471775953976487:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.051704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.059271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471772801009299:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:31.060294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471774058889436:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:31.061329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471775953976487:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:31.080113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:31.080362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471772801009397:2050] [node 3] ICS01 socket: 556 reason# EndOfStream
2025-11-27T18:20:31.080405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471772801009300:2050] [node 3] ICP30 unregister session Session# [4:7577471772801009397:2050] VirtualId# [4:7577471772801009386:0]
2025-11-27T18:20:31.080456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471772801009397:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.080556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:31.080618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:31.081000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:31.081146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471774058889545:2050] [node 3] ICS01 socket: 581 reason# EndOfStream
2025-11-27T18:20:31.081179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471774058889437:2050] [node 3] ICP30 unregister session Session# [5:7577471774058889545:2050] VirtualId# [5:7577471774058889539:0]
2025-11-27T18:20:31.081203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471774058889545:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.081275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:31.081325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:31.081810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:31.081994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471775953976583:2050] [node 3] ICS07 socket disconnect 580 reason# EndOfStream
2025-11-27T18:20:31.082049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471775953976583:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.082157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:31.082210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471775953976583:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:31.082917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471775953976488:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62675) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.082967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471775953976583:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.083001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471775953976488:2050] [node 3] ICP30 unregister session Session# [6:7577471775953976583:2050] VirtualId# [6:7577471775953976577:0]
2025-11-27T18:20:31.083088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:31.083122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471775953976488:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62675) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.083139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62675) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.093073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471775953976488:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:31.120392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:31.120586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471774058889528:2108] [node 4] ICS07 socket disconnect 565 reason# EndOfStream
2025-11-27T18:20:31.120624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471774058889528:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.120711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:31.120750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471774058889528:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:31.121466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:31.121630Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471774058889508:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28528) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.121691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471774058889528:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.121731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471774058889508:2108] [node 4] ICP30 unregister session Session# [5:7577471774058889528:2108] VirtualId# [5:7577471774058889525:0]
2025-11-27T18:20:31.121718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471775953976588:2106] [node 4] ICS01 socket: 575 reason# EndOfStream
2025-11-27T18:20:31.121779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471775953976550:2106] [node 4] ICP30 unregister session Session# [6:7577471775953976588:2106] VirtualId# [6:7577471775953976578:0]
2025-11-27T18:20:31.121830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471775953976588:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.121833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:31.121873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471774058889508:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28528) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.121889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28528) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.121929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:31.122003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:31.145239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:31.145545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471775953976590:2107] [node 5] ICS07 socket disconnect 578 reason# EndOfStream
2025-11-27T18:20:31.145591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471775953976590:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.145720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:31.145787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471775953976590:2107] [node 5] ICS15 start handshake
2025-11-27T18:20:31.146649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471775953976551:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5490) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.146710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471775953976590:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.146749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471775953976551:2107] [node 5] ICP30 unregister session Session# [6:7577471775953976590:2107] VirtualId# [6:7577471775953976574:0]
2025-11-27T18:20:31.146849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:31.146892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471775953976551:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.146906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.156552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471775953976551:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:31.220922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471777767811852:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:31.220944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471777767811853:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:31.240207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471779940506063:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:31.240212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471779940506064:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:31.262847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471778841973297:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:31.262887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471778841973298:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:31.284421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471779759004540:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:31.284457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471779759004541:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:31.284502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471779759004542:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:31.303990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471779621117187:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:31.304056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471779621117189:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:31.304076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471779621117188:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:31.327935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471779147983444:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:31.327973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471779147983446:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:31.328203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471779147983445:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:31.414826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471779147983505:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:31.430623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471777767811915:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:31.430865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471777767811916:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:31.430874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471779759004603:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:31.430856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471779621117252:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:31.431047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471778841973387:2116] [node 5] ICP01 ready to work
2025-11-27T18:20:31.430561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471779940506136:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:31.431158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471779759004604:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:31.431088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471778841973388:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:31.431279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471779621117253:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:31.431289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471779147983508:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:31.431059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471779940506135:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:31.431837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471779940506137:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:31.432156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471777767811917:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:31.432256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471778841973389:2125] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.442784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471778841973398:2141] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22065
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.445570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471778841973404:2143] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22065
2025-11-27T18:20:31.446064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471779940506151:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7536
2025-11-27T18:20:31.446674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471778841973398:2141] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.449415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471779759004613:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14065
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.449911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471779621117262:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51723
2025-11-27T18:20:31.449908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471777767811926:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40530
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.450254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471777767811927:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40530
2025-11-27T18:20:31.450666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471777767811928:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40530
2025-11-27T18:20:31.450635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471779759004613:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.450611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471779621117263:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51723
2025-11-27T18:20:31.450858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471779759004541:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.450892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471779940506151:2124] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.451650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471777767811926:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.453236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471779940506143:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.452784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471779621117263:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.453289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471779940506143:2121] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.454335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471779940506152:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7536
2025-11-27T18:20:31.454346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471777767811928:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.454494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471778841973298:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.454427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471777767811927:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.454840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471779940506153:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7536
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.454910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471777767811934:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40530
2025-11-27T18:20:31.454843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471779940506154:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7536
2025-11-27T18:20:31.454921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471777767811935:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40530
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.455324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471778841973405:2144] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22065
2025-11-27T18:20:31.455422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471779940506143:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.456291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471778841973406:2145] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22065
2025-11-27T18:20:31.455438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471779940506141:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.457665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471779621117262:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.456331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471778841973404:2143] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.457712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471779759004620:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14065
2025-11-27T18:20:31.460543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471777767811935:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.455466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471779940506141:2120] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.457445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471778841973407:2146] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22065
2025-11-27T18:20:31.460550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471779940506135:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.460800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471779621117189:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.461217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471778841973405:2144] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.460603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.460856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471779940506135:2107] [node 4] ICP22 created new session: [2:7577471779940506158:2107]
2025-11-27T18:20:31.461228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471779940506158:2107] [node 4] ICS09 handshake done sender: [2:7577471779940506143:2121] self: [2:7577471779940506142:0] peer: [4:7577471779759004619:0] socket: 524 qp: -1
2025-11-27T18:20:31.461321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471779940506153:2126] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.461624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471779759004621:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14065
2025-11-27T18:20:31.461444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471779940506158:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:31.462180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471778841973395:2139] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.461491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:31.462233Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471778841973395:2139] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.462019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471779940506141:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.463744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471779759004613:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:31.463817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471779940506152:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.464058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471779759004541:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.464041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471779940506064:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.464024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471778841973410:2147] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22065
2025-11-27T18:20:31.463983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471777767811934:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.464025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471779621117271:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51723
2025-11-27T18:20:31.464107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:31.464211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:31.464052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471778841973395:2139] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:31.464227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471779940506064:2049] [node 3] ICP22 created new session: [2:7577471779940506161:2049]
2025-11-27T18:20:31.464646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471779759004541:2049] [node 2] ICP22 created new session: [4:7577471779759004622:2049]
2025-11-27T18:20:31.464661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471779940506161:2049] [node 3] ICS09 handshake done sender: [2:7577471779940506141:2120] self: [2:7577471779940506140:0] peer: [3:7577471778841973408:0] socket: 523 qp: -1
2025-11-27T18:20:31.464804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471779759004622:2049] [node 2] ICS09 handshake done sender: [4:7577471779759004613:2114] self: [4:7577471779759004619:0] peer: [2:7577471779940506142:0] socket: 528 qp: -1
2025-11-27T18:20:31.465225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471779940506161:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:31.464836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471779759004622:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:31.466347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471778841973407:2146] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.465594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:31.465278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:31.466831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471778841973406:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.466850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471778841973387:2116] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.467040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471779621117272:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51723
2025-11-27T18:20:31.466908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:31.466986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471778841973387:2116] [node 5] ICP22 created new session: [3:7577471778841973413:2116]
2025-11-27T18:20:31.466999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471778841973398:2141] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:31.467177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471778841973413:2116] [node 5] ICS09 handshake done sender: [3:7577471778841973395:2139] self: [3:7577471778841973394:0] peer: [5:7577471779621117270:0] socket: 531 qp: -1
2025-11-27T18:20:31.467221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471778841973413:2116] [node 5] ICS10 traffic start
2025-11-27T18:20:31.467333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:31.467488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471779621117262:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:31.467557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471778841973298:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.467612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:31.467743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471778841973298:2049] [node 2] ICP22 created new session: [3:7577471778841973415:2049]
2025-11-27T18:20:31.467775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471779759004620:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.467943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471779621117189:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.467938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471778841973415:2049] [node 2] ICS09 handshake done sender: [3:7577471778841973398:2141] self: [3:7577471778841973408:0] peer: [2:7577471779940506140:0] socket: 525 qp: -1
2025-11-27T18:20:31.467976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471778841973415:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:31.468017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:31.468062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471779759004540:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.468080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:31.468149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471779621117189:2050] [node 3] ICP22 created new session: [5:7577471779621117273:2050]
2025-11-27T18:20:31.468216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471779621117273:2050] [node 3] ICS09 handshake done sender: [5:7577471779621117262:2116] self: [5:7577471779621117270:0] peer: [3:7577471778841973394:0] socket: 540 qp: -1
2025-11-27T18:20:31.468250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471779621117273:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:31.468291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:31.469451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471778841973387:2116] [node 5] ICP18 handshake (actor [3:7577471778841973407:2146]) is too old
2025-11-27T18:20:31.469634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471779940506154:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.469782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471779621117274:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51723
2025-11-27T18:20:31.469838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471778841973297:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.469910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471777767811924:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.469945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471777767811924:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.470627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471777767811924:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:31.470709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471779940506135:2107] [node 4] ICP18 handshake (actor [2:7577471779940506152:2125]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.471979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471779621117274:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.471986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471777767811915:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.472025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471779759004626:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14065
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.472114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:31.472169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471777767811915:2107] [node 4] ICP22 created new session: [1:7577471777767811942:2107]
2025-11-27T18:20:31.472162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471779759004627:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14065
2025-11-27T18:20:31.472238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471777767811942:2107] [node 4] ICS09 handshake done sender: [1:7577471777767811924:2112] self: [1:7577471777767811923:0] peer: [4:7577471779759004625:0] socket: 560 qp: -1
2025-11-27T18:20:31.472286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471777767811942:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:31.472335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:31.473146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471779759004620:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:31.473201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471779621117252:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.473230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471777767811919:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.473279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471777767811919:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.474313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471779147983522:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19817
2025-11-27T18:20:31.474320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471779940506063:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.474144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471779759004540:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471779759004620:2116] held: no
2025-11-27T18:20:31.474316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471779621117271:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.474460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471779621117188:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.474453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471777767811921:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.474478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471777767811921:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.475013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471779621117278:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51723
2025-11-27T18:20:31.475057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471779940506145:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.475098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471779940506145:2122] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.478530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471779759004540:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.478558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471779759004626:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:31.478642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:31.478671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471779940506145:2122] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:31.478695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471779759004540:2048] [node 1] ICP22 created new session: [4:7577471779759004629:2048]
2025-11-27T18:20:31.478767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471777767811919:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:31.478821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471779147983523:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19817
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:31.478764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471779759004612:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.478772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471779759004629:2048] [node 1] ICS09 handshake done sender: [4:7577471779759004620:2116] self: [4:7577471779759004625:0] peer: [1:7577471777767811923:0] socket: 561 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.480284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471779940506136:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.478789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471779759004612:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.480348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:31.480638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471777767811853:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.478836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471779759004629:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:31.480768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471779621117280:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51723
2025-11-27T18:20:31.480638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471779940506136:2108] [node 5] ICP22 created new session: [2:7577471779940506168:2108]
2025-11-27T18:20:31.480810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:31.479103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:31.480846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471778841973420:2148] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22065
2025-11-27T18:20:31.480882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471777767811853:2049] [node 3] ICP22 created new session: [1:7577471777767811947:2049]
2025-11-27T18:20:31.480875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471779940506168:2108] [node 5] ICS09 handshake done sender: [2:7577471779940506145:2122] self: [2:7577471779940506144:0] peer: [5:7577471779621117279:0] socket: 568 qp: -1
2025-11-27T18:20:31.480905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471779940506168:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:31.480972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471777767811947:2049] [node 3] ICS09 handshake done sender: [1:7577471777767811919:2110] self: [1:7577471777767811918:0] peer: [3:7577471778841973418:0] socket: 550 qp: -1
2025-11-27T18:20:31.481188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471777767811947:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:31.480932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:31.481288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:31.481313Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471779621117271:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:31.481512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471777767811921:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:31.482086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471778841973406:2145] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:31.481916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471779147983522:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.482153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471779621117188:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.482395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471779147983445:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.482669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471778841973297:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.483318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471779147983523:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.483342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:31.483549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471779147983446:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.483359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:31.483532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471779621117188:2049] [node 2] ICP22 created new session: [5:7577471779621117281:2049]
2025-11-27T18:20:31.483515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471778841973297:2048] [node 1] ICP22 created new session: [3:7577471778841973421:2048]
2025-11-27T18:20:31.483652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471778841973421:2048] [node 1] ICS09 handshake done sender: [3:7577471778841973406:2145] self: [3:7577471778841973418:0] peer: [1:7577471777767811918:0] socket: 551 qp: -1
2025-11-27T18:20:31.483690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471778841973421:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:31.483738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:31.483610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471779621117281:2049] [node 2] ICS09 handshake done sender: [5:7577471779621117271:2119] self: [5:7577471779621117279:0] peer: [2:7577471779940506144:0] socket: 569 qp: -1
2025-11-27T18:20:31.483860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471779940506150:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.483895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471779940506150:2123] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.483656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471779621117281:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.483781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:31.484931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471779940506167:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7536
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.485065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471777767811852:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.485701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:31.485786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471779759004628:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14065
2025-11-27T18:20:31.485820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471777767811852:2048] [node 2] ICP22 created new session: [1:7577471777767811949:2048]
2025-11-27T18:20:31.485966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471777767811949:2048] [node 2] ICS09 handshake done sender: [1:7577471777767811921:2111] self: [1:7577471777767811920:0] peer: [2:7577471779940506165:0] socket: 555 qp: -1
2025-11-27T18:20:31.486077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471777767811949:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:31.486184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:31.486341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471779940506154:2127] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:31.486365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471778841973403:2142] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.486467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471779759004612:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:31.486398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471778841973403:2142] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.487634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471779147983526:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19817
2025-11-27T18:20:31.488051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471779940506063:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.488165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:31.488183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471779147983527:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19817
2025-11-27T18:20:31.488186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471779621117284:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51723
2025-11-27T18:20:31.488223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471779940506063:2048] [node 1] ICP22 created new session: [2:7577471779940506171:2048]
2025-11-27T18:20:31.488194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471778841973403:2142] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:31.488266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471779940506171:2048] [node 1] ICS09 handshake done sender: [2:7577471779940506154:2127] self: [2:7577471779940506165:0] peer: [1:7577471777767811920:0] socket: 558 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.488303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471779940506171:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:31.488325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471779940506150:2123] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:31.488276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471779621117188:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:31.488370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471779147983528:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19817
2025-11-27T18:20:31.488511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471779759004603:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.488334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:31.488573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:31.489447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471778841973389:2125] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.488615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471779759004603:2108] [node 5] ICP22 created new session: [4:7577471779759004634:2108]
2025-11-27T18:20:31.489496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:31.489780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471779147983529:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19817
2025-11-27T18:20:31.489644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471778841973389:2125] [node 6] ICP22 created new session: [3:7577471778841973424:2125]
2025-11-27T18:20:31.489245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471779759004628:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.490953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471778841973424:2125] [node 6] ICS09 handshake done sender: [3:7577471778841973403:2142] self: [3:7577471778841973402:0] peer: [6:7577471779147983525:0] socket: 543 qp: -1
2025-11-27T18:20:31.489820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471779147983522:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:31.491533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471779147983445:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.489248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471779759004634:2108] [node 5] ICS09 handshake done sender: [4:7577471779759004612:2113] self: [4:7577471779759004611:0] peer: [5:7577471779621117277:0] socket: 545 qp: -1
2025-11-27T18:20:31.491084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471778841973424:2125] [node 6] ICS10 traffic start
2025-11-27T18:20:31.489469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471779759004634:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:31.491687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:31.491739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471779147983445:2049] [node 2] ICP22 created new session: [6:7577471779147983530:2049]
2025-11-27T18:20:31.489612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:31.491113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:31.491757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471779147983526:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.491196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471779759004542:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.492164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471779147983527:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.492182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471779147983530:2049] [node 2] ICS09 handshake done sender: [6:7577471779147983522:2115] self: [6:7577471779147983524:0] peer: [2:7577471779940506149:0] socket: 553 qp: -1
2025-11-27T18:20:31.492855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471779940506137:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.492843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471779621117274:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:31.492436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471779147983530:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:31.492890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:31.492934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471779940506137:2109] [node 6] ICP22 created new session: [2:7577471779940506173:2109]
2025-11-27T18:20:31.492609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:31.492970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471779940506173:2109] [node 6] ICS09 handshake done sender: [2:7577471779940506150:2123] self: [2:7577471779940506149:0] peer: [6:7577471779147983524:0] socket: 529 qp: -1
2025-11-27T18:20:31.492990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471779940506173:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:31.493020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:31.493138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471779147983523:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:31.493525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471779147983446:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.493557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:31.493600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471779147983446:2050] [node 3] ICP22 created new session: [6:7577471779147983532:2050]
2025-11-27T18:20:31.493640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471779147983532:2050] [node 3] ICS09 handshake done sender: [6:7577471779147983523:2116] self: [6:7577471779147983525:0] peer: [3:7577471778841973402:0] socket: 576 qp: -1
2025-11-27T18:20:31.493668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471779147983532:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:31.493660Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471779621117252:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61750) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471779621117274:2121] held: no
2025-11-27T18:20:31.493687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:31.494398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471779621117252:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.494443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:31.494506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471779621117252:2108] [node 4] ICP22 created new session: [5:7577471779621117285:2108]
2025-11-27T18:20:31.494548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471779621117285:2108] [node 4] ICS09 handshake done sender: [5:7577471779621117274:2121] self: [5:7577471779621117277:0] peer: [4:7577471779759004611:0] socket: 572 qp: -1
2025-11-27T18:20:31.494597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471779621117285:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:31.494592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471778841973397:2140] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.494641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471778841973397:2140] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.494639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:31.495057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471779621117278:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.495592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471779147983505:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.495666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471779147983508:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.495841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471779621117187:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.496019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471777767811925:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.496043Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471777767811925:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.497199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471778841973397:2140] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:31.497479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471778841973388:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.498111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:31.498381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471778841973388:2107] [node 4] ICP22 created new session: [3:7577471778841973427:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.498531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471778841973427:2107] [node 4] ICS09 handshake done sender: [3:7577471778841973397:2140] self: [3:7577471778841973396:0] peer: [4:7577471779759004636:0] socket: 546 qp: -1
2025-11-27T18:20:31.498629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471777767811925:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:31.498835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471777767811916:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.498609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471778841973427:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:31.498869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471779621117265:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.498694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:31.498870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471779759004637:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14065
2025-11-27T18:20:31.499388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.498905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471779621117265:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.499440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471777767811916:2108] [node 5] ICP22 created new session: [1:7577471777767811953:2108]
2025-11-27T18:20:31.501031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471779759004615:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.499493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471779621117288:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51723
2025-11-27T18:20:31.501185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471779147983536:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19817
2025-11-27T18:20:31.499481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471777767811953:2108] [node 5] ICS09 handshake done sender: [1:7577471777767811925:2113] self: [1:7577471777767811922:0] peer: [5:7577471779621117287:0] socket: 559 qp: -1
2025-11-27T18:20:31.501065Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471779759004615:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.502041Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471779621117278:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:31.499555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471777767811953:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:31.501220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471779759004542:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12630) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471779759004628:2120] held: no
2025-11-27T18:20:31.502541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471779147983536:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.499588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:31.502700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471779621117187:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.502969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:31.502967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471779147983444:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:31.503029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471779621117187:2048] [node 1] ICP22 created new session: [5:7577471779621117290:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.503221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471779621117290:2048] [node 1] ICS09 handshake done sender: [5:7577471779621117278:2122] self: [5:7577471779621117287:0] peer: [1:7577471777767811922:0] socket: 575 qp: -1
2025-11-27T18:20:31.503604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471779621117290:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.503626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471777767811930:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.503687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:31.503664Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471777767811930:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:31.503627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471779759004638:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14065
2025-11-27T18:20:31.503863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471779759004615:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:31.503794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471779147983539:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19817
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.505238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471779759004628:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:31.505295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471779621117265:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:31.505445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471779147983540:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19817
2025-11-27T18:20:31.505549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471777767811930:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:31.505862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471779759004638:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:31.505909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471779759004604:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.505934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:31.506144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471779621117253:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.506136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471779759004604:2109] [node 6] ICP22 created new session: [4:7577471779759004640:2109]
2025-11-27T18:20:31.506200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471779759004542:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.506205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:31.506244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471779759004640:2109] [node 6] ICS09 handshake done sender: [4:7577471779759004615:2115] self: [4:7577471779759004614:0] peer: [6:7577471779147983535:0] socket: 585 qp: -1
2025-11-27T18:20:31.506338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471779621117253:2109] [node 6] ICP22 created new session: [5:7577471779621117292:2109]
2025-11-27T18:20:31.506383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:31.506430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471779621117292:2109] [node 6] ICS09 handshake done sender: [5:7577471779621117265:2118] self: [5:7577471779621117264:0] peer: [6:7577471779147983534:0] socket: 534 qp: -1
2025-11-27T18:20:31.506469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471779759004640:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:31.506451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471779621117292:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:31.506697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471777767811917:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:31.506605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471779147983526:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:31.506481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471779759004542:2050] [node 3] ICP22 created new session: [4:7577471779759004641:2050]
2025-11-27T18:20:31.506471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:31.506749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:31.506709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471779147983527:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:31.506517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:31.506796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471777767811917:2109] [node 6] ICP22 created new session: [1:7577471777767811956:2109]
2025-11-27T18:20:31.506582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471779759004641:2050] [node 3] ICS09 handshake done sender: [4:7577471779759004628:2120] self: [4:7577471779759004636:0] peer: [3:7577471778841973396:0] socket: 577 qp: -1
2025-11-27T18:20:31.507027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471777767811956:2109] [node 6] ICS09 handshake done sender: [1:7577471777767811930:2117] self: [1:7577471777767811929:0] peer: [6:7577471779147983538:0] socket: 522 qp: -1
2025-11-27T18:20:31.506622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471779759004641:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:31.507064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471777767811956:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:31.507071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471779147983508:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.506660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:31.507090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:31.507113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:31.507137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471779621117187:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:31.507162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471779147983508:2107] [node 4] ICP22 created new session: [6:7577471779147983542:2107]
2025-11-27T18:20:31.507243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471779147983542:2107] [node 4] ICS09 handshake done sender: [6:7577471779147983527:2118] self: [6:7577471779147983535:0] peer: [4:7577471779759004614:0] socket: 586 qp: -1
2025-11-27T18:20:31.507243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471779147983505:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.507274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471779147983542:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:31.507306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:31.507317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:31.507372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471779147983505:2106] [node 5] ICP22 created new session: [6:7577471779147983544:2106]
2025-11-27T18:20:31.507419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471779147983544:2106] [node 5] ICS09 handshake done sender: [6:7577471779147983526:2117] self: [6:7577471779147983534:0] peer: [5:7577471779621117264:0] socket: 557 qp: -1
2025-11-27T18:20:31.507449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471779147983544:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:31.507484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:31.508114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471779759004604:2109] [node 6] ICP18 handshake (actor [4:7577471779759004638:2122]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:31.509143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471779147983541:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19817
2025-11-27T18:20:31.512013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471779147983444:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471779147983536:2121] held: no
2025-11-27T18:20:31.513158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471779147983536:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:31.513500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471779147983444:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:31.513548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:31.513598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471779147983444:2048] [node 1] ICP22 created new session: [6:7577471779147983546:2048]
2025-11-27T18:20:31.513651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471779147983546:2048] [node 1] ICS09 handshake done sender: [6:7577471779147983536:2121] self: [6:7577471779147983538:0] peer: [1:7577471777767811929:0] socket: 538 qp: -1
2025-11-27T18:20:31.513688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471779147983546:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:31.513730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:31.556831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:31.557023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471779940506171:2048] [node 1] ICS07 socket disconnect 558 reason# EndOfStream
2025-11-27T18:20:31.557062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471779940506171:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.557134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:31.557177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471779940506171:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:31.558212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:31.558436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471778841973421:2048] [node 1] ICS07 socket disconnect 551 reason# EndOfStream
2025-11-27T18:20:31.558482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471778841973421:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.558593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:31.558663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471778841973421:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:31.559516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471778841973297:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.559637Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471779940506063:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:31.559669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471779940506171:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:31.559587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471778841973421:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.559620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471778841973297:2048] [node 1] ICP30 unregister session Session# [3:7577471778841973421:2048] VirtualId# [3:7577471778841973418:0]
2025-11-27T18:20:31.559838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:31.559924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471778841973297:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.559942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.561377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:31.561473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471779940506063:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.561558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471779759004629:2048] [node 1] ICS01 socket: 561 reason# EndOfStream
2025-11-27T18:20:31.561533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471779940506171:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.561573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471779940506063:2048] [node 1] ICP30 unregister session Session# [2:7577471779940506171:2048] VirtualId# [2:7577471779940506165:0]
2025-11-27T18:20:31.561666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471779759004540:2048] [node 1] ICP30 unregister session Session# [4:7577471779759004629:2048] VirtualId# [4:7577471779759004625:0]
2025-11-27T18:20:31.561710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:31.561703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471779759004629:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.561766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:31.561766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471779940506063:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.561812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:31.561789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.562580Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:31.562719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471779621117290:2048] [node 1] ICS07 socket disconnect 575 reason# EndOfStream
2025-11-27T18:20:31.562742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471779621117290:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.562805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:31.562841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471779621117290:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:31.563493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471779621117187:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.563574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471779621117290:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.563628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471779621117187:2048] [node 1] ICP30 unregister session Session# [5:7577471779621117290:2048] VirtualId# [5:7577471779621117287:0]
2025-11-27T18:20:31.563759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:31.563791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471779621117187:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.563815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21150) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.563828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:31.564049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471779147983546:2048] [node 1] ICS01 socket: 538 reason# EndOfStream
2025-11-27T18:20:31.564260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471779147983444:2048] [node 1] ICP30 unregister session Session# [6:7577471779147983546:2048] VirtualId# [6:7577471779147983538:0]
2025-11-27T18:20:31.564293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471779147983546:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.564352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:31.564384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:31.570198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471778841973297:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:31.571238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471779940506063:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:31.573309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471779621117187:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:31.625530Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:31.625703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471778841973415:2049] [node 2] ICS07 socket disconnect 525 reason# EndOfStream
2025-11-27T18:20:31.625730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471778841973415:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.625825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:31.625867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471778841973415:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:31.628338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:31.628422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471778841973298:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:31.628451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471778841973415:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:31.628491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471779759004622:2049] [node 2] ICS07 socket disconnect 528 reason# EndOfStream
2025-11-27T18:20:31.628543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471779759004622:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.628610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:31.628644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471779759004622:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:31.629406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471778841973298:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.629476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471778841973415:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.629512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471778841973298:2049] [node 2] ICP30 unregister session Session# [3:7577471778841973415:2049] VirtualId# [3:7577471778841973408:0]
2025-11-27T18:20:31.629631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:31.629696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471778841973298:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.629713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.629746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471779759004541:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.629815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471779759004622:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.629857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471779759004541:2049] [node 2] ICP30 unregister session Session# [4:7577471779759004622:2049] VirtualId# [4:7577471779759004619:0]
2025-11-27T18:20:31.629995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:31.630020Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471779759004541:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.630028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.630220Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:31.630394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471779621117281:2049] [node 2] ICS07 socket disconnect 569 reason# EndOfStream
2025-11-27T18:20:31.630427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471779621117281:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.630490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:31.630534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471779621117281:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:31.631073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471779621117188:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.631128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471779621117281:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.631155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471779621117188:2049] [node 2] ICP30 unregister session Session# [5:7577471779621117281:2049] VirtualId# [5:7577471779621117279:0]
2025-11-27T18:20:31.631262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:31.631290Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471779621117188:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.631299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.631372Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:31.631560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471779147983530:2049] [node 2] ICS07 socket disconnect 553 reason# EndOfStream
2025-11-27T18:20:31.631600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471779147983530:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.631656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:31.631684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471779147983530:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:31.632218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471779147983445:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.632250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471779147983530:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.632269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471779147983445:2049] [node 2] ICP30 unregister session Session# [6:7577471779147983530:2049] VirtualId# [6:7577471779147983524:0]
2025-11-27T18:20:31.632391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:31.632418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471779147983445:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.632432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28701) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.639841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471779759004541:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:31.640887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471779621117188:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:31.641873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471779147983445:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:31.656587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:31.656817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471779759004641:2050] [node 3] ICS01 socket: 577 reason# EndOfStream
2025-11-27T18:20:31.656850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471779759004542:2050] [node 3] ICP30 unregister session Session# [4:7577471779759004641:2050] VirtualId# [4:7577471779759004636:0]
2025-11-27T18:20:31.656880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471779759004641:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.657008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:31.657062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:31.657832Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:31.657961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471779621117273:2050] [node 3] ICS07 socket disconnect 540 reason# EndOfStream
2025-11-27T18:20:31.657983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471779621117273:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.658045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:31.658082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471779621117273:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:31.658788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471779621117189:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12630) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.658846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471779621117273:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.658872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471779621117189:2050] [node 3] ICP30 unregister session Session# [5:7577471779621117273:2050] VirtualId# [5:7577471779621117270:0]
2025-11-27T18:20:31.658927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:31.658971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:31.659007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471779621117189:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12630) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.659018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12630) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.659143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471779147983532:2050] [node 3] ICS07 socket disconnect 576 reason# EndOfStream
2025-11-27T18:20:31.659173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471779147983532:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.659252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:31.659325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471779147983532:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:31.659977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471779147983446:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12630) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.660038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471779147983532:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.660077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471779147983446:2050] [node 3] ICP30 unregister session Session# [6:7577471779147983532:2050] VirtualId# [6:7577471779147983525:0]
2025-11-27T18:20:31.660193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:31.660229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471779147983446:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12630) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.660244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12630) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.668545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471779621117189:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:31.670588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471779147983446:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:31.721852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:31.722065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471779621117285:2108] [node 4] ICS01 socket: 572 reason# EndOfStream
2025-11-27T18:20:31.722104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471779621117252:2108] [node 4] ICP30 unregister session Session# [5:7577471779621117285:2108] VirtualId# [5:7577471779621117277:0]
2025-11-27T18:20:31.722134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471779621117285:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.722216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:31.722272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:31.722900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:31.723091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471779147983542:2107] [node 4] ICS07 socket disconnect 586 reason# EndOfStream
2025-11-27T18:20:31.723127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471779147983542:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.723202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:31.723250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471779147983542:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:31.724027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471779147983508:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61750) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.724115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471779147983542:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.724197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471779147983508:2107] [node 4] ICP30 unregister session Session# [6:7577471779147983542:2107] VirtualId# [6:7577471779147983535:0]
2025-11-27T18:20:31.724314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:31.724355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471779147983508:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.724380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61750) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.734090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471779147983508:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:31.789927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:31.790197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471779147983544:2106] [node 5] ICS07 socket disconnect 557 reason# EndOfStream
2025-11-27T18:20:31.790227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471779147983544:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:31.790318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:31.790387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471779147983544:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:31.791218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471779147983505:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3018) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:31.791316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471779147983544:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:31.791385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471779147983505:2106] [node 5] ICP30 unregister session Session# [6:7577471779147983544:2106] VirtualId# [6:7577471779147983534:0]
2025-11-27T18:20:31.791541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:31.791586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471779147983505:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3018) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:31.791615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3018) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:31.883617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471779879911240:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:31.883626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471779879911241:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:31.902614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471778903285061:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:31.902843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471778903285062:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:31.924891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471777636880084:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:31.924931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471777636880085:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:31.947479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471777146607382:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:31.947512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471777146607383:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:31.947535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471777146607384:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:31.971238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471779027822621:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:31.971461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471779027822622:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:31.971482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471779027822623:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:31.993684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471776691310627:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:31.993721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471776691310629:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:31.993963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471776691310628:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:32.058645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471783198252421:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:32.097113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471781441574741:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:32.101655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471783322789980:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:32.098912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471783198252422:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:32.097068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471784174878600:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:32.097069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471781931847443:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:32.104120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471783198252423:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:32.102799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471784174878601:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:32.102253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471781931847444:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:32.102607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471783322789981:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:32.109294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471780986277985:2105] [node 4] ICP01 ready to work
2025-11-27T18:20:32.102930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471784174878602:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:32.109362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471780986277986:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:32.109586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471781441574742:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:32.109425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471781931847445:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:32.123077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471781931847460:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1359
2025-11-27T18:20:32.123488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471781931847459:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1359
2025-11-27T18:20:32.125264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471783322789996:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24871
2025-11-27T18:20:32.125224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471781931847462:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1359
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.125951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781441574755:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.125980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471783322789995:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24871
2025-11-27T18:20:32.126953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471781931847461:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1359
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.127837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783198252433:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57460
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:32.127412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781441574756:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:32.127503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781441574754:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.130005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781441574755:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.130829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471784174878612:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26743
2025-11-27T18:20:32.130837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471783322789997:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24871
2025-11-27T18:20:32.130834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783198252434:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57460
2025-11-27T18:20:32.133221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471781931847460:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.130842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783198252435:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57460
2025-11-27T18:20:32.133372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783198252433:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.133600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471777146607384:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.133171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471784174878615:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26743
2025-11-27T18:20:32.133544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471780986278001:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25431
2025-11-27T18:20:32.133652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471784174878613:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26743
2025-11-27T18:20:32.133573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783198252435:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.133692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471784174878612:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.133706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781441574754:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.134114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471784174878619:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26743
2025-11-27T18:20:32.134133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471781931847462:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.134128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471784174878615:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.136315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471783322789995:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.134158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471781931847450:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.136318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471784174878613:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.134186Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471781931847450:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.136743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471783322789980:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.137352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783198252434:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.137782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781441574747:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.136300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471781931847459:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.138739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471780986278001:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.139110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471776691310628:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.139579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783198252429:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.137828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471781441574747:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.139228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781441574756:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.137028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471781931847450:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.139617Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471783198252429:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.139697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471784174878619:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.139716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471781441574747:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:32.140318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471783322789997:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.137279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471781931847461:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.140077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471784174878605:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.138503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471777636880084:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.141596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471777146607382:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.140111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471784174878605:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.140128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471781931847443:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.142112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781441574761:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-27T18:20:32.141624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471779027822622:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.142112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471784174878607:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.140234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:32.142231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471781441574741:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.142145Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471784174878607:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.140283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471781931847443:2107] [node 4] ICP22 created new session: [3:7577471781931847468:2107]
2025-11-27T18:20:32.142740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783198252428:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.142326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471783322789996:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.142380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:32.142603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471784174878605:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:32.143010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471783322789999:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24871
2025-11-27T18:20:32.142784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471781441574741:2108] [node 5] ICP22 created new session: [4:7577471781441574764:2108]
2025-11-27T18:20:32.142779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471783198252428:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.142144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471781931847468:2107] [node 4] ICS09 handshake done sender: [3:7577471781931847450:2111] self: [3:7577471781931847448:0] peer: [4:7577471781441574759:0] socket: 529 qp: -1
2025-11-27T18:20:32.145824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471784174878607:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:32.142207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471781931847468:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:32.142777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471783198252429:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.145949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471784174878608:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.143240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471779027822621:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.145167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471781441574764:2108] [node 5] ICS09 handshake done sender: [4:7577471781441574747:2111] self: [4:7577471781441574744:0] peer: [5:7577471783322789998:0] socket: 533 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.142322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:32.146825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471783198252423:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.148206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471783322789995:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:32.145389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471781441574764:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:32.145977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471784174878608:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.147486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:32.145438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:32.151077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471780986278003:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25431
2025-11-27T18:20:32.150830Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471783322789980:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14036) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471783322789995:2115] held: no
2025-11-27T18:20:32.146478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471779879911241:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.147544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471783198252423:2110] [node 6] ICP22 created new session: [2:7577471783198252445:2110]
2025-11-27T18:20:32.147472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471777146607384:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20229) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471781441574755:2115] held: no
2025-11-27T18:20:32.151082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471780986278004:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25431
2025-11-27T18:20:32.153373Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471777636880084:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471781931847461:2117] held: no
2025-11-27T18:20:32.148396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471783198252428:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:32.151571Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471776691310628:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471780986278001:2115] held: no
2025-11-27T18:20:32.146755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:32.149188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471779879911241:2049] [node 3] ICP22 created new session: [1:7577471784174878626:2049]
2025-11-27T18:20:32.153614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471780986278003:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.149157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471784174878600:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.149726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471784174878626:2049] [node 3] ICS09 handshake done sender: [1:7577471784174878605:2111] self: [1:7577471784174878603:0] peer: [3:7577471781931847467:0] socket: 535 qp: -1
2025-11-27T18:20:32.153500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471781441574755:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:32.149705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471783198252445:2110] [node 6] ICS09 handshake done sender: [2:7577471783198252429:2113] self: [2:7577471783198252424:0] peer: [6:7577471780986278002:0] socket: 561 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.151012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471783198252445:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:32.149781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471784174878626:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:32.154913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471783322789980:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.149823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:32.151053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:32.155048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471776691310629:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.150749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:32.154962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:32.158022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471781931847471:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1359
2025-11-27T18:20:32.151237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471784174878600:2107] [node 4] ICP22 created new session: [1:7577471784174878628:2107]
2025-11-27T18:20:32.158678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471777146607384:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.151170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471783198252422:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.155453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471783322789980:2108] [node 4] ICP22 created new session: [5:7577471783322790004:2108]
2025-11-27T18:20:32.159498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471780986278001:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:32.151497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:32.151789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471784174878628:2107] [node 4] ICS09 handshake done sender: [1:7577471784174878607:2112] self: [1:7577471784174878604:0] peer: [4:7577471781441574763:0] socket: 540 qp: -1
2025-11-27T18:20:32.152508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471783198252422:2109] [node 5] ICP22 created new session: [2:7577471783198252448:2109]
2025-11-27T18:20:32.151824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471784174878628:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:32.157965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471783322790002:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24871
2025-11-27T18:20:32.158730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:32.151859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:32.153314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471783198252448:2109] [node 5] ICS09 handshake done sender: [2:7577471783198252428:2112] self: [2:7577471783198252425:0] peer: [5:7577471783322790000:0] socket: 560 qp: -1
2025-11-27T18:20:32.159476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471777146607384:2050] [node 3] ICP22 created new session: [4:7577471781441574768:2050]
2025-11-27T18:20:32.160596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471776691310628:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.159200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471783322790004:2108] [node 4] ICS09 handshake done sender: [5:7577471783322789995:2115] self: [5:7577471783322789998:0] peer: [4:7577471781441574744:0] socket: 550 qp: -1
2025-11-27T18:20:32.158662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471784174878608:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.160844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:32.153517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471783198252448:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:32.159505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471783322790004:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:32.159803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471781441574768:2050] [node 3] ICS09 handshake done sender: [4:7577471781441574755:2115] self: [4:7577471781441574759:0] peer: [3:7577471781931847448:0] socket: 551 qp: -1
2025-11-27T18:20:32.161821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471781931847458:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.159561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:32.160113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471781441574768:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:32.153564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:32.160893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471776691310628:2049] [node 2] ICP22 created new session: [6:7577471780986278008:2049]
2025-11-27T18:20:32.160959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471783322789997:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:32.161860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471781931847458:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.160964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471784174878602:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.161455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471780986278008:2049] [node 2] ICS09 handshake done sender: [6:7577471780986278001:2115] self: [6:7577471780986278002:0] peer: [2:7577471783198252424:0] socket: 564 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.160153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:32.161018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:32.162875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471779027822622:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.161697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471780986278008:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:32.164295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783198252450:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57460
2025-11-27T18:20:32.164300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471781931847461:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:32.164286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781441574766:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-27T18:20:32.163107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:32.161205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471784174878602:2108] [node 5] ICP22 created new session: [1:7577471784174878634:2108]
2025-11-27T18:20:32.161748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:32.163858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471779027822622:2049] [node 2] ICP22 created new session: [5:7577471783322790007:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.161806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471784174878634:2108] [node 5] ICS09 handshake done sender: [1:7577471784174878608:2113] self: [1:7577471784174878606:0] peer: [5:7577471783322790001:0] socket: 542 qp: -1
2025-11-27T18:20:32.162171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471780986278007:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25431
2025-11-27T18:20:32.161953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471784174878634:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:32.162230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:32.164185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471783322790007:2049] [node 2] ICS09 handshake done sender: [5:7577471783322789997:2117] self: [5:7577471783322790000:0] peer: [2:7577471783198252425:0] socket: 565 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.164230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471783322790007:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:32.167730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781441574769:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-27T18:20:32.167773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471777636880084:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.164993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471783322790006:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24871
2025-11-27T18:20:32.167828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:32.165010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:32.167951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471780986278010:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25431
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.167888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471777636880084:2048] [node 1] ICP22 created new session: [3:7577471781931847475:2048]
2025-11-27T18:20:32.167232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471783322789996:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:32.167984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471781931847475:2048] [node 1] ICS09 handshake done sender: [3:7577471781931847461:2117] self: [3:7577471781931847467:0] peer: [1:7577471784174878603:0] socket: 547 qp: -1
2025-11-27T18:20:32.168155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471781441574756:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:32.167773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471779027822621:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.168017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471781931847443:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(21971794ns) {HandleHandshakeStatus:329(21969069ns) {DropIncomingHandshake:442(42174ns) DropOutgoingHandshake:457(508ns) SwitchToState:179(211ns) ProcessPendingSessionEvents:495(18649929ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17501927ns max# 2591540ns}}}
2025-11-27T18:20:32.167910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:32.168055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471781931847475:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.168161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:32.168006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471779027822621:2048] [node 1] ICP22 created new session: [5:7577471783322790009:2048]
2025-11-27T18:20:32.168176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471783322790009:2048] [node 1] ICS09 handshake done sender: [5:7577471783322789996:2116] self: [5:7577471783322790001:0] peer: [1:7577471784174878606:0] socket: 558 qp: -1
2025-11-27T18:20:32.168210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471781931847472:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1359
2025-11-27T18:20:32.168213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471783322790009:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:32.169085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783198252452:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57460
2025-11-27T18:20:32.168631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471781931847458:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:32.169558Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471781441574741:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(20472101ns) {HandleHandshakeStatus:329(20469760ns) {DropIncomingHandshake:442(86035ns) DropOutgoingHandshake:457(1022ns) SwitchToState:179(264ns) ProcessPendingSessionEvents:495(14748379ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14643577ns max# 2798651ns}}}
2025-11-27T18:20:32.168420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:32.170445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471781931847445:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.170570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:32.170431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781441574772:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-27T18:20:32.170605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471781931847445:2109] [node 6] ICP22 created new session: [3:7577471781931847477:2109]
2025-11-27T18:20:32.170623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781441574769:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.170675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471777146607382:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.170675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471781931847477:2109] [node 6] ICS09 handshake done sender: [3:7577471781931847458:2114] self: [3:7577471781931847457:0] peer: [6:7577471780986278006:0] socket: 574 qp: -1
2025-11-27T18:20:32.170788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471781931847472:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.170810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471781931847477:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:32.170939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:32.170881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471780986278011:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25431
2025-11-27T18:20:32.170919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:32.171016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471777146607382:2048] [node 1] ICP22 created new session: [4:7577471781441574773:2048]
2025-11-27T18:20:32.170947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471783198252423:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(21617954ns) {HandleHandshakeStatus:329(21614735ns) {DropIncomingHandshake:442(15266ns) DropOutgoingHandshake:457(420ns) SwitchToState:179(454ns) ProcessPendingSessionEvents:495(15141035ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15010528ns max# 2593619ns}}}
2025-11-27T18:20:32.171062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471781441574773:2048] [node 1] ICS09 handshake done sender: [4:7577471781441574756:2116] self: [4:7577471781441574763:0] peer: [1:7577471784174878604:0] socket: 557 qp: -1
2025-11-27T18:20:32.171085Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471784174878600:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(18762016ns) {HandleHandshakeStatus:329(18758756ns) {DropIncomingHandshake:442(638ns) DropOutgoingHandshake:457(543ns) SwitchToState:179(1489ns) ProcessPendingSessionEvents:495(13861331ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13758009ns max# 2592614ns}}}
2025-11-27T18:20:32.171098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471781441574773:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:32.171182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471780986278003:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:32.171423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471783198252422:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(16839889ns) {HandleHandshakeStatus:329(16837061ns) {DropIncomingHandshake:442(317ns) DropOutgoingHandshake:457(930ns) SwitchToState:179(295ns) ProcessPendingSessionEvents:495(9644927ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9539434ns max# 1047466ns}}}
2025-11-27T18:20:32.171176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:32.171577Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471779879911241:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(21444526ns) {HandleHandshakeStatus:329(21440970ns) {DropIncomingHandshake:442(19403ns) DropOutgoingHandshake:457(612ns) SwitchToState:179(737ns) ProcessPendingSessionEvents:495(16306975ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15819780ns max# 2600610ns}}}
2025-11-27T18:20:32.171581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471776691310629:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.172389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:32.173019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471776691310629:2050] [node 3] ICP22 created new session: [6:7577471780986278012:2050]
2025-11-27T18:20:32.173932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471780986278012:2050] [node 3] ICS09 handshake done sender: [6:7577471780986278003:2116] self: [6:7577471780986278006:0] peer: [3:7577471781931847457:0] socket: 576 qp: -1
2025-11-27T18:20:32.174272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471780986278012:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:32.174508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.175516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471783198252422:2109] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:32.175623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783198252437:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.175216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471777636880085:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.175524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471783322790012:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24871
2025-11-27T18:20:32.175652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471783198252437:2117] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.176596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471783322790012:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.176593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471780986278014:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25431
2025-11-27T18:20:32.176728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471784174878633:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26743
2025-11-27T18:20:32.177348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471780986278010:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.177541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781441574772:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.177845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471777146607383:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.177857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471780986277985:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.178117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783198252439:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.178132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781441574758:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.178150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471783198252439:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.178163Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471781441574758:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.178462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471784174878600:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:32.179779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471783198252437:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:32.181533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471781441574758:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:32.181368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783198252452:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.181417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471778903285062:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.182333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471780986278007:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.181469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:32.181747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471783198252439:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.182216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471778903285062:2049] [node 3] ICP22 created new session: [2:7577471783198252455:2049]
2025-11-27T18:20:32.182494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471776691310627:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.182447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471783198252455:2049] [node 3] ICS09 handshake done sender: [2:7577471783198252437:2117] self: [2:7577471783198252436:0] peer: [3:7577471781931847479:0] socket: 554 qp: -1
2025-11-27T18:20:32.182672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471781931847480:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1359
2025-11-27T18:20:32.182825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471781441574742:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.182495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471783198252455:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:32.183386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471780986278017:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25431
2025-11-27T18:20:32.183066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:32.182641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:32.184067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471779027822623:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.185526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471780986278014:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.184220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471783198252421:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.183112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471781441574742:2109] [node 6] ICP22 created new session: [4:7577471781441574779:2109]
2025-11-27T18:20:32.184231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471784174878618:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.184282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:32.184686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471783198252421:2107] [node 4] ICP22 created new session: [2:7577471783198252457:2107]
2025-11-27T18:20:32.184273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471781441574779:2109] [node 6] ICS09 handshake done sender: [4:7577471781441574758:2117] self: [4:7577471781441574757:0] peer: [6:7577471780986278016:0] socket: 591 qp: -1
2025-11-27T18:20:32.184279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471784174878618:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.189522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471780986278010:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.185697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471783198252457:2107] [node 4] ICS09 handshake done sender: [2:7577471783198252439:2118] self: [2:7577471783198252438:0] peer: [4:7577471781441574776:0] socket: 590 qp: -1
2025-11-27T18:20:32.189525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471783198252457:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:32.185723Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471777636880085:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471781931847472:2120] held: no
2025-11-27T18:20:32.184691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471781441574779:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:32.189586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.184747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:32.189750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471783322790013:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24871
2025-11-27T18:20:32.189825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471778903285061:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.190377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471781931847456:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.189940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471784174878618:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:32.190926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471784174878601:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.190522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781441574778:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-27T18:20:32.190426Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471781931847456:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.192366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:32.192202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471780986277986:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.192778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471784174878601:2109] [node 6] ICP22 created new session: [1:7577471784174878637:2109]
2025-11-27T18:20:32.192732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471780986277985:2105] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14036) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471780986278010:2119] held: no
2025-11-27T18:20:32.194585Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471781931847472:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:32.194736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471781931847456:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:32.194568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471783322790013:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.194345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471784174878637:2109] [node 6] ICS09 handshake done sender: [1:7577471784174878618:2118] self: [1:7577471784174878617:0] peer: [6:7577471780986278018:0] socket: 583 qp: -1
2025-11-27T18:20:32.195115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471777146607383:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471781441574772:2121] held: no
2025-11-27T18:20:32.195423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471777636880085:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.194398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471784174878637:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:32.194604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:32.196750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471783322789994:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.196085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:32.196033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471784174878616:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.196894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471780986277985:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.197853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471781441574772:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:32.196595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471781931847444:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.196791Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471783322789994:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.196074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471784174878616:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.196651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471777636880085:2049] [node 2] ICP22 created new session: [3:7577471781931847482:2049]
2025-11-27T18:20:32.197874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.196735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:32.199545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471780986277985:2105] [node 4] ICP22 created new session: [6:7577471780986278021:2105]
2025-11-27T18:20:32.197852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471781931847444:2108] [node 5] ICP22 created new session: [3:7577471781931847483:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.199954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471780986278021:2105] [node 4] ICS09 handshake done sender: [6:7577471780986278010:2119] self: [6:7577471780986278016:0] peer: [4:7577471781441574757:0] socket: 592 qp: -1
2025-11-27T18:20:32.201252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471784174878616:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:32.197963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471781931847483:2108] [node 5] ICS09 handshake done sender: [3:7577471781931847456:2113] self: [3:7577471781931847453:0] peer: [5:7577471783322790014:0] socket: 543 qp: -1
2025-11-27T18:20:32.201267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471777146607383:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.197965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471781931847482:2049] [node 2] ICS09 handshake done sender: [3:7577471781931847472:2120] self: [3:7577471781931847479:0] peer: [2:7577471783198252436:0] socket: 556 qp: -1
2025-11-27T18:20:32.201322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:32.201565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471783322790016:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24871
2025-11-27T18:20:32.199987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471780986278021:2105] [node 4] ICS10 traffic start
2025-11-27T18:20:32.201370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471777146607383:2049] [node 2] ICP22 created new session: [4:7577471781441574781:2049]
2025-11-27T18:20:32.199609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471781931847483:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:32.200034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:32.199641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:32.201740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471779027822623:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20229) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471783322790012:2121] held: no
2025-11-27T18:20:32.200950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471776691310627:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471780986278007:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.199948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471781931847482:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:32.201601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471780986278019:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25431
2025-11-27T18:20:32.199991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:32.201890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783198252460:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57460
2025-11-27T18:20:32.202027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471781441574781:2049] [node 2] ICS09 handshake done sender: [4:7577471781441574772:2121] self: [4:7577471781441574776:0] peer: [2:7577471783198252438:0] socket: 593 qp: -1
2025-11-27T18:20:32.203431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471781441574781:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:32.203622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471779879911240:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.203842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:32.203969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471779879911240:2048] [node 2] ICP22 created new session: [1:7577471784174878640:2048]
2025-11-27T18:20:32.203471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:32.204565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471783198252452:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:32.204760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471784174878640:2048] [node 2] ICS09 handshake done sender: [1:7577471784174878616:2117] self: [1:7577471784174878614:0] peer: [2:7577471783198252459:0] socket: 582 qp: -1
2025-11-27T18:20:32.204985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471784174878640:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:32.205025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: 2025-11-27T18:20:32.206243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471783322790012:2121] [node 3] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:32.206435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471781441574742:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(17684571ns) {HandleHandshakeStatus:329(17682238ns) {DropIncomingHandshake:442(16285ns) DropOutgoingHandshake:457(564ns) SwitchToState:179(363ns) ProcessPendingSessionEvents:495(10498525ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10367844ns max# 1556099ns}}}
2025-11-27T18:20:32.206530Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471783198252421:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(17142801ns) {HandleHandshakeStatus:329(17137210ns) {DropIncomingHandshake:442(15730ns) DropOutgoingHandshake:457(526ns) SwitchToState:179(353ns) ProcessPendingSessionEvents:495(10462919ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10376331ns max# 1862206ns}}}
2025-11-27T18:20:32.206382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471783322789994:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:32.206551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471780986278007:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:32.206703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471778903285061:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.206632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471779027822623:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.206993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471780986278023:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25431
2025-11-27T18:20:32.206796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:32.206831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:32.207188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471778903285061:2048] [node 1] ICP22 created new session: [2:7577471783198252461:2048]
2025-11-27T18:20:32.206844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471779027822623:2050] [node 3] ICP22 created new session: [5:7577471783322790018:2050]
2025-11-27T18:20:32.207252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471783198252461:2048] [node 1] ICS09 handshake done sender: [2:7577471783198252452:2120] self: [2:7577471783198252459:0] peer: [1:7577471784174878614:0] socket: 588 qp: -1
2025-11-27T18:20:32.207248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471783322790018:2050] [node 3] ICS09 handshake done sender: [5:7577471783322790012:2121] self: [5:7577471783322790014:0] peer: [3:7577471781931847453:0] socket: 534 qp: -1
2025-11-27T18:20:32.207216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471776691310627:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.207352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:32.207318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471783322789981:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.207460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471776691310627:2048] [node 1] ICP22 created new session: [6:7577471780986278024:2048]
2025-11-27T18:20:32.207290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471783198252461:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:32.207374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471783322790018:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:32.207683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471780986278024:2048] [node 1] ICS09 handshake done sender: [6:7577471780986278007:2118] self: [6:7577471780986278018:0] peer: [1:7577471784174878617:0] socket: 584 qp: -1
2025-11-27T18:20:32.207420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:32.207359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:32.207706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471780986278024:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:32.207448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:32.207735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:32.207696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471783322789981:2109] [node 6] ICP22 created new session: [5:7577471783322790020:2109]
2025-11-27T18:20:32.207993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471780986278014:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:32.207788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471783322790020:2109] [node 6] ICS09 handshake done sender: [5:7577471783322789994:2114] self: [5:7577471783322789993:0] peer: [6:7577471780986278020:0] socket: 536 qp: -1
2025-11-27T18:20:32.207948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471783322790020:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:32.207982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:32.209218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471778903285061:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:32.209318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471780986277986:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.209371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:32.209409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471780986277986:2106] [node 5] ICP22 created new session: [6:7577471780986278026:2106]
2025-11-27T18:20:32.209455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471780986278026:2106] [node 5] ICS09 handshake done sender: [6:7577471780986278014:2121] self: [6:7577471780986278020:0] peer: [5:7577471783322789993:0] socket: 548 qp: -1
2025-11-27T18:20:32.209494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471780986278026:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:32.209548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:32.210366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471780986277986:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:32.255088Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:32.255375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471783198252461:2048] [node 1] ICS07 socket disconnect 588 reason# EndOfStream
2025-11-27T18:20:32.255423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471783198252461:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.255512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:32.255584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471783198252461:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:32.256636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:32.256897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471781931847475:2048] [node 1] ICS01 socket: 547 reason# EndOfStream
2025-11-27T18:20:32.256957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471777636880084:2048] [node 1] ICP30 unregister session Session# [3:7577471781931847475:2048] VirtualId# [3:7577471781931847467:0]
2025-11-27T18:20:32.257002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471781931847475:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.257099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:32.257158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:32.258145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471778903285061:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:32.258178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471783198252461:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:32.259363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471778903285061:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:32.260473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471783198252461:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:32.260519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471778903285061:2048] [node 1] ICP30 unregister session Session# [2:7577471783198252461:2048] VirtualId# [2:7577471783198252459:0]
2025-11-27T18:20:32.260623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:32.260657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471778903285061:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.260674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.260850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:32.261114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471781441574773:2048] [node 1] ICS07 socket disconnect 557 reason# EndOfStream
2025-11-27T18:20:32.261151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471781441574773:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.261242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:32.261329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471781441574773:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:32.262082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471777146607382:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:32.262128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471781441574773:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:32.262104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:32.262150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471777146607382:2048] [node 1] ICP30 unregister session Session# [4:7577471781441574773:2048] VirtualId# [4:7577471781441574763:0]
2025-11-27T18:20:32.262291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:32.262329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471777146607382:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.262340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.262392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471783322790009:2048] [node 1] ICS07 socket disconnect 558 reason# EndOfStream
2025-11-27T18:20:32.262428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471783322790009:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.262501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:32.262546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471783322790009:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:32.263300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471779027822621:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:32.263359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471783322790009:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:32.263445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471779027822621:2048] [node 1] ICP30 unregister session Session# [5:7577471783322790009:2048] VirtualId# [5:7577471783322790001:0]
2025-11-27T18:20:32.263483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:32.263524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471779027822621:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.263539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.263540Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:32.263716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471780986278024:2048] [node 1] ICS01 socket: 584 reason# EndOfStream
2025-11-27T18:20:32.263750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471776691310627:2048] [node 1] ICP30 unregister session Session# [6:7577471780986278024:2048] VirtualId# [6:7577471780986278018:0]
2025-11-27T18:20:32.263782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471780986278024:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.263854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:32.263893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:32.271679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471777146607382:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:32.273723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471779027822621:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:32.339591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:32.339851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471781931847482:2049] [node 2] ICS01 socket: 556 reason# EndOfStream
2025-11-27T18:20:32.339914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471777636880085:2049] [node 2] ICP30 unregister session Session# [3:7577471781931847482:2049] VirtualId# [3:7577471781931847479:0]
2025-11-27T18:20:32.339956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471781931847482:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.340058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:32.340121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:32.342349Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:32.342578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471781441574781:2049] [node 2] ICS01 socket: 593 reason# EndOfStream
2025-11-27T18:20:32.342626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471777146607383:2049] [node 2] ICP30 unregister session Session# [4:7577471781441574781:2049] VirtualId# [4:7577471781441574776:0]
2025-11-27T18:20:32.342666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471781441574781:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.342770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:32.342841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:32.343280Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:32.343497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471783322790007:2049] [node 2] ICS07 socket disconnect 565 reason# EndOfStream
2025-11-27T18:20:32.343552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471783322790007:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.343649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:32.343698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471783322790007:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:32.344116Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:32.344278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471780986278008:2049] [node 2] ICS01 socket: 564 reason# EndOfStream
2025-11-27T18:20:32.344309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471776691310628:2049] [node 2] ICP30 unregister session Session# [6:7577471780986278008:2049] VirtualId# [6:7577471780986278002:0]
2025-11-27T18:20:32.344370Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471779027822622:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29920) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:32.344355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471780986278008:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.344432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:32.344454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471783322790007:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:32.344466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:32.344489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471779027822622:2049] [node 2] ICP30 unregister session Session# [5:7577471783322790007:2049] VirtualId# [5:7577471783322790000:0]
2025-11-27T18:20:32.344556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:32.344590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471779027822622:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.344608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29920) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.354626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471779027822622:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:32.395068Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:32.395295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471781441574768:2050] [node 3] ICS01 socket: 551 reason# EndOfStream
2025-11-27T18:20:32.395348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471777146607384:2050] [node 3] ICP30 unregister session Session# [4:7577471781441574768:2050] VirtualId# [4:7577471781441574759:0]
2025-11-27T18:20:32.395394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471781441574768:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.395509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:32.395582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:32.396011Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:32.396233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471783322790018:2050] [node 3] ICS01 socket: 534 reason# EndOfStream
2025-11-27T18:20:32.396279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471779027822623:2050] [node 3] ICP30 unregister session Session# [5:7577471783322790018:2050] VirtualId# [5:7577471783322790014:0]
2025-11-27T18:20:32.396325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471783322790018:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.396414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:32.396464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:32.396774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:32.396942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471780986278012:2050] [node 3] ICS07 socket disconnect 576 reason# EndOfStream
2025-11-27T18:20:32.396976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471780986278012:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.397027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:32.397067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471780986278012:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:32.397613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471776691310629:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20229) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:32.397658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471780986278012:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:32.397690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471776691310629:2050] [node 3] ICP30 unregister session Session# [6:7577471780986278012:2050] VirtualId# [6:7577471780986278006:0]
2025-11-27T18:20:32.397781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:32.397807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471776691310629:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20229) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.397818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20229) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.407864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471776691310629:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:32.460934Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:32.461185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471783322790004:2108] [node 4] ICS01 socket: 550 reason# EndOfStream
2025-11-27T18:20:32.461236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471783322789980:2108] [node 4] ICP30 unregister session Session# [5:7577471783322790004:2108] VirtualId# [5:7577471783322789998:0]
2025-11-27T18:20:32.461307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471783322790004:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.461386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:32.461435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:32.462230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:32.462390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471780986278021:2105] [node 4] ICS01 socket: 592 reason# EndOfStream
2025-11-27T18:20:32.462417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471780986277985:2105] [node 4] ICP30 unregister session Session# [6:7577471780986278021:2105] VirtualId# [6:7577471780986278016:0]
2025-11-27T18:20:32.462438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471780986278021:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.462502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:32.462547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:32.523179Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:32.523343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471780986278026:2106] [node 5] ICS07 socket disconnect 548 reason# EndOfStream
2025-11-27T18:20:32.523374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471780986278026:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:32.523447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:32.523479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471780986278026:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:32.524313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471780986277986:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10081) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:32.524430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471780986278026:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:32.524469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471780986277986:2106] [node 5] ICP30 unregister session Session# [6:7577471780986278026:2106] VirtualId# [6:7577471780986278020:0]
2025-11-27T18:20:32.524566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:32.524598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471780986277986:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10081) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.524621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10081) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:32.534838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471780986277986:2106] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:32.608535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471783269576477:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:32.608951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471783269576478:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:32.629679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471783624576117:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:32.629782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471783624576118:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:32.654500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471783128911860:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:32.654554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471783128911861:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:32.680474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471781358445079:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:32.680503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471781358445080:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:32.680528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471781358445081:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:32.703676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471781029574471:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:32.703719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471781029574472:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:32.703913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471781029574473:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:32.725900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471782682985781:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:32.725936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471782682985783:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:32.726216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471782682985782:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:32.819048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471782682985841:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:32.837432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471783269576541:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:32.841304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471783128911923:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:32.841474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471781029574535:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:32.841486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471783624576180:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:32.837497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471783269576542:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:32.844637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471783128911924:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:32.845246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471781358445142:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:32.845321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471783624576181:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:32.844715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471782682985843:2106] [node 4] ICP01 ready to work
2025-11-27T18:20:32.841486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471783269576543:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:32.844634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471781029574536:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:32.845625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471781358445143:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:32.845942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471783128911925:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:32.844659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471783624576182:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:32.863606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471783624576117:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16822761ns) {Configure:104(16816357ns) {ProcessConfigured:135(628951ns) {StartInitialHandshake:157(627325ns) {DropHandshakes:470(80985ns) {DropIncomingHandshake:442(284ns) DropOutgoingHandshake:457(603ns)} GenerateSessionVirtualId:988(682ns) PrepareNewSessionHandshake:477(1349ns) {SwitchToState:179(767ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:32.864994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471783269576552:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44828
2025-11-27T18:20:32.865204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471783128911934:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61495
2025-11-27T18:20:32.865348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781358445151:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53557
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.867717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781358445151:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.867421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471783128911934:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.867360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471783269576552:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.869232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471783128911938:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61495
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.871954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471781029574550:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19954
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.875273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781358445157:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53557
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.878015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471783269576557:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44828
2025-11-27T18:20:32.872056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471781029574551:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19954
2025-11-27T18:20:32.877142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471781029574550:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.878584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471783128911939:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61495
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.878600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781358445156:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53557
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.878810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471781029574552:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19954
2025-11-27T18:20:32.879675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781358445156:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.880131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471783128911931:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.880189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471782682985857:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24836
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.880147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471781358445079:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.880057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471781029574473:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.881777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471783269576548:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.880161Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471783128911931:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.880763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783624576194:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2604
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.881862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471781029574552:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.882567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471781029574551:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.881804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471782682985858:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24836
2025-11-27T18:20:32.882028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471783128911938:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.882455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781358445157:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.881919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783624576195:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2604
2025-11-27T18:20:32.881834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471783269576548:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.882051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471782682985857:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.883869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783624576194:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.884227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471781029574536:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.882087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471782682985859:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24836
2025-11-27T18:20:32.884285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783624576196:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2604
2025-11-27T18:20:32.884695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471783128911930:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.885055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471783269576551:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.883860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471782682985783:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.884706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471783128911931:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:32.885097Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471783269576551:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.885070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781358445149:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.884683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783624576195:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.885117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471781358445149:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.884883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783624576196:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.885121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471783269576548:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:32.883861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471782682985859:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.884730Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471783128911930:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.885021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471783624576117:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.884120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471782682985861:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24836
2025-11-27T18:20:32.885864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471783269576549:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.887404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471783128911930:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:32.884474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471782682985858:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.885832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781358445147:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.887891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471783128911924:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.885880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471783269576549:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.884682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471782682985781:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.885857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471781358445147:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.887942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:32.886747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471783269576557:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.888101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783624576189:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.890653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471781358445149:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:32.891131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471781029574473:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14320) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471781029574550:2116] held: no
2025-11-27T18:20:32.884808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471782682985843:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.888144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471783128911924:2108] [node 5] ICP22 created new session: [3:7577471783128911946:2108]
2025-11-27T18:20:32.890721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471781358445147:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:32.887873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471783269576541:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.888137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471783624576189:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.891374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471781029574556:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19954
2025-11-27T18:20:32.889319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471783128911946:2108] [node 5] ICS09 handshake done sender: [3:7577471783128911931:2111] self: [3:7577471783128911926:0] peer: [5:7577471781029574553:0] socket: 546 qp: -1
2025-11-27T18:20:32.888454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:32.891781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471783624576189:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:32.891723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781358445163:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53557
2025-11-27T18:20:32.885154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471782682985861:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.892483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471781358445142:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.889320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471783269576541:2107] [node 4] ICP22 created new session: [1:7577471783269576561:2107]
2025-11-27T18:20:32.890679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471783128911946:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:32.886729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471782682985782:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.889694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471783269576561:2107] [node 4] ICS09 handshake done sender: [1:7577471783269576548:2111] self: [1:7577471783269576546:0] peer: [4:7577471781358445161:0] socket: 543 qp: -1
2025-11-27T18:20:32.893099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:32.894358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471781358445142:2109] [node 6] ICP22 created new session: [4:7577471781358445166:2109]
2025-11-27T18:20:32.891117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:32.891529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471782682985867:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24836
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.890716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471783269576561:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:32.895854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471781358445143:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.896693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471782682985869:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24836
2025-11-27T18:20:32.897494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471783624576180:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.891125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471783128911939:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.898661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471781029574550:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.891139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:32.895921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:32.897028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471782682985783:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14320) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471782682985857:2114] held: no
2025-11-27T18:20:32.898488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:32.891157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471783128911923:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.900413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471781029574557:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19954
2025-11-27T18:20:32.896952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471781358445143:2108] [node 5] ICP22 created new session: [4:7577471781358445167:2108]
2025-11-27T18:20:32.891218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:32.899883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471782682985868:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24836
2025-11-27T18:20:32.891348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471783269576551:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:32.899534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471783624576180:2109] [node 6] ICP22 created new session: [2:7577471783624576204:2109]
2025-11-27T18:20:32.902319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471781029574558:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19954
2025-11-27T18:20:32.896964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471781358445166:2109] [node 6] ICS09 handshake done sender: [4:7577471781358445147:2110] self: [4:7577471781358445145:0] peer: [6:7577471782682985865:0] socket: 544 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.892401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471783269576549:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:32.891372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471783128911923:2109] [node 6] ICP22 created new session: [3:7577471783128911949:2109]
2025-11-27T18:20:32.900465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471783624576204:2109] [node 6] ICS09 handshake done sender: [2:7577471783624576189:2112] self: [2:7577471783624576184:0] peer: [6:7577471782682985866:0] socket: 559 qp: -1
2025-11-27T18:20:32.901594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471782682985843:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13777) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471782682985858:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.904552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471781029574473:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.892064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471783128911949:2109] [node 6] ICS09 handshake done sender: [3:7577471783128911930:2110] self: [3:7577471783128911927:0] peer: [6:7577471782682985863:0] socket: 542 qp: -1
2025-11-27T18:20:32.897025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471781358445166:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:32.900524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471783624576204:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:32.895258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471783269576543:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.904755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:32.897072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:32.892415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471783128911949:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:32.907734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471782682985870:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24836
2025-11-27T18:20:32.900565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:32.905406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471781029574551:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:32.908584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471782682985857:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:32.895589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:32.895250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:32.901046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783624576186:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.905462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471781029574536:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13777) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471781029574551:2117] held: no
2025-11-27T18:20:32.898780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471781358445167:2108] [node 5] ICS09 handshake done sender: [4:7577471781358445149:2111] self: [4:7577471781358445144:0] peer: [5:7577471781029574555:0] socket: 545 qp: -1
2025-11-27T18:20:32.909807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471782682985859:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:32.896949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471783128911861:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.895997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471783269576543:2110] [node 6] ICP22 created new session: [1:7577471783269576568:2110]
2025-11-27T18:20:32.898851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471781358445156:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:32.906603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471781029574473:2050] [node 3] ICP22 created new session: [5:7577471781029574560:2050]
2025-11-27T18:20:32.897195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471783269576568:2110] [node 6] ICS09 handshake done sender: [1:7577471783269576551:2114] self: [1:7577471783269576547:0] peer: [6:7577471782682985864:0] socket: 553 qp: -1
2025-11-27T18:20:32.901068Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471783624576186:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.899500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471781358445167:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:32.897234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471783269576568:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:32.907742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471781029574560:2050] [node 3] ICS09 handshake done sender: [5:7577471781029574550:2116] self: [5:7577471781029574553:0] peer: [3:7577471783128911926:0] socket: 552 qp: -1
2025-11-27T18:20:32.900422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:32.911655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471782682985783:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.904484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783624576203:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2604
2025-11-27T18:20:32.898455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471783269576477:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.911704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:32.907786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471781029574560:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:32.904752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471781358445079:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.898478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:32.904765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471783624576117:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7343) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471783624576195:2115] held: no
2025-11-27T18:20:32.912017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471782682985858:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.907830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:32.898522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:32.912087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471782682985783:2050] [node 3] ICP22 created new session: [6:7577471782682985873:2050]
2025-11-27T18:20:32.911743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471781029574558:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.905166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:32.909218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471783624576186:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.899539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471783269576477:2048] [node 2] ICP22 created new session: [1:7577471783269576570:2048]
2025-11-27T18:20:32.912379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471781029574471:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.912133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471782682985873:2050] [node 3] ICS09 handshake done sender: [6:7577471782682985857:2114] self: [6:7577471782682985863:0] peer: [3:7577471783128911927:0] socket: 558 qp: -1
2025-11-27T18:20:32.913125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471783624576118:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.905458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471781358445079:2048] [node 1] ICP22 created new session: [4:7577471781358445173:2048]
2025-11-27T18:20:32.914796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471783128911954:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61495
2025-11-27T18:20:32.906979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471781358445173:2048] [node 1] ICS09 handshake done sender: [4:7577471781358445156:2115] self: [4:7577471781358445161:0] peer: [1:7577471783269576546:0] socket: 556 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.899961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471783269576570:2048] [node 2] ICS09 handshake done sender: [1:7577471783269576549:2112] self: [1:7577471783269576544:0] peer: [2:7577471783624576200:0] socket: 550 qp: -1
2025-11-27T18:20:32.912186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471782682985873:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:32.913474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471781029574536:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.913186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:32.913515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:32.912217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:32.901616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471783269576570:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.915955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471783128911953:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61495
2025-11-27T18:20:32.907029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471781358445173:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.913801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471781029574536:2108] [node 4] ICP22 created new session: [5:7577471781029574564:2108]
2025-11-27T18:20:32.913710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471783624576118:2049] [node 3] ICP22 created new session: [2:7577471783624576209:2049]
2025-11-27T18:20:32.902332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:32.914996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471781029574564:2108] [node 4] ICS09 handshake done sender: [5:7577471781029574551:2117] self: [5:7577471781029574555:0] peer: [4:7577471781358445144:0] socket: 557 qp: -1
No RDMA context found for GID: 2025-11-27T18:20:32.908540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471783269576567:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44828
2025-11-27T18:20:32.915148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471781029574564:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:32.913735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471783624576195:2115] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:32.912248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471783269576567:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.914661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471783624576209:2049] [node 3] ICS09 handshake done sender: [2:7577471783624576186:2110] self: [2:7577471783624576183:0] peer: [3:7577471783128911952:0] socket: 560 qp: -1
2025-11-27T18:20:32.907399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:32.914988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471783269576550:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.915193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:32.914691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471783624576209:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:32.915018Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471783269576550:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.917290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781358445168:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53557
2025-11-27T18:20:32.918062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471781029574565:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19954
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.918168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781358445175:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53557
2025-11-27T18:20:32.915128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:32.912415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471782682985861:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:32.918151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471783128911861:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471783128911939:2116] held: no
2025-11-27T18:20:32.913104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471782682985781:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.918114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471783624576117:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.913460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:32.919816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471783269576541:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(24369952ns) {HandleHandshakeStatus:329(24366847ns) {DropIncomingHandshake:442(309ns) DropOutgoingHandshake:457(556ns) SwitchToState:179(357ns) ProcessPendingSessionEvents:495(20118215ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20012643ns max# 3153484ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.918155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:32.913490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471782682985781:2048] [node 1] ICP22 created new session: [6:7577471782682985876:2048]
2025-11-27T18:20:32.918198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471783624576117:2048] [node 1] ICP22 created new session: [2:7577471783624576213:2048]
2025-11-27T18:20:32.913531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471782682985876:2048] [node 1] ICS09 handshake done sender: [6:7577471782682985859:2116] self: [6:7577471782682985864:0] peer: [1:7577471783269576547:0] socket: 571 qp: -1
2025-11-27T18:20:32.913558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471782682985876:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:32.918960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471783624576213:2048] [node 1] ICS09 handshake done sender: [2:7577471783624576195:2115] self: [2:7577471783624576200:0] peer: [1:7577471783269576544:0] socket: 573 qp: -1
2025-11-27T18:20:32.921048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471783269576573:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44828
2025-11-27T18:20:32.918969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783624576210:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2604
2025-11-27T18:20:32.913579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:32.919067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471783624576213:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:32.914706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471782682985843:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.921156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471783128911939:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:32.914739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:32.920510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:32.921193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781358445168:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.921531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471783128911924:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(25737489ns) {HandleHandshakeStatus:329(25735598ns) {DropIncomingHandshake:442(12820ns) DropOutgoingHandshake:457(305ns) SwitchToState:179(302ns) ProcessPendingSessionEvents:495(21235552ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21126828ns max# 3211216ns}}}
2025-11-27T18:20:32.921968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471781029574544:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.914959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471782682985843:2106] [node 4] ICP22 created new session: [6:7577471782682985878:2106]
2025-11-27T18:20:32.920690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471783624576211:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2604
2025-11-27T18:20:32.921506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471781358445081:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.922035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471783128911923:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(25037190ns) {HandleHandshakeStatus:329(25035326ns) {DropIncomingHandshake:442(13812ns) DropOutgoingHandshake:457(351ns) SwitchToState:179(165ns) ProcessPendingSessionEvents:495(18725035ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18318270ns max# 1360444ns}}}
2025-11-27T18:20:32.917183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471782682985878:2106] [node 4] ICS09 handshake done sender: [6:7577471782682985858:2115] self: [6:7577471782682985865:0] peer: [4:7577471781358445145:0] socket: 561 qp: -1
2025-11-27T18:20:32.922781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471783269576477:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(20859782ns) {HandleHandshakeStatus:329(20857091ns) {DropIncomingHandshake:442(15212ns) DropOutgoingHandshake:457(361ns) SwitchToState:179(260ns) ProcessPendingSessionEvents:495(12901856ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11130737ns max# 2114902ns}}}
2025-11-27T18:20:32.922006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471781029574544:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.922744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471781358445142:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(25631462ns) {HandleHandshakeStatus:329(25628616ns) {DropIncomingHandshake:442(12195ns) DropOutgoingHandshake:457(5812ns) SwitchToState:179(1147ns) ProcessPendingSessionEvents:495(19599521ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18031006ns max# 1378580ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.917229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471782682985878:2106] [node 4] ICS10 traffic start
2025-11-27T18:20:32.923720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783624576210:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.923647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471781358445175:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.923298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471783128911861:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.923599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471783269576550:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:32.924299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471781029574565:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.925754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471783624576180:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(20712269ns) {HandleHandshakeStatus:329(20703055ns) {DropIncomingHandshake:442(841ns) DropOutgoingHandshake:457(977ns) SwitchToState:179(278ns) ProcessPendingSessionEvents:495(13006724ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11904587ns max# 1858337ns}}}
2025-11-27T18:20:32.925309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471783269576543:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(22864191ns) {HandleHandshakeStatus:329(22860877ns) {DropIncomingHandshake:442(504ns) DropOutgoingHandshake:457(649ns) SwitchToState:179(142ns) ProcessPendingSessionEvents:495(16490120ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16364549ns max# 2919735ns}}}
2025-11-27T18:20:32.917280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:32.923613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471783128911955:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61495
2025-11-27T18:20:32.923958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471781358445080:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.926155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471781029574567:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19954
2025-11-27T18:20:32.926967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471783269576574:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44828
2025-11-27T18:20:32.927030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471781358445143:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(25029748ns) {HandleHandshakeStatus:329(25026661ns) {DropIncomingHandshake:442(17220ns) DropOutgoingHandshake:457(693ns) SwitchToState:179(385ns) ProcessPendingSessionEvents:495(13223753ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12456621ns max# 1346178ns}}}
2025-11-27T18:20:32.917554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471782682985782:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.926231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471781029574472:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.927180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783624576190:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.923632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:32.917606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:32.926285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471781029574544:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:32.918049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471782682985871:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24836
2025-11-27T18:20:32.927179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471781029574535:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.927217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471783624576190:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.923857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471783128911942:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.927321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:32.918085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471782682985782:2049] [node 2] ICP22 created new session: [6:7577471782682985880:2049]
2025-11-27T18:20:32.927268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471783624576198:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.923884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471783128911942:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.929354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471781029574535:2109] [node 6] ICP22 created new session: [5:7577471781029574570:2109]
2025-11-27T18:20:32.927290Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471783624576198:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.930164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471783269576542:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.918147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471782682985880:2049] [node 2] ICS09 handshake done sender: [6:7577471782682985861:2117] self: [6:7577471782682985866:0] peer: [2:7577471783624576184:0] socket: 574 qp: -1
2025-11-27T18:20:32.923901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471783128911861:2049] [node 2] ICP22 created new session: [3:7577471783128911956:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.930231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:32.918180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471782682985880:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:32.929356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471781029574558:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:32.924294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471783128911956:2049] [node 2] ICS09 handshake done sender: [3:7577471783128911939:2116] self: [3:7577471783128911952:0] peer: [2:7577471783624576183:0] socket: 565 qp: -1
2025-11-27T18:20:32.930275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471783269576542:2108] [node 5] ICP22 created new session: [1:7577471783269576577:2108]
2025-11-27T18:20:32.930620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471783624576190:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:32.918207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:32.930200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471781029574570:2109] [node 6] ICS09 handshake done sender: [5:7577471781029574544:2114] self: [5:7577471781029574543:0] peer: [6:7577471782682985882:0] socket: 541 qp: -1
2025-11-27T18:20:32.925333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471783128911956:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:32.930594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471783269576577:2108] [node 5] ICS09 handshake done sender: [1:7577471783269576550:2113] self: [1:7577471783269576545:0] peer: [5:7577471781029574563:0] socket: 586 qp: -1
2025-11-27T18:20:32.921033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471782682985871:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.925377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:32.930235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471781029574570:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:32.930737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471783269576577:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:32.933436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471783624576198:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:32.921199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471782682985841:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.934666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471783624576182:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.930264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:32.930157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471783128911954:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.931377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:32.934728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:32.934941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471783624576182:2107] [node 4] ICP22 created new session: [2:7577471783624576218:2107]
2025-11-27T18:20:32.931340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471782682985883:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24836
No RDMA context found for GID: 2025-11-27T18:20:32.930405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471783128911955:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:32.930301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471781029574471:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.934239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471782682985841:2105] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62029) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471782682985871:2123] held: no
2025-11-27T18:20:32.931014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471783128911942:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:32.936528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471783624576218:2107] [node 4] ICS09 handshake done sender: [2:7577471783624576190:2113] self: [2:7577471783624576185:0] peer: [4:7577471781358445177:0] socket: 540 qp: -1
2025-11-27T18:20:32.931732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471783128911860:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:32.935806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781358445179:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53557
2025-11-27T18:20:32.930568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:32.937167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471783269576559:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:32.936577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471783624576218:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:32.933169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471783128911925:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.936572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471781358445178:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53557
2025-11-27T18:20:32.933615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:32.937206Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471783269576559:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:32.933794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471783128911925:2107] [node 4] ICP22 created new session: [3:7577471783128911961:2107]
2025-11-27T18:20:32.930906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471781029574471:2048] [node 1] ICP22 created new session: [5:7577471781029574572:2048]
2025-11-27T18:20:32.936579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471783624576181:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.934213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471783128911961:2107] [node 4] ICS09 handshake done sender: [3:7577471783128911942:2117] self: [3:7577471783128911941:0] peer: [4:7577471781358445176:0] socket: 548 qp: -1
2025-11-27T18:20:32.938610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471781358445081:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14320) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471781358445168:2119] held: no
2025-11-27T18:20:32.931487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471781029574572:2048] [node 1] ICS09 handshake done sender: [5:7577471781029574558:2121] self: [5:7577471781029574563:0] peer: [1:7577471783269576545:0] socket: 588 qp: -1
2025-11-27T18:20:32.936613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:32.934588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471783128911961:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:32.931743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471781029574572:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:32.934621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:32.931787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:32.936633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:32.936669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471783624576181:2108] [node 5] ICP22 created new session: [2:7577471783624576220:2108]
2025-11-27T18:20:32.937186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471783624576220:2108] [node 5] ICS09 handshake done sender: [2:7577471783624576198:2117] self: [2:7577471783624576197:0] peer: [5:7577471781029574569:0] socket: 601 qp: -1
2025-11-27T18:20:32.937838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471783624576220:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:32.937879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:32.941898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471781358445080:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471781358445175:2120] held: no
2025-11-27T18:20:32.941910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471781358445175:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:32.942919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471783269576559:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:32.942920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471782682985871:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:32.943641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471781029574574:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19954
2025-11-27T18:20:32.944629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471782682985841:2105] [node 5] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:32.944687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
::ffff:127.1.0.0
2025-11-27T18:20:32.944272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471781358445168:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:32.944738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471782682985841:2105] [node 5] ICP22 created new session: [6:7577471782682985884:2105]
2025-11-27T18:20:32.944778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471783128911963:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61495
2025-11-27T18:20:32.944879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471782682985884:2105] [node 5] ICS09 handshake done sender: [6:7577471782682985871:2123] self: [6:7577471782682985882:0] peer: [5:7577471781029574543:0] socket: 551 qp: -1
2025-11-27T18:20:32.944922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471781358445081:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.945007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471782682985884:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:32.945331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:32.945397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471783269576478:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:32.945227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:32.945371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471781358445080:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.945465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:32.945628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471781358445081:2050] [node 3] ICP22 created new session: [4:7577471781358445180:2050]
2025-11-27T18:20:32.946041Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471783128911955:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:32.946022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471783269576478:2049] [node 3] ICP22 created new session: [1:7577471783269576580:2049]
2025-11-27T18:20:32.945931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:32.947457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471783128911860:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.946070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471783269576580:2049] [node 3] ICS09 handshake done sender: [1:7577471783269576559:2117] self: [1:7577471783269576558:0] peer: [3:7577471783128911960:0] socket: 597 qp: -1
2025-11-27T18:20:32.945957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471781358445180:2050] [node 3] ICS09 handshake done sender: [4:7577471781358445168:2119] self: [4:7577471781358445176:0] peer: [3:7577471783128911941:0] socket: 594 qp: -1
2025-11-27T18:20:32.947594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:32.947649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471783128911860:2048] [node 1] ICP22 created new session: [3:7577471783128911964:2048]
2025-11-27T18:20:32.947086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471783269576580:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:32.947856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471781029574472:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471781029574565:2122] held: no
2025-11-27T18:20:32.945979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471781358445080:2049] [node 2] ICP22 created new session: [4:7577471781358445181:2049]
2025-11-27T18:20:32.948077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471781029574565:2122] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:32.946028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471781358445181:2049] [node 2] ICS09 handshake done sender: [4:7577471781358445175:2120] self: [4:7577471781358445177:0] peer: [2:7577471783624576185:0] socket: 536 qp: -1
2025-11-27T18:20:32.947687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471783128911964:2048] [node 1] ICS09 handshake done sender: [3:7577471783128911955:2120] self: [3:7577471783128911960:0] peer: [1:7577471783269576558:0] socket: 600 qp: -1
2025-11-27T18:20:32.947553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:32.946060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471781358445180:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:32.947707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471783128911964:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:32.947783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:32.946087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:32.946154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471781358445181:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:32.946948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:32.950356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471781029574472:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:32.950397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:32.950447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471781029574472:2049] [node 2] ICP22 created new session: [5:7577471781029574575:2049]
2025-11-27T18:20:32.950489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471781029574575:2049] [node 2] ICS09 handshake done sender: [5:7577471781029574565:2122] self: [5:7577471781029574569:0] peer: [2:7577471783624576197:0] socket: 602 qp: -1
2025-11-27T18:20:32.950513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471781029574575:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:32.950543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:32.950587Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471783128911860:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:33.017233Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:33.017488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471783624576213:2048] [node 1] ICS01 socket: 573 reason# EndOfStream
2025-11-27T18:20:33.017543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471783624576117:2048] [node 1] ICP30 unregister session Session# [2:7577471783624576213:2048] VirtualId# [2:7577471783624576200:0]
2025-11-27T18:20:33.017579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471783624576213:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.017709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:33.017776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:33.018470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:33.018707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471783128911964:2048] [node 1] ICS07 socket disconnect 600 reason# EndOfStream
2025-11-27T18:20:33.018749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471783128911964:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.018835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:33.018885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471783128911964:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:33.020799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:33.021190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471781358445173:2048] [node 1] ICS07 socket disconnect 556 reason# EndOfStream
2025-11-27T18:20:33.021216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471781358445173:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.021289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:33.021335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471781358445173:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:33.021412Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471783128911860:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7343) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:33.021459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471783128911964:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:33.022414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471781358445079:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7343) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.022513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471781358445173:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.022509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471783128911860:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7343) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.022604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471783128911964:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.022595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471781358445079:2048] [node 1] ICP30 unregister session Session# [4:7577471781358445173:2048] VirtualId# [4:7577471781358445161:0]
2025-11-27T18:20:33.022651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471783128911860:2048] [node 1] ICP30 unregister session Session# [3:7577471783128911964:2048] VirtualId# [3:7577471783128911960:0]
2025-11-27T18:20:33.022658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:33.022701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471781358445079:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7343) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.022724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7343) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.022750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:33.022793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471783128911860:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7343) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.022830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:33.022812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7343) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.022963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471781029574572:2048] [node 1] ICS07 socket disconnect 588 reason# EndOfStream
2025-11-27T18:20:33.022988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471781029574572:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.023048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:33.023108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471781029574572:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:33.023779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471781029574471:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7343) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.023835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471781029574572:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.023861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471781029574471:2048] [node 1] ICP30 unregister session Session# [5:7577471781029574572:2048] VirtualId# [5:7577471781029574563:0]
2025-11-27T18:20:33.024017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:33.024061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471781029574471:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7343) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.024077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7343) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.024075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:33.024273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471782682985876:2048] [node 1] ICS01 socket: 571 reason# EndOfStream
2025-11-27T18:20:33.024320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471782682985781:2048] [node 1] ICP30 unregister session Session# [6:7577471782682985876:2048] VirtualId# [6:7577471782682985864:0]
2025-11-27T18:20:33.024353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471782682985876:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.024444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:33.024490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:33.032489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471781358445079:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:33.032514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471783128911860:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:33.033502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471781029574471:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:33.089452Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:33.089701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471783128911956:2049] [node 2] ICS01 socket: 565 reason# EndOfStream
2025-11-27T18:20:33.089755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471783128911861:2049] [node 2] ICP30 unregister session Session# [3:7577471783128911956:2049] VirtualId# [3:7577471783128911952:0]
2025-11-27T18:20:33.089798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471783128911956:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.089896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:33.089954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:33.092887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:33.093092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471781358445181:2049] [node 2] ICS01 socket: 536 reason# EndOfStream
2025-11-27T18:20:33.093136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471781358445080:2049] [node 2] ICP30 unregister session Session# [4:7577471781358445181:2049] VirtualId# [4:7577471781358445177:0]
2025-11-27T18:20:33.093167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471781358445181:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.093239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:33.093311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:33.094162Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:33.094303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471781029574575:2049] [node 2] ICS01 socket: 602 reason# EndOfStream
2025-11-27T18:20:33.094330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471781029574472:2049] [node 2] ICP30 unregister session Session# [5:7577471781029574575:2049] VirtualId# [5:7577471781029574569:0]
2025-11-27T18:20:33.094349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471781029574575:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.094419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:33.094458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:33.095095Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:33.095327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471782682985880:2049] [node 2] ICS07 socket disconnect 574 reason# EndOfStream
2025-11-27T18:20:33.095364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471782682985880:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.095451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:33.095503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471782682985880:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:33.096257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471782682985782:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11274) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.096354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471782682985880:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.096390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471782682985782:2049] [node 2] ICP30 unregister session Session# [6:7577471782682985880:2049] VirtualId# [6:7577471782682985866:0]
2025-11-27T18:20:33.096481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:33.096524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471782682985782:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.096543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11274) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.106256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471782682985782:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:33.174932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:33.175091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471781358445180:2050] [node 3] ICS01 socket: 594 reason# EndOfStream
2025-11-27T18:20:33.175130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471781358445081:2050] [node 3] ICP30 unregister session Session# [4:7577471781358445180:2050] VirtualId# [4:7577471781358445176:0]
2025-11-27T18:20:33.175158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471781358445180:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.175225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:33.175261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:33.175912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:33.176049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471781029574560:2050] [node 3] ICS01 socket: 552 reason# EndOfStream
2025-11-27T18:20:33.176076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471781029574473:2050] [node 3] ICP30 unregister session Session# [5:7577471781029574560:2050] VirtualId# [5:7577471781029574553:0]
2025-11-27T18:20:33.176094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471781029574560:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.176144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:33.176187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:33.177101Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:33.177347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471782682985873:2050] [node 3] ICS01 socket: 558 reason# EndOfStream
2025-11-27T18:20:33.177393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471782682985783:2050] [node 3] ICP30 unregister session Session# [6:7577471782682985873:2050] VirtualId# [6:7577471782682985863:0]
2025-11-27T18:20:33.177431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471782682985873:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.177538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:33.177597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:33.218596Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:33.218762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471781029574564:2108] [node 4] ICS01 socket: 557 reason# EndOfStream
2025-11-27T18:20:33.218801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471781029574536:2108] [node 4] ICP30 unregister session Session# [5:7577471781029574564:2108] VirtualId# [5:7577471781029574555:0]
2025-11-27T18:20:33.218836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471781029574564:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.218896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:33.218927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:33.219498Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:33.219732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471782682985878:2106] [node 4] ICS01 socket: 561 reason# EndOfStream
2025-11-27T18:20:33.219781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471782682985843:2106] [node 4] ICP30 unregister session Session# [6:7577471782682985878:2106] VirtualId# [6:7577471782682985865:0]
2025-11-27T18:20:33.219812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471782682985878:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.219904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:33.219969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:33.283788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:33.284031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471782682985884:2105] [node 5] ICS01 socket: 551 reason# EndOfStream
2025-11-27T18:20:33.284092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471782682985841:2105] [node 5] ICP30 unregister session Session# [6:7577471782682985884:2105] VirtualId# [6:7577471782682985882:0]
2025-11-27T18:20:33.284133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471782682985884:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.284255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:33.284307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:33.400094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471788371686626:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:33.400216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471788371686625:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:33.419536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471786927455189:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:33.419569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471786927455190:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:33.438575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471787159992080:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:33.438598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471787159992081:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:33.457988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471787749544358:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:33.458166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471787749544359:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:33.458272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471787749544360:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:33.477082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471785513488808:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:33.477084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471785513488809:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:33.477124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471785513488810:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:33.496602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471786089239295:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:33.496658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471786089239296:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:33.497078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471786089239297:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:33.542226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471786089239356:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:33.550122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471788371686688:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:33.583214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471787159992142:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:33.567716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471788371686689:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:33.587868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471785513488871:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:33.589306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471787749544422:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:33.583330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471788371686690:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:33.590086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471786927455252:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:33.590499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471787159992145:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:33.590928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471785513488872:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:33.590497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471787159992144:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:33.590471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471786927455253:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:33.591286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471786089239359:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:33.590498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471786927455254:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:33.591081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471787749544423:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:33.596065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471788371686626:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(53841367ns) {Configure:104(53838602ns) {ProcessConfigured:135(169151ns) {StartInitialHandshake:157(167327ns) {DropHandshakes:470(1256ns) {DropIncomingHandshake:442(390ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(382ns) PrepareNewSessionHandshake:477(1966ns) {SwitchToState:179(996ns)}}}}}
2025-11-27T18:20:33.596277Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471788371686625:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(53889514ns) {Configure:104(53884203ns) {ProcessConfigured:135(215212ns) {StartInitialHandshake:157(213279ns) {DropHandshakes:470(1098ns) {DropIncomingHandshake:442(346ns) DropOutgoingHandshake:457(174ns)} GenerateSessionVirtualId:988(438ns) PrepareNewSessionHandshake:477(1342ns) {SwitchToState:179(600ns)}}}}}
2025-11-27T18:20:33.597522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471786089239356:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(54501726ns) {Configure:104(54495999ns) {ProcessConfigured:135(284834ns) {StartInitialHandshake:157(282696ns) {DropHandshakes:470(1289ns) {DropIncomingHandshake:442(409ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(835ns) PrepareNewSessionHandshake:477(1675ns) {SwitchToState:179(667ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.602005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7577471787749544358:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16754842ns) {Configure:104(16746725ns) {ProcessConfigured:135(562985ns) {StartInitialHandshake:157(560863ns) {DropHandshakes:470(2193ns) {DropIncomingHandshake:442(554ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(745ns) PrepareNewSessionHandshake:477(1153ns) {SwitchToState:179(126ns)}}}}}
2025-11-27T18:20:33.601981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471786089239370:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1901
2025-11-27T18:20:33.605396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471786089239370:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.607339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471786089239295:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.609335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471788371686696:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.611069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471786927455268:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22822
2025-11-27T18:20:33.609375Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471788371686696:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.611221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471785513488882:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59709
2025-11-27T18:20:33.611398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471787749544434:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12293
2025-11-27T18:20:33.611420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471786089239374:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1901
2025-11-27T18:20:33.611093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471786927455265:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22822
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.612223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471786927455265:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.612517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471787749544434:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.612291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471788371686696:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:33.612568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471785513488882:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: 2025-11-27T18:20:33.612602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471788371686704:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:33.612597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471786927455189:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:33.612615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471788371686705:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
2025-11-27T18:20:33.612604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471787749544435:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12293
2025-11-27T18:20:33.612610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471786927455268:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.612882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471786089239376:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1901
2025-11-27T18:20:33.612745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471787749544439:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12293
No RDMA context found for GID: 2025-11-27T18:20:33.612893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471785513488884:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59709
::ffff:127.1.0.02025-11-27T18:20:33.612951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471786089239377:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1901
2025-11-27T18:20:33.612917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471786927455270:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22822
2025-11-27T18:20:33.612890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471787749544358:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.612937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471786927455271:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22822
2025-11-27T18:20:33.613184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471787749544435:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.612954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471786089239378:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1901
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.612960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471785513488888:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59709
2025-11-27T18:20:33.613198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471788371686690:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.614801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471786089239379:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1901
2025-11-27T18:20:33.613276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:33.615925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471785513488884:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.613348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471788371686700:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.616910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471785513488809:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.613377Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471788371686700:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.613393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471788371686690:2109] [node 6] ICP22 created new session: [1:7577471788371686708:2109]
2025-11-27T18:20:33.620229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471786927455264:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.620720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471786089239370:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:33.622043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471785513488888:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.620266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471786927455264:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.621794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471787749544439:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.615095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471788371686697:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.621752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471786089239374:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.615127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471788371686697:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.623465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471785513488810:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.620279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471786927455271:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.623660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471787159992151:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.622036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471786089239376:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.624007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471787749544442:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12293
2025-11-27T18:20:33.615148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471788371686708:2109] [node 6] ICS09 handshake done sender: [1:7577471788371686696:2110] self: [1:7577471788371686692:0] peer: [6:7577471786089239375:0] socket: 541 qp: -1
2025-11-27T18:20:33.622069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471786089239295:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.621337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471786927455270:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.623686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471787159992151:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.615174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471788371686708:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:33.615502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:33.624790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471786927455264:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:33.624740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471787749544428:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:33.622117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
::ffff:127.1.0.0
2025-11-27T18:20:33.627984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471787159992159:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9202
2025-11-27T18:20:33.628182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471785513488879:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.623105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471786089239295:2048] [node 1] ICP22 created new session: [6:7577471786089239380:2048]
2025-11-27T18:20:33.626927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471786927455261:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.624761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471787749544428:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.628993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471787159992151:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:33.615873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471788371686707:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
2025-11-27T18:20:33.628228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471785513488879:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.626965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471786927455261:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.616519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471788371686704:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.629726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471787749544428:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:33.623277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471786089239359:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.630439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471785513488891:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59709
2025-11-27T18:20:33.629304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471787159992150:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.623446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471786089239377:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:33.630462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471787749544434:2116] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:33.629714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471786927455261:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:33.616827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471788371686697:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:33.630451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471785513488879:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:33.629332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471787159992150:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.631276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471787749544422:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.630444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471786927455275:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22822
2025-11-27T18:20:33.623592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471786089239378:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.620364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471788371686688:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.632892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471787159992159:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.632962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471785513488893:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59709
2025-11-27T18:20:33.630889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471786927455189:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471786927455265:2115] held: no
2025-11-27T18:20:33.631392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471787749544358:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.623748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471786089239380:2048] [node 1] ICS09 handshake done sender: [6:7577471786089239370:2115] self: [6:7577471786089239375:0] peer: [1:7577471788371686692:0] socket: 544 qp: -1
2025-11-27T18:20:33.632766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471786927455254:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.620401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:33.633040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471787159992150:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.633947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471785513488891:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.621375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471788371686700:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:33.623750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471786089239296:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.632359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:33.633944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471787159992145:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.632997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:33.633985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471785513488871:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.632360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:33.622013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471788371686688:2107] [node 4] ICP22 created new session: [1:7577471788371686714:2107]
2025-11-27T18:20:33.633046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471786927455254:2108] [node 5] ICP22 created new session: [2:7577471786927455279:2108]
2025-11-27T18:20:33.623790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471786089239380:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:33.634046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:33.634244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471787159992080:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.622027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471788371686707:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.632802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471787749544358:2048] [node 1] ICP22 created new session: [4:7577471787749544446:2048]
2025-11-27T18:20:33.633690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471786927455252:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.635000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471785513488871:2109] [node 6] ICP22 created new session: [5:7577471785513488896:2109]
2025-11-27T18:20:33.623838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:33.633954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:33.634269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:33.624990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471786089239356:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.634549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471786927455252:2109] [node 6] ICP22 created new session: [2:7577471786927455280:2109]
2025-11-27T18:20:33.632819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471787749544422:2110] [node 6] ICP22 created new session: [4:7577471787749544445:2110]
2025-11-27T18:20:33.622743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471788371686705:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.635258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471785513488896:2109] [node 6] ICS09 handshake done sender: [5:7577471785513488879:2111] self: [5:7577471785513488877:0] peer: [6:7577471786089239385:0] socket: 574 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.633000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471787749544446:2048] [node 1] ICS09 handshake done sender: [4:7577471787749544434:2116] self: [4:7577471787749544440:0] peer: [1:7577471788371686694:0] socket: 546 qp: -1
2025-11-27T18:20:33.625003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471786089239297:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.634944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471787159992161:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9202
2025-11-27T18:20:33.635495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471785513488808:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-27T18:20:33.631157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471786089239389:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1901
::ffff:127.1.0.02025-11-27T18:20:33.623069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471788371686714:2107] [node 4] ICS09 handshake done sender: [1:7577471788371686697:2111] self: [1:7577471788371686694:0] peer: [4:7577471787749544440:0] socket: 540 qp: -1
No RDMA context found for GID: 
2025-11-27T18:20:33.635552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471785513488896:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:33.634977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471786927455279:2108] [node 5] ICS09 handshake done sender: [2:7577471786927455264:2114] self: [2:7577471786927455258:0] peer: [5:7577471785513488890:0] socket: 556 qp: -1
2025-11-27T18:20:33.635001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471787159992144:2109] [node 6] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.635488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471786089239378:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:33.623104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471788371686714:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:33.635801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.635302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471786927455280:2109] [node 6] ICS09 handshake done sender: [2:7577471786927455261:2111] self: [2:7577471786927455259:0] peer: [6:7577471786089239382:0] socket: 553 qp: -1
2025-11-27T18:20:33.633904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471787749544445:2110] [node 6] ICS09 handshake done sender: [4:7577471787749544428:2111] self: [4:7577471787749544424:0] peer: [6:7577471786089239381:0] socket: 561 qp: -1
2025-11-27T18:20:33.638580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471786089239387:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1901
2025-11-27T18:20:33.635307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471787159992145:2108] [node 5] ICP22 created new session: [3:7577471787159992164:2108]
2025-11-27T18:20:33.635976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471785513488884:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:33.623151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: 2025-11-27T18:20:33.633930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471787749544446:2048] [node 1] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:20:33.640703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471786089239388:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1901
2025-11-27T18:20:33.635514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:33.640342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471785513488895:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59709
2025-11-27T18:20:33.635321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471786927455279:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:33.633968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:33.623249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471788371686625:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.641494Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471786089239359:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1328) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471786089239376:2117] held: no
2025-11-27T18:20:33.635764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471787159992144:2109] [node 6] ICP22 created new session: [3:7577471787159992166:2109]
2025-11-27T18:20:33.634241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471787749544445:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:33.635999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471787159992164:2108] [node 5] ICS09 handshake done sender: [3:7577471787159992151:2111] self: [3:7577471787159992146:0] peer: [5:7577471785513488892:0] socket: 575 qp: -1
2025-11-27T18:20:33.634282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:33.623506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:33.640703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471785513488809:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.636759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471787159992164:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:33.642392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471786089239356:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.645689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471787749544451:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12293
2025-11-27T18:20:33.623921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471788371686625:2048] [node 2] ICP22 created new session: [1:7577471788371686717:2048]
2025-11-27T18:20:33.636795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:33.635376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:33.640750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:33.646807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471787749544452:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12293
2025-11-27T18:20:33.642875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:33.623977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471788371686717:2048] [node 2] ICS09 handshake done sender: [1:7577471788371686700:2114] self: [1:7577471788371686693:0] peer: [2:7577471786927455272:0] socket: 542 qp: -1
2025-11-27T18:20:33.637203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471787159992166:2109] [node 6] ICS09 handshake done sender: [3:7577471787159992150:2110] self: [3:7577471787159992147:0] peer: [6:7577471786089239386:0] socket: 568 qp: -1
2025-11-27T18:20:33.635511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471786927455280:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:20:33.647689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471787749544452:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:33.641510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471785513488809:2049] [node 2] ICP22 created new session: [5:7577471785513488901:2049]
2025-11-27T18:20:33.643815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471786089239356:2106] [node 5] ICP22 created new session: [6:7577471786089239391:2106]
2025-11-27T18:20:33.624007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471788371686717:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:33.637247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471787159992161:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.635541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:33.648180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471787749544451:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.641982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471785513488901:2049] [node 2] ICS09 handshake done sender: [5:7577471785513488884:2116] self: [5:7577471785513488890:0] peer: [2:7577471786927455258:0] socket: 559 qp: -1
2025-11-27T18:20:33.644498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471786089239391:2106] [node 5] ICS09 handshake done sender: [6:7577471786089239378:2119] self: [6:7577471786089239385:0] peer: [5:7577471785513488877:0] socket: 578 qp: -1
2025-11-27T18:20:33.648484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471787749544359:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.637566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471787159992166:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:33.638274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471786927455265:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:33.643405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471785513488901:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:33.649332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471787749544360:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.638239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:33.640981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471786927455189:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.646623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471787159992168:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9202
2025-11-27T18:20:33.644738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471786089239391:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:33.624038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:33.643836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:33.641654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:33.646765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471787159992171:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9202
2025-11-27T18:20:33.641984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471786927455189:2048] [node 1] ICP22 created new session: [2:7577471786927455285:2048]
2025-11-27T18:20:33.651771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471787749544433:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.644998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:33.635749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471788371686698:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.644754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471785513488888:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:33.651793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471787749544433:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.645696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471785513488900:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59709
2025-11-27T18:20:33.645525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471786089239390:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1901
2025-11-27T18:20:33.646943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471787159992173:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9202
2025-11-27T18:20:33.635775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471788371686698:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.642473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471786927455285:2048] [node 1] ICS09 handshake done sender: [2:7577471786927455265:2115] self: [2:7577471786927455272:0] peer: [1:7577471788371686693:0] socket: 549 qp: -1
2025-11-27T18:20:33.652949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471787749544433:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:33.646856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471786089239374:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:33.648411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471787159992159:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:33.638215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471788371686699:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.654923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471787749544423:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.642517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471786927455285:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:33.646145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471785513488810:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.638244Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471788371686699:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.647736Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471786089239296:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9817) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471786089239374:2116] held: no
2025-11-27T18:20:33.648883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471787159992172:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9202
2025-11-27T18:20:33.642921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:33.649234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471786089239376:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:33.654972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:33.646218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:33.649442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471786927455263:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.649505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471786089239377:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:33.655254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471787749544423:2108] [node 5] ICP22 created new session: [4:7577471787749544457:2108]
2025-11-27T18:20:33.649289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471787159992168:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.651179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471786089239359:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.649468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471786927455263:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.641705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471788371686698:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:33.656361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471787749544457:2108] [node 5] ICS09 handshake done sender: [4:7577471787749544433:2115] self: [4:7577471787749544427:0] peer: [5:7577471785513488906:0] socket: 555 qp: -1
2025-11-27T18:20:33.649330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471787159992080:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.646625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471785513488810:2050] [node 3] ICP22 created new session: [5:7577471785513488904:2050]
2025-11-27T18:20:33.652083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471786927455263:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:33.644898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471788371686699:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:33.649546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:33.651630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:33.656392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471787749544457:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:33.646959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471785513488903:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59709
2025-11-27T18:20:33.656649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:33.654166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471786927455253:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.657837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471787749544358:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18832808ns) {HandleHandshakeStatus:329(18830329ns) {DropIncomingHandshake:442(576ns) DropOutgoingHandshake:457(4558ns) SwitchToState:179(746ns) ProcessPendingSessionEvents:495(14856911ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14777257ns max# 2104297ns}}}
2025-11-27T18:20:33.654232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:33.649769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471787159992080:2048] [node 1] ICP22 created new session: [3:7577471787159992174:2048]
2025-11-27T18:20:33.644983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471788371686626:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.658018Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471787749544359:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9817) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471787749544452:2121] held: no
2025-11-27T18:20:33.650221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471787159992174:2048] [node 1] ICS09 handshake done sender: [3:7577471787159992159:2115] self: [3:7577471787159992163:0] peer: [1:7577471788371686695:0] socket: 586 qp: -1
2025-11-27T18:20:33.654557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471786927455253:2107] [node 4] ICP22 created new session: [2:7577471786927455288:2107]
2025-11-27T18:20:33.651841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471786089239296:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.659107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471787749544422:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(20537759ns) {HandleHandshakeStatus:329(20535784ns) {DropIncomingHandshake:442(11606ns) DropOutgoingHandshake:457(466ns) SwitchToState:179(1787ns) ProcessPendingSessionEvents:495(16262523ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15472230ns max# 1754506ns}}}
2025-11-27T18:20:33.645689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:33.650266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471787159992174:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:33.655203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471786927455288:2107] [node 4] ICS09 handshake done sender: [2:7577471786927455263:2113] self: [2:7577471786927455257:0] peer: [4:7577471787749544453:0] socket: 609 qp: -1
No RDMA context found for GID: 2025-11-27T18:20:33.646588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471788371686626:2049] [node 3] ICP22 created new session: [1:7577471788371686723:2049]
2025-11-27T18:20:33.646991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471785513488904:2050] [node 3] ICS09 handshake done sender: [5:7577471785513488888:2117] self: [5:7577471785513488892:0] peer: [3:7577471787159992146:0] socket: 577 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:33.651588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:33.655497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471786927455288:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:33.646829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471788371686723:2049] [node 3] ICS09 handshake done sender: [1:7577471788371686698:2112] self: [1:7577471788371686695:0] peer: [3:7577471787159992163:0] socket: 584 qp: -1
2025-11-27T18:20:33.647797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471785513488904:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:33.652904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471787159992172:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.655698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:33.646856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471788371686723:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:33.647846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:33.652006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:33.657010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471786927455262:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.652939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471787159992155:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.649543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471785513488900:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:33.657033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471786927455262:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.646893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:33.652964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471787159992155:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:33.659224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471787749544456:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12293
2025-11-27T18:20:33.649843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471785513488891:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:33.648394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471788371686689:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.654975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471787159992081:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.649947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471785513488872:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:33.651851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471786089239359:2109] [node 4] ICP22 created new session: [6:7577471786089239393:2109]
2025-11-27T18:20:33.648631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:33.655196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471787159992155:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:33.652796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471786089239296:2049] [node 2] ICP22 created new session: [6:7577471786089239394:2049]
2025-11-27T18:20:33.650478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471785513488808:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.659754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471786927455252:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(19492565ns) {HandleHandshakeStatus:329(19489756ns) {DropIncomingHandshake:442(10096ns) DropOutgoingHandshake:457(944ns) SwitchToState:179(205ns) ProcessPendingSessionEvents:495(14561803ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14466054ns max# 2173199ns}}}
2025-11-27T18:20:33.656495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471787159992142:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.653101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471786089239297:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.648962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471788371686689:2108] [node 5] ICP22 created new session: [1:7577471788371686725:2108]
2025-11-27T18:20:33.651636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:33.659972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471786927455254:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(21604001ns) {HandleHandshakeStatus:329(20579684ns) {DropIncomingHandshake:442(928ns) DropOutgoingHandshake:457(505ns) SwitchToState:179(334ns) ProcessPendingSessionEvents:495(15359741ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15269282ns max# 3417633ns}}}
2025-11-27T18:20:33.649467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471788371686725:2108] [node 5] ICS09 handshake done sender: [1:7577471788371686699:2113] self: [1:7577471788371686691:0] peer: [5:7577471785513488897:0] socket: 590 qp: -1
2025-11-27T18:20:33.653164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471786089239393:2109] [node 4] ICS09 handshake done sender: [6:7577471786089239376:2117] self: [6:7577471786089239381:0] peer: [4:7577471787749544424:0] socket: 562 qp: -1
2025-11-27T18:20:33.660228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471786927455262:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:33.652035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471785513488808:2048] [node 1] ICP22 created new session: [5:7577471785513488907:2048]
2025-11-27T18:20:33.657221Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471787159992145:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17239490ns) {HandleHandshakeStatus:329(17238058ns) {DropIncomingHandshake:442(93ns) DropOutgoingHandshake:457(181ns) SwitchToState:179(164ns) ProcessPendingSessionEvents:495(10794259ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10305750ns max# 907999ns}}}
2025-11-27T18:20:33.653178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471786089239394:2049] [node 2] ICS09 handshake done sender: [6:7577471786089239374:2116] self: [6:7577471786089239382:0] peer: [2:7577471786927455259:0] socket: 554 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.649802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471788371686725:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:33.652069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471785513488907:2048] [node 1] ICS09 handshake done sender: [5:7577471785513488891:2118] self: [5:7577471785513488897:0] peer: [1:7577471788371686691:0] socket: 591 qp: -1
2025-11-27T18:20:33.657769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:33.654084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:33.652725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471785513488907:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:33.654120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471786089239297:2050] [node 3] ICP22 created new session: [6:7577471786089239395:2050]
2025-11-27T18:20:33.649918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471788371686690:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(30806171ns) {HandleHandshakeStatus:329(30804260ns) {DropIncomingHandshake:442(476ns) DropOutgoingHandshake:457(501ns) SwitchToState:179(146ns) ProcessPendingSessionEvents:495(24090039ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23961353ns max# 2105547ns}}}
2025-11-27T18:20:33.652764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:33.650136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:33.657911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471787159992142:2107] [node 4] ICP22 created new session: [3:7577471787159992179:2107]
2025-11-27T18:20:33.659495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471785513488872:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1328) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471785513488900:2121] held: no
2025-11-27T18:20:33.654153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471786089239393:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:33.659023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471787159992179:2107] [node 4] ICS09 handshake done sender: [3:7577471787159992155:2114] self: [3:7577471787159992149:0] peer: [4:7577471787749544454:0] socket: 552 qp: -1
2025-11-27T18:20:33.659654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471785513488871:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(17862545ns) {HandleHandshakeStatus:329(17860290ns) {DropIncomingHandshake:442(16842ns) DropOutgoingHandshake:457(500ns) SwitchToState:179(185ns) ProcessPendingSessionEvents:495(10820069ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10734409ns max# 769247ns}}}
2025-11-27T18:20:33.654167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471786089239394:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:33.659056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471787159992179:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.651590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471788371686720:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23338
2025-11-27T18:20:33.659101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:33.662430Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471785513488809:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17739435ns) {HandleHandshakeStatus:329(17737262ns) {DropIncomingHandshake:442(1517ns) DropOutgoingHandshake:457(7073ns) SwitchToState:179(256ns) ProcessPendingSessionEvents:495(11185103ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10737362ns max# 1367354ns}}}
2025-11-27T18:20:33.654179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:33.652003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471788371686688:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(25813572ns) {HandleHandshakeStatus:329(25811551ns) {DropIncomingHandshake:442(220ns) DropOutgoingHandshake:457(427ns) SwitchToState:179(225ns) ProcessPendingSessionEvents:495(21157504ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20482104ns max# 3203786ns}}}
2025-11-27T18:20:33.662555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471785513488909:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59709
2025-11-27T18:20:33.654213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:33.659832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471787159992144:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(19583574ns) {HandleHandshakeStatus:329(19581351ns) {DropIncomingHandshake:442(278ns) DropOutgoingHandshake:457(274ns) SwitchToState:179(1368ns) ProcessPendingSessionEvents:495(13084830ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11744068ns max# 1286532ns}}}
2025-11-27T18:20:33.662662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471786927455190:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:33.655208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471788371686688:2107] [node 4] ICP18 handshake (actor [1:7577471788371686705:2116]) is too old
2025-11-27T18:20:33.654728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471786089239295:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(26073446ns) {HandleHandshakeStatus:329(26071224ns) {DropIncomingHandshake:442(539ns) DropOutgoingHandshake:457(5140ns) SwitchToState:179(217ns) ProcessPendingSessionEvents:495(19886784ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19779143ns max# 2094566ns}}}
2025-11-27T18:20:33.659995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471787159992144:2109] [node 6] ICP18 handshake (actor [3:7577471787159992161:2116]) is too old
2025-11-27T18:20:33.655427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471788371686625:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(25401626ns) {HandleHandshakeStatus:329(25400488ns) {DropIncomingHandshake:442(14081ns) DropOutgoingHandshake:457(236ns) SwitchToState:179(116ns) ProcessPendingSessionEvents:495(21493824ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21401966ns max# 1739569ns}}}
2025-11-27T18:20:33.654991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471786089239395:2050] [node 3] ICS09 handshake done sender: [6:7577471786089239377:2118] self: [6:7577471786089239386:0] peer: [3:7577471787159992147:0] socket: 569 qp: -1
2025-11-27T18:20:33.660679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471787159992145:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:33.655321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471786089239395:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:33.662575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471788371686626:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:33.662709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471787159992181:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9202
2025-11-27T18:20:33.655504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:33.662807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:33.663022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471786927455190:2049] [node 3] ICP22 created new session: [2:7577471786927455291:2049]
2025-11-27T18:20:33.663278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471786927455291:2049] [node 3] ICS09 handshake done sender: [2:7577471786927455262:2112] self: [2:7577471786927455255:0] peer: [3:7577471787159992177:0] socket: 610 qp: -1
2025-11-27T18:20:33.664117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471786089239356:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(17201512ns) {HandleHandshakeStatus:329(17199445ns) {DropIncomingHandshake:442(1551ns) DropOutgoingHandshake:457(4820ns) SwitchToState:179(316ns) ProcessPendingSessionEvents:495(12255245ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12157377ns max# 1104642ns}}}
2025-11-27T18:20:33.663638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471786927455291:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:33.663841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:33.663870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471787159992172:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:33.663841Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471787749544360:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61987) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471787749544451:2120] held: no
2025-11-27T18:20:33.664211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471786089239356:2106] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:33.664454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471787749544452:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:33.665643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471785513488900:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:33.665789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471787749544459:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12293
2025-11-27T18:20:33.666225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471787749544359:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.666390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:33.666382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471787159992081:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.666567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471787749544451:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:33.666595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471787749544359:2049] [node 2] ICP22 created new session: [4:7577471787749544460:2049]
2025-11-27T18:20:33.666416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471785513488872:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.666735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471787749544460:2049] [node 2] ICS09 handshake done sender: [4:7577471787749544452:2121] self: [4:7577471787749544453:0] peer: [2:7577471786927455257:0] socket: 548 qp: -1
2025-11-27T18:20:33.666464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:33.666770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471787749544460:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:33.666742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:33.666607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471785513488872:2108] [node 4] ICP22 created new session: [5:7577471785513488910:2108]
2025-11-27T18:20:33.666915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:33.666785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471787159992081:2049] [node 2] ICP22 created new session: [3:7577471787159992182:2049]
2025-11-27T18:20:33.667138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471787749544360:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:33.666902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471787159992182:2049] [node 2] ICS09 handshake done sender: [3:7577471787159992172:2119] self: [3:7577471787159992177:0] peer: [2:7577471786927455255:0] socket: 612 qp: -1
2025-11-27T18:20:33.667109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471785513488910:2108] [node 4] ICS09 handshake done sender: [5:7577471785513488900:2121] self: [5:7577471785513488906:0] peer: [4:7577471787749544427:0] socket: 606 qp: -1
2025-11-27T18:20:33.667121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471787159992182:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:33.667231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471785513488910:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:33.667225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:33.667225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:33.667272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:33.667271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471787749544360:2050] [node 3] ICP22 created new session: [4:7577471787749544462:2050]
2025-11-27T18:20:33.667385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471787749544462:2050] [node 3] ICS09 handshake done sender: [4:7577471787749544451:2120] self: [4:7577471787749544454:0] peer: [3:7577471787159992149:0] socket: 608 qp: -1
2025-11-27T18:20:33.667413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471787749544462:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:33.667567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:33.668104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471787159992081:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:33.715668Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:33.715863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471786927455285:2048] [node 1] ICS01 socket: 549 reason# EndOfStream
2025-11-27T18:20:33.715897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471786927455189:2048] [node 1] ICP30 unregister session Session# [2:7577471786927455285:2048] VirtualId# [2:7577471786927455272:0]
2025-11-27T18:20:33.715922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471786927455285:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.715989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:33.716034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:33.716964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:33.717117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471787159992174:2048] [node 1] ICS07 socket disconnect 586 reason# EndOfStream
2025-11-27T18:20:33.717144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471787159992174:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.717205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:33.717243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471787159992174:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:33.720077Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471787159992080:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:33.720061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:33.720111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471787159992174:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:33.720507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471787749544446:2048] [node 1] ICS07 socket disconnect 546 reason# EndOfStream
2025-11-27T18:20:33.720536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471787749544446:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.720611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:33.720660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471787749544446:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:33.721043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471787159992080:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.721148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471787159992174:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.721195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471787159992080:2048] [node 1] ICP30 unregister session Session# [3:7577471787159992174:2048] VirtualId# [3:7577471787159992163:0]
2025-11-27T18:20:33.721314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:33.721354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471787749544358:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.721348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471787159992080:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.721389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471787749544446:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.721365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.721411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471787749544358:2048] [node 1] ICP30 unregister session Session# [4:7577471787749544446:2048] VirtualId# [4:7577471787749544440:0]
2025-11-27T18:20:33.721542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:33.721510Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:33.721574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471787749544358:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.721586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.721652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471785513488907:2048] [node 1] ICS07 socket disconnect 591 reason# EndOfStream
2025-11-27T18:20:33.721680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471785513488907:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.721731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:33.721776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471785513488907:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:33.722453Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471785513488808:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.722535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471785513488907:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.722584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471785513488808:2048] [node 1] ICP30 unregister session Session# [5:7577471785513488907:2048] VirtualId# [5:7577471785513488897:0]
2025-11-27T18:20:33.722719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:33.722752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471785513488808:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.722772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.722762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:33.722932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471786089239380:2048] [node 1] ICS07 socket disconnect 544 reason# EndOfStream
2025-11-27T18:20:33.722956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471786089239380:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.723014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:33.723086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471786089239380:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:33.723652Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471786089239295:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.723775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471786089239380:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.723801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471786089239295:2048] [node 1] ICP30 unregister session Session# [6:7577471786089239380:2048] VirtualId# [6:7577471786089239375:0]
2025-11-27T18:20:33.723876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:33.723903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471786089239295:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.723911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10843) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.730832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471787159992080:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:33.731843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471787749544358:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:33.732881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471785513488808:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:33.733927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471786089239295:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:33.784411Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:33.784592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471787159992182:2049] [node 2] ICS07 socket disconnect 612 reason# EndOfStream
2025-11-27T18:20:33.784622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471787159992182:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.784691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:33.784742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471787159992182:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:33.786539Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471787159992081:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9817) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:33.786566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471787159992182:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:33.788186Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:33.788285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471787159992081:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9817) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.788380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471787159992182:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.788347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471787749544460:2049] [node 2] ICS01 socket: 548 reason# EndOfStream
2025-11-27T18:20:33.788423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471787159992081:2049] [node 2] ICP30 unregister session Session# [3:7577471787159992182:2049] VirtualId# [3:7577471787159992177:0]
2025-11-27T18:20:33.788383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471787749544359:2049] [node 2] ICP30 unregister session Session# [4:7577471787749544460:2049] VirtualId# [4:7577471787749544453:0]
2025-11-27T18:20:33.788408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471787749544460:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.788471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:33.788519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:33.788511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:33.788551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471787159992081:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9817) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.788569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9817) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.789451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:33.789597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471785513488901:2049] [node 2] ICS07 socket disconnect 559 reason# EndOfStream
2025-11-27T18:20:33.789618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471785513488901:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.789670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:33.789707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471785513488901:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:33.790326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471785513488809:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9817) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.790421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471785513488901:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.790471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471785513488809:2049] [node 2] ICP30 unregister session Session# [5:7577471785513488901:2049] VirtualId# [5:7577471785513488890:0]
2025-11-27T18:20:33.790604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:33.790641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471785513488809:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9817) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.790659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9817) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.790623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:33.790835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471786089239394:2049] [node 2] ICS01 socket: 554 reason# EndOfStream
2025-11-27T18:20:33.790908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471786089239296:2049] [node 2] ICP30 unregister session Session# [6:7577471786089239394:2049] VirtualId# [6:7577471786089239382:0]
2025-11-27T18:20:33.790956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471786089239394:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.791030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:33.791080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:33.800471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471785513488809:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:33.842951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:33.843109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471787749544462:2050] [node 3] ICS01 socket: 608 reason# EndOfStream
2025-11-27T18:20:33.843143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471787749544360:2050] [node 3] ICP30 unregister session Session# [4:7577471787749544462:2050] VirtualId# [4:7577471787749544454:0]
2025-11-27T18:20:33.843178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471787749544462:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.843257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:33.843292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:33.844631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:33.844769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471785513488904:2050] [node 3] ICS07 socket disconnect 577 reason# EndOfStream
2025-11-27T18:20:33.844791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471785513488904:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.844845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:33.844879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471785513488904:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:33.845470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:33.845516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471785513488810:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61987) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.845631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471785513488904:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.845675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471785513488810:2050] [node 3] ICP30 unregister session Session# [5:7577471785513488904:2050] VirtualId# [5:7577471785513488892:0]
2025-11-27T18:20:33.845693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471786089239395:2050] [node 3] ICS07 socket disconnect 569 reason# EndOfStream
2025-11-27T18:20:33.845743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471786089239395:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.845805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:33.845830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:33.845861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471785513488810:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61987) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.845884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61987) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.845899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471786089239395:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:33.846580Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471786089239297:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61987) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.846675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471786089239395:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.846716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471786089239297:2050] [node 3] ICP30 unregister session Session# [6:7577471786089239395:2050] VirtualId# [6:7577471786089239386:0]
2025-11-27T18:20:33.846846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:33.846889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471786089239297:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61987) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.846907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61987) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.855772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471785513488810:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:33.856804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471786089239297:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:33.908772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:33.908954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471785513488910:2108] [node 4] ICS01 socket: 606 reason# EndOfStream
2025-11-27T18:20:33.909004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471785513488872:2108] [node 4] ICP30 unregister session Session# [5:7577471785513488910:2108] VirtualId# [5:7577471785513488906:0]
2025-11-27T18:20:33.909035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471785513488910:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.909127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:33.909179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:33.909691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:33.909919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471786089239393:2109] [node 4] ICS01 socket: 562 reason# EndOfStream
2025-11-27T18:20:33.909976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471786089239359:2109] [node 4] ICP30 unregister session Session# [6:7577471786089239393:2109] VirtualId# [6:7577471786089239381:0]
2025-11-27T18:20:33.910004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471786089239393:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.910096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:33.910158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:33.979104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:33.979353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471786089239391:2106] [node 5] ICS07 socket disconnect 578 reason# EndOfStream
2025-11-27T18:20:33.979386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471786089239391:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:33.979465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:33.979514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471786089239391:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:33.980181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471786089239356:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15849) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:33.980258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471786089239391:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:33.980302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471786089239356:2106] [node 5] ICP30 unregister session Session# [6:7577471786089239391:2106] VirtualId# [6:7577471786089239385:0]
2025-11-27T18:20:33.980401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:33.980432Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471786089239356:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15849) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:33.980443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15849) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:34.074429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471789427759380:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:34.074483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471789427759381:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:34.092690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471793633394804:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:34.092729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471793633394805:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:34.110317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471790516306244:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:34.110366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471790516306245:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:34.128707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471790519447378:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:34.128755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471790519447379:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:34.128784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471790519447380:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:34.146712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471790550446883:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:34.146747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471790550446884:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:34.146778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471790550446885:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:34.164090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471791011204937:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:34.164150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471791011204938:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:34.164209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471791011204939:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:34.242096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471791011204998:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:34.251541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471790516306307:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:34.251591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471789427759443:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:34.251658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471793633394867:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:34.252399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471790516306308:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:34.252414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471789427759444:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:34.252372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471790519447442:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:34.252474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471790550446947:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:34.252656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471793633394868:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:34.252833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471790519447443:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:34.252927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471789427759445:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:34.252949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471790550446948:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:34.252999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471790516306309:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:34.253094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471793633394869:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:34.253047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471791011205001:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.262947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471793633394880:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37138
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.263585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471793633394881:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37138
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.264137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471790516306320:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40252
2025-11-27T18:20:34.264209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471790516306321:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40252
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.264576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471790516306322:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40252
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.267940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471789427759456:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.268300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471789427759457:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.268744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471789427759458:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.268896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471789427759459:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:549
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.269061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790550446962:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6661
2025-11-27T18:20:34.269064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790519447457:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12785
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.269281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471789427759457:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.269297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790519447458:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12785
2025-11-27T18:20:34.269465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471789427759456:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.269476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790519447459:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12785
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.270016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471789427759463:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:549
2025-11-27T18:20:34.271052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471790519447457:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.271445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471793633394880:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.271552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471793633394881:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.271605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471790519447379:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.272050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471790519447458:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.272041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471793633394885:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37138
2025-11-27T18:20:34.272059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471790516306320:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.272148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471790519447380:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.272180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471790516306321:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.272175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471789427759458:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.272252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471789427759459:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.272327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471793633394875:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.272349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471793633394875:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.272429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471790516306322:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.272483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471790516306326:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40252
2025-11-27T18:20:34.272562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471790516306315:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.272582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471790516306315:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.272913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471793633394886:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37138
2025-11-27T18:20:34.272969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471793633394887:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37138
2025-11-27T18:20:34.273090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471793633394885:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.273483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471793633394875:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.274096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790519447462:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12785
2025-11-27T18:20:34.274143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471793633394867:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.274159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471790516306245:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.274213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:34.274253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471793633394867:2107] [node 4] ICP22 created new session: [2:7577471793633394892:2107]
2025-11-27T18:20:34.274292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471793633394892:2107] [node 4] ICS09 handshake done sender: [2:7577471793633394875:2112] self: [2:7577471793633394874:0] peer: [4:7577471790519447460:0] socket: 560 qp: -1
2025-11-27T18:20:34.274322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471793633394892:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:34.274348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471790516306315:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:34.274346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:34.274801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471790519447457:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:34.274794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471790516306307:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.275124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471790519447379:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.274857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:34.274912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471790516306307:2107] [node 4] ICP22 created new session: [3:7577471790516306331:2107]
2025-11-27T18:20:34.275329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:34.275516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471790519447379:2049] [node 2] ICP22 created new session: [4:7577471790519447464:2049]
2025-11-27T18:20:34.274952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471790516306331:2107] [node 4] ICS09 handshake done sender: [3:7577471790516306315:2112] self: [3:7577471790516306314:0] peer: [4:7577471790519447461:0] socket: 570 qp: -1
2025-11-27T18:20:34.275693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471790519447464:2049] [node 2] ICS09 handshake done sender: [4:7577471790519447457:2116] self: [4:7577471790519447460:0] peer: [2:7577471793633394874:0] socket: 562 qp: -1
2025-11-27T18:20:34.274978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471790516306331:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:34.275716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471790519447464:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:34.275001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:34.275741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:34.276114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790550446962:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.276534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471793633394873:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.276561Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471793633394873:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.278091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790519447463:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12785
2025-11-27T18:20:34.278141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790550446964:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6661
2025-11-27T18:20:34.278113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471789427759463:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.278177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471790516306326:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.278630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471793633394887:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.278709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471793633394886:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.279016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471793633394873:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:34.279604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471793633394804:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.279660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471793633394805:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.279694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471790516306244:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.279711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:34.279748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471793633394805:2049] [node 3] ICP22 created new session: [2:7577471793633394898:2049]
2025-11-27T18:20:34.279782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471793633394898:2049] [node 3] ICS09 handshake done sender: [2:7577471793633394873:2111] self: [2:7577471793633394872:0] peer: [3:7577471790516306330:0] socket: 558 qp: -1
2025-11-27T18:20:34.279812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471793633394898:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:34.279782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790550446965:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6661
2025-11-27T18:20:34.279835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.280098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790550446964:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.280094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471790516306334:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40252
2025-11-27T18:20:34.280139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471789427759449:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.280183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471789427759449:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.280224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471790550446885:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.281115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471790516306317:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.281146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471790516306317:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.281231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471790519447380:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15517) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471790519447458:2117] held: no
2025-11-27T18:20:34.281483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471789427759449:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:34.281811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471790516306322:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:34.281948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471790519447458:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:34.282118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471789427759447:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.282159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471789427759447:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.282330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471790519447459:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.283522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471789427759381:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.283580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:34.283752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471789427759381:2049] [node 3] ICP22 created new session: [1:7577471789427759470:2049]
2025-11-27T18:20:34.283812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471789427759470:2049] [node 3] ICS09 handshake done sender: [1:7577471789427759449:2111] self: [1:7577471789427759448:0] peer: [3:7577471790516306335:0] socket: 574 qp: -1
2025-11-27T18:20:34.283806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471790519447380:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.283827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471790516306245:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.283855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471789427759470:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:34.283847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:34.283873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:34.283900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:34.283871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471790519447380:2050] [node 3] ICP22 created new session: [4:7577471790519447466:2050]
2025-11-27T18:20:34.283905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471790516306245:2049] [node 2] ICP22 created new session: [3:7577471790516306338:2049]
2025-11-27T18:20:34.283908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471790519447466:2050] [node 3] ICS09 handshake done sender: [4:7577471790519447458:2117] self: [4:7577471790519447461:0] peer: [3:7577471790516306314:0] socket: 576 qp: -1
2025-11-27T18:20:34.283937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471790516306338:2049] [node 2] ICS09 handshake done sender: [3:7577471790516306322:2117] self: [3:7577471790516306330:0] peer: [2:7577471793633394872:0] socket: 561 qp: -1
2025-11-27T18:20:34.283927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471790519447466:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:34.283955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471790516306338:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:34.283948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:34.283981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:34.284218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790550446965:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.284302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471790550446884:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.284337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791011205015:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51512
2025-11-27T18:20:34.284580Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471790516306245:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:34.285660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471790519447378:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.285729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471793633394877:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:34.285761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471793633394877:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.285831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790550446968:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6661
2025-11-27T18:20:34.285900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791011205016:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51512
2025-11-27T18:20:34.286026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471789427759451:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.286056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471789427759451:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.287279Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471789427759447:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:34.287703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471789427759380:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.287797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:34.287849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471789427759380:2048] [node 2] ICP22 created new session: [1:7577471789427759475:2048]
2025-11-27T18:20:34.287896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471789427759475:2048] [node 2] ICS09 handshake done sender: [1:7577471789427759447:2110] self: [1:7577471789427759446:0] peer: [2:7577471793633394897:0] socket: 573 qp: -1
2025-11-27T18:20:34.287884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790550446969:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6661
2025-11-27T18:20:34.287939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471789427759475:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.287979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:34.288014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790550446970:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6661
2025-11-27T18:20:34.288109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471791011205015:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.288249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471791011204938:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.289089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471793633394879:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.289115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471793633394879:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.289862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790550446968:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.289877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471791011205016:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:34.290096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471791011205001:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.290081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471793633394901:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37138
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.290214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471790516306336:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40252
2025-11-27T18:20:34.290267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790550446971:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6661
2025-11-27T18:20:34.290262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471790519447453:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.290381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471790550446947:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.290282Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471790519447453:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.290947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471793633394877:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:34.291022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471790519447451:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.291048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471790519447451:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.291414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471790516306317:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:34.291593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471789427759451:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:34.292764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471790550446965:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.292914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471790516306244:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471790516306326:2118] held: no
2025-11-27T18:20:34.292959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471793633394868:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.293000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791011205019:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51512
2025-11-27T18:20:34.293027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:34.293074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471793633394868:2108] [node 5] ICP22 created new session: [2:7577471793633394904:2108]
2025-11-27T18:20:34.293063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790519447471:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12785
2025-11-27T18:20:34.293170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471793633394904:2108] [node 5] ICS09 handshake done sender: [2:7577471793633394877:2113] self: [2:7577471793633394876:0] peer: [5:7577471790550446967:0] socket: 593 qp: -1
2025-11-27T18:20:34.293198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471793633394904:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:34.293183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471790550446964:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:34.293220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.293538Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471790519447453:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:34.293605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791011205020:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51512
2025-11-27T18:20:34.293631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471790550446884:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.293829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:34.293930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471790550446884:2049] [node 2] ICP22 created new session: [5:7577471790550446973:2049]
2025-11-27T18:20:34.294018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471790516306308:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.294148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471790550446973:2049] [node 2] ICS09 handshake done sender: [5:7577471790550446965:2118] self: [5:7577471790550446967:0] peer: [2:7577471793633394876:0] socket: 594 qp: -1
2025-11-27T18:20:34.294333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:34.294342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471790550446973:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:34.294325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471790519447451:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:34.294393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:34.294402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471790516306308:2108] [node 5] ICP22 created new session: [3:7577471790516306341:2108]
2025-11-27T18:20:34.294459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471790516306341:2108] [node 5] ICS09 handshake done sender: [3:7577471790516306317:2113] self: [3:7577471790516306316:0] peer: [5:7577471790550446966:0] socket: 552 qp: -1
2025-11-27T18:20:34.294452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471791011205016:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:34.294497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471790516306341:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:34.294550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:34.294670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471790519447443:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.294525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471789427759443:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.294731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:34.294591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:34.294759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471790519447443:2109] [node 6] ICP22 created new session: [4:7577471790519447474:2109]
2025-11-27T18:20:34.294798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471790519447474:2109] [node 6] ICS09 handshake done sender: [4:7577471790519447453:2115] self: [4:7577471790519447452:0] peer: [6:7577471791011205018:0] socket: 597 qp: -1
2025-11-27T18:20:34.294847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471791011205001:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.294701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471789427759443:2107] [node 4] ICP22 created new session: [1:7577471789427759478:2107]
2025-11-27T18:20:34.294820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471790519447474:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:34.294897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:34.294838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:34.294751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471789427759478:2107] [node 4] ICS09 handshake done sender: [1:7577471789427759451:2112] self: [1:7577471789427759450:0] peer: [4:7577471790519447470:0] socket: 575 qp: -1
2025-11-27T18:20:34.294938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471791011205001:2107] [node 4] ICP22 created new session: [6:7577471791011205021:2107]
2025-11-27T18:20:34.294784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471789427759478:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:34.294982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471791011205021:2107] [node 4] ICS09 handshake done sender: [6:7577471791011205016:2116] self: [6:7577471791011205018:0] peer: [4:7577471790519447452:0] socket: 598 qp: -1
2025-11-27T18:20:34.294825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:34.295056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471791011205021:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:34.295099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:34.295240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471793633394886:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:34.295506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471793633394804:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471793633394886:2118] held: no
2025-11-27T18:20:34.295674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471790519447442:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.295707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:34.295729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471790519447442:2108] [node 5] ICP22 created new session: [4:7577471790519447476:2108]
2025-11-27T18:20:34.295756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471790519447476:2108] [node 5] ICS09 handshake done sender: [4:7577471790519447451:2114] self: [4:7577471790519447450:0] peer: [5:7577471790550446972:0] socket: 596 qp: -1
2025-11-27T18:20:34.295775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471790519447476:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:34.295793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:34.295928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471793633394804:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.295958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:34.295984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471793633394804:2048] [node 1] ICP22 created new session: [2:7577471793633394906:2048]
2025-11-27T18:20:34.296014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471793633394906:2048] [node 1] ICS09 handshake done sender: [2:7577471793633394886:2118] self: [2:7577471793633394897:0] peer: [1:7577471789427759446:0] socket: 585 qp: -1
2025-11-27T18:20:34.296045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471793633394906:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:34.296074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:34.296262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471790516306326:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:34.296463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471790550446885:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.296674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:34.296715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471790550446885:2050] [node 3] ICP22 created new session: [5:7577471790550446976:2050]
2025-11-27T18:20:34.296744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471790550446976:2050] [node 3] ICS09 handshake done sender: [5:7577471790550446964:2117] self: [5:7577471790550446966:0] peer: [3:7577471790516306316:0] socket: 556 qp: -1
2025-11-27T18:20:34.296770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471790550446976:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:34.296794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:34.296772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471790519447459:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:34.297304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471790516306244:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.297478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:34.297952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471790516306244:2048] [node 1] ICP22 created new session: [3:7577471790516306343:2048]
2025-11-27T18:20:34.298006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471790516306343:2048] [node 1] ICS09 handshake done sender: [3:7577471790516306326:2118] self: [3:7577471790516306335:0] peer: [1:7577471789427759448:0] socket: 587 qp: -1
2025-11-27T18:20:34.298120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471790516306343:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:34.298167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:34.298204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471793633394879:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:34.298244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471790519447378:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.298278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:34.298305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471790519447378:2048] [node 1] ICP22 created new session: [4:7577471790519447478:2048]
2025-11-27T18:20:34.298844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471790519447478:2048] [node 1] ICS09 handshake done sender: [4:7577471790519447459:2118] self: [4:7577471790519447470:0] peer: [1:7577471789427759450:0] socket: 577 qp: -1
2025-11-27T18:20:34.298870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471790519447478:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:34.299833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:34.300455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471790519447378:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:34.300551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471791011205015:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:34.300597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471793633394869:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.300670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:34.301404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471793633394869:2109] [node 6] ICP22 created new session: [2:7577471793633394908:2109]
2025-11-27T18:20:34.301494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471793633394908:2109] [node 6] ICS09 handshake done sender: [2:7577471793633394879:2114] self: [2:7577471793633394878:0] peer: [6:7577471791011205017:0] socket: 569 qp: -1
2025-11-27T18:20:34.301646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471793633394908:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:34.301674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.301857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791011205023:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51512
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.302021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791011205024:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51512
2025-11-27T18:20:34.302205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471791011204938:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.302246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:34.302296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471791011204938:2049] [node 2] ICP22 created new session: [6:7577471791011205026:2049]
2025-11-27T18:20:34.302353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471791011205026:2049] [node 2] ICS09 handshake done sender: [6:7577471791011205015:2115] self: [6:7577471791011205017:0] peer: [2:7577471793633394878:0] socket: 595 qp: -1
2025-11-27T18:20:34.302383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471791011205026:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:34.302444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.303627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790550446969:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.303699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790550446975:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6661
2025-11-27T18:20:34.304267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471790550446968:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.304821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790519447480:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12785
2025-11-27T18:20:34.304868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471790550446883:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.304932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471790550446947:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.304991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:34.305019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471790550446947:2108] [node 4] ICP22 created new session: [5:7577471790550446981:2108]
2025-11-27T18:20:34.305053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471790550446981:2108] [node 4] ICS09 handshake done sender: [5:7577471790550446968:2119] self: [5:7577471790550446972:0] peer: [4:7577471790519447450:0] socket: 599 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.305081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471790550446981:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:34.305107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:34.305134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471790516306345:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40252
2025-11-27T18:20:34.306118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471791011205023:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.306637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471789427759453:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.306698Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471789427759453:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.307590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471791011205024:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.307616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471789427759453:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:34.307919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471789427759444:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.307976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:34.308017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471789427759444:2108] [node 5] ICP22 created new session: [1:7577471789427759481:2108]
2025-11-27T18:20:34.308071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471789427759481:2108] [node 5] ICS09 handshake done sender: [1:7577471789427759453:2113] self: [1:7577471789427759452:0] peer: [5:7577471790550446980:0] socket: 600 qp: -1
2025-11-27T18:20:34.308111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471789427759481:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:34.308145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:34.308954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471791011204937:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.309069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791011205028:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51512
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.309165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790550446983:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6661
2025-11-27T18:20:34.309522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471791011204939:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.309568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471790516306345:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.309627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471790550446969:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:34.309741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471790516306319:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.309757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471791011205028:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.309778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471790516306319:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.309877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471790550446883:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.309920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471791011204998:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.309909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:34.309936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471790550446883:2048] [node 1] ICP22 created new session: [5:7577471790550446984:2048]
2025-11-27T18:20:34.310349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471790550446984:2048] [node 1] ICS09 handshake done sender: [5:7577471790550446969:2120] self: [5:7577471790550446980:0] peer: [1:7577471789427759452:0] socket: 601 qp: -1
2025-11-27T18:20:34.310383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471790550446984:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:34.310409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:34.310581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471789427759455:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.310620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471789427759455:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.310698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471790516306319:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.312419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791011205032:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51512
2025-11-27T18:20:34.312428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471790516306309:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.312480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:34.312525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471790516306309:2109] [node 6] ICP22 created new session: [3:7577471790516306348:2109]
2025-11-27T18:20:34.312578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471790516306348:2109] [node 6] ICS09 handshake done sender: [3:7577471790516306319:2114] self: [3:7577471790516306318:0] peer: [6:7577471791011205030:0] socket: 564 qp: -1
2025-11-27T18:20:34.312613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471790516306348:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:34.312647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:34.312857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471791011205024:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:34.313118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471791011204939:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.313164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:34.313206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471791011204939:2050] [node 3] ICP22 created new session: [6:7577471791011205033:2050]
2025-11-27T18:20:34.313282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471791011205033:2050] [node 3] ICS09 handshake done sender: [6:7577471791011205024:2120] self: [6:7577471791011205030:0] peer: [3:7577471790516306318:0] socket: 553 qp: -1
2025-11-27T18:20:34.313321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471791011205033:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:34.313356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:34.314252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790550446958:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.314276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471790550446958:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.314391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471789427759455:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:34.314903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471789427759445:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.314969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:34.315015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471789427759445:2109] [node 6] ICP22 created new session: [1:7577471789427759484:2109]
2025-11-27T18:20:34.315064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471789427759484:2109] [node 6] ICS09 handshake done sender: [1:7577471789427759455:2114] self: [1:7577471789427759454:0] peer: [6:7577471791011205029:0] socket: 611 qp: -1
2025-11-27T18:20:34.315101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471789427759484:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:34.315134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:34.315427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471790550446958:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:34.315686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471790550446948:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.315723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:34.315751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471790550446948:2109] [node 6] ICP22 created new session: [5:7577471790550446987:2109]
2025-11-27T18:20:34.315783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471790550446987:2109] [node 6] ICS09 handshake done sender: [5:7577471790550446958:2115] self: [5:7577471790550446957:0] peer: [6:7577471791011205031:0] socket: 586 qp: -1
2025-11-27T18:20:34.315805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471790550446987:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:34.315830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:34.319700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471791011204937:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471791011205023:2119] held: no
2025-11-27T18:20:34.320274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471791011204998:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471791011205028:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.321545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791011205035:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51512
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.321950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791011205036:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51512
2025-11-27T18:20:34.322320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471791011205023:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:34.322518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471791011205028:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:34.322694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471791011204937:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.322734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:34.322786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471791011204937:2048] [node 1] ICP22 created new session: [6:7577471791011205037:2048]
2025-11-27T18:20:34.322831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471791011205037:2048] [node 1] ICS09 handshake done sender: [6:7577471791011205023:2119] self: [6:7577471791011205029:0] peer: [1:7577471789427759454:0] socket: 546 qp: -1
2025-11-27T18:20:34.322868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471791011205037:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:34.322922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:34.323784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471791011204998:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.323819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:34.323860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471791011204998:2106] [node 5] ICP22 created new session: [6:7577471791011205039:2106]
2025-11-27T18:20:34.323905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471791011205039:2106] [node 5] ICS09 handshake done sender: [6:7577471791011205028:2121] self: [6:7577471791011205031:0] peer: [5:7577471790550446957:0] socket: 581 qp: -1
2025-11-27T18:20:34.323939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471791011205039:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:34.323971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:34.344315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:34.344468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471793633394906:2048] [node 1] ICS01 socket: 585 reason# EndOfStream
2025-11-27T18:20:34.344500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471793633394804:2048] [node 1] ICP30 unregister session Session# [2:7577471793633394906:2048] VirtualId# [2:7577471793633394897:0]
2025-11-27T18:20:34.344524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471793633394906:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.344590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:34.344628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:34.345453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:34.345652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471790516306343:2048] [node 1] ICS01 socket: 587 reason# EndOfStream
2025-11-27T18:20:34.345692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471790516306244:2048] [node 1] ICP30 unregister session Session# [3:7577471790516306343:2048] VirtualId# [3:7577471790516306335:0]
2025-11-27T18:20:34.345727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471790516306343:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.345828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:34.345886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:34.347743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:34.347985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471790519447478:2048] [node 1] ICS07 socket disconnect 577 reason# EndOfStream
2025-11-27T18:20:34.348031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471790519447478:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.348110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:34.348177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471790519447478:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:34.348447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:34.348574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471790550446984:2048] [node 1] ICS01 socket: 601 reason# EndOfStream
2025-11-27T18:20:34.348605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471790550446883:2048] [node 1] ICP30 unregister session Session# [5:7577471790550446984:2048] VirtualId# [5:7577471790550446980:0]
2025-11-27T18:20:34.348631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471790550446984:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.348684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:34.348731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:34.348950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471790519447378:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.349020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471790519447478:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.349064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471790519447378:2048] [node 1] ICP30 unregister session Session# [4:7577471790519447478:2048] VirtualId# [4:7577471790519447470:0]
2025-11-27T18:20:34.349131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:34.349181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:34.349232Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471790519447378:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.349271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9474) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.349301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471791011205037:2048] [node 1] ICS01 socket: 546 reason# EndOfStream
2025-11-27T18:20:34.349346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471791011204937:2048] [node 1] ICP30 unregister session Session# [6:7577471791011205037:2048] VirtualId# [6:7577471791011205029:0]
2025-11-27T18:20:34.349364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471791011205037:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.349424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:34.349468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:34.359622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471790519447378:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:34.390264Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:34.390495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471790516306338:2049] [node 2] ICS07 socket disconnect 561 reason# EndOfStream
2025-11-27T18:20:34.390525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471790516306338:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.390619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:34.390677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471790516306338:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:34.392903Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:34.393124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471790519447464:2049] [node 2] ICS07 socket disconnect 562 reason# EndOfStream
2025-11-27T18:20:34.393170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471790519447464:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.393272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:34.393338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471790519447464:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:34.393781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471790516306245:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:34.393811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471790516306338:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:34.393985Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:34.394115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471790550446973:2049] [node 2] ICS07 socket disconnect 594 reason# EndOfStream
2025-11-27T18:20:34.394138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471790550446973:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.394191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:34.394233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471790550446973:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:34.394387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471790516306245:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.394444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471790516306338:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.394414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471790519447379:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.394496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471790516306245:2049] [node 2] ICP30 unregister session Session# [3:7577471790516306338:2049] VirtualId# [3:7577471790516306330:0]
2025-11-27T18:20:34.394486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471790519447464:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.394532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471790519447379:2049] [node 2] ICP30 unregister session Session# [4:7577471790519447464:2049] VirtualId# [4:7577471790519447460:0]
2025-11-27T18:20:34.394608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:34.394641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:34.394645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471790516306245:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.394660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.394686Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471790519447379:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.394702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.395050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:34.395159Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471790550446884:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.395189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471790550446973:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.395211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471790550446884:2049] [node 2] ICP30 unregister session Session# [5:7577471790550446973:2049] VirtualId# [5:7577471790550446967:0]
2025-11-27T18:20:34.395266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471791011205026:2049] [node 2] ICS07 socket disconnect 595 reason# EndOfStream
2025-11-27T18:20:34.395275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:34.395295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471790550446884:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.395304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471791011205026:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.395303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.395381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:34.395450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471791011205026:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:34.396173Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471791011204938:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.396238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471791011205026:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.396271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471791011204938:2049] [node 2] ICP30 unregister session Session# [6:7577471791011205026:2049] VirtualId# [6:7577471791011205017:0]
2025-11-27T18:20:34.396366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:34.396401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471791011204938:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.396433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4753) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.404633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471790550446884:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:34.404650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471790519447379:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:34.406677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471791011204938:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:34.415906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:34.416126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471790519447466:2050] [node 3] ICS01 socket: 576 reason# EndOfStream
2025-11-27T18:20:34.416199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471790519447380:2050] [node 3] ICP30 unregister session Session# [4:7577471790519447466:2050] VirtualId# [4:7577471790519447461:0]
2025-11-27T18:20:34.416247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471790519447466:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.416387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:34.416449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:34.416809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:34.416965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471790550446976:2050] [node 3] ICS01 socket: 556 reason# EndOfStream
2025-11-27T18:20:34.416995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471790550446885:2050] [node 3] ICP30 unregister session Session# [5:7577471790550446976:2050] VirtualId# [5:7577471790550446966:0]
2025-11-27T18:20:34.417014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471790550446976:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.417064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:34.417118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:34.417601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:34.417761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471791011205033:2050] [node 3] ICS07 socket disconnect 553 reason# EndOfStream
2025-11-27T18:20:34.417787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471791011205033:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.417875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:34.417932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471791011205033:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:34.418512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471791011204939:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15517) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.418548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471791011205033:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.418572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471791011204939:2050] [node 3] ICP30 unregister session Session# [6:7577471791011205033:2050] VirtualId# [6:7577471791011205030:0]
2025-11-27T18:20:34.418659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:34.418678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471791011204939:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15517) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.418687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15517) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.428213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471791011204939:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:34.460793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:34.461035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471790550446981:2108] [node 4] ICS07 socket disconnect 599 reason# EndOfStream
2025-11-27T18:20:34.461074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471790550446981:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.461180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:34.461247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471790550446981:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:34.461761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:34.462077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471791011205021:2107] [node 4] ICS07 socket disconnect 598 reason# EndOfStream
2025-11-27T18:20:34.462132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471791011205021:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.462183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471790550446947:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61745) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.462249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471790550446981:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.462278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:34.462288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471790550446947:2108] [node 4] ICP30 unregister session Session# [5:7577471790550446981:2108] VirtualId# [5:7577471790550446972:0]
2025-11-27T18:20:34.462336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471791011205021:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:34.462400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:34.462441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471790550446947:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.462459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.463162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471791011205001:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61745) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.463219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471791011205021:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.463260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471791011205001:2107] [node 4] ICP30 unregister session Session# [6:7577471791011205021:2107] VirtualId# [6:7577471791011205018:0]
2025-11-27T18:20:34.463368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:34.463411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471791011205001:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.463426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.473295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471791011205001:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:34.490151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:34.490323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471791011205039:2106] [node 5] ICS01 socket: 581 reason# EndOfStream
2025-11-27T18:20:34.490354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471791011204998:2106] [node 5] ICP30 unregister session Session# [6:7577471791011205039:2106] VirtualId# [6:7577471791011205031:0]
2025-11-27T18:20:34.490379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471791011205039:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.490465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:34.490516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:34.557375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471793641904253:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:34.557420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471793641904254:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:34.576248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471790276999080:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:34.576324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471790276999081:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:34.594437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471789899973973:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:34.594501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471789899973974:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:34.613424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471790763158958:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:34.613494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471790763158960:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:34.613591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471790763158959:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:34.632090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471790520941983:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:34.632122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471790520941984:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:34.632171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471790520941985:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:34.651752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471791483802484:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:34.651796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471791483802485:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:34.651827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471791483802486:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:34.725954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471793641904316:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:34.733622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471790276999144:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:34.733219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471793641904317:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:34.733677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471789899974036:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:34.733626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471793641904318:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:34.733887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471790520942046:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:34.733960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471791483802546:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:34.733706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471790276999145:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:34.733710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471789899974037:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:34.734052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471790520942047:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:34.734269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471791483802547:2106] [node 4] ICP01 ready to work
2025-11-27T18:20:34.734285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471790276999146:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:34.734306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471790763159022:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:34.734317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471790763159021:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:34.734434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471789899974038:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.745402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471789899974047:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9993
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.745866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790763159031:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1055
2025-11-27T18:20:34.746326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471789899974047:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.747356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471790763159031:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.747660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471793641904332:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23904
2025-11-27T18:20:34.747755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471790276999155:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60192
2025-11-27T18:20:34.747868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790763159037:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1055
2025-11-27T18:20:34.747980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471790763158958:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.748272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790520942061:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61507
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.749529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793641904324:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.749561Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471793641904324:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.750371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471793641904333:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23904
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.750912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471789899973973:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.750874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471793641904334:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23904
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.751065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471790276999155:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.751713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471789899974053:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9993
2025-11-27T18:20:34.750930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471793641904335:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23904
2025-11-27T18:20:34.752446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471790276999158:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60192
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.751976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471793641904324:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:34.754545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471789899974054:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9993
2025-11-27T18:20:34.753624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471790276999162:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60192
2025-11-27T18:20:34.754812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471790276999163:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60192
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.755101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793641904322:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.754923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471790276999164:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60192
2025-11-27T18:20:34.755128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471793641904322:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.755198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471790276999080:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.755173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790763159039:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1055
2025-11-27T18:20:34.757113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790520942061:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.755666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471789899974056:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9993
2025-11-27T18:20:34.757133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471790763159037:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.755216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471790276999158:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.757335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793641904332:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.756320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471790276999163:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.757489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471793641904316:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.757344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471790276999162:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.757596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:34.757654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471793641904316:2107] [node 4] ICP22 created new session: [1:7577471793641904338:2107]
2025-11-27T18:20:34.757697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471793641904338:2107] [node 4] ICS09 handshake done sender: [1:7577471793641904324:2112] self: [1:7577471793641904323:0] peer: [4:7577471790763159038:0] socket: 554 qp: -1
2025-11-27T18:20:34.757726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790763159040:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1055
2025-11-27T18:20:34.757727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471793641904338:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:34.757744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471790763158960:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.757761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471789899974054:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.757807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471790276999164:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.757756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793641904320:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.758361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471790763159031:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:34.757789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471793641904320:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.757802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:34.759069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471789899974053:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.759199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471790763158958:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.759086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471789899974044:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.759372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:34.759110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471789899974044:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.759439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471790763158958:2048] [node 1] ICP22 created new session: [4:7577471790763159042:2048]
2025-11-27T18:20:34.759404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471793641904336:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23904
2025-11-27T18:20:34.760610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793641904335:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.760669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471790763159042:2048] [node 1] ICS09 handshake done sender: [4:7577471790763159031:2114] self: [4:7577471790763159038:0] peer: [1:7577471793641904323:0] socket: 557 qp: -1
2025-11-27T18:20:34.760731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471790763159042:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:34.760772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:34.761043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793641904334:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.761150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471789899974044:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:34.761686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793641904333:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.762696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471789899974037:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.762775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.762823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471789899974037:2107] [node 4] ICP22 created new session: [3:7577471789899974061:2107]
2025-11-27T18:20:34.762903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790763159044:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1055
2025-11-27T18:20:34.762898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471789899974061:2107] [node 4] ICS09 handshake done sender: [3:7577471789899974044:2112] self: [3:7577471789899974043:0] peer: [4:7577471790763159041:0] socket: 561 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.762952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471789899974061:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:34.763001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:34.763064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471793641904322:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:34.763077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471790276999170:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60192
2025-11-27T18:20:34.763128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471793641904316:2107] [node 4] ICP18 handshake (actor [1:7577471793641904334:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.764081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471793641904320:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:34.764092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471789899974056:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.764107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790520942063:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61507
2025-11-27T18:20:34.764214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471789899974060:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9993
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.764296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793641904336:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.765367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790520942064:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61507
2025-11-27T18:20:34.765218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471790276999155:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:34.765704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471793641904254:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.765695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471790276999080:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.765992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:34.766210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471793641904254:2049] [node 3] ICP22 created new session: [1:7577471793641904346:2049]
2025-11-27T18:20:34.765996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:34.766183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471790276999080:2048] [node 1] ICP22 created new session: [2:7577471790276999171:2048]
2025-11-27T18:20:34.766280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471793641904346:2049] [node 3] ICS09 handshake done sender: [1:7577471793641904322:2111] self: [1:7577471793641904321:0] peer: [3:7577471789899974055:0] socket: 553 qp: -1
2025-11-27T18:20:34.766311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471789899973974:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.766314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471793641904346:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:34.766313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471790276999171:2048] [node 1] ICS09 handshake done sender: [2:7577471790276999155:2115] self: [2:7577471790276999165:0] peer: [1:7577471793641904319:0] socket: 558 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.766844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471789899974047:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:34.766344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:34.766342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471790276999171:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:34.767714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471789899973973:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.766373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:34.767712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790520942065:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61507
2025-11-27T18:20:34.767768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:34.767822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471789899973973:2048] [node 1] ICP22 created new session: [3:7577471789899974065:2048]
2025-11-27T18:20:34.767852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471790763159037:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:34.767880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471789899974065:2048] [node 1] ICS09 handshake done sender: [3:7577471789899974047:2114] self: [3:7577471789899974055:0] peer: [1:7577471793641904321:0] socket: 556 qp: -1
2025-11-27T18:20:34.767910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471789899974065:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:34.767924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471790276999148:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.767959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471790276999148:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.767940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:34.768764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471790763158960:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2343) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471790763159037:2116] held: no
2025-11-27T18:20:34.769055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471793641904253:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.769112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:34.769163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471793641904253:2048] [node 2] ICP22 created new session: [1:7577471793641904348:2048]
2025-11-27T18:20:34.769190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471793641904348:2048] [node 2] ICS09 handshake done sender: [1:7577471793641904320:2110] self: [1:7577471793641904319:0] peer: [2:7577471790276999165:0] socket: 552 qp: -1
2025-11-27T18:20:34.769592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471793641904348:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:34.769630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:34.769601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471790276999148:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:34.769906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471790763159039:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.770074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471790276999081:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.769911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471790763158960:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.769948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:34.770116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471790763158960:2050] [node 3] ICP22 created new session: [4:7577471790763159046:2050]
2025-11-27T18:20:34.770142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.770188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471790276999081:2049] [node 3] ICP22 created new session: [2:7577471790276999175:2049]
2025-11-27T18:20:34.770165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471790763159046:2050] [node 3] ICS09 handshake done sender: [4:7577471790763159037:2116] self: [4:7577471790763159041:0] peer: [3:7577471789899974043:0] socket: 563 qp: -1
2025-11-27T18:20:34.770192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471790763159046:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:34.770292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471790276999175:2049] [node 3] ICS09 handshake done sender: [2:7577471790276999148:2111] self: [2:7577471790276999147:0] peer: [3:7577471789899974064:0] socket: 587 qp: -1
2025-11-27T18:20:34.770289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:34.770348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471790276999175:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:34.770360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790520942066:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61507
2025-11-27T18:20:34.770390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.772047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471790763158959:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.772084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471789899974067:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9993
2025-11-27T18:20:34.772372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790520942065:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.773466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471789899973974:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471789899974056:2118] held: no
2025-11-27T18:20:34.773485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471790520941985:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.773789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471793641904253:2048] [node 2] ICP18 handshake (actor [1:7577471793641904336:2119]) is too old
2025-11-27T18:20:34.773814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791483802563:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52570
2025-11-27T18:20:34.773917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791483802564:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52570
2025-11-27T18:20:34.773948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790520942066:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.774041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790520942064:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.774072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471790520941984:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.774028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471790276999152:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.774225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471789899974056:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:34.774060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471790276999152:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.775191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471790520942047:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.776018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471789899974046:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.776551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471790276999154:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.776047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471789899974046:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.776314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471790763159030:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.777371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471789899973974:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.776580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471790276999154:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.777801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:34.776357Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471790763159030:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.778752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471790763159030:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:34.778161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471789899973974:2049] [node 2] ICP22 created new session: [3:7577471789899974069:2049]
2025-11-27T18:20:34.778759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471790276999152:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:34.778661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790520942063:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.778281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471789899974069:2049] [node 2] ICS09 handshake done sender: [3:7577471789899974056:2118] self: [3:7577471789899974064:0] peer: [2:7577471790276999147:0] socket: 592 qp: -1
2025-11-27T18:20:34.778652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471791483802563:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.778312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471789899974069:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:34.778898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471790520941983:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.778798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.778929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791483802565:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52570
2025-11-27T18:20:34.779392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791483802566:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52570
2025-11-27T18:20:34.779887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793641904326:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.779919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471790763159022:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.779435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471791483802547:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.779924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471793641904326:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.780784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:34.780904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471790276999144:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.781149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:34.780844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471790763159033:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.781116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471791483802566:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.780866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471790763159022:2108] [node 5] ICP22 created new session: [4:7577471790763159052:2108]
2025-11-27T18:20:34.781171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790520942071:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61507
2025-11-27T18:20:34.781183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471790276999144:2107] [node 4] ICP22 created new session: [2:7577471790276999180:2107]
2025-11-27T18:20:34.780867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471790763159033:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.781215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471791483802484:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.781219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471790276999180:2107] [node 4] ICS09 handshake done sender: [2:7577471790276999152:2113] self: [2:7577471790276999151:0] peer: [4:7577471790763159049:0] socket: 589 qp: -1
2025-11-27T18:20:34.781086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471790763159052:2108] [node 5] ICS09 handshake done sender: [4:7577471790763159030:2113] self: [4:7577471790763159029:0] peer: [5:7577471790520942069:0] socket: 575 qp: -1
2025-11-27T18:20:34.781243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471790276999180:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:34.781294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:34.781141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471790763159052:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:34.781399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471793641904326:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:34.781173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:34.781695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793641904331:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.781644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471791483802564:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.781731Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471793641904331:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.781951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471790763159051:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1055
2025-11-27T18:20:34.781794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471791483802486:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.782490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471793641904317:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.782514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471789899974049:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.781953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471791483802565:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.782617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:34.782545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471789899974049:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.782651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471793641904317:2108] [node 5] ICP22 created new session: [1:7577471793641904352:2108]
2025-11-27T18:20:34.782363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471791483802485:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.782837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471793641904352:2108] [node 5] ICS09 handshake done sender: [1:7577471793641904326:2113] self: [1:7577471793641904325:0] peer: [5:7577471790520942070:0] socket: 601 qp: -1
2025-11-27T18:20:34.783028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471793641904352:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:34.783060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:34.784145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471793641904331:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:34.783618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471790520942064:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:34.783582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471790276999157:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.784120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471789899974046:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:34.783621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471790276999157:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.784450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471789899974049:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:34.785716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471789899974038:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.786059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471789899974036:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.787021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471790276999154:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:34.786915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471793641904318:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.787124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471790520942047:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.786968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:34.786088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:34.787273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:34.787112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471793641904318:2109] [node 6] ICP22 created new session: [1:7577471793641904355:2109]
2025-11-27T18:20:34.786102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: 2025-11-27T18:20:34.787317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471790520942047:2108] [node 4] ICP22 created new session: [5:7577471790520942075:2108]
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.787287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471790763158959:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471790763159039:2117] held: no
2025-11-27T18:20:34.786821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471789899974038:2109] [node 6] ICP22 created new session: [3:7577471789899974073:2109]
2025-11-27T18:20:34.786911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471789899974036:2108] [node 5] ICP22 created new session: [3:7577471789899974074:2108]
2025-11-27T18:20:34.787286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791483802571:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52570
2025-11-27T18:20:34.787239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471793641904355:2109] [node 6] ICS09 handshake done sender: [1:7577471793641904331:2114] self: [1:7577471793641904330:0] peer: [6:7577471791483802568:0] socket: 591 qp: -1
2025-11-27T18:20:34.787562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471790520942075:2108] [node 4] ICS09 handshake done sender: [5:7577471790520942064:2117] self: [5:7577471790520942069:0] peer: [4:7577471790763159029:0] socket: 603 qp: -1
2025-11-27T18:20:34.787191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471789899974074:2108] [node 5] ICS09 handshake done sender: [3:7577471789899974046:2113] self: [3:7577471789899974045:0] peer: [5:7577471790520942067:0] socket: 604 qp: -1
2025-11-27T18:20:34.787300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471793641904355:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:34.787325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:34.787628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791483802572:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52570
2025-11-27T18:20:34.787664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471790520941983:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24669) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471790520942063:2116] held: no
2025-11-27T18:20:34.787224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471789899974073:2109] [node 6] ICS09 handshake done sender: [3:7577471789899974049:2115] self: [3:7577471789899974048:0] peer: [6:7577471791483802569:0] socket: 577 qp: -1
2025-11-27T18:20:34.787637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791483802573:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52570
2025-11-27T18:20:34.787670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471790520942075:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:34.787231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471789899974074:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:34.787695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:34.787265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:34.787292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471789899974073:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:34.787326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:34.788100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471790276999145:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.788163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:34.788329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471790763159039:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:34.788207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471790276999145:2109] [node 5] ICP22 created new session: [2:7577471790276999183:2109]
2025-11-27T18:20:34.788371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471790276999183:2109] [node 5] ICS09 handshake done sender: [2:7577471790276999154:2114] self: [2:7577471790276999153:0] peer: [5:7577471790520942068:0] socket: 588 qp: -1
2025-11-27T18:20:34.788459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471790276999183:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.788555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471791483802566:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:34.788501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:34.788913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471791483802564:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:34.788949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471790763158959:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.788953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790520942072:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61507
2025-11-27T18:20:34.789008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:34.789579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471791483802484:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.789047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471790763158959:2049] [node 2] ICP22 created new session: [4:7577471790763159055:2049]
2025-11-27T18:20:34.789069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471790763159033:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:34.789776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.789390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471790763159055:2049] [node 2] ICS09 handshake done sender: [4:7577471790763159039:2117] self: [4:7577471790763159049:0] peer: [2:7577471790276999151:0] socket: 590 qp: -1
2025-11-27T18:20:34.789625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471790763159055:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:34.789836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471791483802484:2048] [node 1] ICP22 created new session: [6:7577471791483802575:2048]
2025-11-27T18:20:34.789782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:34.789891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471791483802575:2048] [node 1] ICS09 handshake done sender: [6:7577471791483802566:2119] self: [6:7577471791483802568:0] peer: [1:7577471793641904330:0] socket: 609 qp: -1
2025-11-27T18:20:34.789929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471790276999157:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:34.789927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471791483802575:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:34.789869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790520942073:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61507
2025-11-27T18:20:34.789935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471790763159021:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.790024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:34.790036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.790155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471790763159021:2109] [node 6] ICP22 created new session: [4:7577471790763159057:2109]
2025-11-27T18:20:34.790424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471791483802486:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.790512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471790763159057:2109] [node 6] ICS09 handshake done sender: [4:7577471790763159033:2115] self: [4:7577471790763159032:0] peer: [6:7577471791483802567:0] socket: 562 qp: -1
2025-11-27T18:20:34.790465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:34.790541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471790763159057:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:34.790489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471791483802486:2050] [node 3] ICP22 created new session: [6:7577471791483802577:2050]
2025-11-27T18:20:34.790679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471790520942074:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61507
2025-11-27T18:20:34.790624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:34.790622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471791483802577:2050] [node 3] ICS09 handshake done sender: [6:7577471791483802564:2117] self: [6:7577471791483802569:0] peer: [3:7577471789899974048:0] socket: 607 qp: -1
2025-11-27T18:20:34.790749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471790276999146:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.790724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471791483802577:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:34.790786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:34.790755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:34.790814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471790276999146:2110] [node 6] ICP22 created new session: [2:7577471790276999185:2110]
2025-11-27T18:20:34.791028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471790276999185:2110] [node 6] ICS09 handshake done sender: [2:7577471790276999157:2116] self: [2:7577471790276999156:0] peer: [6:7577471791483802570:0] socket: 559 qp: -1
2025-11-27T18:20:34.791066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471790276999185:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:34.791101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:34.792937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471790520941984:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471790520942066:2119] held: no
2025-11-27T18:20:34.795120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471790520941985:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2343) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471790520942065:2118] held: no
2025-11-27T18:20:34.795351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471791483802563:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:34.795455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471790520942063:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:34.796035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471790520942066:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.796445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791483802574:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52570
2025-11-27T18:20:34.796461Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471791483802485:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471791483802565:2118] held: no
2025-11-27T18:20:34.796561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471790520942065:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:34.796916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471790520941984:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.796923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471790520941983:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.796946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:34.796957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:34.796987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471790520941984:2049] [node 2] ICP22 created new session: [5:7577471790520942078:2049]
2025-11-27T18:20:34.797009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471790520941983:2048] [node 1] ICP22 created new session: [5:7577471790520942079:2048]
2025-11-27T18:20:34.797024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471791483802547:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.797017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471790520942078:2049] [node 2] ICS09 handshake done sender: [5:7577471790520942066:2119] self: [5:7577471790520942068:0] peer: [2:7577471790276999153:0] socket: 605 qp: -1
2025-11-27T18:20:34.797082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:34.797038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471790520942078:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:34.797113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471791483802547:2106] [node 4] ICP22 created new session: [6:7577471791483802579:2106]
2025-11-27T18:20:34.797064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:34.797148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471791483802579:2106] [node 4] ICS09 handshake done sender: [6:7577471791483802563:2116] self: [6:7577471791483802567:0] peer: [4:7577471790763159032:0] socket: 606 qp: -1
2025-11-27T18:20:34.797064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471790520942079:2048] [node 1] ICS09 handshake done sender: [5:7577471790520942063:2116] self: [5:7577471790520942070:0] peer: [1:7577471793641904325:0] socket: 602 qp: -1
2025-11-27T18:20:34.797098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471790520942079:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:34.797167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471791483802579:2106] [node 4] ICS10 traffic start
2025-11-27T18:20:34.797134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:34.797191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:34.797824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471791483802565:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:34.798001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471790520941985:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.798031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:34.798101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471790520941985:2050] [node 3] ICP22 created new session: [5:7577471790520942082:2050]
2025-11-27T18:20:34.798126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471790520942082:2050] [node 3] ICS09 handshake done sender: [5:7577471790520942065:2118] self: [5:7577471790520942067:0] peer: [3:7577471789899974045:0] socket: 578 qp: -1
2025-11-27T18:20:34.798144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471790520942082:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:34.798162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:34.798338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471791483802485:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.798376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:34.798411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471791483802485:2049] [node 2] ICP22 created new session: [6:7577471791483802583:2049]
2025-11-27T18:20:34.798444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471791483802583:2049] [node 2] ICS09 handshake done sender: [6:7577471791483802565:2118] self: [6:7577471791483802570:0] peer: [2:7577471790276999156:0] socket: 608 qp: -1
2025-11-27T18:20:34.798468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471791483802583:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:34.798494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.799337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791483802581:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52570
2025-11-27T18:20:34.800464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471791483802581:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.800620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471791483802546:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:34.800836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471790520942055:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:34.800866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471790520942055:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:34.801922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471790520942055:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:34.802093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471790520942046:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:34.802069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471791483802586:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52570
2025-11-27T18:20:34.802140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:34.802198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471790520942046:2109] [node 6] ICP22 created new session: [5:7577471790520942085:2109]
2025-11-27T18:20:34.802253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471790520942085:2109] [node 6] ICS09 handshake done sender: [5:7577471790520942055:2113] self: [5:7577471790520942054:0] peer: [6:7577471791483802585:0] socket: 566 qp: -1
2025-11-27T18:20:34.802283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471790520942085:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:34.802320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:34.802303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471791483802581:2124] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:34.802556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471791483802546:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:34.802586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:34.802660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471791483802546:2107] [node 5] ICP22 created new session: [6:7577471791483802587:2107]
2025-11-27T18:20:34.802698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471791483802587:2107] [node 5] ICS09 handshake done sender: [6:7577471791483802581:2124] self: [6:7577471791483802585:0] peer: [5:7577471790520942054:0] socket: 568 qp: -1
2025-11-27T18:20:34.802751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471791483802587:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:34.802774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:34.803327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471791483802546:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:34.826211Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:34.826445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471790276999171:2048] [node 1] ICS01 socket: 558 reason# EndOfStream
2025-11-27T18:20:34.826492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471790276999080:2048] [node 1] ICP30 unregister session Session# [2:7577471790276999171:2048] VirtualId# [2:7577471790276999165:0]
2025-11-27T18:20:34.826545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471790276999171:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.826670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:34.826734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:34.827288Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:34.827458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471789899974065:2048] [node 1] ICS07 socket disconnect 556 reason# EndOfStream
2025-11-27T18:20:34.827487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471789899974065:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.827549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:34.827605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471789899974065:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:34.828289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471789899973973:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24669) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.828351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471789899974065:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.828389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471789899973973:2048] [node 1] ICP30 unregister session Session# [3:7577471789899974065:2048] VirtualId# [3:7577471789899974055:0]
2025-11-27T18:20:34.828538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:34.828592Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471789899973973:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24669) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.828611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24669) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.830055Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:34.830270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471790763159042:2048] [node 1] ICS07 socket disconnect 557 reason# EndOfStream
2025-11-27T18:20:34.830301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471790763159042:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.830387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:34.830456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471790763159042:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:34.830833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:34.831071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471790520942079:2048] [node 1] ICS01 socket: 602 reason# EndOfStream
2025-11-27T18:20:34.831079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471790763158958:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24669) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.831192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471790763159042:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.831122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471790520941983:2048] [node 1] ICP30 unregister session Session# [5:7577471790520942079:2048] VirtualId# [5:7577471790520942070:0]
2025-11-27T18:20:34.831164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471790520942079:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.831244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471790763158958:2048] [node 1] ICP30 unregister session Session# [4:7577471790763159042:2048] VirtualId# [4:7577471790763159038:0]
2025-11-27T18:20:34.831237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:34.831292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:34.831368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:34.831407Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471790763158958:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24669) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.831419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24669) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.831786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:34.831924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471791483802575:2048] [node 1] ICS07 socket disconnect 609 reason# EndOfStream
2025-11-27T18:20:34.831962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471791483802575:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.832024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:34.832073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471791483802575:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:34.832587Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471791483802484:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24669) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.832625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471791483802575:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.832674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471791483802484:2048] [node 1] ICP30 unregister session Session# [6:7577471791483802575:2048] VirtualId# [6:7577471791483802568:0]
2025-11-27T18:20:34.832754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:34.832788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471791483802484:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24669) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.832799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24669) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.838844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471789899973973:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:34.840867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471790763158958:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:34.842896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471791483802484:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:34.875659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:34.875882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471789899974069:2049] [node 2] ICS01 socket: 592 reason# EndOfStream
2025-11-27T18:20:34.875925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471789899973974:2049] [node 2] ICP30 unregister session Session# [3:7577471789899974069:2049] VirtualId# [3:7577471789899974064:0]
2025-11-27T18:20:34.875958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471789899974069:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.876052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:34.876119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:34.877989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:34.878184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471790763159055:2049] [node 2] ICS01 socket: 590 reason# EndOfStream
2025-11-27T18:20:34.878220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471790763158959:2049] [node 2] ICP30 unregister session Session# [4:7577471790763159055:2049] VirtualId# [4:7577471790763159049:0]
2025-11-27T18:20:34.878249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471790763159055:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.878312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:34.878382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:34.878894Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:34.879105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471790520942078:2049] [node 2] ICS01 socket: 605 reason# EndOfStream
2025-11-27T18:20:34.879163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471790520941984:2049] [node 2] ICP30 unregister session Session# [5:7577471790520942078:2049] VirtualId# [5:7577471790520942068:0]
2025-11-27T18:20:34.879200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471790520942078:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.879273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:34.879318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:34.879879Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:34.880034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471791483802583:2049] [node 2] ICS01 socket: 608 reason# EndOfStream
2025-11-27T18:20:34.880064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471791483802485:2049] [node 2] ICP30 unregister session Session# [6:7577471791483802583:2049] VirtualId# [6:7577471791483802570:0]
2025-11-27T18:20:34.880089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471791483802583:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.880181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:34.880230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:34.920591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:34.920831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471790763159046:2050] [node 3] ICS01 socket: 563 reason# EndOfStream
2025-11-27T18:20:34.920876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471790763158960:2050] [node 3] ICP30 unregister session Session# [4:7577471790763159046:2050] VirtualId# [4:7577471790763159041:0]
2025-11-27T18:20:34.920912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471790763159046:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.921003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:34.921060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:34.921566Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:34.921814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471790520942082:2050] [node 3] ICS01 socket: 578 reason# EndOfStream
2025-11-27T18:20:34.921887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471790520941985:2050] [node 3] ICP30 unregister session Session# [5:7577471790520942082:2050] VirtualId# [5:7577471790520942067:0]
2025-11-27T18:20:34.921934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471790520942082:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.922007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:34.922053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:34.922300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:34.922427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471791483802577:2050] [node 3] ICS07 socket disconnect 607 reason# EndOfStream
2025-11-27T18:20:34.922447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471791483802577:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.922496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:34.922527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471791483802577:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:34.923117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471791483802486:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2343) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.923204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471791483802577:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.923240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471791483802486:2050] [node 3] ICP30 unregister session Session# [6:7577471791483802577:2050] VirtualId# [6:7577471791483802569:0]
2025-11-27T18:20:34.923348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:34.923382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471791483802486:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2343) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.923395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2343) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.933055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471791483802486:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:34.982509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:34.982783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471790520942075:2108] [node 4] ICS07 socket disconnect 603 reason# EndOfStream
2025-11-27T18:20:34.982820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471790520942075:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.982935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:34.982983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471790520942075:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:34.983340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:34.983469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471791483802579:2106] [node 4] ICS07 socket disconnect 606 reason# EndOfStream
2025-11-27T18:20:34.983493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471791483802579:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:34.983559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:34.983592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471791483802579:2106] [node 4] ICS15 start handshake
2025-11-27T18:20:34.983590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471790520942047:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7940) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.983649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471790520942075:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.983686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471790520942047:2108] [node 4] ICP30 unregister session Session# [5:7577471790520942075:2108] VirtualId# [5:7577471790520942069:0]
2025-11-27T18:20:34.983851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:34.983878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471790520942047:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7940) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.983897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7940) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.984291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471791483802547:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7940) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:34.984403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471791483802579:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:34.984452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471791483802547:2106] [node 4] ICP30 unregister session Session# [6:7577471791483802579:2106] VirtualId# [6:7577471791483802567:0]
2025-11-27T18:20:34.984539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:34.984564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471791483802547:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7940) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.984573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7940) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:34.994524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471791483802547:2106] [node 4] ICP33 wake up from error state
2025-11-27T18:20:35.030698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:35.030859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471791483802587:2107] [node 5] ICS07 socket disconnect 568 reason# EndOfStream
2025-11-27T18:20:35.030887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471791483802587:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.030946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:35.030976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471791483802587:2107] [node 5] ICS15 start handshake
2025-11-27T18:20:35.031664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471791483802546:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17392) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:35.031710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471791483802587:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:35.031756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471791483802546:2107] [node 5] ICP30 unregister session Session# [6:7577471791483802587:2107] VirtualId# [6:7577471791483802585:0]
2025-11-27T18:20:35.031856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:35.031890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471791483802546:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17392) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.031904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17392) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.041583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471791483802546:2107] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:35.124719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471793874925673:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:35.124848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471793874925674:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:35.142957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471795805345073:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:35.142982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471795805345072:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:35.161191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471793879128236:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:35.161224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471793879128237:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:35.185040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471794975395939:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:35.185081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471794975395940:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:35.185142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471794975395941:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:35.208282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471794638790780:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:35.208322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471794638790782:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:35.208381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471794638790781:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:35.230880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471795652582102:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:35.230880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471795652582101:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:35.230956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471795652582103:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:35.316789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471793874925736:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:35.326075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471794975396004:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:35.321845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471795805345136:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:35.324432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471793879128301:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:35.321998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471795805345138:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:35.325697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471795652582163:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:35.326137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471793874925737:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:35.325986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471793879128300:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:35.326383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471794975396003:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:35.326453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471793874925738:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:35.326698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471794638790844:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:35.324410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471795805345137:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:35.326898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471795652582165:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:35.327004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471793879128302:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:35.326943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471794638790845:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.342196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471795805345145:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36216
2025-11-27T18:20:35.343538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471795805345145:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.344560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471793874925745:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.345531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471795805345154:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36216
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.345534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471795805345152:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36216
2025-11-27T18:20:35.347810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793874925745:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.348181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471793874925748:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.350697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793874925748:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.350626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471794975396017:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10742
2025-11-27T18:20:35.351196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471793879128309:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.351205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471793874925755:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
2025-11-27T18:20:35.351542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471795805345152:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.351893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471793874925754:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
2025-11-27T18:20:35.352376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471795805345156:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36216
2025-11-27T18:20:35.353327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793874925754:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:35.355400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471794975396017:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.354557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471793879128309:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.355563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471794975396019:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10742
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.356007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471793879128311:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:20:35.355973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471794975395940:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.356057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471793879128310:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:20:35.357004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471795805345144:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.357040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471795805345144:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.357023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471793879128312:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.358132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471793879128311:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.358137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471793879128310:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.358730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471795805345158:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36216
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.358994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471795652582179:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53753
2025-11-27T18:20:35.358949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471795805345154:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.359002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471793879128312:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.359116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793874925755:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.358958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471795805345144:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:35.359234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471793879128320:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:20:35.359270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471793874925758:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49162
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.359337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471793879128237:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.359541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471795805345136:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.359686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471794975396021:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10742
2025-11-27T18:20:35.359705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471794975396019:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.359553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471795805345156:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.360103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471794975395939:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.359607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:35.360964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793874925742:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.360999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471793874925742:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.359712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471795805345136:2107] [node 4] ICP22 created new session: [2:7577471795805345161:2107]
2025-11-27T18:20:35.359997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471795805345142:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.360023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471795805345142:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.360962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471795805345161:2107] [node 4] ICS09 handshake done sender: [2:7577471795805345144:2113] self: [2:7577471795805345141:0] peer: [4:7577471794975396020:0] socket: 573 qp: -1
2025-11-27T18:20:35.362437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471793874925742:2110] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:35.361010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471795805345161:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:35.362245Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471794975395940:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471794975396017:2115] held: no
2025-11-27T18:20:35.362557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793874925743:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.361196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:35.362440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471794975396017:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:35.365593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471793879128320:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.362584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471793874925743:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.362214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471795805345072:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.366664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471795652582179:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.362586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471795805345142:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:35.367007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471795652582103:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.367178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793874925758:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.367256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471794975395940:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.365735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471795805345073:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.367004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471793879128236:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.367232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471793874925736:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.365777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:35.367308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:35.367614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471793879128306:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.367283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471793874925743:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:35.366193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471795805345073:2049] [node 3] ICP22 created new session: [2:7577471795805345166:2049]
2025-11-27T18:20:35.367657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471793879128306:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.367656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471794638790859:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58468
2025-11-27T18:20:35.367414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471794975395940:2049] [node 2] ICP22 created new session: [4:7577471794975396024:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.367628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:35.368572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471793879128325:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:20:35.366897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471795805345166:2049] [node 3] ICS09 handshake done sender: [2:7577471795805345142:2111] self: [2:7577471795805345140:0] peer: [3:7577471793879128322:0] socket: 570 qp: -1
2025-11-27T18:20:35.367682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471793874925736:2107] [node 4] ICP22 created new session: [1:7577471793874925763:2107]
2025-11-27T18:20:35.367780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471794975396024:2049] [node 2] ICS09 handshake done sender: [4:7577471794975396017:2115] self: [4:7577471794975396020:0] peer: [2:7577471795805345141:0] socket: 579 qp: -1
2025-11-27T18:20:35.366939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471795805345166:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:35.368480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793874925744:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.368473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471794975396024:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:35.371223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471793879128311:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:35.366983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.368558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471794975396023:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10742
2025-11-27T18:20:35.368518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471793874925744:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.371336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471793879128306:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.367883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471795805345158:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.372107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471794638790859:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.373145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471794638790860:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58468
2025-11-27T18:20:35.368622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:35.368601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471793874925763:2107] [node 4] ICS09 handshake done sender: [1:7577471793874925742:2110] self: [1:7577471793874925739:0] peer: [4:7577471794975396022:0] socket: 583 qp: -1
2025-11-27T18:20:35.371992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471793879128237:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.368819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471795805345167:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36216
2025-11-27T18:20:35.370851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471794975396019:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:35.373741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471795652582181:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53753
2025-11-27T18:20:35.368645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471793874925763:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:35.371959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471795805345156:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:35.371902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471794975395939:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471794975396019:2117] held: no
No RDMA context found for GID: 2025-11-27T18:20:35.372046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-11-27T18:20:35.368682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:35.373156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471794638790781:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.374329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471795805345072:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.372337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471793879128237:2049] [node 2] ICP22 created new session: [3:7577471793879128328:2049]
2025-11-27T18:20:35.374371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:35.371040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471793874925673:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.375076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471795652582179:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:35.375371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471794975395939:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.373542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471793879128328:2049] [node 2] ICS09 handshake done sender: [3:7577471793879128311:2116] self: [3:7577471793879128322:0] peer: [2:7577471795805345140:0] socket: 580 qp: -1
2025-11-27T18:20:35.374810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471795805345072:2048] [node 1] ICP22 created new session: [2:7577471795805345170:2048]
2025-11-27T18:20:35.371363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:35.373954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471793879128328:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:35.375366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471795652582182:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53753
2025-11-27T18:20:35.375733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:35.374978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471795805345170:2048] [node 1] ICS09 handshake done sender: [2:7577471795805345156:2119] self: [2:7577471795805345165:0] peer: [1:7577471793874925741:0] socket: 587 qp: -1
2025-11-27T18:20:35.373988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:35.371413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471793874925673:2048] [node 2] ICP22 created new session: [1:7577471793874925765:2048]
2025-11-27T18:20:35.375702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471795652582103:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.375777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471794975395939:2048] [node 1] ICP22 created new session: [4:7577471794975396027:2048]
2025-11-27T18:20:35.376090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471794638790860:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.375370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471795805345170:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:35.376091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471794975396027:2048] [node 1] ICS09 handshake done sender: [4:7577471794975396019:2117] self: [4:7577471794975396022:0] peer: [1:7577471793874925739:0] socket: 589 qp: -1
2025-11-27T18:20:35.371761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471793874925765:2048] [node 2] ICS09 handshake done sender: [1:7577471793874925743:2111] self: [1:7577471793874925741:0] peer: [2:7577471795805345165:0] socket: 586 qp: -1
2025-11-27T18:20:35.376002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:35.376490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471794638790844:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.371890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471793874925765:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:35.376298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471795652582103:2050] [node 3] ICP22 created new session: [6:7577471795652582183:2050]
2025-11-27T18:20:35.376324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471794975396027:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:35.375421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:35.374531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471793879128301:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.374580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:35.371932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:35.376799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471795652582183:2050] [node 3] ICS09 handshake done sender: [6:7577471795652582179:2115] self: [6:7577471795652582180:0] peer: [3:7577471793879128304:0] socket: 596 qp: -1
2025-11-27T18:20:35.376458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:35.377333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471793874925744:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:35.374873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471793879128301:2109] [node 6] ICP22 created new session: [3:7577471793879128330:2109]
2025-11-27T18:20:35.376832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471795652582183:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:35.378325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:35.375451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471793879128330:2109] [node 6] ICS09 handshake done sender: [3:7577471793879128306:2112] self: [3:7577471793879128304:0] peer: [6:7577471795652582180:0] socket: 595 qp: -1
2025-11-27T18:20:35.376958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471794975396015:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.376978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471794975396015:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.375497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471793879128330:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:35.379949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471795805345148:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.380181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471794638790863:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58468
2025-11-27T18:20:35.375956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:35.379983Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471795805345148:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.380295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471793874925674:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.381151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471794975396015:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:35.380341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:35.380866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471793874925674:2049] [node 3] ICP22 created new session: [1:7577471793874925771:2049]
No RDMA context found for GID: 2025-11-27T18:20:35.382014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471794975396004:2108] [node 5] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:35.381360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471793874925771:2049] [node 3] ICS09 handshake done sender: [1:7577471793874925744:2112] self: [1:7577471793874925740:0] peer: [3:7577471793879128326:0] socket: 588 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.381548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471793874925771:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.382815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:35.381885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:35.383056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471794975396004:2108] [node 5] ICP22 created new session: [4:7577471794975396031:2108]
2025-11-27T18:20:35.383127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471795652582185:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53753
2025-11-27T18:20:35.383297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471794638790864:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58468
2025-11-27T18:20:35.383241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471795652582186:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53753
2025-11-27T18:20:35.383466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471794975396031:2108] [node 5] ICS09 handshake done sender: [4:7577471794975396015:2114] self: [4:7577471794975396014:0] peer: [5:7577471794638790862:0] socket: 608 qp: -1
2025-11-27T18:20:35.383514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471794975396031:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:35.384671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.386822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471795652582185:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.385526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471793879128236:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471793879128320:2120] held: no
2025-11-27T18:20:35.387193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471794638790860:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:35.387545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471794638790865:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58468
2025-11-27T18:20:35.387747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471794638790844:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.388175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471795652582165:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.387833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:35.388230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471794638790844:2108] [node 4] ICP22 created new session: [5:7577471794638790867:2108]
2025-11-27T18:20:35.388289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471794638790867:2108] [node 4] ICS09 handshake done sender: [5:7577471794638790860:2117] self: [5:7577471794638790862:0] peer: [4:7577471794975396014:0] socket: 609 qp: -1
2025-11-27T18:20:35.388324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471794638790867:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:35.387868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471795805345073:2049] [node 3] ICP18 handshake (actor [2:7577471795805345158:2120]) is too old
2025-11-27T18:20:35.388563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471795805345148:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.388964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:35.391142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471794638790866:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58468
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.391807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471794638790868:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58468
2025-11-27T18:20:35.392051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471793879128332:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63508
2025-11-27T18:20:35.392142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471795652582182:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.392650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471794638790866:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.392951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471794975396018:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.393041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471794638790863:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.393860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471795805345137:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.392989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471794975396018:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.393716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471794638790859:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:35.393026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471795652582186:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.393729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471794975396033:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10742
2025-11-27T18:20:35.393128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471795652582102:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.394635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:35.394694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471795805345137:2108] [node 5] ICP22 created new session: [2:7577471795805345173:2108]
2025-11-27T18:20:35.395505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471794638790780:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.395508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471795805345173:2108] [node 5] ICS09 handshake done sender: [2:7577471795805345148:2115] self: [2:7577471795805345146:0] peer: [5:7577471794638790861:0] socket: 603 qp: -1
2025-11-27T18:20:35.395614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471795805345173:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:35.393966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471795652582189:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53753
2025-11-27T18:20:35.395659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:35.395904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471794975396018:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:35.395556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471794638790781:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.394829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471795652582101:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.395875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793874925750:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.395949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471795805345153:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.395588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:35.395912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471793874925750:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.397758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471794975396003:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.395830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471794638790781:2049] [node 2] ICP22 created new session: [5:7577471794638790873:2049]
2025-11-27T18:20:35.397772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471795652582189:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.395967Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471795805345153:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.395939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471794638790873:2049] [node 2] ICS09 handshake done sender: [5:7577471794638790859:2116] self: [5:7577471794638790861:0] peer: [2:7577471795805345146:0] socket: 604 qp: -1
2025-11-27T18:20:35.395977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471794638790873:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:35.397458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:35.397921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471795652582163:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.398137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:35.397913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471793879128320:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.398405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471793874925747:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.398440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471794975396003:2109] [node 6] ICP22 created new session: [4:7577471794975396035:2109]
2025-11-27T18:20:35.399845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471794975396035:2109] [node 6] ICS09 handshake done sender: [4:7577471794975396018:2116] self: [4:7577471794975396016:0] peer: [6:7577471795652582188:0] socket: 576 qp: -1
2025-11-27T18:20:35.399909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471795652582192:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53753
2025-11-27T18:20:35.398443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471793874925747:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.399885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471794975396035:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:35.399954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:35.400250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471793874925747:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:35.401362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471795652582185:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:35.401963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471793879128236:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.402312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471793874925750:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:35.402170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:35.402443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471794975396033:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.402646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471795805345153:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:35.402414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471795652582165:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.402358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471793874925737:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.402347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471793879128236:2048] [node 1] ICP22 created new session: [3:7577471793879128335:2048]
2025-11-27T18:20:35.402914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471794975395941:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.402740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:35.403109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471795805345138:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.402704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471793879128335:2048] [node 1] ICS09 handshake done sender: [3:7577471793879128320:2120] self: [3:7577471793879128326:0] peer: [1:7577471793874925740:0] socket: 592 qp: -1
2025-11-27T18:20:35.402973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:35.402796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471795652582165:2108] [node 4] ICP22 created new session: [6:7577471795652582194:2108]
2025-11-27T18:20:35.403261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:35.403343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471793874925737:2108] [node 5] ICP22 created new session: [1:7577471793874925775:2108]
2025-11-27T18:20:35.402855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471793879128335:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:35.405514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471794638790858:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.404661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471795805345138:2109] [node 6] ICP22 created new session: [2:7577471795805345176:2109]
2025-11-27T18:20:35.403328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471795652582194:2108] [node 4] ICS09 handshake done sender: [6:7577471795652582185:2118] self: [6:7577471795652582188:0] peer: [4:7577471794975396016:0] socket: 615 qp: -1
2025-11-27T18:20:35.405551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471794638790858:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.404733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471795805345176:2109] [node 6] ICS09 handshake done sender: [2:7577471795805345153:2117] self: [2:7577471795805345149:0] peer: [6:7577471795652582190:0] socket: 585 qp: -1
2025-11-27T18:20:35.403005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:35.404865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471793874925775:2108] [node 5] ICS09 handshake done sender: [1:7577471793874925747:2114] self: [1:7577471793874925746:0] peer: [5:7577471794638790872:0] socket: 593 qp: -1
2025-11-27T18:20:35.404879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471795805345176:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:35.404672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471795652582194:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:35.403347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471793879128315:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.405168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:35.404902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471793874925775:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:35.404712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:35.403380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471793879128315:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.405180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:35.407524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471793879128315:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:35.405473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471793874925738:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:35.405526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
2025-11-27T18:20:35.408158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471794638790780:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471794638790863:2118] held: no
2025-11-27T18:20:35.406697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471793874925738:2109] [node 6] ICP22 created new session: [1:7577471793874925777:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.409068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471793879128302:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.407398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471793874925777:2109] [node 6] ICS09 handshake done sender: [1:7577471793874925750:2116] self: [1:7577471793874925749:0] peer: [6:7577471795652582191:0] socket: 614 qp: -1
2025-11-27T18:20:35.408979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471795652582195:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53753
2025-11-27T18:20:35.409947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:35.407443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471793874925777:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:35.408916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471794638790865:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.407925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:35.410007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471793879128302:2107] [node 4] ICP22 created new session: [3:7577471793879128338:2107]
2025-11-27T18:20:35.410042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471794975396038:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10742
2025-11-27T18:20:35.410618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471793879128338:2107] [node 4] ICS09 handshake done sender: [3:7577471793879128315:2118] self: [3:7577471793879128313:0] peer: [4:7577471794975396037:0] socket: 598 qp: -1
2025-11-27T18:20:35.410672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471793879128338:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:35.410722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471795652582102:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471795652582182:2117] held: no
2025-11-27T18:20:35.410718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.413224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471794975396033:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.413207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471794638790875:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58468
2025-11-27T18:20:35.413239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471794638790782:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:35.414147Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471795652582101:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471795652582186:2119] held: no
2025-11-27T18:20:35.415082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471794638790858:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:35.415133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471795652582196:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53753
2025-11-27T18:20:35.415841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471794638790863:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:35.416049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471793879128316:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:35.416174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471794638790845:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.416088Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471793879128316:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:35.416488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:35.416622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471794638790845:2110] [node 6] ICP22 created new session: [5:7577471794638790878:2110]
2025-11-27T18:20:35.417280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471794638790878:2110] [node 6] ICS09 handshake done sender: [5:7577471794638790858:2115] self: [5:7577471794638790857:0] peer: [6:7577471795652582193:0] socket: 623 qp: -1
2025-11-27T18:20:35.417323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471794638790878:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:35.417308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471794975395941:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5368) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471794975396033:2120] held: no
2025-11-27T18:20:35.417363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:35.417310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471795652582186:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:35.418270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471795652582101:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.418562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:35.418627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471795652582101:2048] [node 1] ICP22 created new session: [6:7577471795652582199:2048]
2025-11-27T18:20:35.418565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471793879128316:2119] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:35.418688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471795652582199:2048] [node 1] ICS09 handshake done sender: [6:7577471795652582186:2119] self: [6:7577471795652582191:0] peer: [1:7577471793874925749:0] socket: 566 qp: -1
2025-11-27T18:20:35.418704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471794638790780:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.418871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471794975395941:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.418766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471793879128300:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:35.418732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471795652582199:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:35.419005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:35.419000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:35.418765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:35.419043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:35.419052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471794975395941:2050] [node 3] ICP22 created new session: [4:7577471794975396039:2050]
2025-11-27T18:20:35.419252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471794638790780:2048] [node 1] ICP22 created new session: [5:7577471794638790880:2048]
2025-11-27T18:20:35.419108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471793879128300:2108] [node 5] ICP22 created new session: [3:7577471793879128341:2108]
2025-11-27T18:20:35.419295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471793879128341:2108] [node 5] ICS09 handshake done sender: [3:7577471793879128316:2119] self: [3:7577471793879128314:0] peer: [5:7577471794638790877:0] socket: 618 qp: -1
2025-11-27T18:20:35.419296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471794975396039:2050] [node 3] ICS09 handshake done sender: [4:7577471794975396033:2120] self: [4:7577471794975396037:0] peer: [3:7577471793879128313:0] socket: 574 qp: -1
2025-11-27T18:20:35.419321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471794638790880:2048] [node 1] ICS09 handshake done sender: [5:7577471794638790863:2118] self: [5:7577471794638790872:0] peer: [1:7577471793874925746:0] socket: 613 qp: -1
2025-11-27T18:20:35.419337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471793879128341:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:35.419329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471794975396039:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:35.419449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471794638790880:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:35.419491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:35.419492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:35.419533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:35.419875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471795652582182:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:35.421926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471794638790782:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5368) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471794638790865:2120] held: no
2025-11-27T18:20:35.422270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471795652582102:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:35.422279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471795652582163:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25828) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471795652582189:2120] held: no
::ffff:127.1.0.0
2025-11-27T18:20:35.422357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:35.422407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471795652582102:2049] [node 2] ICP22 created new session: [6:7577471795652582201:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:35.422449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471795652582201:2049] [node 2] ICS09 handshake done sender: [6:7577471795652582182:2117] self: [6:7577471795652582190:0] peer: [2:7577471795805345149:0] socket: 612 qp: -1
2025-11-27T18:20:35.422471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471795652582198:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53753
2025-11-27T18:20:35.422476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471795652582201:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:35.422507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:35.422552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471794638790881:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58468
2025-11-27T18:20:35.422839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471794638790865:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:35.423119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471794638790782:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.423183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:35.423254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471794638790782:2050] [node 3] ICP22 created new session: [5:7577471794638790883:2050]
2025-11-27T18:20:35.423293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471794638790883:2050] [node 3] ICS09 handshake done sender: [5:7577471794638790865:2120] self: [5:7577471794638790877:0] peer: [3:7577471793879128314:0] socket: 619 qp: -1
2025-11-27T18:20:35.423347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471794638790883:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:35.423339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471795652582189:2120] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:35.423391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:35.423493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471795652582163:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:35.423582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:35.423632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471795652582163:2106] [node 5] ICP22 created new session: [6:7577471795652582203:2106]
2025-11-27T18:20:35.423694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471795652582203:2106] [node 5] ICS09 handshake done sender: [6:7577471795652582189:2120] self: [6:7577471795652582193:0] peer: [5:7577471794638790857:0] socket: 624 qp: -1
2025-11-27T18:20:35.423734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471795652582203:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:35.423780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:35.477634Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:35.477882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471795805345170:2048] [node 1] ICS07 socket disconnect 587 reason# EndOfStream
2025-11-27T18:20:35.477918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471795805345170:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.478005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:35.478061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471795805345170:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:35.479026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:35.479213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471793879128335:2048] [node 1] ICS01 socket: 592 reason# EndOfStream
2025-11-27T18:20:35.479250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471793879128236:2048] [node 1] ICP30 unregister session Session# [3:7577471793879128335:2048] VirtualId# [3:7577471793879128326:0]
2025-11-27T18:20:35.479272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471793879128335:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.479329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:35.479393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:35.480586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471795805345072:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2752) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:35.480637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471795805345170:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:35.482235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:35.482536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471794975396027:2048] [node 1] ICS01 socket: 589 reason# EndOfStream
2025-11-27T18:20:35.482607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471794975395939:2048] [node 1] ICP30 unregister session Session# [4:7577471794975396027:2048] VirtualId# [4:7577471794975396022:0]
2025-11-27T18:20:35.482634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471794975396027:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.482712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:35.482768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:35.482194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471795805345072:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2752) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:35.482399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471795805345170:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:35.482454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471795805345072:2048] [node 1] ICP30 unregister session Session# [2:7577471795805345170:2048] VirtualId# [2:7577471795805345165:0]
2025-11-27T18:20:35.482568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:35.482610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471795805345072:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2752) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.482625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2752) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.483584Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:35.483833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471794638790880:2048] [node 1] ICS01 socket: 613 reason# EndOfStream
2025-11-27T18:20:35.483881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471794638790780:2048] [node 1] ICP30 unregister session Session# [5:7577471794638790880:2048] VirtualId# [5:7577471794638790872:0]
2025-11-27T18:20:35.483914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471794638790880:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.483978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:35.484021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:35.484748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:35.484933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471795652582199:2048] [node 1] ICS01 socket: 566 reason# EndOfStream
2025-11-27T18:20:35.484974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471795652582101:2048] [node 1] ICP30 unregister session Session# [6:7577471795652582199:2048] VirtualId# [6:7577471795652582191:0]
2025-11-27T18:20:35.485001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471795652582199:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.485072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:35.485152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:35.540429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:35.540662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471793879128328:2049] [node 2] ICS07 socket disconnect 580 reason# EndOfStream
2025-11-27T18:20:35.540698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471793879128328:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.540789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:35.540863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471793879128328:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:35.541630Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471793879128237:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30861) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:35.541697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471793879128328:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:35.541737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471793879128237:2049] [node 2] ICP30 unregister session Session# [3:7577471793879128328:2049] VirtualId# [3:7577471793879128322:0]
2025-11-27T18:20:35.541791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:35.541834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471793879128237:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.541850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.543177Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:35.543361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471794975396024:2049] [node 2] ICS01 socket: 579 reason# EndOfStream
2025-11-27T18:20:35.543396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471794975395940:2049] [node 2] ICP30 unregister session Session# [4:7577471794975396024:2049] VirtualId# [4:7577471794975396020:0]
2025-11-27T18:20:35.543422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471794975396024:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.543491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:35.543553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:35.544157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:35.544355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471794638790873:2049] [node 2] ICS07 socket disconnect 604 reason# EndOfStream
2025-11-27T18:20:35.544388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471794638790873:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.544458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:35.544511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471794638790873:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:35.545136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471794638790781:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30861) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:35.545189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471794638790873:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:35.545280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471794638790781:2049] [node 2] ICP30 unregister session Session# [5:7577471794638790873:2049] VirtualId# [5:7577471794638790861:0]
2025-11-27T18:20:35.545221Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:35.545378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:35.545414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471794638790781:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.545428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.545435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471795652582201:2049] [node 2] ICS01 socket: 612 reason# EndOfStream
2025-11-27T18:20:35.545481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471795652582102:2049] [node 2] ICP30 unregister session Session# [6:7577471795652582201:2049] VirtualId# [6:7577471795652582190:0]
2025-11-27T18:20:35.545511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471795652582201:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.545575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:35.545634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:35.551538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471793879128237:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:35.555630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471794638790781:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:35.607243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:35.607515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471794975396039:2050] [node 3] ICS01 socket: 574 reason# EndOfStream
2025-11-27T18:20:35.607564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471794975395941:2050] [node 3] ICP30 unregister session Session# [4:7577471794975396039:2050] VirtualId# [4:7577471794975396037:0]
2025-11-27T18:20:35.607604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471794975396039:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.607736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:35.607805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:35.608146Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:35.608382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471794638790883:2050] [node 3] ICS01 socket: 619 reason# EndOfStream
2025-11-27T18:20:35.608433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471794638790782:2050] [node 3] ICP30 unregister session Session# [5:7577471794638790883:2050] VirtualId# [5:7577471794638790877:0]
2025-11-27T18:20:35.608473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471794638790883:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.608554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:35.608602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:35.608906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:35.609091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471795652582183:2050] [node 3] ICS07 socket disconnect 596 reason# EndOfStream
2025-11-27T18:20:35.609129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471795652582183:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.609238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:35.609310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471795652582183:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:35.610009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471795652582103:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5368) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:35.610105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471795652582183:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:35.610135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471795652582103:2050] [node 3] ICP30 unregister session Session# [6:7577471795652582183:2050] VirtualId# [6:7577471795652582180:0]
2025-11-27T18:20:35.610170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:35.610194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471795652582103:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5368) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.610203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5368) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.620157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471795652582103:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:35.667888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:35.668084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471794638790867:2108] [node 4] ICS07 socket disconnect 609 reason# EndOfStream
2025-11-27T18:20:35.668124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471794638790867:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.668217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:35.668250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471794638790867:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:35.669005Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471794638790844:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63017) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:35.669051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:35.669121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471794638790867:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:35.669188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471794638790844:2108] [node 4] ICP30 unregister session Session# [5:7577471794638790867:2108] VirtualId# [5:7577471794638790862:0]
2025-11-27T18:20:35.669230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471795652582194:2108] [node 4] ICS07 socket disconnect 615 reason# EndOfStream
2025-11-27T18:20:35.669275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471795652582194:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.669335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:35.669325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:35.669361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471795652582194:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:35.669361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471794638790844:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.669373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.669988Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471795652582165:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63017) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:35.670035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471795652582194:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:35.670117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471795652582165:2108] [node 4] ICP30 unregister session Session# [6:7577471795652582194:2108] VirtualId# [6:7577471795652582188:0]
2025-11-27T18:20:35.670233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:35.670259Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471795652582165:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.670268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:35.680582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471795652582165:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:35.715910Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:35.716094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471795652582203:2106] [node 5] ICS01 socket: 624 reason# EndOfStream
2025-11-27T18:20:35.716132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471795652582163:2106] [node 5] ICP30 unregister session Session# [6:7577471795652582203:2106] VirtualId# [6:7577471795652582193:0]
2025-11-27T18:20:35.716167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471795652582203:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:35.716244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:35.716288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:35.806482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471796288566086:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:35.806504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471796288566087:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:35.827341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471795682117044:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:35.827385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471795682117045:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:35.849604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471794606400821:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:35.849677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471794606400822:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:35.870052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471796287551758:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:35.870084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471796287551759:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:35.870102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471796287551760:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:35.888869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471795554279755:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:35.888899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471795554279756:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:35.889119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471795554279757:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:35.911803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471795474376092:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:35.911858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471795474376094:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:35.912068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471795474376093:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:35.959490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471795474376152:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:35.983144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471796288566149:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:36.000232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471796287551821:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:35.993306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471796288566151:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:36.006157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471799849247115:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:36.005918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471799769343452:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:36.006427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471800583533448:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:36.007558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471798901368180:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:36.007395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471799977084404:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:36.007385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471800582519118:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:36.007568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471799849247116:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:36.007751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471798901368182:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:36.007747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471799977084405:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:36.008194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471799977084406:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:36.007574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471798901368181:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:36.018239Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471794606400821:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17998487ns) {Configure:104(17989205ns) {ProcessConfigured:135(310435ns) {StartInitialHandshake:157(307399ns) {DropHandshakes:470(3480ns) {DropIncomingHandshake:442(1011ns) DropOutgoingHandshake:457(661ns)} GenerateSessionVirtualId:988(859ns) PrepareNewSessionHandshake:477(2312ns) {SwitchToState:179(616ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.028322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471795682117045:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16330047ns) {Configure:104(16325239ns) {ProcessConfigured:135(1479232ns) {StartInitialHandshake:157(1477105ns) {DropHandshakes:470(2363ns) {DropIncomingHandshake:442(489ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(851ns) PrepareNewSessionHandshake:477(1949ns) {SwitchToState:179(738ns)}}}}}
2025-11-27T18:20:36.029073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471795554279757:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17756402ns) {Configure:104(17748904ns) {ProcessConfigured:135(433566ns) {StartInitialHandshake:157(431469ns) {DropHandshakes:470(1254ns) {DropIncomingHandshake:442(369ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(419ns) PrepareNewSessionHandshake:477(1407ns) {SwitchToState:179(520ns)}}}}}
No RDMA context found for GID: 2025-11-27T18:20:36.029209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471800583533457:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34357
::ffff:127.1.0.0
2025-11-27T18:20:36.028749Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471799977084405:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16363669ns) {Configure:104(16358206ns) {ProcessConfigured:135(764272ns) {StartInitialHandshake:157(761683ns) {DropHandshakes:470(1250ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(391ns) PrepareNewSessionHandshake:477(179783ns) {SwitchToState:179(3860ns)}}}}}
2025-11-27T18:20:36.029196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471795554279755:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17669771ns) {Configure:104(17663328ns) {ProcessConfigured:135(689354ns) {StartInitialHandshake:157(687481ns) {DropHandshakes:470(1003ns) {DropIncomingHandshake:442(286ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:477(2762ns) {SwitchToState:179(1732ns)}}}}}
2025-11-27T18:20:36.029570Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471795554279756:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17443195ns) {Configure:104(17439399ns) {ProcessConfigured:135(436914ns) {StartInitialHandshake:157(435339ns) {DropHandshakes:470(1600ns) {DropIncomingHandshake:442(855ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(1366ns) PrepareNewSessionHandshake:477(1347ns) {SwitchToState:179(523ns)}}}}}
2025-11-27T18:20:36.030243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471799769343461:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2897
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.030421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800583533457:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.032047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471800583533461:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34357
2025-11-27T18:20:36.031540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471799769343461:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.032022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471800582519130:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26649
2025-11-27T18:20:36.032792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471799977084413:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56622
2025-11-27T18:20:36.032364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471795474376094:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.033758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471798901368188:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:36.037354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471799977084413:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:36.033799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471798901368188:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.035276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471798901368196:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14617
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.038645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471799977084416:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56622
2025-11-27T18:20:36.037306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471798901368188:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:36.039314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471799977084417:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56622
2025-11-27T18:20:36.039569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471799849247126:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30738
2025-11-27T18:20:36.039664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471799977084416:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.039231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471800582519131:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26649
2025-11-27T18:20:36.039673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800583533461:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.040063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471798901368196:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.039607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471800582519130:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.040191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471799849247125:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30738
2025-11-27T18:20:36.040068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471795682117044:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.040239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471798901368182:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:36.040296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471796287551758:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-27T18:20:36.040947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471799769343468:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2897
2025-11-27T18:20:36.040253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471799977084417:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.040536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471799849247130:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30738
2025-11-27T18:20:36.040531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.041241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471799849247125:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.040943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471798901368198:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14617
2025-11-27T18:20:36.041212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800583533454:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.040534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471800582519131:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.041168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471799977084424:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56622
2025-11-27T18:20:36.046285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471799769343469:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2897
2025-11-27T18:20:36.041828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471799849247126:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.041274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471800583533454:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.041151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471798901368182:2109] [node 6] ICP22 created new session: [3:7577471798901368200:2109]
2025-11-27T18:20:36.048706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471800582519134:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26649
2025-11-27T18:20:36.041372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800583533453:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.046283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471799849247131:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30738
2025-11-27T18:20:36.041529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471798901368200:2109] [node 6] ICS09 handshake done sender: [3:7577471798901368188:2112] self: [3:7577471798901368187:0] peer: [6:7577471799769343467:0] socket: 566 qp: -1
2025-11-27T18:20:36.049707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471799769343461:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:36.041404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471800583533453:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.050301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471799977084410:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.050326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471800582519134:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.046323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471799849247130:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.046305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471798901368200:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:36.050264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471795474376094:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.049811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471800583533453:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:36.048485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471795554279755:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.047033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:36.050337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471799977084410:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.050693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471796287551759:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.048779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471799849247131:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.049955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800583533455:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.050544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:36.052290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471799977084411:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.049789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471795554279756:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.048233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471798901368199:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14617
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.049984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471800583533455:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.051430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471795474376094:2050] [node 3] ICP22 created new session: [6:7577471799769343470:2050]
2025-11-27T18:20:36.052554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471798901368198:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.052332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471799977084411:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.056066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471799849247116:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.057764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471800582519137:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26649
2025-11-27T18:20:36.050282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471800583533454:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:36.057068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471799849247134:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30738
2025-11-27T18:20:36.055142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471799977084414:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.053037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471799769343470:2050] [node 3] ICS09 handshake done sender: [6:7577471799769343461:2114] self: [6:7577471799769343467:0] peer: [3:7577471798901368187:0] socket: 569 qp: -1
2025-11-27T18:20:36.053107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471794606400822:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.057875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471800582519124:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.056844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471798901368199:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.055173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471800583533455:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:36.053079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471799769343470:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:36.055174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471799977084414:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.059219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471799849247134:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.057911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471800582519124:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.055223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471799977084410:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:36.060050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471795554279757:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.056864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471796288566086:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:36.054790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
::ffff:127.1.0.0No RDMA context found for GID: 2025-11-27T18:20:36.057193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471796288566149:2107] [node 4] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:36.060310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471799849247122:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.057833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:36.057325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471799769343469:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.061705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471800582519124:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:36.060351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471799849247122:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.059187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:36.059480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471795474376152:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.061956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471799849247140:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30738
2025-11-27T18:20:36.059175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471799977084427:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56622
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.062030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471800582519130:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:36.061965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471798901368205:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14617
2025-11-27T18:20:36.059190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471796288566086:2048] [node 2] ICP22 created new session: [1:7577471800583533470:2048]
2025-11-27T18:20:36.059234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471799977084414:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:36.061992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471798901368190:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.061957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471799849247122:2112] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:36.062959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471800582519139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26649
2025-11-27T18:20:36.059650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471796288566149:2107] [node 4] ICP22 created new session: [1:7577471800583533471:2107]
2025-11-27T18:20:36.062015Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471798901368190:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.061958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471799849247139:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30738
2025-11-27T18:20:36.064935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471800582519118:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.060008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471800583533470:2048] [node 2] ICS09 handshake done sender: [1:7577471800583533453:2111] self: [1:7577471800583533449:0] peer: [2:7577471799977084425:0] socket: 572 qp: -1
2025-11-27T18:20:36.059335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471799977084411:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:36.064282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471798901368198:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.064473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471799849247142:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30738
2025-11-27T18:20:36.060255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471799977084406:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.064666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471798901368190:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:36.060460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471800583533470:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:36.065093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471796287551758:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.066646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471799769343474:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2897
2025-11-27T18:20:36.060500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471800583533471:2107] [node 4] ICS09 handshake done sender: [1:7577471800583533454:2112] self: [1:7577471800583533450:0] peer: [4:7577471800582519135:0] socket: 571 qp: -1
2025-11-27T18:20:36.060304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:36.060511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471796288566151:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.067192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471794606400822:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11997) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471798901368198:2116] held: no
2025-11-27T18:20:36.060753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471799977084406:2109] [node 5] ICP22 created new session: [2:7577471799977084431:2109]
2025-11-27T18:20:36.060513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:36.067264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471798901368180:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.060554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:36.061385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471795682117045:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.061784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471799977084424:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.060615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471796288566151:2108] [node 5] ICP22 created new session: [1:7577471800583533473:2108]
2025-11-27T18:20:36.065146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.061844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471799977084431:2109] [node 5] ICS09 handshake done sender: [2:7577471799977084410:2111] self: [2:7577471799977084409:0] peer: [5:7577471799849247135:0] socket: 593 qp: -1
2025-11-27T18:20:36.060724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471800583533471:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:36.067367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471799849247115:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.065195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471796287551758:2048] [node 1] ICP22 created new session: [4:7577471800582519141:2048]
2025-11-27T18:20:36.068632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:36.067492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471799849247131:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:36.061525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471800583533473:2108] [node 5] ICS09 handshake done sender: [1:7577471800583533455:2113] self: [1:7577471800583533451:0] peer: [5:7577471799849247132:0] socket: 573 qp: -1
2025-11-27T18:20:36.065431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:36.061854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471799977084404:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.066168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471800582519141:2048] [node 1] ICS09 handshake done sender: [4:7577471800582519130:2114] self: [4:7577471800582519135:0] peer: [1:7577471800583533450:0] socket: 575 qp: -1
2025-11-27T18:20:36.067493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471799849247126:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:36.061545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:36.067555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471799849247125:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:36.061978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:36.066677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471800582519118:2108] [node 5] ICP22 created new session: [4:7577471800582519142:2108]
2025-11-27T18:20:36.068194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:36.061767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471800583533473:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:36.062014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471799977084431:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:36.066875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471800582519141:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:36.069137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471799849247115:2110] [node 6] ICP22 created new session: [5:7577471799849247145:2110]
2025-11-27T18:20:36.061825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:36.068238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471799769343469:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:36.069853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471798901368180:2108] [node 5] ICP22 created new session: [3:7577471798901368208:2108]
2025-11-27T18:20:36.070299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471798901368208:2108] [node 5] ICS09 handshake done sender: [3:7577471798901368190:2113] self: [3:7577471798901368189:0] peer: [5:7577471799849247141:0] socket: 608 qp: -1
2025-11-27T18:20:36.066923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:36.062070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:36.070015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471795474376152:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.069825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471799849247145:2110] [node 6] ICS09 handshake done sender: [5:7577471799849247122:2112] self: [5:7577471799849247120:0] peer: [6:7577471799769343473:0] socket: 600 qp: -1
2025-11-27T18:20:36.069165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471800583533469:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34357
2025-11-27T18:20:36.067227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471800582519142:2108] [node 5] ICS09 handshake done sender: [4:7577471800582519124:2111] self: [4:7577471800582519122:0] peer: [5:7577471799849247138:0] socket: 579 qp: -1
2025-11-27T18:20:36.073066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800583533469:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.067468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471800582519142:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:36.070065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:36.070050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471799849247145:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:36.072910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471798901368208:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:36.067572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:36.070090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:36.073387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:36.067576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471800582519134:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:36.070283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471799849247116:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.074642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471794606400822:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.070005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471796287551759:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.070571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:36.070066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:36.062219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:36.070109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471795474376152:2105] [node 5] ICP22 created new session: [6:7577471799769343475:2105]
2025-11-27T18:20:36.070626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471795554279755:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.073234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471799769343475:2105] [node 5] ICS09 handshake done sender: [6:7577471799769343469:2117] self: [6:7577471799769343473:0] peer: [5:7577471799849247120:0] socket: 601 qp: -1
2025-11-27T18:20:36.070234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471796287551759:2049] [node 2] ICP22 created new session: [4:7577471800582519147:2049]
2025-11-27T18:20:36.062277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471795682117045:2049] [node 3] ICP22 created new session: [2:7577471799977084432:2049]
2025-11-27T18:20:36.072970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:36.074290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471799769343475:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:36.063105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471799977084404:2107] [node 4] ICP22 created new session: [2:7577471799977084434:2107]
2025-11-27T18:20:36.070537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471800582519147:2049] [node 2] ICS09 handshake done sender: [4:7577471800582519134:2117] self: [4:7577471800582519138:0] peer: [2:7577471799977084408:0] socket: 599 qp: -1
2025-11-27T18:20:36.073230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471799849247116:2108] [node 4] ICP22 created new session: [5:7577471799849247147:2108]
2025-11-27T18:20:36.074902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:36.063122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471799977084432:2049] [node 3] ICS09 handshake done sender: [2:7577471799977084414:2115] self: [2:7577471799977084407:0] peer: [3:7577471798901368204:0] socket: 594 qp: -1
2025-11-27T18:20:36.073676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471795554279756:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.070581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471800582519147:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:36.078514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:36.063636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471799977084432:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.074341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471799849247147:2108] [node 4] ICS09 handshake done sender: [5:7577471799849247126:2116] self: [5:7577471799849247138:0] peer: [4:7577471800582519122:0] socket: 590 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.070622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:36.063685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.064500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471799977084434:2107] [node 4] ICS09 handshake done sender: [2:7577471799977084411:2112] self: [2:7577471799977084408:0] peer: [4:7577471800582519138:0] socket: 592 qp: -1
2025-11-27T18:20:36.064957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471799977084434:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:36.064988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:36.079073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471794606400822:2049] [node 2] ICP22 created new session: [3:7577471798901368212:2049]
2025-11-27T18:20:36.080266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471799769343476:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2897
2025-11-27T18:20:36.067592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471799977084416:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:36.070328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471795682117044:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.070365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.070557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471795682117044:2048] [node 1] ICP22 created new session: [2:7577471799977084439:2048]
2025-11-27T18:20:36.070598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471799977084439:2048] [node 1] ICS09 handshake done sender: [2:7577471799977084416:2116] self: [2:7577471799977084425:0] peer: [1:7577471800583533449:0] socket: 586 qp: -1
2025-11-27T18:20:36.072846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471799977084439:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:36.072897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:36.074566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471795554279755:2048] [node 1] ICP22 created new session: [5:7577471799849247148:2048]
2025-11-27T18:20:36.074598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471799849247147:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:36.080724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471799977084406:2109] [node 5] ICP18 handshake (actor [2:7577471799977084424:2119]) is too old
2025-11-27T18:20:36.074925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471799849247148:2048] [node 1] ICS09 handshake done sender: [5:7577471799849247125:2115] self: [5:7577471799849247132:0] peer: [1:7577471800583533451:0] socket: 588 qp: -1
2025-11-27T18:20:36.079384Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471798901368182:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(25991748ns) {HandleHandshakeStatus:329(25988993ns) {DropIncomingHandshake:442(1405ns) DropOutgoingHandshake:457(462ns) SwitchToState:179(306ns) ProcessPendingSessionEvents:495(16566786ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15744608ns max# 1631399ns}}}
2025-11-27T18:20:36.080856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471799769343479:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2897
2025-11-27T18:20:36.080899Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471795474376152:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:36.080927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471799769343477:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2897
2025-11-27T18:20:36.080943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471795474376094:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(18908438ns) {HandleHandshakeStatus:329(18905387ns) {DropIncomingHandshake:442(711ns) DropOutgoingHandshake:457(17951ns) SwitchToState:179(516ns) ProcessPendingSessionEvents:495(12111389ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12001130ns max# 2590197ns}}}
2025-11-27T18:20:36.079849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471798901368212:2049] [node 2] ICS09 handshake done sender: [3:7577471798901368198:2116] self: [3:7577471798901368204:0] peer: [2:7577471799977084407:0] socket: 595 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.074960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471799849247148:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:36.084052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471796287551758:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.079990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471798901368212:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:36.074993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:36.085711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471799769343477:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.075283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:36.087140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471795474376092:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.080134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:36.076760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:36.078233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471795554279756:2049] [node 2] ICP22 created new session: [5:7577471799849247151:2049]
2025-11-27T18:20:36.078502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471799849247151:2049] [node 2] ICS09 handshake done sender: [5:7577471799849247131:2118] self: [5:7577471799849247135:0] peer: [2:7577471799977084409:0] socket: 598 qp: -1
2025-11-27T18:20:36.080927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471798901368210:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14617
2025-11-27T18:20:36.087181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471800583533478:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34357
2025-11-27T18:20:36.078535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471799849247151:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:36.088189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471798901368210:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.088183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471800582519149:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26649
2025-11-27T18:20:36.079007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:36.088566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800583533464:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.088394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471799769343479:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.089753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471795474376093:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.088617Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471796288566149:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(23105965ns) {HandleHandshakeStatus:329(23102880ns) {DropIncomingHandshake:442(19485ns) DropOutgoingHandshake:457(682ns) SwitchToState:179(432ns) ProcessPendingSessionEvents:495(16650889ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16542029ns max# 2403864ns}}}
2025-11-27T18:20:36.089746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471794606400821:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.088624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471800583533464:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.090588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471800582519149:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.091122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800583533456:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.090649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471799977084404:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(18362581ns) {HandleHandshakeStatus:329(18359912ns) {DropIncomingHandshake:442(640ns) DropOutgoingHandshake:457(521ns) SwitchToState:179(404ns) ProcessPendingSessionEvents:495(12974266ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12257753ns max# 2698888ns}}}
2025-11-27T18:20:36.090588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471799769343476:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.090897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471796287551760:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.091160Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471800583533456:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.091126Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471799977084415:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.090914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471799769343452:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.091159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471799977084415:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.093184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471800582519126:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.093198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471799977084415:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:36.093194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471798901368195:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.093233Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471800582519126:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.093550Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471796288566086:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(23244268ns) {HandleHandshakeStatus:329(23240633ns) {DropIncomingHandshake:442(248ns) DropOutgoingHandshake:457(717ns) SwitchToState:179(601ns) ProcessPendingSessionEvents:495(18553230ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17932848ns max# 2204654ns}}}
2025-11-27T18:20:36.093237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471798901368195:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.094866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471800583533464:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:36.095656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471799977084405:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.095660Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471800582519118:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(21843534ns) {HandleHandshakeStatus:329(21841018ns) {DropIncomingHandshake:442(963ns) DropOutgoingHandshake:457(639ns) SwitchToState:179(584ns) ProcessPendingSessionEvents:495(17597316ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16739147ns max# 3079425ns}}}
2025-11-27T18:20:36.098584Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471799849247115:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(20833914ns) {HandleHandshakeStatus:329(20831822ns) {DropIncomingHandshake:442(11558ns) DropOutgoingHandshake:457(146ns) SwitchToState:179(561ns) ProcessPendingSessionEvents:495(13745315ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13644242ns max# 2082792ns}}}
2025-11-27T18:20:36.096072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:36.098519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471799977084405:2110] [node 6] ICP22 created new session: [2:7577471799977084442:2110]
2025-11-27T18:20:36.098715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471799977084442:2110] [node 6] ICS09 handshake done sender: [2:7577471799977084415:2114] self: [2:7577471799977084412:0] peer: [6:7577471799769343481:0] socket: 631 qp: -1
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-27T18:20:36.095802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471800583533448:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.098709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471800582519126:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:36.095846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:36.100063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471799977084442:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:36.100059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471798901368195:2114] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:36.100134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:36.095882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471800583533448:2110] [node 6] ICP22 created new session: [1:7577471800583533482:2110]
2025-11-27T18:20:36.100386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471799769343483:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2897
2025-11-27T18:20:36.100406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471799769343484:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2897
2025-11-27T18:20:36.101618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471798901368181:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.098703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471800583533482:2110] [node 6] ICS09 handshake done sender: [1:7577471800583533464:2117] self: [1:7577471800583533462:0] peer: [6:7577471799769343480:0] socket: 591 qp: -1
2025-11-27T18:20:36.103471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471795554279756:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17160899ns) {HandleHandshakeStatus:329(17157527ns) {DropIncomingHandshake:442(572ns) DropOutgoingHandshake:457(5210ns) SwitchToState:179(396ns) ProcessPendingSessionEvents:495(11055181ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10966011ns max# 3099150ns}}}
2025-11-27T18:20:36.102232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471795682117044:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20271171ns) {HandleHandshakeStatus:329(20268276ns) {DropIncomingHandshake:442(633ns) DropOutgoingHandshake:457(3043ns) SwitchToState:179(369ns) ProcessPendingSessionEvents:495(15662434ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15572263ns max# 3748945ns}}}
2025-11-27T18:20:36.102710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471795474376092:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13702) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471799769343477:2120] held: no
2025-11-27T18:20:36.102766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:36.098748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471800583533482:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:36.104680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471795554279755:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20247691ns) {HandleHandshakeStatus:329(20245737ns) {DropIncomingHandshake:442(1007ns) DropOutgoingHandshake:457(3633ns) SwitchToState:179(362ns) ProcessPendingSessionEvents:495(10977987ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10884857ns max# 2117037ns}}}
2025-11-27T18:20:36.104268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471796287551821:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.100017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:36.104128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471798901368181:2107] [node 4] ICP22 created new session: [3:7577471798901368217:2107]
2025-11-27T18:20:36.103776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471800583533456:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:36.105145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471798901368217:2107] [node 4] ICS09 handshake done sender: [3:7577471798901368195:2114] self: [3:7577471798901368194:0] peer: [4:7577471800582519150:0] socket: 629 qp: -1
2025-11-27T18:20:36.105151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:36.105199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471798901368217:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:36.105210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471796287551821:2109] [node 6] ICP22 created new session: [4:7577471800582519153:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.105378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471795554279757:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6457) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471799849247134:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.105282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:36.105826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471799849247144:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30738
2025-11-27T18:20:36.105594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471800582519153:2109] [node 6] ICS09 handshake done sender: [4:7577471800582519126:2113] self: [4:7577471800582519121:0] peer: [6:7577471799769343482:0] socket: 574 qp: -1
2025-11-27T18:20:36.106247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471798901368216:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14617
2025-11-27T18:20:36.106007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471800582519153:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:36.106431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471795474376093:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11997) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471799769343479:2121] held: no
2025-11-27T18:20:36.106237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471796288566087:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.107559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:36.108069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471799769343477:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:36.106413Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471798901368180:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(23698502ns) {HandleHandshakeStatus:329(23695805ns) {DropIncomingHandshake:442(12693ns) DropOutgoingHandshake:457(453ns) SwitchToState:179(1329ns) ProcessPendingSessionEvents:495(12479826ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12391839ns max# 2316400ns}}}
2025-11-27T18:20:36.108299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471799769343479:2121] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.110152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471799769343485:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2897
2025-11-27T18:20:36.108845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:36.110556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471795474376093:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.109066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471796288566087:2049] [node 3] ICP22 created new session: [1:7577471800583533484:2049]
2025-11-27T18:20:36.110566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471795474376092:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.109234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471800583533484:2049] [node 3] ICS09 handshake done sender: [1:7577471800583533456:2114] self: [1:7577471800583533452:0] peer: [3:7577471798901368214:0] socket: 570 qp: -1
2025-11-27T18:20:36.110945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471799769343476:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:36.109414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471800583533484:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:36.112496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:36.110178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:36.112593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471795474376093:2049] [node 2] ICP22 created new session: [6:7577471799769343486:2049]
2025-11-27T18:20:36.112654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471799769343486:2049] [node 2] ICS09 handshake done sender: [6:7577471799769343479:2121] self: [6:7577471799769343481:0] peer: [2:7577471799977084412:0] socket: 632 qp: -1
2025-11-27T18:20:36.112694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471799769343486:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:36.114319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:36.114351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:36.116362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471794606400821:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13702) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471798901368210:2119] held: no
2025-11-27T18:20:36.114503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471795474376092:2048] [node 1] ICP22 created new session: [6:7577471799769343488:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.115272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471799769343488:2048] [node 1] ICS09 handshake done sender: [6:7577471799769343477:2120] self: [6:7577471799769343480:0] peer: [1:7577471800583533462:0] socket: 630 qp: -1
2025-11-27T18:20:36.116104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471799769343488:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:36.116532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:36.116544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471796287551760:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6457) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471800582519149:2120] held: no
2025-11-27T18:20:36.116606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471800582519152:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26649
2025-11-27T18:20:36.116723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471798901368210:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:36.119171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471799849247134:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:36.118371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471794606400821:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.119617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471800582519149:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:36.119538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:36.119691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471794606400821:2048] [node 1] ICP22 created new session: [3:7577471798901368219:2048]
2025-11-27T18:20:36.119746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471798901368219:2048] [node 1] ICS09 handshake done sender: [3:7577471798901368210:2119] self: [3:7577471798901368214:0] peer: [1:7577471800583533452:0] socket: 606 qp: -1
2025-11-27T18:20:36.119800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471798901368219:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:36.120101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471796287551760:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.119992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:36.120097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471799769343452:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.120412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471795554279757:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.120170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:36.120564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:36.120631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471799769343452:2107] [node 4] ICP22 created new session: [6:7577471799769343490:2107]
2025-11-27T18:20:36.120731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471796287551760:2050] [node 3] ICP22 created new session: [4:7577471800582519155:2050]
2025-11-27T18:20:36.120789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471800582519155:2050] [node 3] ICS09 handshake done sender: [4:7577471800582519149:2120] self: [4:7577471800582519150:0] peer: [3:7577471798901368194:0] socket: 580 qp: -1
2025-11-27T18:20:36.120726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:36.120885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471800582519155:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:36.120784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471795554279757:2050] [node 3] ICP22 created new session: [5:7577471799849247154:2050]
2025-11-27T18:20:36.120950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:36.120985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471799769343490:2107] [node 4] ICS09 handshake done sender: [6:7577471799769343476:2119] self: [6:7577471799769343482:0] peer: [4:7577471800582519121:0] socket: 628 qp: -1
2025-11-27T18:20:36.121026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471799769343490:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:36.121104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:36.122078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471799769343452:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:36.121322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471799849247154:2050] [node 3] ICS09 handshake done sender: [5:7577471799849247134:2119] self: [5:7577471799849247141:0] peer: [3:7577471798901368189:0] socket: 609 qp: -1
2025-11-27T18:20:36.121448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471799849247154:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:36.121502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:36.166695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:36.166979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471799977084439:2048] [node 1] ICS07 socket disconnect 586 reason# EndOfStream
2025-11-27T18:20:36.167022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471799977084439:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.167123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:36.167191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471799977084439:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:36.168239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:36.168470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471798901368219:2048] [node 1] ICS01 socket: 606 reason# EndOfStream
2025-11-27T18:20:36.168516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471794606400821:2048] [node 1] ICP30 unregister session Session# [3:7577471798901368219:2048] VirtualId# [3:7577471798901368214:0]
2025-11-27T18:20:36.168553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471798901368219:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.168631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:36.168685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:36.169630Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471795682117044:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13702) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:36.169682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471799977084439:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:36.171361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471795682117044:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13702) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:36.172229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471799977084439:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:36.172283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471795682117044:2048] [node 1] ICP30 unregister session Session# [2:7577471799977084439:2048] VirtualId# [2:7577471799977084425:0]
2025-11-27T18:20:36.172399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:36.172448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471795682117044:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13702) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.172467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13702) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.172505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:36.172709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471800582519141:2048] [node 1] ICS01 socket: 575 reason# EndOfStream
2025-11-27T18:20:36.172751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471796287551758:2048] [node 1] ICP30 unregister session Session# [4:7577471800582519141:2048] VirtualId# [4:7577471800582519135:0]
2025-11-27T18:20:36.172787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471800582519141:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.172844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:36.172881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:36.173915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:36.174145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471799849247148:2048] [node 1] ICS07 socket disconnect 588 reason# EndOfStream
2025-11-27T18:20:36.174183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471799849247148:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.174255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:36.174313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471799849247148:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:36.174970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471795554279755:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13702) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:36.175016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471799849247148:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:36.175110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471795554279755:2048] [node 1] ICP30 unregister session Session# [5:7577471799849247148:2048] VirtualId# [5:7577471799849247132:0]
2025-11-27T18:20:36.175141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:36.175224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:36.175277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471795554279755:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13702) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.175302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13702) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.175354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471799769343488:2048] [node 1] ICS01 socket: 630 reason# EndOfStream
2025-11-27T18:20:36.175399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471795474376092:2048] [node 1] ICP30 unregister session Session# [6:7577471799769343488:2048] VirtualId# [6:7577471799769343480:0]
2025-11-27T18:20:36.175425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471799769343488:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.175519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:36.175569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:36.185400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471795554279755:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:36.236606Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:36.236761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471798901368212:2049] [node 2] ICS01 socket: 595 reason# EndOfStream
2025-11-27T18:20:36.236829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471794606400822:2049] [node 2] ICP30 unregister session Session# [3:7577471798901368212:2049] VirtualId# [3:7577471798901368204:0]
2025-11-27T18:20:36.236866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471798901368212:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.236943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:36.236989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:36.239430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:36.239600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471800582519147:2049] [node 2] ICS01 socket: 599 reason# EndOfStream
2025-11-27T18:20:36.239638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471796287551759:2049] [node 2] ICP30 unregister session Session# [4:7577471800582519147:2049] VirtualId# [4:7577471800582519138:0]
2025-11-27T18:20:36.239661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471800582519147:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.239724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:36.239765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:36.240688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:36.240854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471799849247151:2049] [node 2] ICS07 socket disconnect 598 reason# EndOfStream
2025-11-27T18:20:36.240880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471799849247151:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.240939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:36.240979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471799849247151:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:36.241768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471795554279756:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11997) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:36.241850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471799849247151:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:36.241799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:36.241896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471795554279756:2049] [node 2] ICP30 unregister session Session# [5:7577471799849247151:2049] VirtualId# [5:7577471799849247135:0]
2025-11-27T18:20:36.242046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471799769343486:2049] [node 2] ICS01 socket: 632 reason# EndOfStream
2025-11-27T18:20:36.242090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471795474376093:2049] [node 2] ICP30 unregister session Session# [6:7577471799769343486:2049] VirtualId# [6:7577471799769343481:0]
2025-11-27T18:20:36.242073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:36.242116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471799769343486:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.242115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471795554279756:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11997) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.242130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11997) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.242180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:36.242233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:36.251921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471795554279756:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:36.322553Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:36.322816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471800582519155:2050] [node 3] ICS01 socket: 580 reason# EndOfStream
2025-11-27T18:20:36.322872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471796287551760:2050] [node 3] ICP30 unregister session Session# [4:7577471800582519155:2050] VirtualId# [4:7577471800582519150:0]
2025-11-27T18:20:36.322909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471800582519155:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.323008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:36.323068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:36.323726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:36.323960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471799849247154:2050] [node 3] ICS01 socket: 609 reason# EndOfStream
2025-11-27T18:20:36.324011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471795554279757:2050] [node 3] ICP30 unregister session Session# [5:7577471799849247154:2050] VirtualId# [5:7577471799849247141:0]
2025-11-27T18:20:36.324046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471799849247154:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.324129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:36.324191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:36.324822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:36.325032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471799769343470:2050] [node 3] ICS07 socket disconnect 569 reason# EndOfStream
2025-11-27T18:20:36.325076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471799769343470:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.325156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:36.325210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471799769343470:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:36.326021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471795474376094:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6457) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:36.326189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471799769343470:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:36.326247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471795474376094:2050] [node 3] ICP30 unregister session Session# [6:7577471799769343470:2050] VirtualId# [6:7577471799769343467:0]
2025-11-27T18:20:36.326360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:36.326403Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471795474376094:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6457) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.326426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6457) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.335916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471795474376094:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:36.388711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:36.388942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471799849247147:2108] [node 4] ICS07 socket disconnect 590 reason# EndOfStream
2025-11-27T18:20:36.388980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471799849247147:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.389073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:36.389124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471799849247147:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:36.389951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:36.390191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471799769343490:2107] [node 4] ICS07 socket disconnect 628 reason# EndOfStream
2025-11-27T18:20:36.390148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471799849247116:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6504) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:36.390234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471799769343490:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.390264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471799849247147:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:36.390311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:36.390360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471799769343490:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:36.390383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471799849247116:2108] [node 4] ICP30 unregister session Session# [5:7577471799849247147:2108] VirtualId# [5:7577471799849247138:0]
2025-11-27T18:20:36.390453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:36.390524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471799849247116:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.390564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.391011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471799769343452:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6504) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:36.391091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471799769343490:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:36.391146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471799769343452:2107] [node 4] ICP30 unregister session Session# [6:7577471799769343490:2107] VirtualId# [6:7577471799769343482:0]
2025-11-27T18:20:36.391249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:36.391275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471799769343452:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.391284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:36.400448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471799849247116:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:36.401430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471799769343452:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:36.476279Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:36.476535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471799769343475:2105] [node 5] ICS01 socket: 601 reason# EndOfStream
2025-11-27T18:20:36.476580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471795474376152:2105] [node 5] ICP30 unregister session Session# [6:7577471799769343475:2105] VirtualId# [6:7577471799769343473:0]
2025-11-27T18:20:36.476611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471799769343475:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.476682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:36.476740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:36.582918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471800888207757:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:36.583038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471800888207758:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:36.605830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471801948682449:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:36.606084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471801948682450:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:36.628701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471802301309471:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:36.628768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471802301309472:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:36.651833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471799494321576:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:36.651870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471799494321577:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:36.651896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471799494321578:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:36.671364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471801901441174:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:36.671362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471801901441173:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:36.671435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471801901441175:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:36.694493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471798275764346:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:36.694560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471798275764348:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:36.694615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471798275764347:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:36.774493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471802301309543:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:36.779039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471801901441236:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:36.779009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471800888207820:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:36.782074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471801948682512:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:36.781112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471798275764407:2105] [node 4] ICP01 ready to work
2025-11-27T18:20:36.782562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471801948682513:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:36.782581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471800888207821:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:36.783079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471800888207822:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:36.783138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471798275764409:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:36.782860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471802301309544:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:36.783169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471801901441237:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:36.782613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471799494321639:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:36.783154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471799494321640:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:36.783778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471802301309545:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:36.782305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471801948682514:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:36.795411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471800888207758:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28206633ns) {Configure:104(28196925ns) {ProcessConfigured:135(576829ns) {StartInitialHandshake:157(574812ns) {DropHandshakes:470(1004ns) {DropIncomingHandshake:442(292ns) DropOutgoingHandshake:457(157ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:477(1867ns) {SwitchToState:179(761ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:36.801933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471799494321655:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0No RDMA context found for GID: 
::ffff:127.1.0.0
2025-11-27T18:20:36.804323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471798275764428:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6777
2025-11-27T18:20:36.804309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471799494321654:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-27T18:20:36.805121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471801948682529:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41226
2025-11-27T18:20:36.805058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471802301309559:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28948
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.805284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471801901441251:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:824
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.804331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471798275764425:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6777
2025-11-27T18:20:36.804337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471799494321656:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-27T18:20:36.805080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471800888207837:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:499
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.804519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471798275764427:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6777
2025-11-27T18:20:36.805251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471801948682528:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41226
2025-11-27T18:20:36.805129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471802301309560:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28948
2025-11-27T18:20:36.805097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471800888207836:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:499
2025-11-27T18:20:36.805368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471799494321657:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-27T18:20:36.811886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471801948682530:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41226
2025-11-27T18:20:36.807974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471802301309560:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.810354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800888207836:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.811114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471798275764426:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6777
2025-11-27T18:20:36.812509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471801948682528:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.811170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471799494321655:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:36.812725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471801901441251:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:36.812118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471798275764425:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.810885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471800888207838:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:499
2025-11-27T18:20:36.811807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471802301309559:2124] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.811711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471799494321654:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.812668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471801948682530:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.813376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471801901441239:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.812127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471799494321656:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.812795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471798275764409:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.811833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800888207837:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.813289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471802301309550:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.813095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471801948682529:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.813413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471801901441239:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.813280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471798275764428:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.813310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471802301309550:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.813041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800888207832:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.815133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471801948682449:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.812240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471799494321578:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.815441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471801901441174:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.813069Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471800888207832:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.815543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471802301309550:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:36.812452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471799494321657:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.815510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471798275764427:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.813069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800888207838:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.815477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471798275764407:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.815155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471801948682531:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41226
2025-11-27T18:20:36.820725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471801901441239:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:36.812511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471799494321576:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.815161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471801948682521:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.822880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471801901441237:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.816392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471798275764426:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.822184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471802301309544:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.817567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800888207827:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.812779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471799494321577:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.816517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471798275764346:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.817603Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471800888207827:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.822265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:36.815195Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471801948682521:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.823414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:36.817563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471799494321644:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.817600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471798275764347:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.824070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471801901441237:2109] [node 6] ICP22 created new session: [5:7577471801901441255:2109]
2025-11-27T18:20:36.816292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471801948682520:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.819551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471800888207832:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:36.822865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471802301309544:2107] [node 4] ICP22 created new session: [3:7577471802301309564:2107]
2025-11-27T18:20:36.825193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471798275764434:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6777
::ffff:127.1.0.0
2025-11-27T18:20:36.817604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471799494321644:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.819734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800888207828:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.824554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471801901441255:2109] [node 6] ICS09 handshake done sender: [5:7577471801901441239:2110] self: [5:7577471801901441238:0] peer: [6:7577471798275764429:0] socket: 571 qp: -1
2025-11-27T18:20:36.816328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471801948682520:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.824525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471802301309564:2107] [node 4] ICS09 handshake done sender: [3:7577471802301309550:2120] self: [3:7577471802301309547:0] peer: [4:7577471799494321659:0] socket: 579 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.823495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471799494321578:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5233) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471799494321656:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.819762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471800888207828:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.824715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471801901441255:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:36.824710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471802301309564:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:36.818178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471801948682521:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:36.824963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471799494321663:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-27T18:20:36.824771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:36.824438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471800888207827:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:36.819565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471801948682523:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.830134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471798275764433:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6777
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.825592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:36.825648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471799494321662:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-27T18:20:36.830581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471798275764435:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6777
2025-11-27T18:20:36.819585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471801948682523:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.828493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471801901441254:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:824
2025-11-27T18:20:36.824931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471800888207757:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.828057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471799494321644:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:36.833927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471798275764437:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6777
2025-11-27T18:20:36.825217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471800888207821:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.819623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471801948682531:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.828611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471801901441174:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2721) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471801901441251:2115] held: no
2025-11-27T18:20:36.834010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471798275764427:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.828197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471799494321664:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-27T18:20:36.826438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:36.822787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471801948682520:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:36.830395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471799494321640:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.827326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:36.830678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:36.836980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471798275764347:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.824111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471801948682523:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.831711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471799494321654:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:36.827557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471800888207757:2048] [node 2] ICP22 created new session: [1:7577471800888207846:2048]
2025-11-27T18:20:36.824393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471801948682512:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.837224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:36.834000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471799494321640:2109] [node 6] ICP22 created new session: [4:7577471799494321667:2109]
2025-11-27T18:20:36.842904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471802301309567:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28948
2025-11-27T18:20:36.837278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471798275764347:2049] [node 2] ICP22 created new session: [6:7577471798275764438:2049]
2025-11-27T18:20:36.843510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471802301309568:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28948
2025-11-27T18:20:36.842994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471801901441251:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:36.828002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471800888207821:2107] [node 4] ICP22 created new session: [1:7577471800888207845:2107]
2025-11-27T18:20:36.837331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471798275764436:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6777
2025-11-27T18:20:36.824582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-11-27T18:20:36.840432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471798275764438:2049] [node 2] ICS09 handshake done sender: [6:7577471798275764427:2118] self: [6:7577471798275764431:0] peer: [2:7577471801948682516:0] socket: 604 qp: -1
2025-11-27T18:20:36.843547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471801901441259:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:824
2025-11-27T18:20:36.828063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471800888207828:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:36.834333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471799494321576:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471799494321655:2116] held: no
::ffff:127.1.0.0
2025-11-27T18:20:36.824859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471801948682512:2107] [node 4] ICP22 created new session: [2:7577471801948682540:2107]
2025-11-27T18:20:36.841195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471798275764438:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:36.835337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471799494321667:2109] [node 6] ICS09 handshake done sender: [4:7577471799494321644:2110] self: [4:7577471799494321642:0] peer: [6:7577471798275764430:0] socket: 582 qp: -1
2025-11-27T18:20:36.828247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471800888207845:2107] [node 4] ICS09 handshake done sender: [1:7577471800888207832:2114] self: [1:7577471800888207825:0] peer: [4:7577471799494321658:0] socket: 576 qp: -1
2025-11-27T18:20:36.836310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471799494321667:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:36.824905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471801948682513:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.828541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471800888207846:2048] [node 2] ICS09 handshake done sender: [1:7577471800888207827:2110] self: [1:7577471800888207823:0] peer: [2:7577471801948682534:0] socket: 580 qp: -1
2025-11-27T18:20:36.841207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471798275764425:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:36.836363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:36.845367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471801901441174:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.828811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471800888207845:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:36.825934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471801948682540:2107] [node 4] ICS09 handshake done sender: [2:7577471801948682521:2111] self: [2:7577471801948682518:0] peer: [4:7577471799494321660:0] socket: 572 qp: -1
2025-11-27T18:20:36.836801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471799494321656:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:36.838790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471799494321655:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:36.842203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:36.841250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471799494321577:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.828837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471800888207846:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:36.825953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:36.842962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471798275764346:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471798275764426:2117] held: no
2025-11-27T18:20:36.843977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471798275764428:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:36.828880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:36.825996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471801948682514:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.841318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:36.846262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471802301309567:2126] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.845403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471798275764426:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:36.829851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:36.846286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:36.826102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471801948682540:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:36.843170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471799494321577:2049] [node 2] ICP22 created new session: [4:7577471799494321669:2049]
2025-11-27T18:20:36.846316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471798275764409:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.832057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471800888207822:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.846352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471801901441174:2049] [node 2] ICP22 created new session: [5:7577471801901441260:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.826175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471801948682513:2108] [node 5] ICP22 created new session: [2:7577471801948682542:2108]
2025-11-27T18:20:36.834009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:36.844019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471799494321669:2049] [node 2] ICS09 handshake done sender: [4:7577471799494321654:2115] self: [4:7577471799494321660:0] peer: [2:7577471801948682518:0] socket: 593 qp: -1
2025-11-27T18:20:36.846629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471801901441260:2049] [node 2] ICS09 handshake done sender: [5:7577471801901441251:2115] self: [5:7577471801901441252:0] peer: [2:7577471801948682517:0] socket: 586 qp: -1
2025-11-27T18:20:36.826451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:36.834066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471800888207822:2109] [node 6] ICP22 created new session: [1:7577471800888207851:2109]
2025-11-27T18:20:36.846968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471801901441260:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:36.844047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471799494321578:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.826476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:36.846720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471802301309568:2127] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.846945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:36.836034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471800888207851:2109] [node 6] ICS09 handshake done sender: [1:7577471800888207828:2111] self: [1:7577471800888207824:0] peer: [6:7577471798275764432:0] socket: 573 qp: -1
2025-11-27T18:20:36.847018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:36.826499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471801948682514:2109] [node 6] ICP22 created new session: [2:7577471801948682543:2109]
2025-11-27T18:20:36.836447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471800888207851:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:36.847029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471798275764409:2106] [node 5] ICP22 created new session: [6:7577471798275764440:2106]
2025-11-27T18:20:36.844065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471799494321669:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:36.846957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471802301309472:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.847222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471801901441259:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.847132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471798275764436:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.828215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471801948682542:2108] [node 5] ICS09 handshake done sender: [2:7577471801948682520:2110] self: [2:7577471801948682517:0] peer: [5:7577471801901441252:0] socket: 570 qp: -1
2025-11-27T18:20:36.836793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:36.845363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:36.847373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471798275764440:2106] [node 5] ICS09 handshake done sender: [6:7577471798275764425:2116] self: [6:7577471798275764429:0] peer: [5:7577471801901441238:0] socket: 590 qp: -1
2025-11-27T18:20:36.847449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471801901441173:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.846000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471800888207852:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:499
2025-11-27T18:20:36.828230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471801948682543:2109] [node 6] ICS09 handshake done sender: [2:7577471801948682523:2113] self: [2:7577471801948682516:0] peer: [6:7577471798275764431:0] socket: 578 qp: -1
2025-11-27T18:20:36.847348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471802301309471:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.845785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:36.847495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471798275764407:2105] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5677) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471798275764428:2119] held: no
2025-11-27T18:20:36.847113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471800888207854:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:499
2025-11-27T18:20:36.828261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471801948682542:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:36.845987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471799494321578:2050] [node 3] ICP22 created new session: [4:7577471799494321672:2050]
2025-11-27T18:20:36.847515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800888207852:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.828271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471801948682543:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:36.846037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471799494321672:2050] [node 3] ICS09 handshake done sender: [4:7577471799494321656:2117] self: [4:7577471799494321659:0] peer: [3:7577471802301309547:0] socket: 598 qp: -1
2025-11-27T18:20:36.828839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:36.847563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471798275764440:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:36.828861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:36.846332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471799494321672:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:36.847620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:36.846381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:36.835127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471801948682539:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41226
2025-11-27T18:20:36.847649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471798275764346:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.846526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471799494321576:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.847687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:36.847668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800888207831:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.846898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:36.842213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471801948682529:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:36.847684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471800888207830:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.846945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471799494321576:2048] [node 1] ICP22 created new session: [4:7577471799494321675:2048]
2025-11-27T18:20:36.847727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471798275764346:2048] [node 1] ICP22 created new session: [6:7577471798275764442:2048]
2025-11-27T18:20:36.847884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471801901441262:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:824
2025-11-27T18:20:36.847703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471800888207831:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.846995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471799494321675:2048] [node 1] ICS09 handshake done sender: [4:7577471799494321655:2116] self: [4:7577471799494321658:0] peer: [1:7577471800888207825:0] socket: 597 qp: -1
2025-11-27T18:20:36.847882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471798275764442:2048] [node 1] ICS09 handshake done sender: [6:7577471798275764426:2117] self: [6:7577471798275764432:0] peer: [1:7577471800888207824:0] socket: 596 qp: -1
2025-11-27T18:20:36.844022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471801948682449:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.847014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471799494321666:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-27T18:20:36.847718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471800888207830:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.849197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471798275764442:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:36.847144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471799494321675:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:36.847180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:36.844072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:36.849701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:36.847729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471799494321666:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.850744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471800888207831:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:36.845843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471801948682449:2048] [node 1] ICP22 created new session: [2:7577471801948682548:2048]
2025-11-27T18:20:36.850990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471798275764407:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.846039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471801948682548:2048] [node 1] ICS09 handshake done sender: [2:7577471801948682529:2116] self: [2:7577471801948682534:0] peer: [1:7577471800888207823:0] socket: 605 qp: -1
2025-11-27T18:20:36.846318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471801948682548:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:36.851150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:36.846364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:36.851192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471798275764407:2105] [node 4] ICP22 created new session: [6:7577471798275764446:2105]
2025-11-27T18:20:36.851118Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471801948682512:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22865983ns) {HandleHandshakeStatus:329(22863799ns) {DropIncomingHandshake:442(1382ns) DropOutgoingHandshake:457(392ns) SwitchToState:179(491ns) ProcessPendingSessionEvents:495(17750945ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17646053ns max# 2098991ns}}}
2025-11-27T18:20:36.851452Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471802301309544:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(24201966ns) {HandleHandshakeStatus:329(24198301ns) {DropIncomingHandshake:442(21581ns) DropOutgoingHandshake:457(594ns) SwitchToState:179(364ns) ProcessPendingSessionEvents:495(18107303ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17999297ns max# 1707501ns}}}
2025-11-27T18:20:36.851368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471798275764446:2105] [node 4] ICS09 handshake done sender: [6:7577471798275764428:2119] self: [6:7577471798275764430:0] peer: [4:7577471799494321642:0] socket: 608 qp: -1
2025-11-27T18:20:36.851334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471801948682522:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.851397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471798275764347:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:36.851548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7577471801901441237:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(23482817ns) {HandleHandshakeStatus:329(23479676ns) {DropIncomingHandshake:442(11777ns) DropOutgoingHandshake:457(652ns) SwitchToState:179(404ns) ProcessPendingSessionEvents:495(20633447ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20309463ns max# 3278751ns}}}
2025-11-27T18:20:36.851358Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471801948682522:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.851406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471798275764446:2105] [node 4] ICS10 traffic start
2025-11-27T18:20:36.851495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:36.851387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471801948682513:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(22061536ns) {HandleHandshakeStatus:329(22059463ns) {DropIncomingHandshake:442(13652ns) DropOutgoingHandshake:457(647ns) SwitchToState:179(460ns) ProcessPendingSessionEvents:495(12974150ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12900797ns max# 2070704ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.851453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471801948682514:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(20611604ns) {HandleHandshakeStatus:329(20609723ns) {DropIncomingHandshake:442(9939ns) DropOutgoingHandshake:457(386ns) SwitchToState:179(1169ns) ProcessPendingSessionEvents:495(15678365ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15590363ns max# 3357885ns}}}
2025-11-27T18:20:36.852190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471800888207820:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.852158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471798275764348:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.852241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:36.852493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471801901441264:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:824
2025-11-27T18:20:36.852490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471801948682522:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:36.852349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471800888207830:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:36.852498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471801901441262:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.852410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471800888207820:2108] [node 5] ICP22 created new session: [1:7577471800888207858:2108]
2025-11-27T18:20:36.852679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471802301309549:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.852489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471800888207757:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(22054101ns) {HandleHandshakeStatus:329(22051118ns) {DropIncomingHandshake:442(733ns) DropOutgoingHandshake:457(359ns) SwitchToState:179(359ns) ProcessPendingSessionEvents:495(13599480ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12531604ns max# 2002113ns}}}
2025-11-27T18:20:36.853040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471799494321640:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(17246890ns) {HandleHandshakeStatus:329(17244524ns) {DropIncomingHandshake:442(21995ns) DropOutgoingHandshake:457(189ns) SwitchToState:179(590ns) ProcessPendingSessionEvents:495(5482976ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5126314ns max# 1252086ns}}}
2025-11-27T18:20:36.852721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471802301309549:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.853011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471801901441236:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.852555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471800888207858:2108] [node 5] ICS09 handshake done sender: [1:7577471800888207831:2113] self: [1:7577471800888207829:0] peer: [5:7577471801901441263:0] socket: 589 qp: -1
2025-11-27T18:20:36.854905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471802301309571:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28948
2025-11-27T18:20:36.853268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471801948682450:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.852595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471800888207858:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:36.854562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471799494321645:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.854601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471801901441259:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:36.854608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471799494321645:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.852967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471800888207821:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(22150148ns) {HandleHandshakeStatus:329(22146902ns) {DropIncomingHandshake:442(12149ns) DropOutgoingHandshake:457(299ns) SwitchToState:179(319ns) ProcessPendingSessionEvents:495(12054537ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10674711ns max# 2085254ns}}}
2025-11-27T18:20:36.855585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:36.855807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471801901441173:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471801901441259:2117] held: no
2025-11-27T18:20:36.853004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: 2025-11-27T18:20:36.855652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471801948682450:2049] [node 3] ICP22 created new session: [2:7577471801948682551:2049]
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:36.853022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471800888207758:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.856099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471801948682551:2049] [node 3] ICS09 handshake done sender: [2:7577471801948682522:2112] self: [2:7577471801948682519:0] peer: [3:7577471802301309569:0] socket: 600 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:36.853245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:36.856120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471801948682551:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:36.853320Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471800888207822:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16922613ns) {HandleHandshakeStatus:329(16919936ns) {DropIncomingHandshake:442(18453ns) DropOutgoingHandshake:457(846ns) SwitchToState:179(523ns) ProcessPendingSessionEvents:495(5456082ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4913061ns max# 1260889ns}}}
2025-11-27T18:20:36.856153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471802301309549:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:36.855554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471800888207758:2049] [node 3] ICP22 created new session: [1:7577471800888207860:2049]
2025-11-27T18:20:36.856207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471802301309573:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28948
2025-11-27T18:20:36.856278Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471799494321645:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:36.857160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:36.857172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471802301309572:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28948
2025-11-27T18:20:36.858314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471799494321639:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.855769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471800888207860:2049] [node 3] ICS09 handshake done sender: [1:7577471800888207830:2112] self: [1:7577471800888207826:0] peer: [3:7577471802301309570:0] socket: 633 qp: -1
2025-11-27T18:20:36.856086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471800888207860:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:36.859483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:36.856124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:36.860005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471799494321639:2108] [node 5] ICP22 created new session: [4:7577471799494321679:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.857302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471802301309471:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471802301309568:2127] held: no
2025-11-27T18:20:36.859362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471802301309545:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.860658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471798275764449:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6777
2025-11-27T18:20:36.860252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471799494321679:2108] [node 5] ICS09 handshake done sender: [4:7577471799494321645:2111] self: [4:7577471799494321643:0] peer: [5:7577471801901441265:0] socket: 591 qp: -1
2025-11-27T18:20:36.860509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471801901441173:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.859492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471802301309571:2128] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.860710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:36.860638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471799494321679:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:36.860222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471802301309567:2126] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:36.862308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471798275764436:2123] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:36.860751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:36.861050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471801901441173:2048] [node 1] ICP22 created new session: [5:7577471801901441267:2048]
2025-11-27T18:20:36.860240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:36.862179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471801901441267:2048] [node 1] ICS09 handshake done sender: [5:7577471801901441259:2117] self: [5:7577471801901441263:0] peer: [1:7577471800888207829:0] socket: 634 qp: -1
2025-11-27T18:20:36.860295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471802301309545:2109] [node 6] ICP22 created new session: [3:7577471802301309575:2109]
2025-11-27T18:20:36.863780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471798275764348:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.862352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471801901441267:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:36.861077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471802301309575:2109] [node 6] ICS09 handshake done sender: [3:7577471802301309549:2119] self: [3:7577471802301309546:0] peer: [6:7577471798275764448:0] socket: 627 qp: -1
2025-11-27T18:20:36.863920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.864535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471798275764348:2050] [node 3] ICP22 created new session: [6:7577471798275764450:2050]
2025-11-27T18:20:36.862388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:36.861113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471802301309575:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:36.862116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:36.863200Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471801901441236:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5677) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471801901441262:2118] held: no
2025-11-27T18:20:36.864918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471798275764450:2050] [node 3] ICS09 handshake done sender: [6:7577471798275764436:2123] self: [6:7577471798275764448:0] peer: [3:7577471802301309546:0] socket: 628 qp: -1
2025-11-27T18:20:36.862743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471802301309472:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2721) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471802301309567:2126] held: no
2025-11-27T18:20:36.864947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471798275764450:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:36.863443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471802301309568:2127] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:36.864070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471802301309471:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.864973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:36.865545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471801901441266:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:824
2025-11-27T18:20:36.865199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:36.865454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471802301309471:2048] [node 1] ICP22 created new session: [3:7577471802301309578:2048]
2025-11-27T18:20:36.865612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471802301309578:2048] [node 1] ICS09 handshake done sender: [3:7577471802301309568:2127] self: [3:7577471802301309570:0] peer: [1:7577471800888207826:0] socket: 574 qp: -1
2025-11-27T18:20:36.865643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471802301309578:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:36.865750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:36.865762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471800888207757:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:36.865756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471802301309472:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.865863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:36.865916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471802301309472:2049] [node 2] ICP22 created new session: [3:7577471802301309580:2049]
2025-11-27T18:20:36.866073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471801901441262:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:36.866418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471801901441236:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.866229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471802301309580:2049] [node 2] ICS09 handshake done sender: [3:7577471802301309567:2126] self: [3:7577471802301309569:0] peer: [2:7577471801948682519:0] socket: 632 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.866257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471802301309580:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:36.866513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:36.866390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:36.866794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471801901441236:2108] [node 4] ICP22 created new session: [5:7577471801901441270:2108]
2025-11-27T18:20:36.866399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471802301309545:2109] [node 6] ICP18 handshake (actor [3:7577471802301309571:2128]) is too old
2025-11-27T18:20:36.867215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471801901441269:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:824
2025-11-27T18:20:36.867655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471801901441270:2108] [node 4] ICS09 handshake done sender: [5:7577471801901441262:2118] self: [5:7577471801901441265:0] peer: [4:7577471799494321643:0] socket: 584 qp: -1
2025-11-27T18:20:36.867744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471801901441270:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:36.868015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:36.868570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471801901441269:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.868725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471801901441175:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:36.868916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471802301309555:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:36.868941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471802301309555:2123] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:36.870344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471802301309555:2123] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:36.870574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471802301309543:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:36.870660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:36.870630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471801901441273:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:824
2025-11-27T18:20:36.870719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471802301309543:2108] [node 5] ICP22 created new session: [3:7577471802301309583:2108]
2025-11-27T18:20:36.870769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471802301309583:2108] [node 5] ICS09 handshake done sender: [3:7577471802301309555:2123] self: [3:7577471802301309552:0] peer: [5:7577471801901441272:0] socket: 575 qp: -1
2025-11-27T18:20:36.870882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471801901441269:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:36.870799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471802301309583:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:36.870832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:36.871030Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471801901441175:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5233) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471801901441269:2121] held: no
2025-11-27T18:20:36.871832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471801901441175:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:36.872424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:36.872462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471801901441175:2050] [node 3] ICP22 created new session: [5:7577471801901441274:2050]
2025-11-27T18:20:36.872494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471801901441274:2050] [node 3] ICS09 handshake done sender: [5:7577471801901441269:2121] self: [5:7577471801901441272:0] peer: [3:7577471802301309552:0] socket: 581 qp: -1
2025-11-27T18:20:36.872515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471801901441274:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:36.872542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:36.922782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:36.922949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471801948682548:2048] [node 1] ICS01 socket: 605 reason# EndOfStream
2025-11-27T18:20:36.922994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471801948682449:2048] [node 1] ICP30 unregister session Session# [2:7577471801948682548:2048] VirtualId# [2:7577471801948682534:0]
2025-11-27T18:20:36.923026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471801948682548:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.923087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:36.923129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:36.923827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:36.924071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471802301309578:2048] [node 1] ICS01 socket: 574 reason# EndOfStream
2025-11-27T18:20:36.924130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471802301309471:2048] [node 1] ICP30 unregister session Session# [3:7577471802301309578:2048] VirtualId# [3:7577471802301309570:0]
2025-11-27T18:20:36.924177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471802301309578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.924275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:36.924329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:36.926517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:36.926732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471799494321675:2048] [node 1] ICS01 socket: 597 reason# EndOfStream
2025-11-27T18:20:36.926772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471799494321576:2048] [node 1] ICP30 unregister session Session# [4:7577471799494321675:2048] VirtualId# [4:7577471799494321658:0]
2025-11-27T18:20:36.926809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471799494321675:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.926865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:36.926908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:36.927484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:36.927664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471801901441267:2048] [node 1] ICS01 socket: 634 reason# EndOfStream
2025-11-27T18:20:36.927719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471801901441173:2048] [node 1] ICP30 unregister session Session# [5:7577471801901441267:2048] VirtualId# [5:7577471801901441263:0]
2025-11-27T18:20:36.927771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471801901441267:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.927839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:36.927896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:36.928177Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:36.928295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471798275764442:2048] [node 1] ICS01 socket: 596 reason# EndOfStream
2025-11-27T18:20:36.928325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471798275764346:2048] [node 1] ICP30 unregister session Session# [6:7577471798275764442:2048] VirtualId# [6:7577471798275764432:0]
2025-11-27T18:20:36.928348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471798275764442:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:36.928419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:36.928470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:37.019314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:37.019567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471802301309580:2049] [node 2] ICS01 socket: 632 reason# EndOfStream
2025-11-27T18:20:37.019617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471802301309472:2049] [node 2] ICP30 unregister session Session# [3:7577471802301309580:2049] VirtualId# [3:7577471802301309569:0]
2025-11-27T18:20:37.019654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471802301309580:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.019778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:37.019849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:37.021864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:37.022085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471799494321669:2049] [node 2] ICS07 socket disconnect 593 reason# EndOfStream
2025-11-27T18:20:37.022133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471799494321669:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.022245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:37.022295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471799494321669:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:37.022603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:37.022726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471801901441260:2049] [node 2] ICS01 socket: 586 reason# EndOfStream
2025-11-27T18:20:37.022758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471801901441174:2049] [node 2] ICP30 unregister session Session# [5:7577471801901441260:2049] VirtualId# [5:7577471801901441252:0]
2025-11-27T18:20:37.022782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471801901441260:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.022836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:37.022869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:37.023017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471799494321577:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2721) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:37.023262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471799494321669:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:37.023312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471799494321577:2049] [node 2] ICP30 unregister session Session# [4:7577471799494321669:2049] VirtualId# [4:7577471799494321660:0]
2025-11-27T18:20:37.023351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:37.023387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471799494321577:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2721) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.023406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2721) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.023833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:37.024019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471798275764438:2049] [node 2] ICS07 socket disconnect 604 reason# EndOfStream
2025-11-27T18:20:37.024061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471798275764438:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.024128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:37.024193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471798275764438:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:37.024741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471798275764347:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2721) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:37.024782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471798275764438:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:37.024826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471798275764347:2049] [node 2] ICP30 unregister session Session# [6:7577471798275764438:2049] VirtualId# [6:7577471798275764431:0]
2025-11-27T18:20:37.024929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:37.024956Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471798275764347:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2721) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.024966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2721) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.033301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471799494321577:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:37.034257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471798275764347:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:37.065206Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:37.065477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471799494321672:2050] [node 3] ICS01 socket: 598 reason# EndOfStream
2025-11-27T18:20:37.065530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471799494321578:2050] [node 3] ICP30 unregister session Session# [4:7577471799494321672:2050] VirtualId# [4:7577471799494321659:0]
2025-11-27T18:20:37.065573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471799494321672:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.065658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:37.065728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:37.066278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:37.066477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471801901441274:2050] [node 3] ICS01 socket: 581 reason# EndOfStream
2025-11-27T18:20:37.066519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471801901441175:2050] [node 3] ICP30 unregister session Session# [5:7577471801901441274:2050] VirtualId# [5:7577471801901441272:0]
2025-11-27T18:20:37.066562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471801901441274:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.066652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:37.066695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:37.066976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:37.067114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471798275764450:2050] [node 3] ICS01 socket: 628 reason# EndOfStream
2025-11-27T18:20:37.067145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471798275764348:2050] [node 3] ICP30 unregister session Session# [6:7577471798275764450:2050] VirtualId# [6:7577471798275764448:0]
2025-11-27T18:20:37.067163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471798275764450:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.067223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:37.067284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:37.149300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:37.149543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471801901441270:2108] [node 4] ICS01 socket: 584 reason# EndOfStream
2025-11-27T18:20:37.149594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471801901441236:2108] [node 4] ICP30 unregister session Session# [5:7577471801901441270:2108] VirtualId# [5:7577471801901441265:0]
2025-11-27T18:20:37.149629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471801901441270:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.149700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:37.149749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:37.150407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:37.150596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471798275764446:2105] [node 4] ICS01 socket: 608 reason# EndOfStream
2025-11-27T18:20:37.150642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471798275764407:2105] [node 4] ICP30 unregister session Session# [6:7577471798275764446:2105] VirtualId# [6:7577471798275764430:0]
2025-11-27T18:20:37.150677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471798275764446:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.150736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:37.150776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:37.227847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:37.228110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471798275764440:2106] [node 5] ICS07 socket disconnect 590 reason# EndOfStream
2025-11-27T18:20:37.228157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471798275764440:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.228244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:37.228308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471798275764440:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:37.229050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471798275764409:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14339) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:37.229106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471798275764440:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:37.229161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471798275764409:2106] [node 5] ICP30 unregister session Session# [6:7577471798275764440:2106] VirtualId# [6:7577471798275764429:0]
2025-11-27T18:20:37.229282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:37.229314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471798275764409:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14339) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.229327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14339) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:37.342643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471805298951884:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:37.342709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471805298951885:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:37.365034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471806022687077:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:37.365106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471806022687078:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:37.388593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471804744798938:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:37.388651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471804744798939:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:37.409554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471803023927123:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:37.409618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471803023927124:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:37.409668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471803023927125:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:37.428182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471803847602782:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:37.428235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471803847602783:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:37.428264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471803847602784:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:37.447409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471805348762485:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:37.447492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471805348762486:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:37.447526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471805348762487:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:37.498070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471805348762546:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.544293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471803847602843:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7236
2025-11-27T18:20:37.545353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471804744799001:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:37.545814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471804744799002:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:37.546048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471806022687140:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:37.546244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7577471805298951885:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45855068ns) {Configure:104(45852081ns) {ProcessConfigured:135(580691ns) {StartInitialHandshake:157(575239ns) {DropHandshakes:470(3156ns) {DropIncomingHandshake:442(794ns) DropOutgoingHandshake:457(986ns)} GenerateSessionVirtualId:988(904ns) PrepareNewSessionHandshake:477(1865ns) {SwitchToState:179(981ns)}}}}}
2025-11-27T18:20:37.546686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471806022687141:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:37.546749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471805298951951:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:37.546737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471803023927187:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:37.546774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471804744799003:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:37.546858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471805348762553:2110] [node 4] ICP01 ready to work
2025-11-27T18:20:37.547162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471806022687142:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:37.547144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471803847602846:2109] [node 4] ICP01 ready to work
2025-11-27T18:20:37.547371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471805298951952:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:37.547415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471803023927188:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:37.547966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471803847602847:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:37.548089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471805298951953:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:37.548640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471803847602843:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.555855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471805298951962:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48494
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.556449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806022687153:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28263
2025-11-27T18:20:37.556501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806022687154:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28263
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.559385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471804744799014:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46442
2025-11-27T18:20:37.559657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471804744799015:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46442
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.560567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471803023927199:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57391
2025-11-27T18:20:37.560640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471803023927200:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57391
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.562108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471803023927203:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57391
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.562315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471805298951962:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.562521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471804744799019:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46442
2025-11-27T18:20:37.562899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806022687154:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.564106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471805298951964:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48494
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.564889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806022687158:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28263
2025-11-27T18:20:37.564940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806022687159:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28263
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.565012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806022687153:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.565063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806022687160:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28263
2025-11-27T18:20:37.565217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471806022687077:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.566781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471805298951965:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48494
2025-11-27T18:20:37.566873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471805298951966:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48494
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.566983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471805298951967:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48494
2025-11-27T18:20:37.567098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471805298951964:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.567175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471805298951945:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.567194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471805298951945:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.569131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471805298951945:2107] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:37.569107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471804744799014:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.569187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471804744799015:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.569211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806022687159:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.569299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806022687163:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28263
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.569608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471804744799020:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46442
2025-11-27T18:20:37.569592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471805298951966:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.569642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471805298951884:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.569769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471804744798938:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.569818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:37.569880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471805298951884:2048] [node 2] ICP22 created new session: [1:7577471805298951971:2048]
2025-11-27T18:20:37.569861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471804744799019:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.569928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471804744798939:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.569923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471805298951971:2048] [node 2] ICS09 handshake done sender: [1:7577471805298951945:2107] self: [1:7577471805298951944:0] peer: [2:7577471806022687162:0] socket: 579 qp: -1
2025-11-27T18:20:37.569964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471805298951971:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:37.570000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:37.569981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806022687158:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.570085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471806022687153:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:37.570642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806022687146:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.570670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471803023927199:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.570670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471806022687146:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.571106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471806022687077:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.571160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:37.571199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471806022687077:2048] [node 1] ICP22 created new session: [2:7577471806022687165:2048]
2025-11-27T18:20:37.571229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471806022687165:2048] [node 1] ICS09 handshake done sender: [2:7577471806022687153:2115] self: [2:7577471806022687162:0] peer: [1:7577471805298951944:0] socket: 584 qp: -1
2025-11-27T18:20:37.571213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471805298951965:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.571246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471806022687165:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:37.571220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471804744799020:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.571268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:37.571289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471803023927203:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.571390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471803023927124:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.571586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471803023927125:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.571672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471805298951950:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.571705Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471805298951950:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.571789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471804744799009:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.571813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471804744799009:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.572877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471805298951967:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.572937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471803023927200:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.573034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471803023927123:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.573239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471804744799009:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.573414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471803023927208:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57391
2025-11-27T18:20:37.573630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806022687160:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.573706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471804744799001:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.573774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:37.573828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471804744799001:2107] [node 4] ICP22 created new session: [3:7577471804744799026:2107]
2025-11-27T18:20:37.573927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471804744799026:2107] [node 4] ICS09 handshake done sender: [3:7577471804744799009:2112] self: [3:7577471804744799008:0] peer: [4:7577471803023927206:0] socket: 601 qp: -1
2025-11-27T18:20:37.573915Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471803023927203:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:37.573966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471804744799026:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:37.574024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:37.574555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471803023927125:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.574608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:37.574646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471803023927125:2050] [node 3] ICP22 created new session: [4:7577471803023927209:2050]
2025-11-27T18:20:37.574719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471803023927209:2050] [node 3] ICS09 handshake done sender: [4:7577471803023927203:2118] self: [4:7577471803023927206:0] peer: [3:7577471804744799008:0] socket: 604 qp: -1
2025-11-27T18:20:37.574756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471803023927209:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:37.574785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:37.574750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806022687148:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.574772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471806022687148:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.574829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471805298951956:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.574850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471805298951956:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.576058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471806022687146:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:37.576405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471805298951950:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:37.576444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471806022687078:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.576479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:37.576508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471806022687078:2049] [node 3] ICP22 created new session: [2:7577471806022687172:2049]
2025-11-27T18:20:37.576537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471806022687172:2049] [node 3] ICS09 handshake done sender: [2:7577471806022687146:2111] self: [2:7577471806022687145:0] peer: [3:7577471804744799022:0] socket: 588 qp: -1
2025-11-27T18:20:37.576620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471803847602862:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7236
2025-11-27T18:20:37.576583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471806022687172:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:37.576609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:37.576766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471805298951885:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.576807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:37.576835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471805298951885:2049] [node 3] ICP22 created new session: [1:7577471805298951979:2049]
2025-11-27T18:20:37.576883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471805298951979:2049] [node 3] ICS09 handshake done sender: [1:7577471805298951950:2111] self: [1:7577471805298951949:0] peer: [3:7577471804744799021:0] socket: 583 qp: -1
2025-11-27T18:20:37.576902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471805298951979:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:37.576922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:37.577423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471806022687148:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.578035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471804744799028:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46442
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.578391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471804744799029:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46442
2025-11-27T18:20:37.578375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471803847602863:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7236
2025-11-27T18:20:37.578459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471805298951956:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.578858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471803847602864:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7236
2025-11-27T18:20:37.579007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471806022687140:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.579057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:37.579406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471806022687140:2107] [node 4] ICP22 created new session: [2:7577471806022687174:2107]
2025-11-27T18:20:37.579521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471806022687174:2107] [node 4] ICS09 handshake done sender: [2:7577471806022687148:2112] self: [2:7577471806022687147:0] peer: [4:7577471803023927205:0] socket: 589 qp: -1
2025-11-27T18:20:37.579583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471806022687174:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:37.579654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.579702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471804744799015:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:37.579743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471803023927211:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57391
2025-11-27T18:20:37.579920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471804744798939:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26478) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471804744799015:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.580207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471803023927212:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57391
2025-11-27T18:20:37.580385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471804744798939:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.580413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:37.580439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471804744798939:2049] [node 2] ICP22 created new session: [3:7577471804744799031:2049]
2025-11-27T18:20:37.580711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471804744799031:2049] [node 2] ICS09 handshake done sender: [3:7577471804744799015:2116] self: [3:7577471804744799022:0] peer: [2:7577471806022687145:0] socket: 594 qp: -1
2025-11-27T18:20:37.580783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471804744799031:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:37.580806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:37.580847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471805298951951:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.580897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:37.580960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471805298951951:2108] [node 4] ICP22 created new session: [1:7577471805298951981:2108]
2025-11-27T18:20:37.581005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471805298951981:2108] [node 4] ICS09 handshake done sender: [1:7577471805298951956:2112] self: [1:7577471805298951955:0] peer: [4:7577471803023927207:0] socket: 591 qp: -1
2025-11-27T18:20:37.580970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471803023927199:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:37.581045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471805298951981:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:37.581075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.581274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471803847602865:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7236
2025-11-27T18:20:37.581315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471803023927200:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:37.581572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471803023927124:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.581634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:37.581719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471803023927124:2049] [node 2] ICP22 created new session: [4:7577471803023927214:2049]
2025-11-27T18:20:37.581781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471803023927214:2049] [node 2] ICS09 handshake done sender: [4:7577471803023927199:2116] self: [4:7577471803023927205:0] peer: [2:7577471806022687147:0] socket: 595 qp: -1
2025-11-27T18:20:37.581813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471803023927214:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:37.581846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:37.583604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471804744799014:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.583893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471805348762564:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42309
2025-11-27T18:20:37.584385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471803847602862:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.584559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471803847602782:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.584934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471805348762565:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42309
2025-11-27T18:20:37.585740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471805298951958:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.585773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471805298951958:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.586040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471803023927123:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.586083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:37.586142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471803023927123:2048] [node 1] ICP22 created new session: [4:7577471803023927216:2048]
2025-11-27T18:20:37.586181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471803023927216:2048] [node 1] ICS09 handshake done sender: [4:7577471803023927200:2117] self: [4:7577471803023927207:0] peer: [1:7577471805298951955:0] socket: 596 qp: -1
2025-11-27T18:20:37.586205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471803023927216:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:37.586242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:37.586219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471804744798938:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28349) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471804744799014:2115] held: no
2025-11-27T18:20:37.586441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471805348762565:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.586568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471805348762485:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.586601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471805298951958:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:37.586602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471803847602864:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.586607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471804744798938:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.586671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:37.586709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471804744798938:2048] [node 1] ICP22 created new session: [3:7577471804744799033:2048]
2025-11-27T18:20:37.586743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471804744799033:2048] [node 1] ICS09 handshake done sender: [3:7577471804744799014:2115] self: [3:7577471804744799021:0] peer: [1:7577471805298951949:0] socket: 593 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.586804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471804744799033:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:37.586848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:37.586915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471803847602867:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7236
2025-11-27T18:20:37.586946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471805298951960:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.586967Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471805298951960:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.587328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471803847602783:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.587466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471803847602862:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:37.587665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471805298951952:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.587718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:37.587750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471805298951952:2109] [node 5] ICP22 created new session: [1:7577471805298951985:2109]
2025-11-27T18:20:37.587721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471803847602782:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.587780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471805298951985:2109] [node 5] ICS09 handshake done sender: [1:7577471805298951958:2113] self: [1:7577471805298951957:0] peer: [5:7577471803847602866:0] socket: 623 qp: -1
2025-11-27T18:20:37.587758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:37.587812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471805298951985:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:37.587782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471803847602782:2048] [node 1] ICP22 created new session: [5:7577471803847602869:2048]
2025-11-27T18:20:37.587834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:37.587809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471803847602869:2048] [node 1] ICS09 handshake done sender: [5:7577471803847602862:2116] self: [5:7577471803847602866:0] peer: [1:7577471805298951957:0] socket: 625 qp: -1
2025-11-27T18:20:37.587827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471803847602869:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:37.587894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:37.588517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806022687150:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.588554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471806022687150:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.589946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471805348762564:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.590036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471805348762567:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42309
2025-11-27T18:20:37.590111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471805348762486:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.590192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471806022687150:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:37.591342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806022687152:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.591378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471806022687152:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.591404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471805348762567:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.591508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471805348762487:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.591669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471804744799013:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.591690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471804744799013:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.592841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471806022687141:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.593009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:37.593081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471806022687141:2108] [node 5] ICP22 created new session: [2:7577471806022687178:2108]
2025-11-27T18:20:37.593089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471803847602863:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.593131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471806022687178:2108] [node 5] ICS09 handshake done sender: [2:7577471806022687150:2113] self: [2:7577471806022687149:0] peer: [5:7577471803847602868:0] socket: 629 qp: -1
2025-11-27T18:20:37.593140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471804744799013:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:37.593165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471806022687178:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:37.593228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.593621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471804744799003:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.593391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471805348762570:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42309
2025-11-27T18:20:37.593662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:37.593698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471804744799003:2109] [node 6] ICP22 created new session: [3:7577471804744799037:2109]
2025-11-27T18:20:37.593932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471804744799037:2109] [node 6] ICS09 handshake done sender: [3:7577471804744799013:2114] self: [3:7577471804744799012:0] peer: [6:7577471805348762569:0] socket: 634 qp: -1
2025-11-27T18:20:37.593962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471804744799037:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:37.594010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:37.594088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471805348762567:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:37.594203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471803023927123:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:37.594295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471805348762487:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.594373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:37.594400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471805348762487:2050] [node 3] ICP22 created new session: [6:7577471805348762572:2050]
2025-11-27T18:20:37.594445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471805348762572:2050] [node 3] ICS09 handshake done sender: [6:7577471805348762567:2118] self: [6:7577471805348762569:0] peer: [3:7577471804744799012:0] socket: 635 qp: -1
2025-11-27T18:20:37.594473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471805348762572:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.594491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:37.594476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471803847602784:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.594559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471803847602871:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7236
2025-11-27T18:20:37.595208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471805298951960:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:37.595242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471804744799011:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.595208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471803847602865:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.595271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471804744799011:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.595294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471803847602864:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:37.596205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471805298951953:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.596248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:37.596669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471805298951953:2110] [node 6] ICP22 created new session: [1:7577471805298951988:2110]
2025-11-27T18:20:37.596695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471805298951988:2110] [node 6] ICS09 handshake done sender: [1:7577471805298951960:2114] self: [1:7577471805298951959:0] peer: [6:7577471805348762566:0] socket: 597 qp: -1
2025-11-27T18:20:37.596716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471805298951988:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:37.596735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.596835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471806022687152:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:37.596953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471803023927219:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57391
2025-11-27T18:20:37.598316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471806022687142:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.598384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:37.598439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471806022687142:2109] [node 6] ICP22 created new session: [2:7577471806022687181:2109]
2025-11-27T18:20:37.598487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471806022687181:2109] [node 6] ICS09 handshake done sender: [2:7577471806022687152:2114] self: [2:7577471806022687151:0] peer: [6:7577471805348762568:0] socket: 615 qp: -1
2025-11-27T18:20:37.598521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471806022687181:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.598572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:37.598555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471803847602783:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.598609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:37.598650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471803847602783:2049] [node 2] ICP22 created new session: [5:7577471803847602874:2049]
2025-11-27T18:20:37.598687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471803847602874:2049] [node 2] ICS09 handshake done sender: [5:7577471803847602864:2118] self: [5:7577471803847602868:0] peer: [2:7577471806022687149:0] socket: 631 qp: -1
2025-11-27T18:20:37.598714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471803847602874:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:37.598745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:37.598623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471805348762571:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42309
2025-11-27T18:20:37.598884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471804744799011:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:37.600073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471804744799002:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.600129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:37.600173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471804744799002:2108] [node 5] ICP22 created new session: [3:7577471804744799040:2108]
2025-11-27T18:20:37.600226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471804744799040:2108] [node 5] ICS09 handshake done sender: [3:7577471804744799011:2113] self: [3:7577471804744799010:0] peer: [5:7577471803847602873:0] socket: 627 qp: -1
2025-11-27T18:20:37.600258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471804744799040:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:37.600313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:37.600287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471803847602783:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:37.601897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471805348762565:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:37.602088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471805348762485:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28349) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471805348762565:2117] held: no
2025-11-27T18:20:37.602560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471805348762485:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.602613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:37.602643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471805348762485:2048] [node 1] ICP22 created new session: [6:7577471805348762576:2048]
2025-11-27T18:20:37.602627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471803847602846:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.602666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471805348762576:2048] [node 1] ICS09 handshake done sender: [6:7577471805348762565:2117] self: [6:7577471805348762566:0] peer: [1:7577471805298951959:0] socket: 613 qp: -1
2025-11-27T18:20:37.602690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471805348762576:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:37.602711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:37.604806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471805348762486:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26478) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471805348762564:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.605334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471805348762574:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42309
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.605500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471805348762575:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42309
2025-11-27T18:20:37.605784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471803023927196:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.605827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471803023927196:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.605762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471803847602784:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27317) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471803847602863:2117] held: no
2025-11-27T18:20:37.606436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471803023927219:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.606801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471803023927196:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.606878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471805348762574:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.606976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471803847602876:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7236
2025-11-27T18:20:37.607102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471805348762564:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:37.607265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471803023927187:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.607320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:37.607361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471803023927187:2108] [node 5] ICP22 created new session: [4:7577471803023927223:2108]
2025-11-27T18:20:37.607417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471803023927223:2108] [node 5] ICS09 handshake done sender: [4:7577471803023927196:2114] self: [4:7577471803023927195:0] peer: [5:7577471803847602877:0] socket: 599 qp: -1
2025-11-27T18:20:37.607456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471803023927223:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:37.607490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:37.607478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471805348762486:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.607509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:37.607546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471805348762486:2049] [node 2] ICP22 created new session: [6:7577471805348762580:2049]
2025-11-27T18:20:37.607581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471805348762580:2049] [node 2] ICS09 handshake done sender: [6:7577471805348762564:2116] self: [6:7577471805348762568:0] peer: [2:7577471806022687151:0] socket: 617 qp: -1
2025-11-27T18:20:37.607604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471805348762580:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:37.607624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:37.608039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471803847602863:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:37.608304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471803847602784:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.608393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:37.608454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471803847602784:2050] [node 3] ICP22 created new session: [5:7577471803847602880:2050]
2025-11-27T18:20:37.608511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471803847602880:2050] [node 3] ICS09 handshake done sender: [5:7577471803847602863:2117] self: [5:7577471803847602873:0] peer: [3:7577471804744799010:0] socket: 628 qp: -1
2025-11-27T18:20:37.608498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471805348762553:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.608556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471803847602880:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:37.608586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.609120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471803023927222:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57391
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.609493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471803847602879:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7236
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.610098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471805348762579:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42309
2025-11-27T18:20:37.610319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471803023927198:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.610342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471803023927198:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.610991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471803023927222:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.611480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471803023927198:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.611647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471805348762583:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42309
2025-11-27T18:20:37.611711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471803023927188:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.611745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:37.611774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471803023927188:2110] [node 6] ICP22 created new session: [4:7577471803023927227:2110]
2025-11-27T18:20:37.611800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471803023927227:2110] [node 6] ICS09 handshake done sender: [4:7577471803023927198:2115] self: [4:7577471803023927197:0] peer: [6:7577471805348762582:0] socket: 638 qp: -1
2025-11-27T18:20:37.611829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471803023927227:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:37.611859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:37.612264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471805348762574:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:37.612473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471803847602846:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12256) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471803847602865:2119] held: no
2025-11-27T18:20:37.612556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471805348762553:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.612616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:37.612651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471805348762553:2110] [node 4] ICP22 created new session: [6:7577471805348762584:2110]
2025-11-27T18:20:37.612681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471805348762584:2110] [node 4] ICS09 handshake done sender: [6:7577471805348762574:2121] self: [6:7577471805348762582:0] peer: [4:7577471803023927197:0] socket: 582 qp: -1
2025-11-27T18:20:37.612706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471805348762584:2110] [node 4] ICS10 traffic start
2025-11-27T18:20:37.612727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:37.613006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471803847602865:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:37.613463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471803847602846:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.613510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:37.613547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471803847602846:2109] [node 4] ICP22 created new session: [5:7577471803847602882:2109]
2025-11-27T18:20:37.613587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471803847602882:2109] [node 4] ICS09 handshake done sender: [5:7577471803847602865:2119] self: [5:7577471803847602877:0] peer: [4:7577471803023927195:0] socket: 611 qp: -1
2025-11-27T18:20:37.613623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471803847602882:2109] [node 4] ICS10 traffic start
2025-11-27T18:20:37.613660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:37.614721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471805348762579:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.615085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471805348762546:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:37.615257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471803847602857:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:37.615276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471803847602857:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:37.615906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471803847602857:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:37.616115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471805348762587:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42309
2025-11-27T18:20:37.616144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471803847602847:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:37.616194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:37.616232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471803847602847:2110] [node 6] ICP22 created new session: [5:7577471803847602885:2110]
2025-11-27T18:20:37.616272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471803847602885:2110] [node 6] ICS09 handshake done sender: [5:7577471803847602857:2115] self: [5:7577471803847602856:0] peer: [6:7577471805348762586:0] socket: 602 qp: -1
2025-11-27T18:20:37.616301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471803847602885:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:37.616336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:37.616499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471805348762579:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:37.616679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471805348762546:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:37.616738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:37.616783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471805348762546:2106] [node 5] ICP22 created new session: [6:7577471805348762588:2106]
2025-11-27T18:20:37.616820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471805348762588:2106] [node 5] ICS09 handshake done sender: [6:7577471805348762579:2123] self: [6:7577471805348762586:0] peer: [5:7577471803847602856:0] socket: 603 qp: -1
2025-11-27T18:20:37.616849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471805348762588:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:37.616880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:37.638813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:37.639044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471806022687165:2048] [node 1] ICS01 socket: 584 reason# EndOfStream
2025-11-27T18:20:37.639096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471806022687077:2048] [node 1] ICP30 unregister session Session# [2:7577471806022687165:2048] VirtualId# [2:7577471806022687162:0]
2025-11-27T18:20:37.639135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471806022687165:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.639228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:37.639291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:37.640107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:37.640263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471804744799033:2048] [node 1] ICS01 socket: 593 reason# EndOfStream
2025-11-27T18:20:37.640294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471804744798938:2048] [node 1] ICP30 unregister session Session# [3:7577471804744799033:2048] VirtualId# [3:7577471804744799021:0]
2025-11-27T18:20:37.640321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471804744799033:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.640377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:37.640439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:37.642557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:37.642794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471803023927216:2048] [node 1] ICS07 socket disconnect 596 reason# EndOfStream
2025-11-27T18:20:37.642835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471803023927216:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.642910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:37.642952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471803023927216:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:37.643597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:37.643789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471803847602869:2048] [node 1] ICS07 socket disconnect 625 reason# EndOfStream
2025-11-27T18:20:37.643841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471803847602869:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.643913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:37.643882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471803023927123:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28349) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:37.643938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471803023927216:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:37.643956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471803847602869:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:37.643973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471803023927123:2048] [node 1] ICP30 unregister session Session# [4:7577471803023927216:2048] VirtualId# [4:7577471803023927207:0]
2025-11-27T18:20:37.644098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:37.644146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471803023927123:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28349) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.644163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28349) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.644725Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471803847602782:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28349) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:37.644774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471803847602869:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:37.644806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471803847602782:2048] [node 1] ICP30 unregister session Session# [5:7577471803847602869:2048] VirtualId# [5:7577471803847602866:0]
2025-11-27T18:20:37.644849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:37.644908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:37.644944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471803847602782:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28349) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.644961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28349) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.645060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471805348762576:2048] [node 1] ICS01 socket: 613 reason# EndOfStream
2025-11-27T18:20:37.645105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471805348762485:2048] [node 1] ICP30 unregister session Session# [6:7577471805348762576:2048] VirtualId# [6:7577471805348762566:0]
2025-11-27T18:20:37.645148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471805348762576:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.645233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:37.645301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:37.654858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471803847602782:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:37.654858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471803023927123:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:37.695615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:37.695795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471804744799031:2049] [node 2] ICS01 socket: 594 reason# EndOfStream
2025-11-27T18:20:37.695829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471804744798939:2049] [node 2] ICP30 unregister session Session# [3:7577471804744799031:2049] VirtualId# [3:7577471804744799022:0]
2025-11-27T18:20:37.695859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471804744799031:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.695926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:37.695966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:37.698392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:37.698568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471803023927214:2049] [node 2] ICS07 socket disconnect 595 reason# EndOfStream
2025-11-27T18:20:37.698593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471803023927214:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.698649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:37.698681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471803023927214:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:37.699230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:37.699419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471803847602874:2049] [node 2] ICS07 socket disconnect 631 reason# EndOfStream
2025-11-27T18:20:37.699440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471803847602874:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.699420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471803023927124:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26478) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:37.699454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471803023927214:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:37.699499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:37.699478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471803023927124:2049] [node 2] ICP30 unregister session Session# [4:7577471803023927214:2049] VirtualId# [4:7577471803023927205:0]
2025-11-27T18:20:37.699539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471803847602874:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:37.699566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:37.699598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471803023927124:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26478) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.699613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26478) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.700166Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471803847602783:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26478) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:37.700250Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:37.700204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471803847602874:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:37.700224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471803847602783:2049] [node 2] ICP30 unregister session Session# [5:7577471803847602874:2049] VirtualId# [5:7577471803847602868:0]
2025-11-27T18:20:37.700253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:37.700283Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471803847602783:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26478) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.700441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471805348762580:2049] [node 2] ICS01 socket: 617 reason# EndOfStream
2025-11-27T18:20:37.700297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26478) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.700498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471805348762486:2049] [node 2] ICP30 unregister session Session# [6:7577471805348762580:2049] VirtualId# [6:7577471805348762568:0]
2025-11-27T18:20:37.700521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471805348762580:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.700596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:37.700663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:37.709094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471803023927124:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:37.710133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471803847602783:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:37.722911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:37.723178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471803023927209:2050] [node 3] ICS07 socket disconnect 604 reason# EndOfStream
2025-11-27T18:20:37.723210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471803023927209:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.723295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:37.723352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471803023927209:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:37.723845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:37.723999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471803847602880:2050] [node 3] ICS01 socket: 628 reason# EndOfStream
2025-11-27T18:20:37.724036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471803847602784:2050] [node 3] ICP30 unregister session Session# [5:7577471803847602880:2050] VirtualId# [5:7577471803847602873:0]
2025-11-27T18:20:37.724032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471803023927125:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27317) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:37.724086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471803847602880:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.724071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471803023927209:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:37.724098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471803023927125:2050] [node 3] ICP30 unregister session Session# [4:7577471803023927209:2050] VirtualId# [4:7577471803023927206:0]
2025-11-27T18:20:37.724157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:37.724194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:37.724190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:37.724218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471803023927125:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27317) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.724237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27317) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.724937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:37.725106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471805348762572:2050] [node 3] ICS07 socket disconnect 635 reason# EndOfStream
2025-11-27T18:20:37.725138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471805348762572:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.725201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:37.725248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471805348762572:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:37.725998Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471805348762487:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27317) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:37.726044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471805348762572:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:37.726072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471805348762487:2050] [node 3] ICP30 unregister session Session# [6:7577471805348762572:2050] VirtualId# [6:7577471805348762569:0]
2025-11-27T18:20:37.726159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:37.726194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471805348762487:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27317) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.726207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27317) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.733690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471803023927125:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:37.735787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471805348762487:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:37.768913Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:37.769190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471803847602882:2109] [node 4] ICS01 socket: 611 reason# EndOfStream
2025-11-27T18:20:37.769242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471803847602846:2109] [node 4] ICP30 unregister session Session# [5:7577471803847602882:2109] VirtualId# [5:7577471803847602877:0]
2025-11-27T18:20:37.769298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471803847602882:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.769385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:37.769444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:37.770165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:37.770416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471805348762584:2110] [node 4] ICS07 socket disconnect 582 reason# EndOfStream
2025-11-27T18:20:37.770455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471805348762584:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.770541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:37.770593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471805348762584:2110] [node 4] ICS15 start handshake
2025-11-27T18:20:37.771391Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471805348762553:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12256) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:37.771447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471805348762584:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:37.771490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471805348762553:2110] [node 4] ICP30 unregister session Session# [6:7577471805348762584:2110] VirtualId# [6:7577471805348762582:0]
2025-11-27T18:20:37.771584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:37.771617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471805348762553:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12256) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.771629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12256) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.781830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471805348762553:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:20:37.801465Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:37.801653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471805348762588:2106] [node 5] ICS07 socket disconnect 603 reason# EndOfStream
2025-11-27T18:20:37.801682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471805348762588:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:37.801748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:37.801783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471805348762588:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:37.802474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471805348762546:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17436) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:37.802530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471805348762588:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:37.802572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471805348762546:2106] [node 5] ICP30 unregister session Session# [6:7577471805348762588:2106] VirtualId# [6:7577471805348762586:0]
2025-11-27T18:20:37.802679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:37.802712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471805348762546:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17436) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.802727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17436) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:37.812532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471805348762546:2106] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:37.896734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471805860149712:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:37.896976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471805860149713:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:37.916036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471803570758432:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:37.916046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471803570758433:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:37.937937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471806296013213:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:37.937952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471806296013212:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:37.958999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471804424661245:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:37.959041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471804424661247:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:37.959285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471804424661246:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:37.979593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471805718886789:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:37.979661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471805718886791:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:37.979738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471805718886790:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:37.997280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471804540414854:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:37.997298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471804540414855:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:37.997354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471804540414856:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:38.067661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471808835382210:2105] [node 5] ICP01 ready to work
2025-11-27T18:20:38.073962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471807865725791:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:38.080329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471810155117071:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:38.083677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471810013854149:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:38.080566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471810590980573:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:38.085248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471810155117072:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:38.086053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471807865725793:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:38.085284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471808835382212:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:38.083530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471810590980572:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:38.086246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471807865725792:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:38.086389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471808719628605:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:38.086226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471810013854150:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:38.085270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471810155117073:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:38.086229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471810590980574:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:38.086399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471808719628604:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:38.098117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471810013854164:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7549
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.098475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471810155117088:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
2025-11-27T18:20:38.098571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471810590980589:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36122
2025-11-27T18:20:38.098637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471810155117087:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
2025-11-27T18:20:38.098651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471810590980588:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36122
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.100448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471810155117089:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.104639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471807865725803:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6729
2025-11-27T18:20:38.106220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471808719628614:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60400
2025-11-27T18:20:38.104650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471807865725802:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6729
2025-11-27T18:20:38.106196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471807865725804:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6729
2025-11-27T18:20:38.107195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471808719628615:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60400
2025-11-27T18:20:38.108039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471810013854164:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.106999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471807865725805:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6729
2025-11-27T18:20:38.108865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471808719628615:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.108320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471807865725803:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.108948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471810590980589:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.108955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471807865725805:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.109002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471804424661247:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.109629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471810013854165:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7549
2025-11-27T18:20:38.109835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471807865725811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6729
2025-11-27T18:20:38.109679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471808719628614:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.110088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471804424661245:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.110087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471810590980588:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.110581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471807865725804:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.110654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471810590980590:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36122
2025-11-27T18:20:38.110665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471810590980580:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.111286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471810155117088:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.110689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471810590980580:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.111459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471810155117087:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:38.111795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471810590980591:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36122
2025-11-27T18:20:38.111647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471810155117089:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.111834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471806296013212:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.111806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471810155117079:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.113228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471810590980580:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:38.111845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471810155117079:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.114109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471810155117090:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
2025-11-27T18:20:38.114367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471810155117091:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
2025-11-27T18:20:38.114629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471810155117077:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.114721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471810590980573:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.114670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471810155117077:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.114710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471807865725802:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.114766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:38.114806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471810590980573:2107] [node 4] ICP22 created new session: [3:7577471810590980595:2107]
No RDMA context found for GID: 2025-11-27T18:20:38.115127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471810590980595:2107] [node 4] ICS09 handshake done sender: [3:7577471810590980580:2113] self: [3:7577471810590980579:0] peer: [4:7577471808719628621:0] socket: 603 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:38.115292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471810155117091:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.115292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471810590980595:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:38.115328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:38.116623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471810155117090:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.116609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471810590980590:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.116828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471807865725811:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.116824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471808719628623:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60400
2025-11-27T18:20:38.116903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471810155117079:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:38.117058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471803570758432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.117053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471810590980591:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.117352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471806296013213:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.117561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471808719628615:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:38.117667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471810155117075:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.117863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471807865725795:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.117697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471810155117075:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.118299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471810013854165:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.117890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471807865725795:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.119508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471805718886790:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.119450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471804424661247:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.119450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471808719628624:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60400
2025-11-27T18:20:38.119512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:38.119584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471804424661247:2050] [node 3] ICP22 created new session: [4:7577471808719628625:2050]
2025-11-27T18:20:38.119734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471810155117077:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:38.119751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471810013854167:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7549
2025-11-27T18:20:38.119765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471807865725800:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.119663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471808719628625:2050] [node 3] ICS09 handshake done sender: [4:7577471808719628615:2115] self: [4:7577471808719628621:0] peer: [3:7577471810590980579:0] socket: 608 qp: -1
2025-11-27T18:20:38.119799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471807865725800:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.119687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471808719628625:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:38.119712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:38.119822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471807865725795:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:38.120741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471810155117071:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.121453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471803570758433:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.120803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:38.121403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471805860149713:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.121851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:38.121451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:38.121536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471810155117071:2107] [node 4] ICP22 created new session: [1:7577471810155117099:2107]
2025-11-27T18:20:38.121901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471803570758433:2049] [node 3] ICP22 created new session: [2:7577471807865725818:2049]
2025-11-27T18:20:38.121633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471805860149713:2049] [node 3] ICP22 created new session: [1:7577471810155117100:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.121859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471810155117100:2049] [node 3] ICS09 handshake done sender: [1:7577471810155117077:2111] self: [1:7577471810155117076:0] peer: [3:7577471810590980593:0] socket: 584 qp: -1
2025-11-27T18:20:38.121951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471807865725818:2049] [node 3] ICS09 handshake done sender: [2:7577471807865725795:2110] self: [2:7577471807865725794:0] peer: [3:7577471810590980598:0] socket: 610 qp: -1
2025-11-27T18:20:38.121876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471810155117099:2107] [node 4] ICS09 handshake done sender: [1:7577471810155117079:2112] self: [1:7577471810155117078:0] peer: [4:7577471808719628622:0] socket: 585 qp: -1
2025-11-27T18:20:38.121899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471810155117100:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:38.121989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471807865725818:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:38.121911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471810155117099:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:38.122026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471810590980599:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36122
2025-11-27T18:20:38.122026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:38.121925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:38.121948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:38.123261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471807865725800:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:38.123253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471810590980573:2107] [node 4] ICP18 handshake (actor [3:7577471810590980591:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.123444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471810013854169:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7549
2025-11-27T18:20:38.123600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471808719628614:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:38.123562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471810013854167:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.123650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471805718886791:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.123763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471810013854165:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:38.124248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471805718886790:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.124778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471804424661245:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.124246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471807865725791:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.124702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:38.124833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:38.124703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:38.124884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471804424661245:2048] [node 1] ICP22 created new session: [4:7577471808719628627:2048]
2025-11-27T18:20:38.124833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471805718886790:2049] [node 2] ICP22 created new session: [5:7577471810013854171:2049]
2025-11-27T18:20:38.124732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471807865725791:2108] [node 5] ICP22 created new session: [2:7577471807865725822:2108]
2025-11-27T18:20:38.124933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471808719628627:2048] [node 1] ICS09 handshake done sender: [4:7577471808719628614:2114] self: [4:7577471808719628622:0] peer: [1:7577471810155117078:0] socket: 605 qp: -1
2025-11-27T18:20:38.125102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471808719628627:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:38.124948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471807865725822:2108] [node 5] ICS09 handshake done sender: [2:7577471807865725800:2112] self: [2:7577471807865725798:0] peer: [5:7577471810013854168:0] socket: 612 qp: -1
2025-11-27T18:20:38.125042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471810013854171:2049] [node 2] ICS09 handshake done sender: [5:7577471810013854165:2117] self: [5:7577471810013854168:0] peer: [2:7577471807865725798:0] socket: 617 qp: -1
2025-11-27T18:20:38.125270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:38.125058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471807865725822:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:38.125638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471810590980588:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:38.125068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471810013854171:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:38.125093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:38.125764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471806296013213:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18714) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471810590980590:2118] held: no
2025-11-27T18:20:38.125091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:38.127004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471810590980582:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.127037Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471810590980582:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.127332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471810155117075:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.129149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471810590980600:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36122
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.129195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471806296013212:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.130505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471808835382229:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8032
2025-11-27T18:20:38.129306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:38.131235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471808835382228:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8032
2025-11-27T18:20:38.129469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471806296013212:2048] [node 1] ICP22 created new session: [3:7577471810590980601:2048]
2025-11-27T18:20:38.130555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471810590980601:2048] [node 1] ICS09 handshake done sender: [3:7577471810590980588:2116] self: [3:7577471810590980593:0] peer: [1:7577471810155117076:0] socket: 593 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.130814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471810590980601:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:38.130849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:38.131937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471810013854174:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7549
2025-11-27T18:20:38.131985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471805860149712:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.132054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:38.132094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471805860149712:2048] [node 2] ICP22 created new session: [1:7577471810155117106:2048]
2025-11-27T18:20:38.132142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471810155117106:2048] [node 2] ICS09 handshake done sender: [1:7577471810155117075:2110] self: [1:7577471810155117074:0] peer: [2:7577471807865725815:0] socket: 582 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.132168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471810155117106:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:38.132216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.132264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471807865725824:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6729
2025-11-27T18:20:38.132351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471808835382230:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8032
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.132482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471808719628630:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60400
2025-11-27T18:20:38.132486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471808719628629:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60400
2025-11-27T18:20:38.132572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471807865725802:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:38.133171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471806296013212:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:38.133564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471803570758432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.133597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471808719628630:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.133603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:38.133634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471803570758432:2048] [node 1] ICP22 created new session: [2:7577471807865725826:2048]
2025-11-27T18:20:38.133660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471807865725826:2048] [node 1] ICS09 handshake done sender: [2:7577471807865725802:2114] self: [2:7577471807865725815:0] peer: [1:7577471810155117074:0] socket: 594 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.133684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471807865725826:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:38.133704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:38.133750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471810013854175:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7549
2025-11-27T18:20:38.133729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471808719628629:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.133942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471804424661246:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.134290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471810590980590:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:38.134582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471808835382228:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.135024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471804540414854:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.135311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471810590980582:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.135606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471806296013213:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.135701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471808835382230:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:38.135655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-11-27T18:20:38.135721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471806296013213:2049] [node 2] ICP22 created new session: [3:7577471810590980605:2049]
2025-11-27T18:20:38.135848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471810013854176:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7549
2025-11-27T18:20:38.135965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471808835382212:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.135872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471810013854174:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.136064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471808835382231:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8032
2025-11-27T18:20:38.135861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471810590980572:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.135972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471810013854175:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.135888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471810590980605:2049] [node 2] ICS09 handshake done sender: [3:7577471810590980590:2118] self: [3:7577471810590980598:0] peer: [2:7577471807865725794:0] socket: 614 qp: -1
2025-11-27T18:20:38.136102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471810155117086:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.136148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471810155117086:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.136113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471805718886789:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.136190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471808719628613:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.135898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:38.136199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471810013854149:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.136230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471808719628613:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.135923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471810590980605:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:38.137487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471810013854167:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:38.135961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:38.138363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471805718886791:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.136011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471810590980572:2108] [node 5] ICP22 created new session: [3:7577471810590980606:2108]
2025-11-27T18:20:38.138355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471810155117081:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.138409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:38.136104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471810590980606:2108] [node 5] ICS09 handshake done sender: [3:7577471810590980582:2114] self: [3:7577471810590980581:0] peer: [5:7577471810013854170:0] socket: 628 qp: -1
2025-11-27T18:20:38.138389Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471810155117081:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.138777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471808719628613:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:38.136140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471810590980606:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:38.138446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471805718886791:2050] [node 3] ICP22 created new session: [5:7577471810013854179:2050]
2025-11-27T18:20:38.138811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471808719628617:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.136196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:38.138866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471810013854179:2050] [node 3] ICS09 handshake done sender: [5:7577471810013854167:2118] self: [5:7577471810013854170:0] peer: [3:7577471810590980581:0] socket: 630 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:38.138936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471810013854179:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:38.138834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471808719628617:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.139099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:38.139550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471808719628605:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.139572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471807865725801:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.139588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:38.139606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471807865725801:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.139625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471808719628605:2109] [node 6] ICP22 created new session: [4:7577471808719628637:2109]
2025-11-27T18:20:38.139649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471808835382234:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8032
2025-11-27T18:20:38.139663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471808719628637:2109] [node 6] ICS09 handshake done sender: [4:7577471808719628613:2113] self: [4:7577471808719628612:0] peer: [6:7577471808835382233:0] socket: 613 qp: -1
2025-11-27T18:20:38.140027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471808719628637:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:38.140058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:38.140123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471808719628617:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:38.141529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471810155117081:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:38.141804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471808835382229:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.141890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471808835382230:2117] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:38.142190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471808835382212:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.142084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471810155117086:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:38.142236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:38.142262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471808719628604:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.142308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:38.142415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471808835382212:2107] [node 4] ICP22 created new session: [6:7577471808835382236:2107]
2025-11-27T18:20:38.142349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471808719628604:2108] [node 5] ICP22 created new session: [4:7577471808719628639:2108]
2025-11-27T18:20:38.142620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471807865725801:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:38.142677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471808835382236:2107] [node 4] ICS09 handshake done sender: [6:7577471808835382230:2117] self: [6:7577471808835382233:0] peer: [4:7577471808719628612:0] socket: 640 qp: -1
2025-11-27T18:20:38.142504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471808719628639:2108] [node 5] ICS09 handshake done sender: [4:7577471808719628617:2116] self: [4:7577471808719628616:0] peer: [5:7577471810013854178:0] socket: 641 qp: -1
2025-11-27T18:20:38.142748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471808835382236:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:20:38.142730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471808719628639:2108] [node 5] ICS10 traffic start
::ffff:127.1.0.0
2025-11-27T18:20:38.142904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471810155117072:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.143090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471807865725792:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.142771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:38.142796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:38.142946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:38.143146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:38.143402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471808835382235:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8032
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.143091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471810155117073:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.143275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471807865725792:2107] [node 4] ICP22 created new session: [2:7577471807865725829:2107]
2025-11-27T18:20:38.143202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471810155117072:2108] [node 5] ICP22 created new session: [1:7577471810155117110:2108]
2025-11-27T18:20:38.143633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471810013854181:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7549
2025-11-27T18:20:38.143479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471807865725829:2107] [node 4] ICS09 handshake done sender: [2:7577471807865725801:2113] self: [2:7577471807865725799:0] peer: [4:7577471808719628634:0] socket: 616 qp: -1
2025-11-27T18:20:38.143224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:38.143265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471810155117073:2109] [node 6] ICP22 created new session: [1:7577471810155117111:2109]
2025-11-27T18:20:38.143500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471807865725829:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:38.143622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:38.143350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471810155117111:2109] [node 6] ICS09 handshake done sender: [1:7577471810155117086:2114] self: [1:7577471810155117085:0] peer: [6:7577471808835382232:0] socket: 635 qp: -1
2025-11-27T18:20:38.143375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471810155117110:2108] [node 5] ICS09 handshake done sender: [1:7577471810155117081:2113] self: [1:7577471810155117080:0] peer: [5:7577471810013854177:0] socket: 621 qp: -1
2025-11-27T18:20:38.143378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471810155117111:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.143400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471810155117110:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:38.143406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:38.144324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471804540414855:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.144327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471810013854182:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7549
2025-11-27T18:20:38.143431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:38.144841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471807865725807:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.144843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471808835382228:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:38.144869Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471807865725807:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.145620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471804540414854:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.145661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:38.145668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471810013854175:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:38.145703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471804540414854:2048] [node 1] ICP22 created new session: [6:7577471808835382241:2048]
2025-11-27T18:20:38.145749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471808835382241:2048] [node 1] ICS09 handshake done sender: [6:7577471808835382228:2115] self: [6:7577471808835382232:0] peer: [1:7577471810155117085:0] socket: 636 qp: -1
2025-11-27T18:20:38.145783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471808835382241:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:38.145930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:38.146332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471810013854174:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:38.146395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471810013854149:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.146413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471807865725807:2118] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:38.146517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:38.146646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471810013854149:2108] [node 4] ICP22 created new session: [5:7577471810013854184:2108]
2025-11-27T18:20:38.147173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471807865725793:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.146904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471810013854184:2108] [node 4] ICS09 handshake done sender: [5:7577471810013854175:2121] self: [5:7577471810013854178:0] peer: [4:7577471808719628616:0] socket: 596 qp: -1
2025-11-27T18:20:38.147497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:38.146930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471810013854184:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:38.147541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471807865725793:2109] [node 6] ICP22 created new session: [2:7577471807865725832:2109]
2025-11-27T18:20:38.146957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:38.147784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471807865725832:2109] [node 6] ICS09 handshake done sender: [2:7577471807865725807:2118] self: [2:7577471807865725806:0] peer: [6:7577471808835382238:0] socket: 638 qp: -1
2025-11-27T18:20:38.148860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471807865725832:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:38.148910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:38.149136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471804424661246:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18714) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471808719628629:2119] held: no
2025-11-27T18:20:38.149217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471805718886789:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.149273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:38.149321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471805718886789:2048] [node 1] ICP22 created new session: [5:7577471810013854186:2048]
2025-11-27T18:20:38.149357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471810013854186:2048] [node 1] ICS09 handshake done sender: [5:7577471810013854174:2120] self: [5:7577471810013854177:0] peer: [1:7577471810155117080:0] socket: 637 qp: -1
2025-11-27T18:20:38.149388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471810013854186:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:38.149426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:38.149509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471810013854149:2108] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.150028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471808719628641:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60400
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.150701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471810590980609:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36122
2025-11-27T18:20:38.150901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471808719628629:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:38.151108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471808835382231:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.151311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471808835382240:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8032
2025-11-27T18:20:38.151407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471804424661246:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.151486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:38.152010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471804424661246:2049] [node 2] ICP22 created new session: [4:7577471808719628642:2049]
2025-11-27T18:20:38.152059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471808719628642:2049] [node 2] ICS09 handshake done sender: [4:7577471808719628629:2119] self: [4:7577471808719628634:0] peer: [2:7577471807865725799:0] socket: 619 qp: -1
2025-11-27T18:20:38.152086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471808719628642:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:38.152114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:38.152761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471804540414855:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18714) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471808835382229:2116] held: no
2025-11-27T18:20:38.152769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471805718886789:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.153401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471808835382243:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8032
2025-11-27T18:20:38.153861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471804540414856:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.154326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471808835382229:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:38.154409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471810590980584:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.154450Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471810590980584:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.155176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471804540414855:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.155228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:38.155263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471804540414855:2049] [node 2] ICP22 created new session: [6:7577471808835382245:2049]
2025-11-27T18:20:38.155306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471808835382245:2049] [node 2] ICS09 handshake done sender: [6:7577471808835382229:2116] self: [6:7577471808835382238:0] peer: [2:7577471807865725806:0] socket: 639 qp: -1
2025-11-27T18:20:38.155333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471808835382245:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:38.155340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471810590980584:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:38.155353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:38.155346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471810590980609:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.155448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471808835382240:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.155457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471810590980574:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.155574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:38.155622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471810590980574:2110] [node 6] ICP22 created new session: [3:7577471810590980612:2110]
2025-11-27T18:20:38.155693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471810590980612:2110] [node 6] ICS09 handshake done sender: [3:7577471810590980584:2115] self: [3:7577471810590980583:0] peer: [6:7577471808835382244:0] socket: 644 qp: -1
2025-11-27T18:20:38.155724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471810590980612:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:38.155751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.155930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471808835382247:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8032
2025-11-27T18:20:38.156398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471808835382210:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.156464Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471804540414856:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6797) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471808835382231:2118] held: no
2025-11-27T18:20:38.156949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471808835382231:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:38.157135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471810013854163:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.157181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471810013854163:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.157649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471804540414856:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.157679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:38.157710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471804540414856:2050] [node 3] ICP22 created new session: [6:7577471808835382249:2050]
2025-11-27T18:20:38.157731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471808835382249:2050] [node 3] ICS09 handshake done sender: [6:7577471808835382231:2118] self: [6:7577471808835382244:0] peer: [3:7577471810590980583:0] socket: 645 qp: -1
2025-11-27T18:20:38.157759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471808835382249:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:38.157783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.158815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471810013854163:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:38.158864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471808835382251:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8032
2025-11-27T18:20:38.158976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471810013854150:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.159122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:38.159160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471810013854150:2109] [node 6] ICP22 created new session: [5:7577471810013854189:2109]
2025-11-27T18:20:38.159161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471808835382240:2121] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:38.159284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471810013854189:2109] [node 6] ICS09 handshake done sender: [5:7577471810013854163:2115] self: [5:7577471810013854162:0] peer: [6:7577471808835382248:0] socket: 618 qp: -1
2025-11-27T18:20:38.159318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471810013854189:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:38.159357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:38.159440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471808835382210:2105] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.159485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:38.159529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471808835382210:2105] [node 5] ICP22 created new session: [6:7577471808835382252:2105]
2025-11-27T18:20:38.159634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471808835382252:2105] [node 5] ICS09 handshake done sender: [6:7577471808835382240:2121] self: [6:7577471808835382248:0] peer: [5:7577471810013854162:0] socket: 651 qp: -1
2025-11-27T18:20:38.159686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471808835382252:2105] [node 5] ICS10 traffic start
2025-11-27T18:20:38.159715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:38.198796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:38.198968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471807865725826:2048] [node 1] ICS07 socket disconnect 594 reason# EndOfStream
2025-11-27T18:20:38.199000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471807865725826:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.199064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:38.199102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471807865725826:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.200286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:38.200461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471810590980601:2048] [node 1] ICS07 socket disconnect 593 reason# EndOfStream
2025-11-27T18:20:38.200493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471810590980601:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.200553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:38.200593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471810590980601:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.201971Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471803570758432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:38.202007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471807865725826:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.203328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471806296013212:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:38.203353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471810590980601:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.204086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471803570758432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.204128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471807865725826:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.204086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:38.204386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471803570758432:2048] [node 1] ICP30 unregister session Session# [2:7577471807865725826:2048] VirtualId# [2:7577471807865725815:0]
2025-11-27T18:20:38.204543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471808719628627:2048] [node 1] ICS07 socket disconnect 605 reason# EndOfStream
2025-11-27T18:20:38.204568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:38.204609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471808719628627:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.204594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471803570758432:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.204603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.204691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:38.204724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471806296013212:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.204756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471808719628627:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.204816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471810590980601:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.204851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471806296013212:2048] [node 1] ICP30 unregister session Session# [3:7577471810590980601:2048] VirtualId# [3:7577471810590980593:0]
2025-11-27T18:20:38.204932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:38.204982Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471806296013212:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.204997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.205501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471804424661245:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.205534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471808719628627:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.205636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471804424661245:2048] [node 1] ICP30 unregister session Session# [4:7577471808719628627:2048] VirtualId# [4:7577471808719628622:0]
2025-11-27T18:20:38.205714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:38.205732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471804424661245:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.205745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.205849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:38.206038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471810013854186:2048] [node 1] ICS07 socket disconnect 637 reason# EndOfStream
2025-11-27T18:20:38.206066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471810013854186:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.206120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:38.206166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471810013854186:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.206761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471805718886789:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.206850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471810013854186:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.206883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471805718886789:2048] [node 1] ICP30 unregister session Session# [5:7577471810013854186:2048] VirtualId# [5:7577471810013854177:0]
2025-11-27T18:20:38.206978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:38.207009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471805718886789:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.206997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:38.207025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.207153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471808835382241:2048] [node 1] ICS07 socket disconnect 636 reason# EndOfStream
2025-11-27T18:20:38.207175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471808835382241:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.207229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:38.207279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471808835382241:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.207919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471804540414854:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.207968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471808835382241:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.207999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471804540414854:2048] [node 1] ICP30 unregister session Session# [6:7577471808835382241:2048] VirtualId# [6:7577471808835382232:0]
2025-11-27T18:20:38.208079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:38.208109Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471804540414854:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.208122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.213923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471803570758432:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:38.214920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471806296013212:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:38.215979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471804424661245:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:38.216973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471805718886789:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:38.218014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471804540414854:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:38.283744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:38.283924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471810590980605:2049] [node 2] ICS01 socket: 614 reason# EndOfStream
2025-11-27T18:20:38.283984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471806296013213:2049] [node 2] ICP30 unregister session Session# [3:7577471810590980605:2049] VirtualId# [3:7577471810590980598:0]
2025-11-27T18:20:38.284018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471810590980605:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.284087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:38.284142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:38.287237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:38.287448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471808719628642:2049] [node 2] ICS01 socket: 619 reason# EndOfStream
2025-11-27T18:20:38.287489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471804424661246:2049] [node 2] ICP30 unregister session Session# [4:7577471808719628642:2049] VirtualId# [4:7577471808719628634:0]
2025-11-27T18:20:38.287520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471808719628642:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.287615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:38.287680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:38.288304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:38.288442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471810013854171:2049] [node 2] ICS01 socket: 617 reason# EndOfStream
2025-11-27T18:20:38.288474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471805718886790:2049] [node 2] ICP30 unregister session Session# [5:7577471810013854171:2049] VirtualId# [5:7577471810013854168:0]
2025-11-27T18:20:38.288516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471810013854171:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.288574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:38.288610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:38.289483Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:38.289681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471808835382245:2049] [node 2] ICS01 socket: 639 reason# EndOfStream
2025-11-27T18:20:38.289719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471804540414855:2049] [node 2] ICP30 unregister session Session# [6:7577471808835382245:2049] VirtualId# [6:7577471808835382238:0]
2025-11-27T18:20:38.289747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471808835382245:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.289813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:38.289862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:38.330461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:38.330723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471808719628625:2050] [node 3] ICS07 socket disconnect 608 reason# EndOfStream
2025-11-27T18:20:38.330773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471808719628625:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.330875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:38.330945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471808719628625:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:38.331389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:38.331582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471810013854179:2050] [node 3] ICS07 socket disconnect 630 reason# EndOfStream
2025-11-27T18:20:38.331614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471810013854179:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.331677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:38.331714Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471804424661247:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6797) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.331722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471810013854179:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:38.331815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471808719628625:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.331869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471804424661247:2050] [node 3] ICP30 unregister session Session# [4:7577471808719628625:2050] VirtualId# [4:7577471808719628621:0]
2025-11-27T18:20:38.332025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:38.332060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471804424661247:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6797) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.332080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6797) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.332472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471805718886791:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6797) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.332539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471810013854179:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.332563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471805718886791:2050] [node 3] ICP30 unregister session Session# [5:7577471810013854179:2050] VirtualId# [5:7577471810013854170:0]
2025-11-27T18:20:38.332645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:38.332672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471805718886791:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6797) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.332622Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:38.332687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6797) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.332779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471808835382249:2050] [node 3] ICS01 socket: 645 reason# EndOfStream
2025-11-27T18:20:38.332807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471804540414856:2050] [node 3] ICP30 unregister session Session# [6:7577471808835382249:2050] VirtualId# [6:7577471808835382244:0]
2025-11-27T18:20:38.332835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471808835382249:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.332906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:38.332951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:38.341939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471804424661247:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:38.342967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471805718886791:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:38.397454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:38.397635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471810013854184:2108] [node 4] ICS07 socket disconnect 596 reason# EndOfStream
2025-11-27T18:20:38.397661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471810013854184:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.397740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:38.397773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471810013854184:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:38.398655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471810013854149:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61675) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.398717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471810013854184:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.398681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:38.398759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471810013854149:2108] [node 4] ICP30 unregister session Session# [5:7577471810013854184:2108] VirtualId# [5:7577471810013854178:0]
2025-11-27T18:20:38.398842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471808835382236:2107] [node 4] ICS07 socket disconnect 640 reason# EndOfStream
2025-11-27T18:20:38.398877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471808835382236:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.398856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:38.398891Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471810013854149:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61675) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.398905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61675) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.398944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:38.398985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471808835382236:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:38.399683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471808835382212:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61675) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.399725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471808835382236:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.399770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471808835382212:2107] [node 4] ICP30 unregister session Session# [6:7577471808835382236:2107] VirtualId# [6:7577471808835382233:0]
2025-11-27T18:20:38.399858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:38.399882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471808835382212:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61675) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.399897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61675) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.408503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471810013854149:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:38.409495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471808835382212:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:38.443910Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:38.444179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471808835382252:2105] [node 5] ICS07 socket disconnect 651 reason# EndOfStream
2025-11-27T18:20:38.444226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471808835382252:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.444303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:38.444353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471808835382252:2105] [node 5] ICS15 start handshake
2025-11-27T18:20:38.445098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471808835382210:2105] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32029) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.445168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471808835382252:2105] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.445207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471808835382210:2105] [node 5] ICP30 unregister session Session# [6:7577471808835382252:2105] VirtualId# [6:7577471808835382248:0]
2025-11-27T18:20:38.445282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:38.445320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471808835382210:2105] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.445335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32029) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:38.539325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471807110939129:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:38.539485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471807110939130:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:38.558608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471806731084353:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:38.558649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471806731084352:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:38.578189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471809130857587:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:38.578229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471809130857588:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:38.599726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471809232381738:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:38.599770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471809232381739:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:38.599880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471809232381740:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:38.621122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471808689101501:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:38.621162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471808689101502:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:38.621186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471808689101503:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:38.641874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471807263523128:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:38.641903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471807263523129:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:38.641936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471807263523130:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:38.686068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471807263523190:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:38.686224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471807110939192:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:38.686794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471807110939193:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:38.687164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471806731084416:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:38.687410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471807110939194:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:38.687728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471806731084417:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:38.688105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471809130857649:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:38.688227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471806731084418:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:38.688792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471809130857651:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:38.689326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471809232381801:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:38.689423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471809130857652:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:38.690648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471808689101565:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:38.693623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471808689101566:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:38.694096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471807263523197:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.709668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471809232381808:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:38.735285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471808689101563:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17476
2025-11-27T18:20:38.744463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471806731084417:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50147859ns) {Configure:104(50141514ns) {ProcessConfigured:135(916591ns) {StartInitialHandshake:157(913446ns) {DropHandshakes:470(4116ns) {DropIncomingHandshake:442(1207ns) DropOutgoingHandshake:457(731ns)} GenerateSessionVirtualId:988(574ns) PrepareNewSessionHandshake:477(2067ns) {SwitchToState:179(625ns)}}}}}
2025-11-27T18:20:38.744578Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471809130857652:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(49695828ns) {Configure:104(49691799ns) {ProcessConfigured:135(252392ns) {StartInitialHandshake:157(250250ns) {DropHandshakes:470(892ns) {DropIncomingHandshake:442(266ns) DropOutgoingHandshake:457(241ns)} GenerateSessionVirtualId:988(290ns) PrepareNewSessionHandshake:477(1446ns) {SwitchToState:179(815ns)}}}}}
2025-11-27T18:20:38.744668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471806731084418:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(50338232ns) {Configure:104(50334600ns) {ProcessConfigured:135(725301ns) {StartInitialHandshake:157(722786ns) {DropHandshakes:470(1478ns) {DropIncomingHandshake:442(436ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(448ns) PrepareNewSessionHandshake:477(1667ns) {SwitchToState:179(627ns)}}}}}
2025-11-27T18:20:38.744720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471808689101503:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(49902891ns) {Configure:104(49899919ns) {ProcessConfigured:135(202256ns) {StartInitialHandshake:157(200562ns) {DropHandshakes:470(4355ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(484ns) PrepareNewSessionHandshake:477(982ns) {SwitchToState:179(384ns)}}}}}
2025-11-27T18:20:38.744782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7577471809130857651:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50416748ns) {Configure:104(50283222ns) {ProcessConfigured:135(395723ns) {StartInitialHandshake:157(392664ns) {DropHandshakes:470(2527ns) {DropIncomingHandshake:442(592ns) DropOutgoingHandshake:457(460ns)} GenerateSessionVirtualId:988(1138ns) PrepareNewSessionHandshake:477(2520ns) {SwitchToState:179(839ns)}}}}}
2025-11-27T18:20:38.744906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471808689101563:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.745752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806731084425:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.746163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471807110939208:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15600
2025-11-27T18:20:38.746328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806731084425:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.746376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471809232381809:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-27T18:20:38.746381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471809130857662:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7736
2025-11-27T18:20:38.746397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7577471808689101565:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(50491027ns) {Configure:104(50486880ns) {ProcessConfigured:135(492743ns) {StartInitialHandshake:157(490641ns) {DropHandshakes:470(6173ns) {DropIncomingHandshake:442(274ns) DropOutgoingHandshake:457(243ns)} GenerateSessionVirtualId:988(592ns) PrepareNewSessionHandshake:477(1570ns) {SwitchToState:179(582ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.747105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471809130857666:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7736
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.747704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471809130857668:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7736
2025-11-27T18:20:38.746805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7577471807263523130:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47860837ns) {Configure:104(41039799ns) {ProcessConfigured:135(608214ns) {StartInitialHandshake:157(606422ns) {DropHandshakes:470(1215ns) {DropIncomingHandshake:442(425ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(464ns) PrepareNewSessionHandshake:477(1615ns) {SwitchToState:179(875ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.748788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471807110939209:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15600
2025-11-27T18:20:38.748758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471809130857662:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.749108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471809130857587:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.750052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471807110939210:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15600
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.750321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471806731084352:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.750207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471809232381809:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.750458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471807110939197:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.750501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471807110939197:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.752593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806731084433:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-27T18:20:38.754351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471809232381814:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-27T18:20:38.754000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471807110939211:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15600
2025-11-27T18:20:38.754220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471807110939208:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.755149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471807110939197:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:38.756014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471809232381738:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.757764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471809232381816:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-27T18:20:38.757896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806731084435:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
No RDMA context found for GID: 2025-11-27T18:20:38.758129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471807110939212:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15600
::ffff:127.1.0.0
2025-11-27T18:20:38.757692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471809130857670:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7736
2025-11-27T18:20:38.758719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471809130857666:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.758440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471809232381816:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.757907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806731084433:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.758402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471807110939210:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.758793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471809232381814:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.758137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806731084434:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-27T18:20:38.758982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471809130857670:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.758860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471807110939200:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.759110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471809130857588:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.758874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471809232381740:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.759008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806731084435:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.758890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471807110939200:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.759724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471809232381739:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.759817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471809130857657:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.759071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806731084436:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-27T18:20:38.759511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471807110939198:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.759762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806731084421:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.759535Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471807110939198:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.759838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471809130857657:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.759782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471806731084421:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.761057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471807110939130:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.760766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806731084436:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.761923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471809130857671:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7736
2025-11-27T18:20:38.761865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806731084424:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.761242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:38.765143Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471809130857657:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:38.761882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471806731084424:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.762186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471807110939130:2049] [node 3] ICP22 created new session: [1:7577471807110939215:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.764154Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471806731084421:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:38.765787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471809130857662:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:38.762215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471807110939209:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.764113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471807110939212:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.765958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471806731084424:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:38.767732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471809130857668:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.767731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471807263523206:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51534
2025-11-27T18:20:38.764140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471807110939215:2049] [node 3] ICS09 handshake done sender: [1:7577471807110939197:2110] self: [1:7577471807110939196:0] peer: [3:7577471809130857669:0] socket: 591 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.767802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471809130857587:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.766181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471806731084353:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.764595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471807110939215:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:38.767803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471809130857649:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.766244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:38.764699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:38.767805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471809232381822:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-27T18:20:38.767858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:38.765212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471807110939200:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:38.767987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471809232381824:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-27T18:20:38.766411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471806731084353:2049] [node 3] ICP22 created new session: [2:7577471806731084444:2049]
2025-11-27T18:20:38.767909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471809130857587:2048] [node 1] ICP22 created new session: [3:7577471809130857676:2048]
2025-11-27T18:20:38.765460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471807110939198:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:38.767152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471806731084444:2049] [node 3] ICS09 handshake done sender: [2:7577471806731084421:2111] self: [2:7577471806731084419:0] peer: [3:7577471809130857672:0] socket: 601 qp: -1
2025-11-27T18:20:38.767984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471809232381823:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-27T18:20:38.767225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471807110939129:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.767979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:38.767188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471806731084444:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:38.768027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471809130857676:2048] [node 1] ICS09 handshake done sender: [3:7577471809130857662:2114] self: [3:7577471809130857669:0] peer: [1:7577471807110939196:0] socket: 597 qp: -1
2025-11-27T18:20:38.768996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471809232381809:2113] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:38.767428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:38.769309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471809232381816:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:38.767559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471807110939211:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.767225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:38.768044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471809130857649:2107] [node 4] ICP22 created new session: [3:7577471809130857677:2107]
2025-11-27T18:20:38.769684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471809232381814:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:38.767808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471806731084416:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.768151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471809130857676:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:38.767747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471807110939129:2048] [node 2] ICP22 created new session: [1:7577471807110939222:2048]
2025-11-27T18:20:38.770442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471809232381738:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.767754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471807110939192:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.770634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:38.770740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471809232381740:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.768197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:38.770781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:38.767811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471806731084443:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41982
2025-11-27T18:20:38.767784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:38.767794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471807110939222:2048] [node 2] ICS09 handshake done sender: [1:7577471807110939198:2111] self: [1:7577471807110939195:0] peer: [2:7577471806731084437:0] socket: 592 qp: -1
2025-11-27T18:20:38.770822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471809232381738:2048] [node 1] ICP22 created new session: [4:7577471809232381825:2048]
2025-11-27T18:20:38.767848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:38.768798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471809130857677:2107] [node 4] ICS09 handshake done sender: [3:7577471809130857657:2111] self: [3:7577471809130857656:0] peer: [4:7577471809232381820:0] socket: 610 qp: -1
2025-11-27T18:20:38.767964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471806731084416:2107] [node 4] ICP22 created new session: [2:7577471806731084446:2107]
2025-11-27T18:20:38.770905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471809232381740:2050] [node 3] ICP22 created new session: [4:7577471809232381826:2050]
2025-11-27T18:20:38.767836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471807110939222:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:38.768825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471809130857677:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:38.767838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471807110939192:2107] [node 4] ICP22 created new session: [1:7577471807110939223:2107]
2025-11-27T18:20:38.768013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471806731084446:2107] [node 4] ICS09 handshake done sender: [2:7577471806731084424:2113] self: [2:7577471806731084423:0] peer: [4:7577471809232381821:0] socket: 603 qp: -1
2025-11-27T18:20:38.771045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471809232381825:2048] [node 1] ICS09 handshake done sender: [4:7577471809232381809:2113] self: [4:7577471809232381819:0] peer: [1:7577471807110939199:0] socket: 596 qp: -1
2025-11-27T18:20:38.770341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:38.768049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471806731084446:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:38.767876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:38.771246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471809232381825:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:38.773299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471809130857588:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65187) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471809130857666:2116] held: no
2025-11-27T18:20:38.767893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471807110939223:2107] [node 4] ICS09 handshake done sender: [1:7577471807110939200:2112] self: [1:7577471807110939199:0] peer: [4:7577471809232381819:0] socket: 593 qp: -1
2025-11-27T18:20:38.768164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: 2025-11-27T18:20:38.771382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471809232381826:2050] [node 3] ICS09 handshake done sender: [4:7577471809232381816:2116] self: [4:7577471809232381820:0] peer: [3:7577471809130857656:0] socket: 625 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:38.771482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:38.767992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471807110939223:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:38.771500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471809232381826:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:38.774364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471808689101581:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17476
2025-11-27T18:20:38.770337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806731084434:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.768023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:38.771642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:38.771725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471806731084425:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:38.774417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471809232381739:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.774535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:38.773784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471806731084352:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.774584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471809232381739:2049] [node 2] ICP22 created new session: [4:7577471809232381829:2049]
2025-11-27T18:20:38.774597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471807110939130:2049] [node 3] ICP18 handshake (actor [1:7577471807110939210:2117]) is too old
2025-11-27T18:20:38.773825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:38.774633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471809232381829:2049] [node 2] ICS09 handshake done sender: [4:7577471809232381814:2114] self: [4:7577471809232381821:0] peer: [2:7577471806731084423:0] socket: 606 qp: -1
2025-11-27T18:20:38.773944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471806731084352:2048] [node 1] ICP22 created new session: [2:7577471806731084450:2048]
2025-11-27T18:20:38.774729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471809232381829:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:38.774293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471806731084450:2048] [node 1] ICS09 handshake done sender: [2:7577471806731084425:2114] self: [2:7577471806731084437:0] peer: [1:7577471807110939195:0] socket: 595 qp: -1
2025-11-27T18:20:38.774933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:38.774329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471806731084450:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:38.774478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.775340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471809130857675:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7736
2025-11-27T18:20:38.775938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471809130857649:2107] [node 4] ICP18 handshake (actor [3:7577471809130857668:2117]) is too old
2025-11-27T18:20:38.776753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471806731084416:2107] [node 4] ICP18 handshake (actor [2:7577471806731084434:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.777270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471807263523207:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51534
2025-11-27T18:20:38.778099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471809130857666:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.779985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471807263523208:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51534
2025-11-27T18:20:38.780624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471808689101581:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.780698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471809130857588:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.780833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:38.781048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471808689101502:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.780883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471809130857588:2049] [node 2] ICP22 created new session: [3:7577471809130857682:2049]
2025-11-27T18:20:38.781111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471807263523206:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.781026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471809130857682:2049] [node 2] ICS09 handshake done sender: [3:7577471809130857666:2116] self: [3:7577471809130857672:0] peer: [2:7577471806731084419:0] socket: 607 qp: -1
2025-11-27T18:20:38.781056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471809130857682:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.781155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.781241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471807263523207:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.781281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471807263523128:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.781296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471807110939192:2107] [node 4] ICP18 handshake (actor [1:7577471807110939211:2118]) is too old
2025-11-27T18:20:38.781333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471807263523129:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.781302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471808689101583:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17476
2025-11-27T18:20:38.781438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471808689101582:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17476
2025-11-27T18:20:38.782826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806731084428:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.782879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471806731084428:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.785424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471807110939202:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.785475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471807110939202:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.786451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471806731084429:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.786493Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471806731084429:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.787539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471807110939202:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:38.787422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471808689101583:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.787603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471808689101501:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.788218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471806731084428:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:38.788421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471808689101582:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.788758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471807263523208:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.788266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471806731084429:2116] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:38.788745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471808689101503:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.789386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471807110939204:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.789243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471809130857663:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.790249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471806731084417:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.789250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471807263523130:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.789431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471807110939204:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.789298Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471809130857663:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.790863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471807263523211:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51534
2025-11-27T18:20:38.790722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:38.791508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471808689101586:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17476
2025-11-27T18:20:38.790318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471807110939194:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.791033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471809130857661:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.791053Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471809130857661:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.790776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471806731084417:2108] [node 5] ICP22 created new session: [2:7577471806731084454:2108]
2025-11-27T18:20:38.790901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:38.791133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471809130857663:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:38.791174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471807110939194:2109] [node 6] ICP22 created new session: [1:7577471807110939228:2109]
2025-11-27T18:20:38.791109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471806731084454:2108] [node 5] ICS09 handshake done sender: [2:7577471806731084428:2115] self: [2:7577471806731084426:0] peer: [5:7577471808689101584:0] socket: 608 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.793684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471807263523212:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51534
2025-11-27T18:20:38.791351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471807110939228:2109] [node 6] ICS09 handshake done sender: [1:7577471807110939202:2113] self: [1:7577471807110939201:0] peer: [6:7577471807263523209:0] socket: 639 qp: -1
2025-11-27T18:20:38.793808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471809130857651:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.791154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471806731084454:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:38.791153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471806731084418:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.791585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471807110939228:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:38.794447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471808689101588:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17476
2025-11-27T18:20:38.793993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:38.794553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471807263523128:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471807263523206:2115] held: no
2025-11-27T18:20:38.791208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:38.791623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:38.791493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471806731084418:2109] [node 6] ICP22 created new session: [2:7577471806731084455:2109]
2025-11-27T18:20:38.794040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471809130857651:2108] [node 5] ICP22 created new session: [3:7577471809130857686:2108]
2025-11-27T18:20:38.791611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:38.792916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471807110939204:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:38.794578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471809130857686:2108] [node 5] ICS09 handshake done sender: [3:7577471809130857663:2115] self: [3:7577471809130857660:0] peer: [5:7577471808689101587:0] socket: 613 qp: -1
2025-11-27T18:20:38.792018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471806731084455:2109] [node 6] ICS09 handshake done sender: [2:7577471806731084429:2116] self: [2:7577471806731084427:0] peer: [6:7577471807263523210:0] socket: 604 qp: -1
2025-11-27T18:20:38.795866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471808689101502:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65187) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471808689101581:2116] held: no
2025-11-27T18:20:38.794062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471807110939193:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.792429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471806731084455:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:38.795862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471809130857686:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:38.794116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:38.792944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:38.795955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:38.796154Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471807263523206:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:38.796109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471809130857661:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:38.794802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471807110939193:2108] [node 5] ICP22 created new session: [1:7577471807110939231:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.796373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471808689101581:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:38.796752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471809130857652:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.795876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471807110939231:2108] [node 5] ICS09 handshake done sender: [1:7577471807110939204:2114] self: [1:7577471807110939203:0] peer: [5:7577471808689101585:0] socket: 619 qp: -1
2025-11-27T18:20:38.796789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:38.796087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471807110939231:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:38.796145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:38.797054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471807263523214:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51534
2025-11-27T18:20:38.796992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471809130857652:2109] [node 6] ICP22 created new session: [3:7577471809130857688:2109]
2025-11-27T18:20:38.797045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471809130857688:2109] [node 6] ICS09 handshake done sender: [3:7577471809130857661:2113] self: [3:7577471809130857659:0] peer: [6:7577471807263523213:0] socket: 602 qp: -1
2025-11-27T18:20:38.797076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471809130857688:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:38.798593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471808689101502:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.797210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:38.798859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471808689101588:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.798593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471807263523128:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:38.800013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-11-27T18:20:38.798735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:38.798890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471807263523128:2048] [node 1] ICP22 created new session: [6:7577471807263523216:2048]
2025-11-27T18:20:38.800111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471808689101502:2049] [node 2] ICP22 created new session: [5:7577471808689101591:2049]
2025-11-27T18:20:38.800032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471807263523216:2048] [node 1] ICS09 handshake done sender: [6:7577471807263523206:2115] self: [6:7577471807263523209:0] peer: [1:7577471807110939201:0] socket: 641 qp: -1
2025-11-27T18:20:38.800237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471808689101589:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17476
2025-11-27T18:20:38.800065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471807263523216:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:38.800310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471808689101591:2049] [node 2] ICS09 handshake done sender: [5:7577471808689101581:2116] self: [5:7577471808689101584:0] peer: [2:7577471806731084426:0] socket: 614 qp: -1
2025-11-27T18:20:38.800097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:38.800139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471807263523207:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:38.800434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471808689101591:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:38.800466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:38.800922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471807263523129:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65187) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471807263523207:2116] held: no
2025-11-27T18:20:38.800972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471807263523214:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.801118Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471808689101503:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14366) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471808689101582:2117] held: no
2025-11-27T18:20:38.801283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471807263523215:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51534
2025-11-27T18:20:38.801521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471807263523129:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.802226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:38.802288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471807263523129:2049] [node 2] ICP22 created new session: [6:7577471807263523220:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.802391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471807263523220:2049] [node 2] ICS09 handshake done sender: [6:7577471807263523207:2116] self: [6:7577471807263523210:0] peer: [2:7577471806731084427:0] socket: 615 qp: -1
2025-11-27T18:20:38.802433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471807263523220:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:38.802474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:38.802467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471808689101590:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17476
2025-11-27T18:20:38.802548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471808689101501:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471808689101583:2118] held: no
2025-11-27T18:20:38.803152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471807263523197:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.803258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471807263523208:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:38.803258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471808689101582:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.803263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471809130857690:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7736
2025-11-27T18:20:38.803382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471809232381833:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-27T18:20:38.803531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471809232381818:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.803563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471809232381818:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.803574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471808689101565:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.803647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471808689101503:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.803629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471807263523130:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.803678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:38.803719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471808689101503:2050] [node 3] ICP22 created new session: [5:7577471808689101596:2050]
2025-11-27T18:20:38.803738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:38.803721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471808689101583:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:38.803793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471807263523130:2050] [node 3] ICP22 created new session: [6:7577471807263523223:2050]
2025-11-27T18:20:38.804296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471807263523223:2050] [node 3] ICS09 handshake done sender: [6:7577471807263523208:2117] self: [6:7577471807263523213:0] peer: [3:7577471809130857659:0] socket: 609 qp: -1
2025-11-27T18:20:38.804446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471809232381815:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.804326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471807263523223:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:38.803746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471808689101596:2050] [node 3] ICS09 handshake done sender: [5:7577471808689101582:2117] self: [5:7577471808689101587:0] peer: [3:7577471809130857660:0] socket: 618 qp: -1
2025-11-27T18:20:38.804484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:38.804487Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471809232381815:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.803976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471808689101501:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.804006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471808689101596:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:38.804012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:38.806203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471809232381834:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-27T18:20:38.804043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:38.806217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471809232381818:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:38.804484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471808689101501:2048] [node 1] ICP22 created new session: [5:7577471808689101597:2048]
2025-11-27T18:20:38.804562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471808689101597:2048] [node 1] ICS09 handshake done sender: [5:7577471808689101583:2118] self: [5:7577471808689101585:0] peer: [1:7577471807110939203:0] socket: 644 qp: -1
2025-11-27T18:20:38.804676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471808689101597:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:38.804712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:38.806509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471809232381808:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.806606Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471809232381815:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:38.806658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:38.806691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471809232381808:2109] [node 6] ICP22 created new session: [4:7577471809232381837:2109]
2025-11-27T18:20:38.806735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471809232381837:2109] [node 6] ICS09 handshake done sender: [4:7577471809232381818:2117] self: [4:7577471809232381817:0] peer: [6:7577471807263523222:0] socket: 650 qp: -1
2025-11-27T18:20:38.806781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471809232381837:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:38.806813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:38.806937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471809232381801:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.806974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:38.807027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471809232381801:2108] [node 5] ICP22 created new session: [4:7577471809232381839:2108]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.807085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471809232381839:2108] [node 5] ICS09 handshake done sender: [4:7577471809232381815:2115] self: [4:7577471809232381813:0] peer: [5:7577471808689101595:0] socket: 648 qp: -1
2025-11-27T18:20:38.807141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471809232381839:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:38.807577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.807774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471807263523225:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51534
2025-11-27T18:20:38.808156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471807263523226:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51534
2025-11-27T18:20:38.808348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471807263523214:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:38.808792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471807263523197:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.808923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471809232381833:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.808828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:38.808919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471807263523197:2110] [node 4] ICP22 created new session: [6:7577471807263523227:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.808964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471807263523227:2110] [node 4] ICS09 handshake done sender: [6:7577471807263523214:2120] self: [6:7577471807263523222:0] peer: [4:7577471809232381817:0] socket: 651 qp: -1
2025-11-27T18:20:38.809156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471807263523227:2110] [node 4] ICS10 traffic start
2025-11-27T18:20:38.809156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471807263523225:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.809287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471808689101600:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17476
2025-11-27T18:20:38.809235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:38.809454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471809232381808:2109] [node 6] ICP18 handshake (actor [4:7577471809232381833:2121]) is too old
2025-11-27T18:20:38.809509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471808689101588:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:38.811830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471808689101565:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.811910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:38.812013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471809232381834:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.812053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471808689101565:2108] [node 4] ICP22 created new session: [5:7577471808689101601:2108]
2025-11-27T18:20:38.812200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471808689101601:2108] [node 4] ICS09 handshake done sender: [5:7577471808689101588:2120] self: [5:7577471808689101595:0] peer: [4:7577471809232381813:0] socket: 649 qp: -1
2025-11-27T18:20:38.812259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471808689101601:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:38.812299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:38.812608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471807263523190:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:38.812623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471809232381801:2108] [node 5] ICP18 handshake (actor [4:7577471809232381834:2122]) is too old
2025-11-27T18:20:38.812916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471808689101579:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:38.812950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471808689101579:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:38.814601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471808689101579:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:38.814852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471808689101566:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:38.814878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471807263523230:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51534
2025-11-27T18:20:38.814957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:38.815001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471808689101566:2110] [node 6] ICP22 created new session: [5:7577471808689101604:2110]
2025-11-27T18:20:38.815058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471808689101604:2110] [node 6] ICS09 handshake done sender: [5:7577471808689101579:2115] self: [5:7577471808689101578:0] peer: [6:7577471807263523229:0] socket: 626 qp: -1
2025-11-27T18:20:38.815091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471808689101604:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:38.815100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471807263523225:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:38.815123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:38.815220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471807263523190:2108] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:38.815332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:38.815384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471807263523190:2108] [node 5] ICP22 created new session: [6:7577471807263523231:2108]
2025-11-27T18:20:38.815523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471807263523231:2108] [node 5] ICS09 handshake done sender: [6:7577471807263523225:2122] self: [6:7577471807263523229:0] peer: [5:7577471808689101578:0] socket: 598 qp: -1
2025-11-27T18:20:38.815605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471807263523231:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:38.815642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:38.862210Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:38.862399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471806731084450:2048] [node 1] ICS07 socket disconnect 595 reason# EndOfStream
2025-11-27T18:20:38.862426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471806731084450:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.862490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:38.862532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471806731084450:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.863348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:38.863559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471809130857676:2048] [node 1] ICS07 socket disconnect 597 reason# EndOfStream
2025-11-27T18:20:38.863595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471809130857676:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.863677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:38.863732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471809130857676:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.864742Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471806731084352:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:38.864799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471806731084450:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.865587Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471806731084352:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.866030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471806731084450:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.866067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471806731084352:2048] [node 1] ICP30 unregister session Session# [2:7577471806731084450:2048] VirtualId# [2:7577471806731084437:0]
2025-11-27T18:20:38.867009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:38.867036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471806731084352:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.867045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.867131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471809130857587:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:38.867135Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:38.867180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471809130857676:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.867595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471809232381825:2048] [node 1] ICS07 socket disconnect 596 reason# EndOfStream
2025-11-27T18:20:38.867633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471809232381825:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.867710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:38.867758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471809232381825:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:38.868151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471809130857587:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.868257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471809130857676:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.868313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471809130857587:2048] [node 1] ICP30 unregister session Session# [3:7577471809130857676:2048] VirtualId# [3:7577471809130857669:0]
2025-11-27T18:20:38.868394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:38.868422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471809130857587:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.868430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.868480Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:38.868536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471809232381738:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.868787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471808689101597:2048] [node 1] ICS01 socket: 644 reason# EndOfStream
2025-11-27T18:20:38.868830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471808689101501:2048] [node 1] ICP30 unregister session Session# [5:7577471808689101597:2048] VirtualId# [5:7577471808689101585:0]
2025-11-27T18:20:38.868857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471808689101597:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.868688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471809232381825:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.868940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:38.868768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471809232381738:2048] [node 1] ICP30 unregister session Session# [4:7577471809232381825:2048] VirtualId# [4:7577471809232381819:0]
2025-11-27T18:20:38.868984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:38.868861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:38.868905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471809232381738:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.868924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.869360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:38.869529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471807263523216:2048] [node 1] ICS01 socket: 641 reason# EndOfStream
2025-11-27T18:20:38.869567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471807263523128:2048] [node 1] ICP30 unregister session Session# [6:7577471807263523216:2048] VirtualId# [6:7577471807263523209:0]
2025-11-27T18:20:38.869593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471807263523216:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.869658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:38.869712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:38.878511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471809130857587:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:38.878529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471809232381738:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:38.905043Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:38.905192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471809130857682:2049] [node 2] ICS01 socket: 607 reason# EndOfStream
2025-11-27T18:20:38.905227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471809130857588:2049] [node 2] ICP30 unregister session Session# [3:7577471809130857682:2049] VirtualId# [3:7577471809130857672:0]
2025-11-27T18:20:38.905270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471809130857682:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.905340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:38.905390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:38.908271Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:38.908460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471809232381829:2049] [node 2] ICS07 socket disconnect 606 reason# EndOfStream
2025-11-27T18:20:38.908503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471809232381829:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.908660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:38.908740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471809232381829:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:38.909400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:38.909566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471808689101591:2049] [node 2] ICS01 socket: 614 reason# EndOfStream
2025-11-27T18:20:38.909593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471808689101502:2049] [node 2] ICP30 unregister session Session# [5:7577471808689101591:2049] VirtualId# [5:7577471808689101584:0]
2025-11-27T18:20:38.909614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471808689101591:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.909675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:38.909741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:38.909744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471809232381739:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65187) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.909789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471809232381829:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.909818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471809232381739:2049] [node 2] ICP30 unregister session Session# [4:7577471809232381829:2049] VirtualId# [4:7577471809232381821:0]
2025-11-27T18:20:38.909909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:38.909961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471809232381739:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65187) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.909974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65187) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.910458Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:38.910622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471807263523220:2049] [node 2] ICS01 socket: 615 reason# EndOfStream
2025-11-27T18:20:38.910661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471807263523129:2049] [node 2] ICP30 unregister session Session# [6:7577471807263523220:2049] VirtualId# [6:7577471807263523210:0]
2025-11-27T18:20:38.910689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471807263523220:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.910754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:38.910814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:38.919455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471809232381739:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:38.977778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:38.977936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471809232381826:2050] [node 3] ICS07 socket disconnect 625 reason# EndOfStream
2025-11-27T18:20:38.977958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471809232381826:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.978019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:38.978055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471809232381826:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:38.978648Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471809232381740:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14366) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.978623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:38.978733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471809232381826:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.978759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471809232381740:2050] [node 3] ICP30 unregister session Session# [4:7577471809232381826:2050] VirtualId# [4:7577471809232381820:0]
2025-11-27T18:20:38.978862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:38.978860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471808689101596:2050] [node 3] ICS01 socket: 618 reason# EndOfStream
2025-11-27T18:20:38.978893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471809232381740:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14366) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.978908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471808689101503:2050] [node 3] ICP30 unregister session Session# [5:7577471808689101596:2050] VirtualId# [5:7577471808689101587:0]
2025-11-27T18:20:38.978902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14366) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.978944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471808689101596:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.979022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:38.979068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:38.979393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:38.979528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471807263523223:2050] [node 3] ICS07 socket disconnect 609 reason# EndOfStream
2025-11-27T18:20:38.979551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471807263523223:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:38.979600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:38.979642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471807263523223:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:38.980228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471807263523130:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14366) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:38.980292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471807263523223:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:38.980331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471807263523130:2050] [node 3] ICP30 unregister session Session# [6:7577471807263523223:2050] VirtualId# [6:7577471807263523213:0]
2025-11-27T18:20:38.980435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:38.980462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471807263523130:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14366) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.980473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14366) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:38.989117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471809232381740:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:38.990098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471807263523130:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:39.038468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:39.038642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471808689101601:2108] [node 4] ICS07 socket disconnect 649 reason# EndOfStream
2025-11-27T18:20:39.038667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471808689101601:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.038730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:39.038760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471808689101601:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:39.039251Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:39.039303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471808689101565:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23624) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:39.039344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471808689101601:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:39.039417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471807263523227:2110] [node 4] ICS07 socket disconnect 651 reason# EndOfStream
2025-11-27T18:20:39.039424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471808689101565:2108] [node 4] ICP30 unregister session Session# [5:7577471808689101601:2108] VirtualId# [5:7577471808689101595:0]
2025-11-27T18:20:39.039438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471807263523227:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.039494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:39.039531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471807263523227:2110] [node 4] ICS15 start handshake
2025-11-27T18:20:39.039518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:39.039544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471808689101565:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.039556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.040084Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471807263523197:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23624) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:39.040142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471807263523227:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:39.040208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471807263523197:2110] [node 4] ICP30 unregister session Session# [6:7577471807263523227:2110] VirtualId# [6:7577471807263523222:0]
2025-11-27T18:20:39.040331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:39.040366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471807263523197:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.040379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.050507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471807263523197:2110] [node 4] ICP33 wake up from error state
2025-11-27T18:20:39.081517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:39.081690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471807263523231:2108] [node 5] ICS07 socket disconnect 598 reason# EndOfStream
2025-11-27T18:20:39.081714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471807263523231:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.081779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:39.081812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471807263523231:2108] [node 5] ICS15 start handshake
2025-11-27T18:20:39.082531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471807263523190:2108] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17476) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:39.082604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471807263523231:2108] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:39.082634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471807263523190:2108] [node 5] ICP30 unregister session Session# [6:7577471807263523231:2108] VirtualId# [6:7577471807263523229:0]
2025-11-27T18:20:39.082698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:39.082726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471807263523190:2108] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17476) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.082735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17476) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.092482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471807263523190:2108] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:39.179395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471813889825185:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:39.179433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471813889825186:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:39.199920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471812409014895:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:39.200146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471812409014896:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:39.218583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471814938710550:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:39.218619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471814938710551:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:39.237752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471813883860732:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:39.237781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471813883860734:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:39.237874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471813883860733:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:39.256386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471814394669644:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:39.256419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471814394669646:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:39.256647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471814394669645:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:39.274738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471811856197102:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:39.274774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471811856197103:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:39.275191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471811856197104:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:39.346989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471811856197164:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:39.354871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471813889825249:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:39.356714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471812409014959:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:39.355041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471814938710614:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:39.356237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471813889825248:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:39.356386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471813883860796:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:39.356780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471813889825250:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:39.356228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471812409014960:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:39.356189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471814938710613:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:39.356784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471812409014961:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:39.356944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471811856197165:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:39.356800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471814394669707:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:39.357152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471814938710615:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:39.356879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471813883860795:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:39.357145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471814394669708:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:39.364617Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471812409014895:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16866825ns) {Configure:104(16854836ns) {ProcessConfigured:135(436616ns) {StartInitialHandshake:157(434162ns) {DropHandshakes:470(2199ns) {DropIncomingHandshake:442(575ns) DropOutgoingHandshake:457(287ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:477(1113ns) {SwitchToState:179(502ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.371807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471814938710624:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26453
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.372745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471812409014974:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63530
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:39.375348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471812409014973:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63530
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.376632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471813883860808:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45383
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.379042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471811856197178:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45609
2025-11-27T18:20:39.379464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471814394669720:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14631
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.379451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471814938710628:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26453
2025-11-27T18:20:39.379844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471813889825263:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22336
2025-11-27T18:20:39.380015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471814938710629:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26453
2025-11-27T18:20:39.380133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471811856197179:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45609
2025-11-27T18:20:39.379878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471813889825262:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22336
2025-11-27T18:20:39.381203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471814394669721:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14631
2025-11-27T18:20:39.382699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471814938710629:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.383543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471811856197178:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.382692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471814394669722:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14631
2025-11-27T18:20:39.381219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471813889825264:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22336
2025-11-27T18:20:39.382772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471813883860808:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.383518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471812409014974:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.382723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471814938710628:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.383101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471813889825263:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.383123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471813883860734:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.383545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471812409014973:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.383088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471814394669720:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.383868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471811856197179:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.383324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471814394669645:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.383536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471814938710624:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.383567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471813889825262:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.384589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471813889825256:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.383704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471814394669721:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.384266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471812409014966:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.383570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471814938710550:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.384610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471813889825264:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.386545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471811856197104:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.383956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471814394669644:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.384291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471812409014966:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.386747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471811856197102:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.383603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471814938710551:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.386315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471814394669722:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.384616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471813889825256:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.386818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471811856197180:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45609
2025-11-27T18:20:39.383611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471814938710630:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26453
2025-11-27T18:20:39.384320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471812409014976:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63530
2025-11-27T18:20:39.386501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471813889825257:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.386524Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471813889825257:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.384317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471814938710622:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.384331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471812409014969:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.390812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471811856197180:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.388742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471813889825256:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:39.384344Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471814938710622:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.391068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471811856197165:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.384352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471812409014969:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.388808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471813889825255:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.391746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471813883860812:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45383
2025-11-27T18:20:39.385422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471812409014975:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63530
2025-11-27T18:20:39.384626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471814938710630:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.388839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471813889825255:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.391956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471813883860802:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.386884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471814938710622:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:39.386316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471812409014976:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:39.391024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471813889825255:2110] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-27T18:20:39.391991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471813883860802:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.391127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471813889825186:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.388810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471814938710621:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.388953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471812409014966:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.393719Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471813883860802:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:39.388844Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471814938710621:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.394127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471811856197185:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45609
2025-11-27T18:20:39.389670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471812409014975:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.391305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.395257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471811856197187:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45609
2025-11-27T18:20:39.393959Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471813883860734:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21863) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471813883860808:2114] held: no
2025-11-27T18:20:39.396869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471814394669728:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14631
2025-11-27T18:20:39.391792Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471812409014969:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.391535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471813889825186:2049] [node 3] ICP22 created new session: [1:7577471813889825273:2049]
2025-11-27T18:20:39.391331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471814938710613:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.396346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471813883860796:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.391392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:39.397603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471814394669645:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11000) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471814394669720:2114] held: no
2025-11-27T18:20:39.391979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471812409014896:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.391968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471813889825273:2049] [node 3] ICS09 handshake done sender: [1:7577471813889825256:2111] self: [1:7577471813889825253:0] peer: [3:7577471814938710633:0] socket: 603 qp: -1
2025-11-27T18:20:39.399128Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471811856197102:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16471) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471811856197179:2115] held: no
2025-11-27T18:20:39.399215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471814394669729:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14631
2025-11-27T18:20:39.398657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:39.393109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:39.392063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471813889825248:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.391971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471814938710613:2107] [node 4] ICP22 created new session: [3:7577471814938710641:2107]
2025-11-27T18:20:39.399063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471813883860796:2109] [node 6] ICP22 created new session: [4:7577471813883860815:2109]
2025-11-27T18:20:39.400236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471811856197178:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:39.392116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471813889825273:2049] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.393156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471812409014896:2049] [node 3] ICP22 created new session: [2:7577471812409014985:2049]
2025-11-27T18:20:39.392383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471814938710641:2107] [node 4] ICS09 handshake done sender: [3:7577471814938710622:2113] self: [3:7577471814938710619:0] peer: [4:7577471813883860811:0] socket: 606 qp: -1
2025-11-27T18:20:39.399631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471813883860815:2109] [node 6] ICS09 handshake done sender: [4:7577471813883860802:2111] self: [4:7577471813883860798:0] peer: [6:7577471811856197186:0] socket: 627 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.393904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471812409014985:2049] [node 3] ICS09 handshake done sender: [2:7577471812409014966:2112] self: [2:7577471812409014963:0] peer: [3:7577471814938710634:0] socket: 598 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.392170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:39.392970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471814938710641:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:39.400471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471813883860808:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:39.392204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471813889825257:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:39.393945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471812409014959:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.403544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471814394669721:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:39.403448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471811856197188:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45609
2025-11-27T18:20:39.400924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471813883860815:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:39.393120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.393175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:39.394116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:39.403640Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471814394669644:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16471) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471814394669721:2115] held: no
2025-11-27T18:20:39.404013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471811856197104:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.400971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:39.393156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471814938710621:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.393924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471813889825248:2109] [node 6] ICP22 created new session: [1:7577471813889825277:2109]
2025-11-27T18:20:39.394334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471812409014985:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:39.395912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471814938710638:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26453
2025-11-27T18:20:39.403986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471814394669720:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:39.404051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:39.395244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471813889825277:2109] [node 6] ICS09 handshake done sender: [1:7577471813889825255:2110] self: [1:7577471813889825252:0] peer: [6:7577471811856197184:0] socket: 599 qp: -1
2025-11-27T18:20:39.394342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471812409014959:2108] [node 5] ICP22 created new session: [2:7577471812409014987:2108]
2025-11-27T18:20:39.405650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471814394669645:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.395953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471813889825249:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.403430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471813883860814:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45383
2025-11-27T18:20:39.404698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471811856197104:2050] [node 3] ICP22 created new session: [6:7577471811856197191:2050]
2025-11-27T18:20:39.396955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471814938710639:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26453
2025-11-27T18:20:39.394373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:39.395976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471813889825277:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:39.403761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471813883860734:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.404773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471811856197179:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:39.395907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471812409014987:2108] [node 5] ICS09 handshake done sender: [2:7577471812409014969:2114] self: [2:7577471812409014967:0] peer: [5:7577471814394669725:0] socket: 607 qp: -1
2025-11-27T18:20:39.396004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:39.397615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471814938710551:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11000) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471814938710628:2115] held: no
2025-11-27T18:20:39.403815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:39.405565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471811856197191:2050] [node 3] ICS09 handshake done sender: [6:7577471811856197178:2114] self: [6:7577471811856197183:0] peer: [3:7577471814938710617:0] socket: 613 qp: -1
2025-11-27T18:20:39.396183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471812409014987:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:39.404195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471813883860734:2050] [node 3] ICP22 created new session: [4:7577471813883860820:2050]
2025-11-27T18:20:39.405881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:39.398286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471814938710614:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.405857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471811856197191:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:39.396889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:39.406286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471814394669645:2049] [node 2] ICP22 created new session: [5:7577471814394669732:2049]
2025-11-27T18:20:39.398321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:39.396015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:39.401130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471812409014990:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63530
2025-11-27T18:20:39.407190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471814394669732:2049] [node 2] ICS09 handshake done sender: [5:7577471814394669720:2114] self: [5:7577471814394669725:0] peer: [2:7577471812409014967:0] socket: 611 qp: -1
2025-11-27T18:20:39.406263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:39.398482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471814938710614:2109] [node 6] ICP22 created new session: [3:7577471814938710644:2109]
2025-11-27T18:20:39.407230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471814394669732:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:39.406846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471811856197102:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.396377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471813889825249:2108] [node 5] ICP22 created new session: [1:7577471813889825279:2108]
2025-11-27T18:20:39.407517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471814394669644:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.406874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:39.397814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471813889825279:2108] [node 5] ICS09 handshake done sender: [1:7577471813889825257:2112] self: [1:7577471813889825254:0] peer: [5:7577471814394669726:0] socket: 608 qp: -1
2025-11-27T18:20:39.399348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471814938710644:2109] [node 6] ICS09 handshake done sender: [3:7577471814938710621:2111] self: [3:7577471814938710617:0] peer: [6:7577471811856197183:0] socket: 600 qp: -1
2025-11-27T18:20:39.404252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471813883860820:2050] [node 3] ICS09 handshake done sender: [4:7577471813883860808:2114] self: [4:7577471813883860811:0] peer: [3:7577471814938710619:0] socket: 626 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.407553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:39.398787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471813889825279:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:39.406920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471811856197102:2048] [node 1] ICP22 created new session: [6:7577471811856197193:2048]
2025-11-27T18:20:39.399500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471814938710644:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:39.407600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471814394669644:2048] [node 1] ICP22 created new session: [5:7577471814394669733:2048]
2025-11-27T18:20:39.398828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:39.404756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471813883860814:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.407640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471814394669733:2048] [node 1] ICS09 handshake done sender: [5:7577471814394669721:2115] self: [5:7577471814394669726:0] peer: [1:7577471813889825254:0] socket: 620 qp: -1
2025-11-27T18:20:39.407027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471811856197189:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45609
2025-11-27T18:20:39.403729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471813889825278:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22336
2025-11-27T18:20:39.405015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471813883860820:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:39.399690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:39.407381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471811856197193:2048] [node 1] ICS09 handshake done sender: [6:7577471811856197179:2115] self: [6:7577471811856197184:0] peer: [1:7577471813889825252:0] socket: 624 qp: -1
2025-11-27T18:20:39.407666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:39.405624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:39.403659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471814938710629:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:39.407426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471811856197193:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:39.404181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471814938710628:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:39.407466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:39.407833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471814394669733:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:39.405837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471813883860816:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45383
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.407560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471813883860733:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.408195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471812409014968:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.406898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471814938710550:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.407544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471811856197180:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:39.408229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:39.406985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:39.409589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471813883860822:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45383
2025-11-27T18:20:39.408267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471811856197189:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.407038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471814938710551:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.408320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471814394669730:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14631
2025-11-27T18:20:39.408235Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471812409014968:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.408306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471811856197165:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.407375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471814938710550:2048] [node 1] ICP22 created new session: [3:7577471814938710647:2048]
2025-11-27T18:20:39.408465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:39.407187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:39.409323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471811856197165:2108] [node 4] ICP22 created new session: [6:7577471811856197195:2108]
2025-11-27T18:20:39.407416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471814938710647:2048] [node 1] ICS09 handshake done sender: [3:7577471814938710629:2116] self: [3:7577471814938710633:0] peer: [1:7577471813889825253:0] socket: 614 qp: -1
2025-11-27T18:20:39.410093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471811856197195:2108] [node 4] ICS09 handshake done sender: [6:7577471811856197180:2116] self: [6:7577471811856197186:0] peer: [4:7577471813883860798:0] socket: 635 qp: -1
2025-11-27T18:20:39.410137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471811856197195:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:39.410804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471814394669730:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.410846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:39.411041Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471812409014968:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.407441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471814938710647:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:39.407464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:39.411332Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471813889825186:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(16440480ns) {HandleHandshakeStatus:329(16438387ns) {DropIncomingHandshake:442(569ns) DropOutgoingHandshake:457(676ns) SwitchToState:179(161ns) ProcessPendingSessionEvents:495(14164012ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13360699ns max# 1602821ns}}}
2025-11-27T18:20:39.411702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471813889825282:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22336
2025-11-27T18:20:39.407471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471814938710551:2049] [node 2] ICP22 created new session: [3:7577471814938710649:2049]
2025-11-27T18:20:39.411559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471811856197103:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.411190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471812409014990:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.411922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471813883860822:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.412301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471813883860816:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.412474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471814394669708:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.411575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471812409014895:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.407510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471814938710649:2049] [node 2] ICS09 handshake done sender: [3:7577471814938710628:2115] self: [3:7577471814938710634:0] peer: [2:7577471812409014963:0] socket: 601 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.412536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471813889825258:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.407547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471814938710649:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:39.411740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471812409014960:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.413244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471811856197198:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45609
2025-11-27T18:20:39.412567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471813889825258:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.413158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471813883860824:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45383
2025-11-27T18:20:39.412266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.413428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471813889825248:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16531632ns) {HandleHandshakeStatus:329(16529857ns) {DropIncomingHandshake:442(9574ns) DropOutgoingHandshake:457(270ns) SwitchToState:179(146ns) ProcessPendingSessionEvents:495(9770503ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9672708ns max# 1248884ns}}}
2025-11-27T18:20:39.407574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:39.413562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471813883860732:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.412387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471812409014960:2107] [node 4] ICP22 created new session: [2:7577471812409014994:2107]
2025-11-27T18:20:39.413713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471813889825282:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.414755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471811856197198:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.413319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471814938710613:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16210377ns) {HandleHandshakeStatus:329(16207837ns) {DropIncomingHandshake:442(15545ns) DropOutgoingHandshake:457(498ns) SwitchToState:179(178ns) ProcessPendingSessionEvents:495(13404724ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12792529ns max# 1235512ns}}}
2025-11-27T18:20:39.413582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471813883860810:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.416742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471814938710648:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26453
2025-11-27T18:20:39.416768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471814394669737:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14631
2025-11-27T18:20:39.412505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471812409014994:2107] [node 4] ICS09 handshake done sender: [2:7577471812409014968:2113] self: [2:7577471812409014964:0] peer: [4:7577471813883860823:0] socket: 651 qp: -1
2025-11-27T18:20:39.416981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471811856197164:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.413606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471813883860810:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.418190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471814394669724:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.416999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471813883860814:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:39.412540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471812409014994:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:39.418207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471813889825266:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.418216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471814394669724:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.417376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471813883860733:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11000) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471813883860814:2117] held: no
2025-11-27T18:20:39.412809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.419267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471813883860810:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:39.418242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471813889825266:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.414733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471812409014978:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.414765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471812409014978:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.418438Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7577471813889825186:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:39.419882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471811856197201:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45609
2025-11-27T18:20:39.419906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471813889825258:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:39.418021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471812409014978:2119] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:39.420477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471814394669724:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:39.420462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471813883860795:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.419536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471812409014961:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.420732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471813889825185:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.420529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:39.419588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:39.420774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:39.421407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471813889825185:2048] [node 2] ICP22 created new session: [1:7577471813889825286:2048]
2025-11-27T18:20:39.420698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471813883860795:2108] [node 5] ICP22 created new session: [4:7577471813883860828:2108]
2025-11-27T18:20:39.419633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471812409014961:2110] [node 6] ICP22 created new session: [2:7577471812409014997:2110]
2025-11-27T18:20:39.421788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471814394669707:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.421413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471813883860828:2108] [node 5] ICS09 handshake done sender: [4:7577471813883860810:2115] self: [4:7577471813883860809:0] peer: [5:7577471814394669738:0] socket: 595 qp: -1
2025-11-27T18:20:39.421814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471813889825286:2048] [node 2] ICS09 handshake done sender: [1:7577471813889825258:2113] self: [1:7577471813889825251:0] peer: [2:7577471812409014993:0] socket: 622 qp: -1
2025-11-27T18:20:39.421847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471813889825286:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:39.419807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471812409014997:2110] [node 6] ICS09 handshake done sender: [2:7577471812409014978:2119] self: [2:7577471812409014977:0] peer: [6:7577471811856197199:0] socket: 602 qp: -1
2025-11-27T18:20:39.420431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471812409014997:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:39.421835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:39.420474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:39.421851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471813889825266:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:39.422066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471814394669707:2109] [node 6] ICP22 created new session: [5:7577471814394669741:2109]
::ffff:127.1.0.0
2025-11-27T18:20:39.421815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471813883860828:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:39.422275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:39.422247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471814394669741:2109] [node 6] ICS09 handshake done sender: [5:7577471814394669724:2117] self: [5:7577471814394669723:0] peer: [6:7577471811856197200:0] socket: 655 qp: -1
2025-11-27T18:20:39.423663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471813889825250:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.422388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471814394669741:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:39.421855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:39.422410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:39.423645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471812409014998:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63530
2025-11-27T18:20:39.422127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471813883860733:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.425914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471814394669739:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14631
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.425372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:39.427403Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471814394669708:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18353) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471814394669730:2120] held: no
2025-11-27T18:20:39.426182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:39.425966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471813883860733:2049] [node 2] ICP22 created new session: [4:7577471813883860831:2049]
2025-11-27T18:20:39.426390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471813889825250:2107] [node 4] ICP22 created new session: [1:7577471813889825288:2107]
2025-11-27T18:20:39.426800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471813889825288:2107] [node 4] ICS09 handshake done sender: [1:7577471813889825266:2117] self: [1:7577471813889825265:0] peer: [4:7577471813883860826:0] socket: 652 qp: -1
2025-11-27T18:20:39.426201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471813883860831:2049] [node 2] ICS09 handshake done sender: [4:7577471813883860814:2117] self: [4:7577471813883860823:0] peer: [2:7577471812409014964:0] socket: 653 qp: -1
2025-11-27T18:20:39.428417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471811856197189:2121] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:39.428031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471812409014895:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16471) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471812409014990:2120] held: no
2025-11-27T18:20:39.427357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471813889825288:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:39.426244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471813883860831:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:39.427804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:39.426759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:39.428770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471814394669737:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.427303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471813883860830:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45383
2025-11-27T18:20:39.430291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471814394669646:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:39.430708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471811856197103:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11000) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471811856197189:2121] held: no
2025-11-27T18:20:39.430836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471814394669730:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:39.430962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471814938710632:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:39.431036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471812409014990:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:39.431005Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471814938710632:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:39.431084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471814394669708:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.431828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471813883860816:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:39.432718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471811856197103:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.432685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471813883860732:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16471) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471813883860816:2118] held: no
2025-11-27T18:20:39.431960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:39.432771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471812409014895:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.432907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:39.431996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471814394669708:2108] [node 4] ICP22 created new session: [5:7577471814394669744:2108]
2025-11-27T18:20:39.432826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:39.433283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471811856197103:2049] [node 2] ICP22 created new session: [6:7577471811856197203:2049]
2025-11-27T18:20:39.433213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471812409014895:2048] [node 1] ICP22 created new session: [2:7577471812409015000:2048]
2025-11-27T18:20:39.432704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471814394669744:2108] [node 4] ICS09 handshake done sender: [5:7577471814394669730:2120] self: [5:7577471814394669738:0] peer: [4:7577471813883860809:0] socket: 656 qp: -1
2025-11-27T18:20:39.434082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471811856197203:2049] [node 2] ICS09 handshake done sender: [6:7577471811856197189:2121] self: [6:7577471811856197199:0] peer: [2:7577471812409014977:0] socket: 609 qp: -1
2025-11-27T18:20:39.434023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471812409015000:2048] [node 1] ICS09 handshake done sender: [2:7577471812409014990:2120] self: [2:7577471812409014993:0] peer: [1:7577471813889825251:0] socket: 604 qp: -1
2025-11-27T18:20:39.432738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471814394669744:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:39.434108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471811856197164:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471811856197198:2122] held: no
2025-11-27T18:20:39.434071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471812409015000:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:39.433223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:39.434122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471811856197203:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:39.434121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:39.434155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:39.434426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471814938710632:2118] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:39.434467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471813883860732:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.434564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:39.434614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471813883860732:2048] [node 1] ICP22 created new session: [4:7577471813883860833:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:39.434666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471813883860833:2048] [node 1] ICS09 handshake done sender: [4:7577471813883860816:2118] self: [4:7577471813883860826:0] peer: [1:7577471813889825265:0] socket: 629 qp: -1
2025-11-27T18:20:39.434663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471814938710615:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:39.434784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471813883860833:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:39.434826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:39.434853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471814394669746:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14631
2025-11-27T18:20:39.434821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:39.434862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471814938710615:2108] [node 5] ICP22 created new session: [3:7577471814938710653:2108]
2025-11-27T18:20:39.434911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471811856197202:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45609
2025-11-27T18:20:39.434950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471814938710653:2108] [node 5] ICS09 handshake done sender: [3:7577471814938710632:2118] self: [3:7577471814938710631:0] peer: [5:7577471814394669743:0] socket: 663 qp: -1
2025-11-27T18:20:39.434997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471814938710653:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:39.435081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471814394669737:2121] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:39.435046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:39.435495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471814394669646:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21863) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471814394669737:2121] held: no
2025-11-27T18:20:39.436370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471811856197198:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:39.436622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471811856197164:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.436666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:39.436844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471811856197164:2107] [node 5] ICP22 created new session: [6:7577471811856197205:2107]
2025-11-27T18:20:39.437098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471811856197205:2107] [node 5] ICS09 handshake done sender: [6:7577471811856197198:2122] self: [6:7577471811856197200:0] peer: [5:7577471814394669723:0] socket: 660 qp: -1
2025-11-27T18:20:39.437145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471811856197205:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:39.437195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:39.437311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471814394669646:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:39.437350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:39.437390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471814394669646:2050] [node 3] ICP22 created new session: [5:7577471814394669747:2050]
2025-11-27T18:20:39.437445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471814394669747:2050] [node 3] ICS09 handshake done sender: [5:7577471814394669737:2121] self: [5:7577471814394669743:0] peer: [3:7577471814938710631:0] socket: 664 qp: -1
2025-11-27T18:20:39.437476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471814394669747:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:39.437511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:39.481331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:39.481570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471812409015000:2048] [node 1] ICS01 socket: 604 reason# EndOfStream
2025-11-27T18:20:39.481618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471812409014895:2048] [node 1] ICP30 unregister session Session# [2:7577471812409015000:2048] VirtualId# [2:7577471812409014993:0]
2025-11-27T18:20:39.481656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471812409015000:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.481741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:39.481797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:39.482704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:39.482898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471814938710647:2048] [node 1] ICS07 socket disconnect 614 reason# EndOfStream
2025-11-27T18:20:39.482930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471814938710647:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.482997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:39.483050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471814938710647:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:39.485132Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:39.485304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471814938710550:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16471) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:39.485343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471814938710647:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:39.485328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471813883860833:2048] [node 1] ICS01 socket: 629 reason# EndOfStream
2025-11-27T18:20:39.485369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471813883860732:2048] [node 1] ICP30 unregister session Session# [4:7577471813883860833:2048] VirtualId# [4:7577471813883860826:0]
2025-11-27T18:20:39.485396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471813883860833:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.485474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:39.485844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:39.486466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471814938710550:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16471) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:39.486554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471814938710647:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:39.486522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:39.486610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471814938710550:2048] [node 1] ICP30 unregister session Session# [3:7577471814938710647:2048] VirtualId# [3:7577471814938710633:0]
2025-11-27T18:20:39.486716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:39.486753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471814938710550:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16471) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.486777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471814394669733:2048] [node 1] ICS01 socket: 620 reason# EndOfStream
2025-11-27T18:20:39.486768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16471) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.486819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471814394669644:2048] [node 1] ICP30 unregister session Session# [5:7577471814394669733:2048] VirtualId# [5:7577471814394669726:0]
2025-11-27T18:20:39.486852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471814394669733:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.486934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:39.486996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:39.487410Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:39.487617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471811856197193:2048] [node 1] ICS01 socket: 624 reason# EndOfStream
2025-11-27T18:20:39.487660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471811856197102:2048] [node 1] ICP30 unregister session Session# [6:7577471811856197193:2048] VirtualId# [6:7577471811856197184:0]
2025-11-27T18:20:39.487720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471811856197193:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.487805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:39.487876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:39.497035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471814938710550:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:39.540498Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:39.540715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471814938710649:2049] [node 2] ICS01 socket: 601 reason# EndOfStream
2025-11-27T18:20:39.540761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471814938710551:2049] [node 2] ICP30 unregister session Session# [3:7577471814938710649:2049] VirtualId# [3:7577471814938710634:0]
2025-11-27T18:20:39.540793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471814938710649:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.540868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:39.540965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:39.542797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:39.542961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471813883860831:2049] [node 2] ICS01 socket: 653 reason# EndOfStream
2025-11-27T18:20:39.542998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471813883860733:2049] [node 2] ICP30 unregister session Session# [4:7577471813883860831:2049] VirtualId# [4:7577471813883860823:0]
2025-11-27T18:20:39.543023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471813883860831:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.543115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:39.543174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:39.543598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:39.543817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471814394669732:2049] [node 2] ICS01 socket: 611 reason# EndOfStream
2025-11-27T18:20:39.543874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471814394669645:2049] [node 2] ICP30 unregister session Session# [5:7577471814394669732:2049] VirtualId# [5:7577471814394669725:0]
2025-11-27T18:20:39.543921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471814394669732:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.543992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:39.544049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:39.544323Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:39.544492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471811856197203:2049] [node 2] ICS01 socket: 609 reason# EndOfStream
2025-11-27T18:20:39.544530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471811856197103:2049] [node 2] ICP30 unregister session Session# [6:7577471811856197203:2049] VirtualId# [6:7577471811856197199:0]
2025-11-27T18:20:39.544559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471811856197203:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.544630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:39.544705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:39.610004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:39.610201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471813883860820:2050] [node 3] ICS01 socket: 626 reason# EndOfStream
2025-11-27T18:20:39.610244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471813883860734:2050] [node 3] ICP30 unregister session Session# [4:7577471813883860820:2050] VirtualId# [4:7577471813883860811:0]
2025-11-27T18:20:39.610270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471813883860820:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.610386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:39.610435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:39.610962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:39.611193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471814394669747:2050] [node 3] ICS01 socket: 664 reason# EndOfStream
2025-11-27T18:20:39.611239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471814394669646:2050] [node 3] ICP30 unregister session Session# [5:7577471814394669747:2050] VirtualId# [5:7577471814394669743:0]
2025-11-27T18:20:39.611284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471814394669747:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.611365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:39.611410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:39.611708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:39.611930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471811856197191:2050] [node 3] ICS07 socket disconnect 613 reason# EndOfStream
2025-11-27T18:20:39.611962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471811856197191:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.612030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:39.612071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471811856197191:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:39.612727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471811856197104:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21863) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:39.612832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471811856197191:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:39.612878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471811856197104:2050] [node 3] ICP30 unregister session Session# [6:7577471811856197191:2050] VirtualId# [6:7577471811856197183:0]
2025-11-27T18:20:39.612930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:39.612962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471811856197104:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21863) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.612976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21863) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.622955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471811856197104:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:39.657094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:39.657352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471814394669744:2108] [node 4] ICS01 socket: 656 reason# EndOfStream
2025-11-27T18:20:39.657398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471814394669708:2108] [node 4] ICP30 unregister session Session# [5:7577471814394669744:2108] VirtualId# [5:7577471814394669738:0]
2025-11-27T18:20:39.657432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471814394669744:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.657549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:39.657614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:39.658107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:39.658272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471811856197195:2108] [node 4] ICS07 socket disconnect 635 reason# EndOfStream
2025-11-27T18:20:39.658311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471811856197195:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.658411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:39.658466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471811856197195:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:39.659260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471811856197165:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18353) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:39.659355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471811856197195:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:39.659449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471811856197165:2108] [node 4] ICP30 unregister session Session# [6:7577471811856197195:2108] VirtualId# [6:7577471811856197186:0]
2025-11-27T18:20:39.659548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:39.659583Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471811856197165:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18353) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.659599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18353) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:39.669036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471811856197165:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:39.730459Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:39.730694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471811856197205:2107] [node 5] ICS01 socket: 660 reason# EndOfStream
2025-11-27T18:20:39.730734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471811856197164:2107] [node 5] ICP30 unregister session Session# [6:7577471811856197205:2107] VirtualId# [6:7577471811856197200:0]
2025-11-27T18:20:39.730768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471811856197205:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:39.730831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:39.730870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:39.813910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471812483495847:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:39.813940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471812483495848:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:39.833039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471813576584077:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:39.833084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471813576584078:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:39.851393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471812573436686:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:39.851708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471812573436687:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:39.869663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471813112810714:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:39.869703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471813112810715:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:39.870124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471813112810713:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:39.890065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471812471364812:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:39.890085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471812471364814:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:39.890090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471812471364813:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:39.910156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471812276607681:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:39.910185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471812276607682:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:39.910205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471812276607683:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:39.986480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471813112810776:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:39.992625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471813112810777:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:39.991781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471812471364875:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:39.992356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471813576584142:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:39.992603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471812276607741:2105] [node 4] ICP01 ready to work
2025-11-27T18:20:39.991861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471812483495910:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:39.992784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471812573436752:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:39.992538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471812471364876:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:39.992589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471813576584141:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:39.992585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471812483495912:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:39.992765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471812276607742:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:39.993002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471812573436751:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:39.992770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471812573436750:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:39.992580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471812483495911:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:39.992942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471813576584143:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.014042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471816766332188:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-27T18:20:40.012238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471816778463219:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24338
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.014234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471817407778087:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54909
2025-11-27T18:20:40.014840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471816571575052:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46679
2025-11-27T18:20:40.016332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471817871551452:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55881
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.014834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471816778463223:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24338
2025-11-27T18:20:40.016341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471817871551454:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55881
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:40.017241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471817407778087:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.018063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471817871551452:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.017968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471816778463223:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.017115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471816766332186:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-27T18:20:40.017628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471816571575052:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.017636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471817407778088:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54909
2025-11-27T18:20:40.017613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471816766332187:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-27T18:20:40.017789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471816571575053:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46679
2025-11-27T18:20:40.018024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471816778463219:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.019534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471816868404064:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33339
2025-11-27T18:20:40.019534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471817407778080:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.017965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471816571575054:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46679
2025-11-27T18:20:40.017983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471816766332188:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.020453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471816778463218:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.020801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471817871551444:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.020464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471816868404062:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33339
2025-11-27T18:20:40.019561Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471817407778080:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.018388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471812471364812:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.018364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471812276607741:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.020839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471817871551444:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.020478Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471816778463218:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.022262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471816868404063:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33339
2025-11-27T18:20:40.019260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471816766332187:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.021146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471817407778089:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54909
2025-11-27T18:20:40.019074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471816571575054:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.021533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471816778463215:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.025384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471817871551454:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.022963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471817407778088:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.019723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471816766332186:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.022665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471816868404062:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.019541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471816571575055:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46679
2025-11-27T18:20:40.021550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471816778463215:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.026209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471817871551444:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:40.024513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471816868404063:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.021139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471812471364813:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.023981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471817407778079:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.020492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471812276607681:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.024254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471816778463215:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:40.025361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471812573436686:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.024010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471817407778079:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.026239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471817871551449:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.021201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471816571575055:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.021591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471812471364875:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.029389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471816778463214:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.021545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471816571575053:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.024039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471817407778080:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:40.025509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471812573436687:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.026264Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471817871551449:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.024458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471816766332189:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-27T18:20:40.029398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471816778463218:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:40.022941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471812276607683:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.025313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471813112810715:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.029416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471816778463214:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.028697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471816766332189:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.024484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471812276607682:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.026048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471816868404056:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.026493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471817871551448:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.026525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471817871551448:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.029626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471817407778089:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.026071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471816868404056:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.029503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471812483495911:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.032071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471816571575064:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46679
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.029636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471817407778079:2110] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:40.030963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471817871551449:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:40.026531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471816868404064:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.032487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471816571575063:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46679
2025-11-27T18:20:40.027662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471816868404055:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.030108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:40.030409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471813112810776:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.033727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471816766332193:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-27T18:20:40.031102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471817871551448:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:40.030367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471812483495911:2109] [node 6] ICP22 created new session: [1:7577471816778463228:2109]
2025-11-27T18:20:40.027683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471816868404055:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.030885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471813112810777:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.035911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471812276607741:2105] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32214) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471816571575052:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.030189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471816868404055:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:40.033742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471816766332195:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-27T18:20:40.031558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471813576584141:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.030892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471816778463228:2109] [node 6] ICS09 handshake done sender: [1:7577471816778463215:2111] self: [1:7577471816778463211:0] peer: [6:7577471816571575060:0] socket: 618 qp: -1
2025-11-27T18:20:40.030926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:40.033990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471816766332194:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-27T18:20:40.031616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:40.030973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471812483495910:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.038592Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471812276607683:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15234) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471816571575053:2116] held: no
2025-11-27T18:20:40.030238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471816868404056:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:40.031404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:40.039619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471816571575066:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46679
2025-11-27T18:20:40.039604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471812471364813:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18906) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471816766332187:2116] held: no
2025-11-27T18:20:40.031922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471812573436750:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.032160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471813576584141:2108] [node 5] ICP22 created new session: [2:7577471817871551460:2108]
2025-11-27T18:20:40.031420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471816778463228:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:40.031408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471813112810777:2109] [node 6] ICP22 created new session: [4:7577471817407778097:2109]
2025-11-27T18:20:40.031434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:40.031958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:40.039895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471816571575065:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46679
2025-11-27T18:20:40.031456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:40.032428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471812573436752:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.031894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471813112810776:2108] [node 5] ICP22 created new session: [4:7577471817407778096:2108]
2025-11-27T18:20:40.041580Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471816571575054:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:40.032496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471813576584142:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.032509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471812573436750:2109] [node 6] ICP22 created new session: [3:7577471816868404071:2109]
2025-11-27T18:20:40.039631Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471812471364875:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32214) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471816766332186:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.039666Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471812471364812:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471816766332188:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.031609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471812483495910:2108] [node 5] ICP22 created new session: [1:7577471816778463231:2108]
2025-11-27T18:20:40.040723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471816766332188:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:40.033723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:40.033713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471817407778097:2109] [node 6] ICS09 handshake done sender: [4:7577471817407778080:2111] self: [4:7577471817407778074:0] peer: [6:7577471816571575059:0] socket: 601 qp: -1
2025-11-27T18:20:40.033799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:40.032547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471816778463231:2108] [node 5] ICS09 handshake done sender: [1:7577471816778463218:2114] self: [1:7577471816778463213:0] peer: [5:7577471816766332190:0] socket: 620 qp: -1
2025-11-27T18:20:40.035150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471816868404071:2109] [node 6] ICS09 handshake done sender: [3:7577471816868404056:2113] self: [3:7577471816868404050:0] peer: [6:7577471816571575061:0] socket: 609 qp: -1
2025-11-27T18:20:40.033994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471817407778096:2108] [node 5] ICS09 handshake done sender: [4:7577471817407778079:2110] self: [4:7577471817407778075:0] peer: [5:7577471816766332192:0] socket: 602 qp: -1
2025-11-27T18:20:40.032593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471816778463231:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:40.033828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471813576584078:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.035690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471812573436752:2107] [node 4] ICP22 created new session: [3:7577471816868404073:2107]
2025-11-27T18:20:40.042781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471816571575055:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:40.034026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471817407778096:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:40.043569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471812276607681:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.035882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471816868404071:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:40.035910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471816868404073:2107] [node 4] ICS09 handshake done sender: [3:7577471816868404055:2112] self: [3:7577471816868404053:0] peer: [4:7577471817407778092:0] socket: 613 qp: -1
2025-11-27T18:20:40.033835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:40.034056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:40.035925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:40.034011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471816778463214:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:40.033909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:40.037115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471816868404073:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:40.043620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:40.034854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471817407778097:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:40.037736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471812483495848:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:40.037149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:40.034885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:40.034013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471817871551460:2108] [node 5] ICS09 handshake done sender: [2:7577471817871551444:2111] self: [2:7577471817871551442:0] peer: [5:7577471816766332191:0] socket: 619 qp: -1
2025-11-27T18:20:40.043814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471812276607681:2048] [node 1] ICP22 created new session: [6:7577471816571575068:2048]
2025-11-27T18:20:40.037762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:40.043961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471812471364812:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.039866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471816868404070:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33339
2025-11-27T18:20:40.034021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471813576584142:2110] [node 6] ICP22 created new session: [2:7577471817871551461:2110]
2025-11-27T18:20:40.044111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471816571575053:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:40.038864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471812483495848:2049] [node 3] ICP22 created new session: [1:7577471816778463235:2049]
2025-11-27T18:20:40.035869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471817407778094:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54909
2025-11-27T18:20:40.044282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:40.041560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471812573436687:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18906) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471816868404063:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.034054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471817871551460:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:40.044321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471816571575068:2048] [node 1] ICS09 handshake done sender: [6:7577471816571575054:2117] self: [6:7577471816571575060:0] peer: [1:7577471816778463211:0] socket: 631 qp: -1
2025-11-27T18:20:40.044595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471812471364812:2048] [node 1] ICP22 created new session: [5:7577471816766332197:2048]
2025-11-27T18:20:40.039938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471817407778088:2116] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:40.040223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471816778463235:2049] [node 3] ICS09 handshake done sender: [1:7577471816778463214:2110] self: [1:7577471816778463210:0] peer: [3:7577471816868404065:0] socket: 616 qp: -1
2025-11-27T18:20:40.043897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471816868404072:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33339
2025-11-27T18:20:40.044588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471816571575052:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:40.034091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:40.040751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471816778463235:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:40.042290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471813112810715:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.034849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471813576584078:2049] [node 3] ICP22 created new session: [2:7577471817871551462:2049]
2025-11-27T18:20:40.040981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:40.044628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471816571575068:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:40.044625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471816766332197:2048] [node 1] ICS09 handshake done sender: [5:7577471816766332188:2117] self: [5:7577471816766332190:0] peer: [1:7577471816778463213:0] socket: 627 qp: -1
2025-11-27T18:20:40.048164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471816868404078:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33339
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.043782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:40.043733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471816778463236:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24338
2025-11-27T18:20:40.035185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471817871551461:2110] [node 6] ICS09 handshake done sender: [2:7577471817871551448:2114] self: [2:7577471817871551441:0] peer: [6:7577471816571575062:0] socket: 628 qp: -1
2025-11-27T18:20:40.044387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471813112810715:2050] [node 3] ICP22 created new session: [4:7577471817407778103:2050]
2025-11-27T18:20:40.044670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.045666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471816778463238:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24338
2025-11-27T18:20:40.044649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471816766332197:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:40.035759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471817871551461:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:40.044679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:40.049008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471816778463239:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24338
2025-11-27T18:20:40.045243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471817407778103:2050] [node 3] ICS09 handshake done sender: [4:7577471817407778088:2116] self: [4:7577471817407778092:0] peer: [3:7577471816868404053:0] socket: 617 qp: -1
2025-11-27T18:20:40.044771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471812276607682:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.045720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471816766332186:2115] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:40.035890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471817871551462:2049] [node 3] ICS09 handshake done sender: [2:7577471817871551449:2115] self: [2:7577471817871551443:0] peer: [3:7577471816868404066:0] socket: 621 qp: -1
2025-11-27T18:20:40.049459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471816778463236:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.047314Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471816766332187:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:40.045224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:40.045655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471817407778102:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54909
2025-11-27T18:20:40.035902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:40.049988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471816868404062:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.045705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471817407778103:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:40.045252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471812276607682:2049] [node 2] ICP22 created new session: [6:7577471816571575071:2049]
2025-11-27T18:20:40.035928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471817871551462:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:40.050872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471816778463217:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.047596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471812471364875:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.037128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:40.047078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471812276607683:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.050849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471816868404063:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:40.047643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:40.047510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471816571575071:2049] [node 2] ICS09 handshake done sender: [6:7577471816571575055:2118] self: [6:7577471816571575062:0] peer: [2:7577471817871551441:0] socket: 632 qp: -1
2025-11-27T18:20:40.047823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471812471364875:2108] [node 4] ICP22 created new session: [5:7577471816766332202:2108]
2025-11-27T18:20:40.050909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471816778463217:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.047671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471812276607741:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.047127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:40.043344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471817871551466:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55881
2025-11-27T18:20:40.048600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471816766332202:2108] [node 4] ICS09 handshake done sender: [5:7577471816766332186:2115] self: [5:7577471816766332192:0] peer: [4:7577471817407778075:0] socket: 603 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.052349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471816778463216:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.047743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:40.052376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471816778463216:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.049007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471816766332202:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:40.047940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471817407778102:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.047758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471816571575071:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:40.052400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471812573436687:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.045153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471817871551468:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55881
2025-11-27T18:20:40.049055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471812471364813:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.049440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:40.047899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:40.049437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471813112810713:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.049498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:40.046014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471817871551466:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.047985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:40.049540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471812471364813:2049] [node 2] ICP22 created new session: [5:7577471816766332204:2049]
2025-11-27T18:20:40.052672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:40.052315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471817407778105:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54909
2025-11-27T18:20:40.049005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471813576584077:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.048195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471812276607683:2050] [node 3] ICP22 created new session: [6:7577471816571575072:2050]
2025-11-27T18:20:40.050813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471816766332198:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-27T18:20:40.049892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471817871551468:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.053764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471812573436687:2049] [node 2] ICP22 created new session: [3:7577471816868404080:2049]
2025-11-27T18:20:40.052765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471816778463217:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:40.048285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471812276607741:2105] [node 4] ICP22 created new session: [6:7577471816571575073:2105]
2025-11-27T18:20:40.050815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471816766332204:2049] [node 2] ICS09 handshake done sender: [5:7577471816766332187:2116] self: [5:7577471816766332191:0] peer: [2:7577471817871551442:0] socket: 623 qp: -1
2025-11-27T18:20:40.052649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471817871551470:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55881
2025-11-27T18:20:40.053770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471816778463238:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.053778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471812573436686:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.050884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471816766332204:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:40.048596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471816571575073:2105] [node 4] ICS09 handshake done sender: [6:7577471816571575052:2115] self: [6:7577471816571575059:0] peer: [4:7577471817407778074:0] socket: 606 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.053827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471816868404080:2049] [node 2] ICS09 handshake done sender: [3:7577471816868404063:2117] self: [3:7577471816868404066:0] peer: [2:7577471817871551443:0] socket: 633 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.048631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471816571575072:2050] [node 3] ICS09 handshake done sender: [6:7577471816571575053:2116] self: [6:7577471816571575061:0] peer: [3:7577471816868404050:0] socket: 611 qp: -1
2025-11-27T18:20:40.050924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:40.053883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471817407778105:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.053869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471816868404080:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:40.053696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471816766332198:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.048983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471816571575073:2105] [node 4] ICS10 traffic start
2025-11-27T18:20:40.053875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471812471364814:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.054119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471813112810776:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18180371ns) {HandleHandshakeStatus:329(18178788ns) {DropIncomingHandshake:442(9296ns) DropOutgoingHandshake:457(197ns) SwitchToState:179(185ns) ProcessPendingSessionEvents:495(12657642ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12577065ns max# 2942381ns}}}
2025-11-27T18:20:40.049011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:40.053888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:40.049052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471816571575072:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:40.054449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471816778463216:2112] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:40.054384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471817407778107:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54909
2025-11-27T18:20:40.053908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:40.049086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:40.054590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471812483495911:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(19537300ns) {HandleHandshakeStatus:329(19534918ns) {DropIncomingHandshake:442(167ns) DropOutgoingHandshake:457(346ns) SwitchToState:179(136ns) ProcessPendingSessionEvents:495(16494826ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15977801ns max# 3403127ns}}}
2025-11-27T18:20:40.054073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471812573436686:2048] [node 1] ICP22 created new session: [3:7577471816868404081:2048]
2025-11-27T18:20:40.054551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471813112810714:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.054338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471816571575077:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46679
2025-11-27T18:20:40.054809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471812483495912:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.054925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471817407778102:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:40.055720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471817871551447:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.054242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471816868404081:2048] [node 1] ICS09 handshake done sender: [3:7577471816868404062:2116] self: [3:7577471816868404065:0] peer: [1:7577471816778463210:0] socket: 630 qp: -1
2025-11-27T18:20:40.055868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471813112810777:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18568202ns) {HandleHandshakeStatus:329(18566613ns) {DropIncomingHandshake:442(6963ns) DropOutgoingHandshake:457(282ns) SwitchToState:179(112ns) ProcessPendingSessionEvents:495(9380617ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8713187ns max# 1081114ns}}}
2025-11-27T18:20:40.054924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471816571575077:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.054272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471816868404081:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:40.055755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471817871551447:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.056325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471813112810713:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.054819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471812483495910:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(18103061ns) {HandleHandshakeStatus:329(18100866ns) {DropIncomingHandshake:442(12730ns) DropOutgoingHandshake:457(364ns) SwitchToState:179(1421ns) ProcessPendingSessionEvents:495(13774116ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13667458ns max# 1220575ns}}}
2025-11-27T18:20:40.054419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:40.055796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471813576584142:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(16521643ns) {HandleHandshakeStatus:329(16519256ns) {DropIncomingHandshake:442(285ns) DropOutgoingHandshake:457(218ns) SwitchToState:179(364ns) ProcessPendingSessionEvents:495(9079500ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8850627ns max# 1322168ns}}}
2025-11-27T18:20:40.057323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471816766332182:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.056365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:40.054865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:40.056117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471812276607742:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.057052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471813576584141:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(19986805ns) {HandleHandshakeStatus:329(19858060ns) {DropIncomingHandshake:442(12020ns) DropOutgoingHandshake:457(217ns) SwitchToState:179(178ns) ProcessPendingSessionEvents:495(15451686ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14890099ns max# 1823085ns}}}
2025-11-27T18:20:40.057014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471812573436750:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(17125195ns) {HandleHandshakeStatus:329(17122657ns) {DropIncomingHandshake:442(7980ns) DropOutgoingHandshake:457(238ns) SwitchToState:179(136ns) ProcessPendingSessionEvents:495(10671281ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10076861ns max# 1112270ns}}}
2025-11-27T18:20:40.057344Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471816766332182:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.055693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471812483495912:2107] [node 4] ICP22 created new session: [1:7577471816778463243:2107]
2025-11-27T18:20:40.057511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471813112810713:2048] [node 1] ICP22 created new session: [4:7577471817407778109:2048]
2025-11-27T18:20:40.060465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471817407778109:2048] [node 1] ICS09 handshake done sender: [4:7577471817407778102:2119] self: [4:7577471817407778106:0] peer: [1:7577471816778463212:0] socket: 614 qp: -1
2025-11-27T18:20:40.057052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471812573436752:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(18643402ns) {HandleHandshakeStatus:329(18641795ns) {DropIncomingHandshake:442(148ns) DropOutgoingHandshake:457(195ns) SwitchToState:179(1401ns) ProcessPendingSessionEvents:495(11459582ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11352374ns max# 1219058ns}}}
2025-11-27T18:20:40.055749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471816778463243:2107] [node 4] ICS09 handshake done sender: [1:7577471816778463217:2113] self: [1:7577471816778463212:0] peer: [4:7577471817407778106:0] socket: 662 qp: -1
2025-11-27T18:20:40.059176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471817871551447:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:40.057300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471816868404058:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.055884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471816778463243:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:40.060500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471817407778109:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:40.060310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471817871551473:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55881
2025-11-27T18:20:40.062072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471816766332182:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:40.056134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:40.057317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471816868404058:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.060465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471812483495847:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.061486Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471813576584078:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(19626938ns) {HandleHandshakeStatus:329(19624534ns) {DropIncomingHandshake:442(13955ns) DropOutgoingHandshake:457(288ns) SwitchToState:179(2834ns) ProcessPendingSessionEvents:495(11631286ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11128218ns max# 1098130ns}}}
2025-11-27T18:20:40.060301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471816868404079:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33339
2025-11-27T18:20:40.060754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471812483495848:2049] [node 3] ICP18 handshake (actor [1:7577471816778463238:2118]) is too old
2025-11-27T18:20:40.061738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471813576584143:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.060692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471816868404058:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:40.061793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:40.060832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:40.060684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.061717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471816868404079:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.061301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471812483495847:2048] [node 2] ICP22 created new session: [1:7577471816778463245:2048]
2025-11-27T18:20:40.062157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471817871551466:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:40.061464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471816778463245:2048] [node 2] ICS09 handshake done sender: [1:7577471816778463216:2112] self: [1:7577471816778463209:0] peer: [2:7577471817871551471:0] socket: 610 qp: -1
2025-11-27T18:20:40.061751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471812573436751:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.062242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471813576584143:2107] [node 4] ICP22 created new session: [2:7577471817871551475:2107]
2025-11-27T18:20:40.061620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471816778463245:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:40.062281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471812471364876:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.062099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:40.062704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471817871551475:2107] [node 4] ICS09 handshake done sender: [2:7577471817871551447:2113] self: [2:7577471817871551445:0] peer: [4:7577471817407778108:0] socket: 664 qp: -1
2025-11-27T18:20:40.061766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.062176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471812573436751:2108] [node 5] ICP22 created new session: [3:7577471816868404085:2108]
2025-11-27T18:20:40.062922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471817871551475:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:40.062567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:40.062234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471816868404085:2108] [node 5] ICS09 handshake done sender: [3:7577471816868404058:2115] self: [3:7577471816868404054:0] peer: [5:7577471816766332206:0] socket: 622 qp: -1
2025-11-27T18:20:40.063242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.062734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471812471364876:2109] [node 6] ICP22 created new session: [5:7577471816766332209:2109]
2025-11-27T18:20:40.062271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471816868404085:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:40.063081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471816766332207:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22300
2025-11-27T18:20:40.063121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471816766332209:2109] [node 6] ICS09 handshake done sender: [5:7577471816766332182:2114] self: [5:7577471816766332176:0] peer: [6:7577471816571575078:0] socket: 666 qp: -1
2025-11-27T18:20:40.062301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:40.063634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471816571575079:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46679
2025-11-27T18:20:40.063149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471816766332209:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:40.064435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471817407778111:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54909
2025-11-27T18:20:40.063264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471812573436752:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:40.063322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:40.065945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471813112810714:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18906) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471817407778105:2120] held: no
2025-11-27T18:20:40.066324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471813576584077:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.066380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:40.066541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471813576584077:2048] [node 1] ICP22 created new session: [2:7577471817871551477:2048]
2025-11-27T18:20:40.067357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471817871551477:2048] [node 1] ICS09 handshake done sender: [2:7577471817871551466:2118] self: [2:7577471817871551471:0] peer: [1:7577471816778463209:0] socket: 624 qp: -1
2025-11-27T18:20:40.067836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471817871551477:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:40.067874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:40.067949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471816766332198:2122] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:40.069622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471812276607742:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7255) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471816571575077:2123] held: no
2025-11-27T18:20:40.069995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471817407778105:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:40.070334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471813112810714:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.070353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7577471812573436751:2108] [node 5] ICP18 handshake (actor [3:7577471816868404079:2122]) is too old
2025-11-27T18:20:40.070641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:40.070533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471812471364814:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.070693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471813112810714:2049] [node 2] ICP22 created new session: [4:7577471817407778112:2049]
2025-11-27T18:20:40.070827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471817407778112:2049] [node 2] ICS09 handshake done sender: [4:7577471817407778105:2120] self: [4:7577471817407778108:0] peer: [2:7577471817871551445:0] socket: 667 qp: -1
2025-11-27T18:20:40.070862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471817407778112:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:40.071050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:40.071131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:40.071238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471816571575077:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:40.071275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471812471364814:2050] [node 3] ICP22 created new session: [5:7577471816766332211:2050]
2025-11-27T18:20:40.071422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471816766332211:2050] [node 3] ICS09 handshake done sender: [5:7577471816766332198:2122] self: [5:7577471816766332206:0] peer: [3:7577471816868404054:0] socket: 665 qp: -1
2025-11-27T18:20:40.073001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471816766332211:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:40.072983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471812276607742:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.073052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:40.073032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:40.073081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471812276607742:2106] [node 5] ICP22 created new session: [6:7577471816571575080:2106]
2025-11-27T18:20:40.073196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471816571575080:2106] [node 5] ICS09 handshake done sender: [6:7577471816571575077:2123] self: [6:7577471816571575078:0] peer: [5:7577471816766332176:0] socket: 608 qp: -1
2025-11-27T18:20:40.073227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471816571575080:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:40.073224Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471813576584077:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:40.073289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:40.133187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:40.133434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471817871551477:2048] [node 1] ICS07 socket disconnect 624 reason# EndOfStream
2025-11-27T18:20:40.133470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471817871551477:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.133561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:40.133629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471817871551477:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:40.134556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:40.134701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471816868404081:2048] [node 1] ICS07 socket disconnect 630 reason# EndOfStream
2025-11-27T18:20:40.134724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471816868404081:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.134774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:40.134810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471816868404081:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:40.135970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471813576584077:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:40.136031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471817871551477:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:40.137037Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471812573436686:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:40.137167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471813576584077:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.137064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471816868404081:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:40.138773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471817871551477:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.138823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471813576584077:2048] [node 1] ICP30 unregister session Session# [2:7577471817871551477:2048] VirtualId# [2:7577471817871551471:0]
2025-11-27T18:20:40.138915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:40.138952Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471813576584077:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.138965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.138997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:40.139142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471817407778109:2048] [node 1] ICS07 socket disconnect 614 reason# EndOfStream
2025-11-27T18:20:40.139214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471812573436686:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.139173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471817407778109:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.139228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:40.139262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471817407778109:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:40.139253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471816868404081:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.139273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471812573436686:2048] [node 1] ICP30 unregister session Session# [3:7577471816868404081:2048] VirtualId# [3:7577471816868404065:0]
2025-11-27T18:20:40.139429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:40.139453Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471812573436686:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.139461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.139954Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471813112810713:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.140022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471817407778109:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.140062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471813112810713:2048] [node 1] ICP30 unregister session Session# [4:7577471817407778109:2048] VirtualId# [4:7577471817407778106:0]
2025-11-27T18:20:40.140159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:40.140192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471813112810713:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.140206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.140242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:40.140458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471816766332197:2048] [node 1] ICS01 socket: 627 reason# EndOfStream
2025-11-27T18:20:40.140507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471812471364812:2048] [node 1] ICP30 unregister session Session# [5:7577471816766332197:2048] VirtualId# [5:7577471816766332190:0]
2025-11-27T18:20:40.140534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471816766332197:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.140600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:40.140645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:40.141357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:40.141601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471816571575068:2048] [node 1] ICS07 socket disconnect 631 reason# EndOfStream
2025-11-27T18:20:40.141639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471816571575068:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.141721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:40.141777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471816571575068:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:40.142438Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471812276607681:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.142503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471816571575068:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.142555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471812276607681:2048] [node 1] ICP30 unregister session Session# [6:7577471816571575068:2048] VirtualId# [6:7577471816571575060:0]
2025-11-27T18:20:40.142617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:40.142647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471812276607681:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.142661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4703) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.149270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471812573436686:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:40.150296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471813112810713:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:40.152332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471812276607681:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:40.212689Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:40.212868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471816868404080:2049] [node 2] ICS01 socket: 633 reason# EndOfStream
2025-11-27T18:20:40.212905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471812573436687:2049] [node 2] ICP30 unregister session Session# [3:7577471816868404080:2049] VirtualId# [3:7577471816868404066:0]
2025-11-27T18:20:40.212930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471816868404080:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.212988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:40.213025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:40.215224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:40.215408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471817407778112:2049] [node 2] ICS01 socket: 667 reason# EndOfStream
2025-11-27T18:20:40.215443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471813112810714:2049] [node 2] ICP30 unregister session Session# [4:7577471817407778112:2049] VirtualId# [4:7577471817407778108:0]
2025-11-27T18:20:40.215465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471817407778112:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.215518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:40.215550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:40.216315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:40.216502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471816766332204:2049] [node 2] ICS01 socket: 623 reason# EndOfStream
2025-11-27T18:20:40.216555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471812471364813:2049] [node 2] ICP30 unregister session Session# [5:7577471816766332204:2049] VirtualId# [5:7577471816766332191:0]
2025-11-27T18:20:40.216587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471816766332204:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.216661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:40.216702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:40.217392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:40.217529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471816571575071:2049] [node 2] ICS07 socket disconnect 632 reason# EndOfStream
2025-11-27T18:20:40.217553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471816571575071:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.217604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:40.217634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471816571575071:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:40.218201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471812276607682:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18906) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.218287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471816571575071:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.218320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471812276607682:2049] [node 2] ICP30 unregister session Session# [6:7577471816571575071:2049] VirtualId# [6:7577471816571575062:0]
2025-11-27T18:20:40.218352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:40.218371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471812276607682:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18906) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.218394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18906) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.228140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471812276607682:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:40.263937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:40.264093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471817407778103:2050] [node 3] ICS01 socket: 617 reason# EndOfStream
2025-11-27T18:20:40.264141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471813112810715:2050] [node 3] ICP30 unregister session Session# [4:7577471817407778103:2050] VirtualId# [4:7577471817407778092:0]
2025-11-27T18:20:40.264174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471817407778103:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.264239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:40.264277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:40.265019Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:40.265206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471816766332211:2050] [node 3] ICS07 socket disconnect 665 reason# EndOfStream
2025-11-27T18:20:40.265246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471816766332211:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.265336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:40.265379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471816766332211:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:40.266034Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:40.266106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471812471364814:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15234) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.266191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471816571575072:2050] [node 3] ICS01 socket: 611 reason# EndOfStream
2025-11-27T18:20:40.266219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471816766332211:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.266221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471812276607683:2050] [node 3] ICP30 unregister session Session# [6:7577471816571575072:2050] VirtualId# [6:7577471816571575061:0]
2025-11-27T18:20:40.266256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471812471364814:2050] [node 3] ICP30 unregister session Session# [5:7577471816766332211:2050] VirtualId# [5:7577471816766332206:0]
2025-11-27T18:20:40.266245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471816571575072:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.266295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:40.266333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:40.266352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:40.266397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471812471364814:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15234) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.266417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15234) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.276278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471812471364814:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:40.351639Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:40.351896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471816766332202:2108] [node 4] ICS01 socket: 603 reason# EndOfStream
2025-11-27T18:20:40.351944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471812471364875:2108] [node 4] ICP30 unregister session Session# [5:7577471816766332202:2108] VirtualId# [5:7577471816766332192:0]
2025-11-27T18:20:40.351983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471816766332202:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.352059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:40.352130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:40.352765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:40.352905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471816571575073:2105] [node 4] ICS01 socket: 606 reason# EndOfStream
2025-11-27T18:20:40.352932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471812276607741:2105] [node 4] ICP30 unregister session Session# [6:7577471816571575073:2105] VirtualId# [6:7577471816571575059:0]
2025-11-27T18:20:40.352961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471816571575073:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.353014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:40.353045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:40.415649Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:40.415817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471816571575080:2106] [node 5] ICS01 socket: 608 reason# EndOfStream
2025-11-27T18:20:40.415847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471812276607742:2106] [node 5] ICP30 unregister session Session# [6:7577471816571575080:2106] VirtualId# [6:7577471816571575078:0]
2025-11-27T18:20:40.415873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471816571575080:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.415933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:40.415965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:40.473383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471815231088719:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:40.473427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471815231088720:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:40.488762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471815831741260:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:40.488799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471815831741261:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:40.505224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471815711104798:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:40.505280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471815711104799:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:40.521677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471816418253552:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:40.521722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471816418253553:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:40.521765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471816418253554:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:40.537499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471817088786455:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:40.537540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471817088786456:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:40.537567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471817088786457:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:40.553434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471815230661368:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:40.553472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471815230661369:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:40.553499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471815230661370:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:40.630862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471815230661430:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:40.634003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471815231088782:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:40.633987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471815831741323:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:40.634390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471815711104861:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:40.634618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471815831741324:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:40.634712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471815231088783:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:40.634972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471815711104862:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:40.635487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471815231088784:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:40.635463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471817088786518:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:40.635521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471815831741325:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:40.635503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471816418253615:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:40.635773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471815711104863:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:40.635775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471815230661433:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:40.636020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471817088786519:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:40.636192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471816418253616:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.643182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471815231088798:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19704
2025-11-27T18:20:40.643198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471815831741339:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.645337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471816418253627:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-27T18:20:40.645410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471816418253628:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.646721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471817088786530:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1653
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.651259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471815831741340:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.651421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471815231088799:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19704
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.651508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471815831741339:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:40.651563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471815831741341:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
::ffff:127.1.0.0
2025-11-27T18:20:40.651564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471815231088798:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.651540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471815711104874:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34831
2025-11-27T18:20:40.651810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471815231088800:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19704
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.651801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471815831741342:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
No RDMA context found for GID: 2025-11-27T18:20:40.651910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471815711104875:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34831
::ffff:127.1.0.0
2025-11-27T18:20:40.652270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471815831741260:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.652498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471815231088801:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19704
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.652542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471815711104876:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34831
2025-11-27T18:20:40.652587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471815711104877:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34831
2025-11-27T18:20:40.652752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471815231088786:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.652773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471815231088786:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.652927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471815711104875:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.653010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471815711104874:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.653025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471815831741344:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-27T18:20:40.653101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471815831741340:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.653215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471815711104799:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.653279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471815711104798:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.653299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471815231088799:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.653640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471815831741329:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:40.653664Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471815831741329:2111] [node 3] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-11-27T18:20:40.654338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471815231088803:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19704
2025-11-27T18:20:40.654443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471815231088786:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.654553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471815831741346:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57906
2025-11-27T18:20:40.654576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471817088786530:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.654876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471815231088788:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.654900Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471815231088788:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.655692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471815231088801:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.655824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471815831741342:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.655884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471815231088719:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.655932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:40.655864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471816418253632:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-27T18:20:40.656012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471816418253627:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.656013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471815231088719:2048] [node 2] ICP22 created new session: [1:7577471815231088806:2048]
2025-11-27T18:20:40.656028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471815831741329:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:40.656050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471815231088806:2048] [node 2] ICS09 handshake done sender: [1:7577471815231088786:2110] self: [1:7577471815231088785:0] peer: [2:7577471815831741343:0] socket: 607 qp: -1
2025-11-27T18:20:40.656078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471815231088806:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:40.656215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471816418253628:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.656101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:40.656548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471815831741341:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.656546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471816418253552:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.656612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471815831741261:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.656594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471816418253553:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.656652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:40.656685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471815831741261:2049] [node 3] ICP22 created new session: [2:7577471815831741348:2049]
2025-11-27T18:20:40.656716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471815831741348:2049] [node 3] ICS09 handshake done sender: [2:7577471815831741329:2111] self: [2:7577471815831741328:0] peer: [3:7577471815711104881:0] socket: 609 qp: -1
2025-11-27T18:20:40.656740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471815831741348:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:40.656758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:40.656985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471815231088800:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.657676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471815231088790:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.657694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471815231088790:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.658007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471815831741339:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:40.658415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471815831741331:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.658433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471815831741331:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.659177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471815231088788:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.659400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471817088786535:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1653
2025-11-27T18:20:40.659823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471816418253635:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-27T18:20:40.659874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471815231088720:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.659948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:40.659981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471815231088720:2049] [node 3] ICP22 created new session: [1:7577471815231088813:2049]
2025-11-27T18:20:40.660020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471815231088813:2049] [node 3] ICS09 handshake done sender: [1:7577471815231088788:2111] self: [1:7577471815231088787:0] peer: [3:7577471815711104882:0] socket: 608 qp: -1
2025-11-27T18:20:40.660042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471815231088813:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:40.660063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:40.660060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471815831741260:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.660180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:40.660212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471815831741260:2048] [node 1] ICP22 created new session: [2:7577471815831741350:2048]
2025-11-27T18:20:40.660326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471815831741350:2048] [node 1] ICS09 handshake done sender: [2:7577471815831741339:2115] self: [2:7577471815831741343:0] peer: [1:7577471815231088785:0] socket: 613 qp: -1
2025-11-27T18:20:40.660345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471815831741350:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:40.660361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:40.660599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471815711104877:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.660676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471815711104876:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.660865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471815831741260:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.661074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471816418253632:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.661113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471815711104883:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34831
2025-11-27T18:20:40.661198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471816418253554:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.661507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471815231088790:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:40.661752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471815711104869:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.661775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471815711104869:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.661987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471816418253627:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.662345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471816418253552:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.662402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471815711104884:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34831
2025-11-27T18:20:40.662426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:40.662475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471816418253552:2048] [node 1] ICP22 created new session: [4:7577471816418253637:2048]
2025-11-27T18:20:40.662545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471816418253637:2048] [node 1] ICS09 handshake done sender: [4:7577471816418253627:2115] self: [4:7577471816418253633:0] peer: [1:7577471815231088789:0] socket: 618 qp: -1
2025-11-27T18:20:40.662591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471816418253637:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:40.662628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:40.662752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471815711104875:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.663150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471815831741331:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:40.663180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471817088786536:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1653
2025-11-27T18:20:40.663211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471815711104799:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.663242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:40.663234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471815231088803:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.663266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471815711104799:2049] [node 2] ICP22 created new session: [3:7577471815711104888:2049]
2025-11-27T18:20:40.663353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471815711104888:2049] [node 2] ICS09 handshake done sender: [3:7577471815711104875:2116] self: [3:7577471815711104881:0] peer: [2:7577471815831741328:0] socket: 634 qp: -1
2025-11-27T18:20:40.663382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471815711104888:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:40.663386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471815231088782:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.663401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:40.663454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:40.663503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471815231088782:2107] [node 4] ICP22 created new session: [1:7577471815231088815:2107]
2025-11-27T18:20:40.663553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471815231088815:2107] [node 4] ICS09 handshake done sender: [1:7577471815231088790:2112] self: [1:7577471815231088789:0] peer: [4:7577471816418253633:0] socket: 610 qp: -1
2025-11-27T18:20:40.663589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471815231088815:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:40.663635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471815831741344:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.663619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:40.663684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471815831741323:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.663710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:40.663748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471815831741323:2107] [node 4] ICP22 created new session: [2:7577471815831741357:2107]
2025-11-27T18:20:40.663783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471815831741357:2107] [node 4] ICS09 handshake done sender: [2:7577471815831741331:2112] self: [2:7577471815831741330:0] peer: [4:7577471816418253634:0] socket: 611 qp: -1
2025-11-27T18:20:40.663810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471815831741357:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.663831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:40.663868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471815230661447:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-27T18:20:40.664768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471815711104874:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:40.666384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471815711104798:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.666418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:40.666444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471815711104798:2048] [node 1] ICP22 created new session: [3:7577471815711104890:2048]
2025-11-27T18:20:40.666474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471815711104890:2048] [node 1] ICS09 handshake done sender: [3:7577471815711104874:2115] self: [3:7577471815711104882:0] peer: [1:7577471815231088787:0] socket: 633 qp: -1
2025-11-27T18:20:40.666499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471815711104890:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:40.666517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:40.668144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471816418253553:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13026) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471816418253628:2116] held: no
2025-11-27T18:20:40.668209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471815831741261:2049] [node 3] ICP18 handshake (actor [2:7577471815831741344:2119]) is too old
2025-11-27T18:20:40.668292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471817088786536:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.668395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471817088786456:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.668782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471816418253639:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-27T18:20:40.668832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471815230661448:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-27T18:20:40.668846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471815711104869:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.669680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471816418253640:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-27T18:20:40.670290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471815231088720:2049] [node 3] ICP18 handshake (actor [1:7577471815231088803:2119]) is too old
2025-11-27T18:20:40.670639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471815711104861:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.670646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471815831741333:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.670679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:40.670709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471815711104861:2107] [node 4] ICP22 created new session: [3:7577471815711104892:2107]
2025-11-27T18:20:40.670669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471815831741333:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.670736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471815711104892:2107] [node 4] ICS09 handshake done sender: [3:7577471815711104869:2112] self: [3:7577471815711104868:0] peer: [4:7577471816418253636:0] socket: 643 qp: -1
2025-11-27T18:20:40.670765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471815711104892:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:40.670788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:40.670935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471817088786535:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.671049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471817088786538:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1653
2025-11-27T18:20:40.671137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471817088786455:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.671217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471816418253628:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:40.671294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471815230661447:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.671505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471816418253632:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:40.671406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471815230661369:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.671462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471815831741333:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.671898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471815231088792:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.671922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471817088786540:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1653
2025-11-27T18:20:40.671990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471815831741335:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.671938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471815231088792:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.671991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471816418253553:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.672013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471815831741335:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.672036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:40.672593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471816418253553:2049] [node 2] ICP22 created new session: [4:7577471816418253643:2049]
2025-11-27T18:20:40.672652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471816418253643:2049] [node 2] ICS09 handshake done sender: [4:7577471816418253628:2116] self: [4:7577471816418253634:0] peer: [2:7577471815831741330:0] socket: 620 qp: -1
2025-11-27T18:20:40.672681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471816418253643:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: 2025-11-27T18:20:40.672721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
::ffff:127.1.0.0
2025-11-27T18:20:40.672794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471815230661450:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-27T18:20:40.673073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471815831741324:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.673122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:40.673161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471815831741324:2108] [node 5] ICP22 created new session: [2:7577471815831741360:2108]
2025-11-27T18:20:40.673189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471815831741360:2108] [node 5] ICS09 handshake done sender: [2:7577471815831741333:2113] self: [2:7577471815831741332:0] peer: [5:7577471817088786537:0] socket: 657 qp: -1
2025-11-27T18:20:40.673151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471817088786536:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:40.673209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471815831741360:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:40.673231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:40.673418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471817088786456:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.673488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:40.673559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471817088786456:2049] [node 2] ICP22 created new session: [5:7577471817088786541:2049]
2025-11-27T18:20:40.673728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471817088786541:2049] [node 2] ICS09 handshake done sender: [5:7577471817088786536:2117] self: [5:7577471817088786537:0] peer: [2:7577471815831741332:0] socket: 658 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.673814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471817088786541:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:40.674016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:40.674068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471815230661451:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-27T18:20:40.674303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471816418253554:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.674391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:40.674400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471815230661448:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.674446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471816418253554:2050] [node 3] ICP22 created new session: [4:7577471816418253645:2050]
2025-11-27T18:20:40.674484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471815231088792:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:40.674577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471815230661368:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.674611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471816418253645:2050] [node 3] ICS09 handshake done sender: [4:7577471816418253632:2117] self: [4:7577471816418253636:0] peer: [3:7577471815711104868:0] socket: 650 qp: -1
2025-11-27T18:20:40.674718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471816418253645:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:40.674901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:40.675169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471815231088794:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.675201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471815231088794:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.675301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471815831741335:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.675774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471815230661453:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-27T18:20:40.676007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471815231088783:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.675987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471815831741325:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.676245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:40.676205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:40.676238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471815831741325:2109] [node 6] ICP22 created new session: [2:7577471815831741363:2109]
2025-11-27T18:20:40.676284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471815231088783:2108] [node 5] ICP22 created new session: [1:7577471815231088819:2108]
2025-11-27T18:20:40.676282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471815831741363:2109] [node 6] ICS09 handshake done sender: [2:7577471815831741335:2114] self: [2:7577471815831741334:0] peer: [6:7577471815230661449:0] socket: 659 qp: -1
2025-11-27T18:20:40.676303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471815831741363:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:40.676321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:40.676351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471815231088819:2108] [node 5] ICS09 handshake done sender: [1:7577471815231088792:2113] self: [1:7577471815231088791:0] peer: [5:7577471817088786539:0] socket: 653 qp: -1
2025-11-27T18:20:40.676384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471815231088819:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:40.676411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:40.676759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471815230661447:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:40.677129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471815230661451:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.677162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471815230661369:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.677188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:40.677217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471815230661369:2049] [node 2] ICP22 created new session: [6:7577471815230661454:2049]
2025-11-27T18:20:40.677311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471815230661454:2049] [node 2] ICS09 handshake done sender: [6:7577471815230661447:2116] self: [6:7577471815230661449:0] peer: [2:7577471815831741334:0] socket: 660 qp: -1
2025-11-27T18:20:40.677329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471815230661454:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:40.677348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:40.679582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471815231088794:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:40.679862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471815231088784:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.679921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:40.679973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471815231088784:2109] [node 6] ICP22 created new session: [1:7577471815231088822:2109]
2025-11-27T18:20:40.680022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471815231088822:2109] [node 6] ICS09 handshake done sender: [1:7577471815231088794:2114] self: [1:7577471815231088793:0] peer: [6:7577471815230661452:0] socket: 625 qp: -1
2025-11-27T18:20:40.680065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471815231088822:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:40.680096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:40.681922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471816418253554:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:40.681975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471817088786455:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471817088786535:2116] held: no
2025-11-27T18:20:40.682369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471815230661370:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.682665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471815230661456:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-27T18:20:40.682668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471815711104873:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.682694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471815711104873:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.682923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471817088786543:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1653
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.683556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471817088786544:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1653
2025-11-27T18:20:40.683743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471815711104873:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:40.684571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471815711104863:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.684604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:40.684642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471815711104863:2109] [node 6] ICP22 created new session: [3:7577471815711104897:2109]
2025-11-27T18:20:40.684668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471815711104897:2109] [node 6] ICS09 handshake done sender: [3:7577471815711104873:2114] self: [3:7577471815711104872:0] peer: [6:7577471815230661457:0] socket: 664 qp: -1
2025-11-27T18:20:40.684693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471815711104897:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:40.684715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:40.684763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471815230661368:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471815230661448:2117] held: no
2025-11-27T18:20:40.684903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471817088786543:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.685026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471817088786538:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.685395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471817088786535:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:40.685471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471815230661448:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.685950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471816418253647:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-27T18:20:40.686028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471815711104899:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34831
2025-11-27T18:20:40.686060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471817088786455:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.686120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:40.686170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471817088786455:2048] [node 1] ICP22 created new session: [5:7577471817088786547:2048]
2025-11-27T18:20:40.686221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471817088786547:2048] [node 1] ICS09 handshake done sender: [5:7577471817088786535:2116] self: [5:7577471817088786539:0] peer: [1:7577471815231088791:0] socket: 654 qp: -1
2025-11-27T18:20:40.686256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471817088786547:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:40.686312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:40.686448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471815230661450:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.686494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471815230661368:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.686524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:40.686566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471815230661368:2048] [node 1] ICP22 created new session: [6:7577471815230661461:2048]
2025-11-27T18:20:40.686620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471815230661461:2048] [node 1] ICS09 handshake done sender: [6:7577471815230661448:2117] self: [6:7577471815230661452:0] peer: [1:7577471815231088793:0] socket: 637 qp: -1
2025-11-27T18:20:40.686653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471815230661461:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:40.686683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:40.687219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471817088786457:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.687292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471817088786518:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.687649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471815230661459:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-27T18:20:40.687801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471815711104871:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.687823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471815711104871:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.688455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471816418253624:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.688485Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471816418253624:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.688522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471815230661460:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-27T18:20:40.688655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471815711104871:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:40.689837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471815711104862:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.689877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:40.689904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471815711104862:2108] [node 5] ICP22 created new session: [3:7577471815711104901:2108]
2025-11-27T18:20:40.689984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471815711104901:2108] [node 5] ICS09 handshake done sender: [3:7577471815711104871:2113] self: [3:7577471815711104870:0] peer: [5:7577471817088786549:0] socket: 635 qp: -1
2025-11-27T18:20:40.690007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471815711104901:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:40.690029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:40.690065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471815230661433:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.690144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471815230661451:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.690400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471816418253648:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49980
2025-11-27T18:20:40.690477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471815230661370:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.690492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471816418253624:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:40.690504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:40.690532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471815230661370:2050] [node 3] ICP22 created new session: [6:7577471815230661465:2050]
2025-11-27T18:20:40.690562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471815230661465:2050] [node 3] ICS09 handshake done sender: [6:7577471815230661451:2119] self: [6:7577471815230661457:0] peer: [3:7577471815711104872:0] socket: 665 qp: -1
2025-11-27T18:20:40.690589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471815230661465:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:40.690606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:40.690794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471816418253626:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.690812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471816418253626:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.691614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471816418253615:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.691645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:40.691677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471816418253615:2108] [node 5] ICP22 created new session: [4:7577471816418253650:2108]
2025-11-27T18:20:40.691735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471816418253650:2108] [node 5] ICS09 handshake done sender: [4:7577471816418253624:2113] self: [4:7577471816418253623:0] peer: [5:7577471817088786550:0] socket: 639 qp: -1
2025-11-27T18:20:40.691766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471816418253650:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:40.691787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:40.692733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471816418253647:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.693007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471816418253626:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:40.693465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471817088786457:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471817088786543:2120] held: no
2025-11-27T18:20:40.693536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471816418253615:2108] [node 5] ICP18 handshake (actor [4:7577471816418253647:2121]) is too old
2025-11-27T18:20:40.693574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471816418253616:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.693595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:40.693621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471816418253616:2109] [node 6] ICP22 created new session: [4:7577471816418253653:2109]
2025-11-27T18:20:40.693654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471816418253653:2109] [node 6] ICS09 handshake done sender: [4:7577471816418253626:2114] self: [4:7577471816418253625:0] peer: [6:7577471815230661464:0] socket: 645 qp: -1
2025-11-27T18:20:40.693678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471816418253653:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:40.693701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:40.693914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471815711104863:2109] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.694525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471817088786551:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1653
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.694980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471817088786552:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1653
2025-11-27T18:20:40.695107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471815230661460:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.695294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471815230661467:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-27T18:20:40.695283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471816418253648:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.695347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471816418253616:2109] [node 6] ICP18 handshake (actor [4:7577471816418253648:2122]) is too old
2025-11-27T18:20:40.696305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471817088786518:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471817088786538:2118] held: no
2025-11-27T18:20:40.696721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471815230661430:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:40.696768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471815230661450:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:40.696980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471817088786543:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:40.696994Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471815230661433:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471815230661450:2118] held: no
2025-11-27T18:20:40.697246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471817088786538:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:40.697474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471815230661433:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.697501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:40.697528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471815230661433:2107] [node 4] ICP22 created new session: [6:7577471815230661469:2107]
2025-11-27T18:20:40.697562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471815230661469:2107] [node 4] ICS09 handshake done sender: [6:7577471815230661450:2118] self: [6:7577471815230661464:0] peer: [4:7577471816418253625:0] socket: 663 qp: -1
2025-11-27T18:20:40.697584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471815230661469:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:40.697606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:40.697686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471817088786457:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.697727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:40.697769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471817088786457:2050] [node 3] ICP22 created new session: [5:7577471817088786553:2050]
2025-11-27T18:20:40.697808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471817088786553:2050] [node 3] ICS09 handshake done sender: [5:7577471817088786543:2120] self: [5:7577471817088786549:0] peer: [3:7577471815711104870:0] socket: 668 qp: -1
2025-11-27T18:20:40.697857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471817088786553:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:40.697891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:40.698653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471817088786518:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.698694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:40.698735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471817088786518:2108] [node 4] ICP22 created new session: [5:7577471817088786555:2108]
2025-11-27T18:20:40.698775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471817088786555:2108] [node 4] ICS09 handshake done sender: [5:7577471817088786538:2118] self: [5:7577471817088786550:0] peer: [4:7577471816418253623:0] socket: 641 qp: -1
2025-11-27T18:20:40.698799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471817088786555:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:40.698833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:40.700012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471817088786529:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:40.700040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471817088786529:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:40.701463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471817088786529:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:40.701579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471815230661471:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-27T18:20:40.701721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471817088786519:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:40.701767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:40.701813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471817088786519:2109] [node 6] ICP22 created new session: [5:7577471817088786558:2109]
2025-11-27T18:20:40.701882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471817088786558:2109] [node 6] ICS09 handshake done sender: [5:7577471817088786529:2114] self: [5:7577471817088786528:0] peer: [6:7577471815230661468:0] socket: 630 qp: -1
2025-11-27T18:20:40.701918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471817088786558:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:40.701947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:40.701917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471815230661460:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:40.702053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471815230661430:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:40.702096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:40.702120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471815230661430:2106] [node 5] ICP22 created new session: [6:7577471815230661472:2106]
2025-11-27T18:20:40.702147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471815230661472:2106] [node 5] ICS09 handshake done sender: [6:7577471815230661460:2123] self: [6:7577471815230661468:0] peer: [5:7577471817088786528:0] socket: 670 qp: -1
2025-11-27T18:20:40.702175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471815230661472:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:40.702196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:40.719958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:40.720113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471815831741350:2048] [node 1] ICS07 socket disconnect 613 reason# EndOfStream
2025-11-27T18:20:40.720150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471815831741350:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.720217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:40.720255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471815831741350:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:40.721042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:40.721171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471815711104890:2048] [node 1] ICS07 socket disconnect 633 reason# EndOfStream
2025-11-27T18:20:40.721192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471815711104890:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.721247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:40.721312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471815711104890:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:40.723038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471815831741260:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:40.723068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471815831741350:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:40.724140Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:40.724357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471816418253637:2048] [node 1] ICS07 socket disconnect 618 reason# EndOfStream
2025-11-27T18:20:40.724395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471815831741260:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.724391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471816418253637:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.724436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471815831741350:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.724464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471815831741260:2048] [node 1] ICP30 unregister session Session# [2:7577471815831741350:2048] VirtualId# [2:7577471815831741343:0]
2025-11-27T18:20:40.724462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:40.724464Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471815711104798:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:40.724494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471816418253637:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:40.724494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:40.724496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471815711104890:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:40.724516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471815831741260:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.724525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.725302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471815711104798:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.725323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471816418253552:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.725359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471815711104890:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.725368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471816418253637:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.725394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471816418253552:2048] [node 1] ICP30 unregister session Session# [4:7577471816418253637:2048] VirtualId# [4:7577471816418253633:0]
2025-11-27T18:20:40.725429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:40.725454Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471816418253552:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.725463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.725413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471815711104798:2048] [node 1] ICP30 unregister session Session# [3:7577471815711104890:2048] VirtualId# [3:7577471815711104882:0]
2025-11-27T18:20:40.725461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:40.725496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471815711104798:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.725510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.725790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:40.726012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471817088786547:2048] [node 1] ICS01 socket: 654 reason# EndOfStream
2025-11-27T18:20:40.726053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471817088786455:2048] [node 1] ICP30 unregister session Session# [5:7577471817088786547:2048] VirtualId# [5:7577471817088786539:0]
2025-11-27T18:20:40.726089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471817088786547:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.726153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:40.726197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:40.726928Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:40.727122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471815230661461:2048] [node 1] ICS01 socket: 637 reason# EndOfStream
2025-11-27T18:20:40.727164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471815230661368:2048] [node 1] ICP30 unregister session Session# [6:7577471815230661461:2048] VirtualId# [6:7577471815230661452:0]
2025-11-27T18:20:40.727205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471815230661461:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.727270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:40.727319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:40.733994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471815831741260:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:40.735024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471816418253552:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:40.736040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471815711104798:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:40.771017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:40.771182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471815711104888:2049] [node 2] ICS07 socket disconnect 634 reason# EndOfStream
2025-11-27T18:20:40.771209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471815711104888:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.771283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:40.771329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471815711104888:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:40.774809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:40.774940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471815711104799:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13026) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:40.774974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471815711104888:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:40.775052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471816418253643:2049] [node 2] ICS01 socket: 620 reason# EndOfStream
2025-11-27T18:20:40.775122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471816418253553:2049] [node 2] ICP30 unregister session Session# [4:7577471816418253643:2049] VirtualId# [4:7577471816418253634:0]
2025-11-27T18:20:40.775162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471816418253643:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.775254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:40.775306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:40.775585Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471815711104799:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13026) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.775624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471815711104888:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.775653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471815711104799:2049] [node 2] ICP30 unregister session Session# [3:7577471815711104888:2049] VirtualId# [3:7577471815711104881:0]
2025-11-27T18:20:40.775690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:40.775715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471815711104799:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13026) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.775724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13026) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.776154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:40.776373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471817088786541:2049] [node 2] ICS07 socket disconnect 658 reason# EndOfStream
2025-11-27T18:20:40.776421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471817088786541:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.776503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:40.776549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471817088786541:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:40.777323Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:40.777374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471817088786456:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13026) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.777426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471817088786541:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.777463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471817088786456:2049] [node 2] ICP30 unregister session Session# [5:7577471817088786541:2049] VirtualId# [5:7577471817088786537:0]
2025-11-27T18:20:40.777536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471815230661454:2049] [node 2] ICS01 socket: 660 reason# EndOfStream
2025-11-27T18:20:40.777577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:40.777586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471815230661369:2049] [node 2] ICP30 unregister session Session# [6:7577471815230661454:2049] VirtualId# [6:7577471815230661449:0]
2025-11-27T18:20:40.777623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471815230661454:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.777613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471817088786456:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13026) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.777629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13026) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.777690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:40.777737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:40.787240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471817088786456:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:40.800839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:40.801018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471816418253645:2050] [node 3] ICS07 socket disconnect 650 reason# EndOfStream
2025-11-27T18:20:40.801045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471816418253645:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.801113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:40.801147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471816418253645:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:40.801704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:40.801902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471817088786553:2050] [node 3] ICS01 socket: 668 reason# EndOfStream
2025-11-27T18:20:40.801923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471816418253554:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3976) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.801937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471817088786457:2050] [node 3] ICP30 unregister session Session# [5:7577471817088786553:2050] VirtualId# [5:7577471817088786549:0]
2025-11-27T18:20:40.801960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471816418253645:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.801960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471817088786553:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.801985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471816418253554:2050] [node 3] ICP30 unregister session Session# [4:7577471816418253645:2050] VirtualId# [4:7577471816418253636:0]
2025-11-27T18:20:40.802024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:40.802076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:40.802085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:40.802120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471816418253554:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3976) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.802132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3976) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.802526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:40.802737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471815230661465:2050] [node 3] ICS07 socket disconnect 665 reason# EndOfStream
2025-11-27T18:20:40.802771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471815230661465:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.802863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:40.802915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471815230661465:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:40.803567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471815230661370:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3976) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.803621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471815230661465:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.803654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471815230661370:2050] [node 3] ICP30 unregister session Session# [6:7577471815230661465:2050] VirtualId# [6:7577471815230661457:0]
2025-11-27T18:20:40.803743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:40.803781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471815230661370:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3976) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.803798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3976) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.811801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471816418253554:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:40.813859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471815230661370:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:40.834290Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:40.834450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471817088786555:2108] [node 4] ICS01 socket: 641 reason# EndOfStream
2025-11-27T18:20:40.834486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471817088786518:2108] [node 4] ICP30 unregister session Session# [5:7577471817088786555:2108] VirtualId# [5:7577471817088786550:0]
2025-11-27T18:20:40.834511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471817088786555:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.834565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:40.834602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:40.835141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:40.835339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471815230661469:2107] [node 4] ICS01 socket: 663 reason# EndOfStream
2025-11-27T18:20:40.835378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471815230661433:2107] [node 4] ICP30 unregister session Session# [6:7577471815230661469:2107] VirtualId# [6:7577471815230661464:0]
2025-11-27T18:20:40.835415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471815230661469:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.835512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:40.835561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:40.857428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:40.857665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471815230661472:2106] [node 5] ICS07 socket disconnect 670 reason# EndOfStream
2025-11-27T18:20:40.857702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471815230661472:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:40.857781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:40.857828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471815230661472:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:40.858644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471815230661430:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29958) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:40.858701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471815230661472:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:40.858749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471815230661430:2106] [node 5] ICP30 unregister session Session# [6:7577471815230661472:2106] VirtualId# [6:7577471815230661468:0]
2025-11-27T18:20:40.858862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:40.858898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471815230661430:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29958) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.858921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29958) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:40.869195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471815230661430:2106] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:40.928190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471815320242525:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:40.928258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471815320242526:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:40.945071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471817271781904:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:40.945082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471817271781905:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:40.963112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471816893711257:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:40.963138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471816893711258:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:40.981444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471816620435883:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:40.981443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471816620435884:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:40.981488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471816620435885:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:40.998929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471816240427586:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:40.998973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471816240427587:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:40.999015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471816240427588:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:41.017914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471820244742914:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:41.017970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471820244742915:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:41.017999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471820244742916:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:41.106893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471820244742975:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:41.107803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471820915403242:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:41.107534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471821188678616:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:41.107846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471820535394945:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:41.107660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471821566749263:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:41.108250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471821188678617:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:41.107884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471820244742978:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:41.108210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471819615209885:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:41.108044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471821566749264:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:41.108227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471819615209884:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:41.108004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471820535394946:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:41.108871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471821566749265:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:41.108359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471821188678618:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:41.108945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471820915403243:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:41.108964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471819615209886:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-27T18:20:41.117747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471819615209898:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55640
2025-11-27T18:20:41.118052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471819615209901:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55640
2025-11-27T18:20:41.118047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471820915403257:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48180
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.118820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471819615209902:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55640
2025-11-27T18:20:41.118788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471821188678632:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22578
2025-11-27T18:20:41.119919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471821188678633:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22578
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.120661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471821566749274:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.120879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471821566749275:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
2025-11-27T18:20:41.121094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471821566749276:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.124003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471820535394955:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-27T18:20:41.124292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471820535394958:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.124445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471821566749282:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.124757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821566749276:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.125939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821566749282:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.125965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471819615209903:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55640
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.126740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821566749274:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.127099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471821188678634:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22578
2025-11-27T18:20:41.126760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471820915403257:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.127181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471816620435883:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.127286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471820915403258:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48180
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.128035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471821188678634:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.128233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471821188678635:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22578
2025-11-27T18:20:41.128093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819615209901:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.128270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819615209902:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.128626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471821188678636:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22578
2025-11-27T18:20:41.128470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819615209893:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.129711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471821188678633:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.129709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821566749275:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.128482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819615209898:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.128498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471819615209893:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.130185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471821188678632:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.130159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471819615209904:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55640
2025-11-27T18:20:41.130311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471821188678635:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.130326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471816893711257:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.130399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471817271781904:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.130418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819615209903:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.130385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471820535394955:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.130483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471816893711258:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.130518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819615209890:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.130541Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471819615209890:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.130554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471819615209893:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:41.130687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821566749268:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.130715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471821566749268:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.130779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471821188678636:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.131001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471820535394958:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.131660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819615209888:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.131673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471820915403260:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48180
2025-11-27T18:20:41.131681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471819615209888:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.132020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819615209904:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.131731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471816240427588:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.132070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471819615209885:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.132047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471816240427587:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.132146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:41.132198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471819615209885:2107] [node 4] ICP22 created new session: [1:7577471819615209910:2107]
2025-11-27T18:20:41.132244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471819615209910:2107] [node 4] ICS09 handshake done sender: [1:7577471819615209893:2112] self: [1:7577471819615209892:0] peer: [4:7577471820915403259:0] socket: 619 qp: -1
2025-11-27T18:20:41.132257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821566749271:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.132308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471821188678625:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.132275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471819615209910:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:41.132298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471820915403258:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.132766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:41.132295Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471821566749271:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.132745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471821566749268:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:41.132342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471821188678625:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.132754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471820915403257:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:41.134295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471821566749271:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:41.134914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471817271781905:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.134135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471816620435883:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.134972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.134181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:41.134882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471816620435883:2048] [node 1] ICP22 created new session: [4:7577471820915403262:2048]
2025-11-27T18:20:41.135053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471820915403262:2048] [node 1] ICS09 handshake done sender: [4:7577471820915403257:2115] self: [4:7577471820915403259:0] peer: [1:7577471819615209892:0] socket: 629 qp: -1
2025-11-27T18:20:41.135033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471817271781905:2049] [node 3] ICP22 created new session: [2:7577471821566749289:2049]
2025-11-27T18:20:41.135084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471821566749289:2049] [node 3] ICS09 handshake done sender: [2:7577471821566749268:2110] self: [2:7577471821566749267:0] peer: [3:7577471821188678640:0] socket: 638 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.135128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471820915403262:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:41.135110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471821188678642:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22578
2025-11-27T18:20:41.135227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471821566749289:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:41.135237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.135266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:41.135623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471820535394964:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-27T18:20:41.136323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471821188678643:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22578
2025-11-27T18:20:41.136563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471821188678625:2112] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:41.136653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471821566749263:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.136702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:41.136807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471819615209890:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:41.136744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471821566749263:2108] [node 5] ICP22 created new session: [2:7577471821566749291:2108]
2025-11-27T18:20:41.137003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471821566749291:2108] [node 5] ICS09 handshake done sender: [2:7577471821566749271:2112] self: [2:7577471821566749270:0] peer: [5:7577471820535394963:0] socket: 642 qp: -1
2025-11-27T18:20:41.137005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471820535394958:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:41.137044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471821566749291:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:41.137021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471820535394965:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-27T18:20:41.137080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:41.138074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471816240427587:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.138176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471815320242526:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.138136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:41.138234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:41.138233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471816240427587:2049] [node 2] ICP22 created new session: [5:7577471820535394966:2049]
2025-11-27T18:20:41.138593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471821188678616:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.138284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471820535394966:2049] [node 2] ICS09 handshake done sender: [5:7577471820535394958:2116] self: [5:7577471820535394963:0] peer: [2:7577471821566749270:0] socket: 644 qp: -1
2025-11-27T18:20:41.138479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471815320242526:2049] [node 3] ICP22 created new session: [1:7577471819615209916:2049]
2025-11-27T18:20:41.138326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471820535394966:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:41.139073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471819615209916:2049] [node 3] ICS09 handshake done sender: [1:7577471819615209890:2111] self: [1:7577471819615209889:0] peer: [3:7577471821188678639:0] socket: 620 qp: -1
2025-11-27T18:20:41.139346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:41.138357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:41.139281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471819615209916:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:41.139315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471820535394955:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:41.139314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:41.139480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471821188678616:2108] [node 5] ICP22 created new session: [3:7577471821188678645:2108]
2025-11-27T18:20:41.139604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471821188678635:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:41.139642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471821188678645:2108] [node 5] ICS09 handshake done sender: [3:7577471821188678625:2112] self: [3:7577471821188678623:0] peer: [5:7577471820535394962:0] socket: 614 qp: -1
2025-11-27T18:20:41.139683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471821188678645:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:41.139719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:41.139935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471816240427588:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.140039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:41.140041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471816893711258:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471821188678635:2118] held: no
2025-11-27T18:20:41.140045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471819615209888:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:41.140087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471816240427588:2050] [node 3] ICP22 created new session: [5:7577471820535394969:2050]
2025-11-27T18:20:41.140144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471820535394969:2050] [node 3] ICS09 handshake done sender: [5:7577471820535394955:2114] self: [5:7577471820535394962:0] peer: [3:7577471821188678623:0] socket: 635 qp: -1
2025-11-27T18:20:41.140174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471820535394969:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:41.140208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.140794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471816893711258:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.140842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:41.141114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471821566749293:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
2025-11-27T18:20:41.140885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471816893711258:2049] [node 2] ICP22 created new session: [3:7577471821188678647:2049]
2025-11-27T18:20:41.141000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471821188678647:2049] [node 2] ICS09 handshake done sender: [3:7577471821188678635:2118] self: [3:7577471821188678640:0] peer: [2:7577471821566749267:0] socket: 650 qp: -1
2025-11-27T18:20:41.141216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471815320242525:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.141034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471821188678647:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:41.141192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:41.141310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:41.141372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471815320242525:2048] [node 2] ICP22 created new session: [1:7577471819615209918:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:41.141430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471819615209918:2048] [node 2] ICS09 handshake done sender: [1:7577471819615209888:2110] self: [1:7577471819615209887:0] peer: [2:7577471821566749286:0] socket: 616 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:41.141525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471819615209918:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:41.141567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:41.141555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471820915403265:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48180
2025-11-27T18:20:41.141586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820244742992:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56156
2025-11-27T18:20:41.142016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471821566749275:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.142192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471820535394967:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-27T18:20:41.144217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471821188678632:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.144567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471817271781904:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.144709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:41.144844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820244742993:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56156
2025-11-27T18:20:41.144879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471817271781904:2048] [node 1] ICP22 created new session: [2:7577471821566749296:2048]
2025-11-27T18:20:41.144928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471821566749296:2048] [node 1] ICS09 handshake done sender: [2:7577471821566749275:2115] self: [2:7577471821566749286:0] peer: [1:7577471819615209887:0] socket: 623 qp: -1
2025-11-27T18:20:41.144961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471821566749296:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:41.144995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:41.145029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471820535394967:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.145089Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471816893711257:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471821188678632:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.145296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471820535394971:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.145455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471820535394945:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.145581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820244742994:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56156
2025-11-27T18:20:41.146102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471820915403251:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.146141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471816893711257:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.146186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:41.146149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471820915403251:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.147379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471816893711257:2048] [node 1] ICP22 created new session: [3:7577471821188678651:2048]
2025-11-27T18:20:41.147483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471821188678651:2048] [node 1] ICS09 handshake done sender: [3:7577471821188678632:2115] self: [3:7577471821188678639:0] peer: [1:7577471819615209889:0] socket: 622 qp: -1
2025-11-27T18:20:41.147754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471821188678651:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:41.147757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471820244742992:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.147790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:41.147808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471820244742994:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.147846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471820244742915:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.147882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471820915403251:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.147995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471820915403265:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.148022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471820244742975:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.148026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471817271781904:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:41.148064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471820915403242:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.148062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471820535394974:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-27T18:20:41.148048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821566749281:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.148192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471820244742993:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.148172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471820535394971:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.148189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:41.148069Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471821566749281:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.148654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471820244742914:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.148229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471820915403242:2108] [node 5] ICP22 created new session: [4:7577471820915403267:2108]
2025-11-27T18:20:41.148664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471820535394957:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.148494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471820915403267:2108] [node 5] ICS09 handshake done sender: [4:7577471820915403251:2113] self: [4:7577471820915403250:0] peer: [5:7577471820535394973:0] socket: 617 qp: -1
2025-11-27T18:20:41.148521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471820915403267:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:41.148695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471820535394957:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.148678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:41.149647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471820535394967:2119] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:41.150113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471821566749281:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.150534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819615209900:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.150608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471816620435884:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.150579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471819615209900:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.150648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471816240427586:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.150650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820244742998:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56156
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.151331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471820535394945:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.151599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471819615209900:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:41.151641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:41.151707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471820535394945:2108] [node 4] ICP22 created new session: [5:7577471820535394976:2108]
2025-11-27T18:20:41.151782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471821566749265:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.151785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820244742999:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56156
2025-11-27T18:20:41.152004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:41.151748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471820535394976:2108] [node 4] ICS09 handshake done sender: [5:7577471820535394967:2119] self: [5:7577471820535394973:0] peer: [4:7577471820915403250:0] socket: 663 qp: -1
2025-11-27T18:20:41.151781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471820535394976:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:41.152112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471821566749265:2109] [node 6] ICP22 created new session: [2:7577471821566749299:2109]
2025-11-27T18:20:41.151808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:41.152157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471821566749299:2109] [node 6] ICS09 handshake done sender: [2:7577471821566749281:2117] self: [2:7577471821566749280:0] peer: [6:7577471820244742995:0] socket: 666 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.152181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471821566749299:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:41.152206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:41.152577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471820915403269:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48180
2025-11-27T18:20:41.152603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820244743000:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56156
2025-11-27T18:20:41.152877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471819615209886:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.152938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:41.152983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471819615209886:2109] [node 6] ICP22 created new session: [1:7577471819615209922:2109]
2025-11-27T18:20:41.153466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471820535394957:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:41.153089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471819615209922:2109] [node 6] ICS09 handshake done sender: [1:7577471819615209900:2115] self: [1:7577471819615209899:0] peer: [6:7577471820244742997:0] socket: 641 qp: -1
2025-11-27T18:20:41.153443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471819615209922:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:41.153574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:41.154539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471820535394946:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.154639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:41.154683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471820535394946:2109] [node 6] ICP22 created new session: [5:7577471820535394979:2109]
2025-11-27T18:20:41.154678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471820244742992:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:41.154688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471820244742993:2116] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:41.154727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471820535394979:2109] [node 6] ICS09 handshake done sender: [5:7577471820535394957:2115] self: [5:7577471820535394956:0] peer: [6:7577471820244742996:0] socket: 612 qp: -1
2025-11-27T18:20:41.154751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471820915403269:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.155321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471816620435885:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.154760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471820535394979:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:41.155342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471820244742915:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.154771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471820535394945:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:41.154931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:41.155432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821566749273:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.155444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471820244742914:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.155448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:41.155457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471821188678626:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.155509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819615209894:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.155490Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471821188678626:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.155460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471821566749273:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.155538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471819615209894:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.155483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:41.155509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471820244742915:2049] [node 2] ICP22 created new session: [6:7577471820244743003:2049]
2025-11-27T18:20:41.155586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471820244742914:2048] [node 1] ICP22 created new session: [6:7577471820244743004:2048]
2025-11-27T18:20:41.155901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471820244743004:2048] [node 1] ICS09 handshake done sender: [6:7577471820244742993:2116] self: [6:7577471820244742997:0] peer: [1:7577471819615209899:0] socket: 668 qp: -1
2025-11-27T18:20:41.156289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471820244743003:2049] [node 2] ICS09 handshake done sender: [6:7577471820244742992:2115] self: [6:7577471820244742995:0] peer: [2:7577471821566749280:0] socket: 667 qp: -1
2025-11-27T18:20:41.156498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471820244743003:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:41.156665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471820244743004:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:41.157125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471819615209894:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:41.157184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471821188678626:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:41.156691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:41.156694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:41.157593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471821188678617:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.157685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:41.157745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471821566749273:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:41.157802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471821188678617:2107] [node 4] ICP22 created new session: [3:7577471821188678654:2107]
2025-11-27T18:20:41.157837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471819615209884:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:41.157934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471821188678654:2107] [node 4] ICS09 handshake done sender: [3:7577471821188678626:2113] self: [3:7577471821188678624:0] peer: [4:7577471820915403272:0] socket: 670 qp: -1
::ffff:127.1.0.0
2025-11-27T18:20:41.157883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:41.157922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471819615209884:2108] [node 5] ICP22 created new session: [1:7577471819615209925:2108]
2025-11-27T18:20:41.157969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471821188678654:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:41.157977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471819615209925:2108] [node 5] ICS09 handshake done sender: [1:7577471819615209894:2113] self: [1:7577471819615209891:0] peer: [5:7577471820535394975:0] socket: 613 qp: -1
2025-11-27T18:20:41.158010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:41.157996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471820915403273:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48180
2025-11-27T18:20:41.158037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471821566749264:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.158008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471819615209925:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:41.158094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:41.158122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:41.158134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471821566749264:2107] [node 4] ICP22 created new session: [2:7577471821566749302:2107]
2025-11-27T18:20:41.158198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471821566749302:2107] [node 4] ICS09 handshake done sender: [2:7577471821566749273:2113] self: [2:7577471821566749272:0] peer: [4:7577471820915403270:0] socket: 665 qp: -1
2025-11-27T18:20:41.158252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471821566749302:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:41.158580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:41.160447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471820915403269:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:41.160691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471816240427586:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471820535394971:2120] held: no
2025-11-27T18:20:41.161289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471816620435885:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12888) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471820915403269:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.161491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471816620435884:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471820915403265:2118] held: no
2025-11-27T18:20:41.161591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471820535394981:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
2025-11-27T18:20:41.161769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471820535394971:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:41.161841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471816620435885:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.162291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:41.162406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471816240427586:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.162329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471816620435885:2050] [node 3] ICP22 created new session: [4:7577471820915403275:2050]
2025-11-27T18:20:41.162496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:41.162602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471816240427586:2048] [node 1] ICP22 created new session: [5:7577471820535394982:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.162428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471820915403275:2050] [node 3] ICS09 handshake done sender: [4:7577471820915403269:2119] self: [4:7577471820915403272:0] peer: [3:7577471821188678624:0] socket: 674 qp: -1
2025-11-27T18:20:41.162487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471820915403275:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:41.162725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471820535394982:2048] [node 1] ICS09 handshake done sender: [5:7577471820535394971:2120] self: [5:7577471820535394975:0] peer: [1:7577471819615209891:0] socket: 652 qp: -1
No RDMA context found for GID: 2025-11-27T18:20:41.162540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
::ffff:127.1.0.0
2025-11-27T18:20:41.162759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471820535394982:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:41.162800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:41.162757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820244743002:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56156
2025-11-27T18:20:41.162802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471820915403274:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48180
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.163714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820244743001:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56156
2025-11-27T18:20:41.163802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471820244742998:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.163939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471820244742915:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:41.164063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471820244742914:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:41.164213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471820915403265:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:41.164248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471820244743001:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.164362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471820244742994:2117] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:41.164340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471816620435884:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.164390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:41.164437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471816620435884:2049] [node 2] ICP22 created new session: [4:7577471820915403277:2049]
2025-11-27T18:20:41.164503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471820915403277:2049] [node 2] ICS09 handshake done sender: [4:7577471820915403265:2118] self: [4:7577471820915403270:0] peer: [2:7577471821566749272:0] socket: 615 qp: -1
2025-11-27T18:20:41.164555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471820915403277:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:41.164610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:41.164647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471820244742975:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.164692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:41.164706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471820244742916:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.164733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471820244742975:2106] [node 5] ICP22 created new session: [6:7577471820244743010:2106]
2025-11-27T18:20:41.164771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471820244743010:2106] [node 5] ICS09 handshake done sender: [6:7577471820244742994:2117] self: [6:7577471820244742996:0] peer: [5:7577471820535394956:0] socket: 645 qp: -1
2025-11-27T18:20:41.164800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471820244743010:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:41.164829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.165331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471820535394984:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36614
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.165564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471820244742978:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.165602Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471821188678631:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.165627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471820915403279:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48180
2025-11-27T18:20:41.165625Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471821188678631:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.165922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471820915403255:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.165948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471820915403255:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.166855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471821188678631:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:41.167487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471821188678618:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.167552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471820915403255:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:41.167606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:41.167643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471821188678618:2109] [node 6] ICP22 created new session: [3:7577471821188678657:2109]
2025-11-27T18:20:41.167677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471821188678657:2109] [node 6] ICS09 handshake done sender: [3:7577471821188678631:2114] self: [3:7577471821188678630:0] peer: [6:7577471820244743009:0] socket: 671 qp: -1
2025-11-27T18:20:41.167697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471821188678657:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:41.167718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:41.167786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471820915403243:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.167834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:41.167891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471820915403243:2109] [node 6] ICP22 created new session: [4:7577471820915403281:2109]
2025-11-27T18:20:41.167944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471820915403281:2109] [node 6] ICS09 handshake done sender: [4:7577471820915403255:2114] self: [4:7577471820915403252:0] peer: [6:7577471820244743012:0] socket: 647 qp: -1
2025-11-27T18:20:41.168014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471820915403281:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:41.168064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:41.168226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471820915403279:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.168394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471820244742916:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12888) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471820244742998:2118] held: no
2025-11-27T18:20:41.168876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471820915403243:2109] [node 6] ICP18 handshake (actor [4:7577471820915403279:2122]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.169323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820244743013:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56156
2025-11-27T18:20:41.169340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820244743014:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56156
2025-11-27T18:20:41.169763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471820244742978:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13500) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471820244743001:2121] held: no
2025-11-27T18:20:41.170336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471820244742998:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:41.170548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471820244743001:2121] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:41.170778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471820244742978:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.170784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471820244742916:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.170808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:41.170817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:41.170833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471820244742978:2107] [node 4] ICP22 created new session: [6:7577471820244743015:2107]
2025-11-27T18:20:41.170857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471820244742916:2050] [node 3] ICP22 created new session: [6:7577471820244743016:2050]
2025-11-27T18:20:41.170864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471820244743015:2107] [node 4] ICS09 handshake done sender: [6:7577471820244743001:2121] self: [6:7577471820244743012:0] peer: [4:7577471820915403252:0] socket: 678 qp: -1
2025-11-27T18:20:41.170891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471820244743015:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:41.170899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471820244743016:2050] [node 3] ICS09 handshake done sender: [6:7577471820244742998:2118] self: [6:7577471820244743009:0] peer: [3:7577471821188678630:0] socket: 672 qp: -1
2025-11-27T18:20:41.170916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:41.170928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471820244743016:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:41.170959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:41.206718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:41.206976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471821566749296:2048] [node 1] ICS07 socket disconnect 623 reason# EndOfStream
2025-11-27T18:20:41.207024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471821566749296:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.207097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:41.207144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471821566749296:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:41.208114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:41.208256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471821188678651:2048] [node 1] ICS01 socket: 622 reason# EndOfStream
2025-11-27T18:20:41.208290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471816893711257:2048] [node 1] ICP30 unregister session Session# [3:7577471821188678651:2048] VirtualId# [3:7577471821188678639:0]
2025-11-27T18:20:41.208311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471821188678651:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.208365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:41.208401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:41.209549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471817271781904:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:41.209597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471821566749296:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:41.210419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471817271781904:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.210465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471821566749296:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.210782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471817271781904:2048] [node 1] ICP30 unregister session Session# [2:7577471821566749296:2048] VirtualId# [2:7577471821566749286:0]
2025-11-27T18:20:41.210891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:41.210922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471817271781904:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.210937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.210999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:41.211161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471820915403262:2048] [node 1] ICS07 socket disconnect 629 reason# EndOfStream
2025-11-27T18:20:41.211198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471820915403262:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.211246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:41.211276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471820915403262:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:41.211739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471816620435883:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.211781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471820915403262:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.211851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471816620435883:2048] [node 1] ICP30 unregister session Session# [4:7577471820915403262:2048] VirtualId# [4:7577471820915403259:0]
2025-11-27T18:20:41.212011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:41.212048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471816620435883:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.212061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.212065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:41.212261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471820535394982:2048] [node 1] ICS01 socket: 652 reason# EndOfStream
2025-11-27T18:20:41.212298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471816240427586:2048] [node 1] ICP30 unregister session Session# [5:7577471820535394982:2048] VirtualId# [5:7577471820535394975:0]
2025-11-27T18:20:41.212335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471820535394982:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.212445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:41.212492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:41.213127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:41.213382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471820244743004:2048] [node 1] ICS07 socket disconnect 668 reason# EndOfStream
2025-11-27T18:20:41.213413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471820244743004:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.213462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:41.213497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471820244743004:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:41.214161Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471820244742914:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.214215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471820244743004:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.214291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471820244742914:2048] [node 1] ICP30 unregister session Session# [6:7577471820244743004:2048] VirtualId# [6:7577471820244742997:0]
2025-11-27T18:20:41.214391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:41.214423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471820244742914:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.214436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22745) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.220415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577471817271781904:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:41.221399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471816620435883:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:41.224472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471820244742914:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:41.249926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:41.250088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471821188678647:2049] [node 2] ICS01 socket: 650 reason# EndOfStream
2025-11-27T18:20:41.250121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471816893711258:2049] [node 2] ICP30 unregister session Session# [3:7577471821188678647:2049] VirtualId# [3:7577471821188678640:0]
2025-11-27T18:20:41.250148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471821188678647:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.250210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:41.250242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:41.251778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:41.251918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471820915403277:2049] [node 2] ICS01 socket: 615 reason# EndOfStream
2025-11-27T18:20:41.251957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471816620435884:2049] [node 2] ICP30 unregister session Session# [4:7577471820915403277:2049] VirtualId# [4:7577471820915403270:0]
2025-11-27T18:20:41.251975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471820915403277:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.252026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:41.252062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:41.252495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:41.252625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471820535394966:2049] [node 2] ICS07 socket disconnect 644 reason# EndOfStream
2025-11-27T18:20:41.252647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471820535394966:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.252696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:41.252726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471820535394966:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:41.253253Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:41.253355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471816240427587:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.253389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471820535394966:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.253410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471816240427587:2049] [node 2] ICP30 unregister session Session# [5:7577471820535394966:2049] VirtualId# [5:7577471820535394963:0]
2025-11-27T18:20:41.253504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471820244743003:2049] [node 2] ICS07 socket disconnect 667 reason# EndOfStream
2025-11-27T18:20:41.253535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471820244743003:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.253590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:41.253617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:41.253621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471816240427587:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.253637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.253669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471820244743003:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:41.254291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471820244742915:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.254364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471820244743003:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.254397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471820244742915:2049] [node 2] ICP30 unregister session Session# [6:7577471820244743003:2049] VirtualId# [6:7577471820244742995:0]
2025-11-27T18:20:41.254474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:41.254508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471820244742915:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.254522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.263387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471816240427587:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:41.264415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471820244742915:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:41.285643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:41.285801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471820915403275:2050] [node 3] ICS01 socket: 674 reason# EndOfStream
2025-11-27T18:20:41.285849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471816620435885:2050] [node 3] ICP30 unregister session Session# [4:7577471820915403275:2050] VirtualId# [4:7577471820915403272:0]
2025-11-27T18:20:41.285884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471820915403275:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.285954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:41.285987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:41.286473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:41.286599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471820535394969:2050] [node 3] ICS07 socket disconnect 635 reason# EndOfStream
2025-11-27T18:20:41.286624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471820535394969:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.286675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:41.286706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471820535394969:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:41.287205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:41.287363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471816240427588:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12888) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.287461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471820244743016:2050] [node 3] ICS01 socket: 672 reason# EndOfStream
2025-11-27T18:20:41.287474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471820535394969:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.287509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471820244742916:2050] [node 3] ICP30 unregister session Session# [6:7577471820244743016:2050] VirtualId# [6:7577471820244743009:0]
2025-11-27T18:20:41.287525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471816240427588:2050] [node 3] ICP30 unregister session Session# [5:7577471820535394969:2050] VirtualId# [5:7577471820535394962:0]
2025-11-27T18:20:41.287547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471820244743016:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.287605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:41.287641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:41.287652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:41.287679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471816240427588:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12888) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.287693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12888) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.297199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471816240427588:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:41.344701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:41.344850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471820535394976:2108] [node 4] ICS07 socket disconnect 663 reason# EndOfStream
2025-11-27T18:20:41.344875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471820535394976:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.344940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:41.344984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471820535394976:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:41.345634Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471820535394945:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13500) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.345671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471820535394976:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.345820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471820535394945:2108] [node 4] ICP30 unregister session Session# [5:7577471820535394976:2108] VirtualId# [5:7577471820535394973:0]
2025-11-27T18:20:41.345852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:41.345930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:41.345962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471820535394945:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.345976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.346043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471820244743015:2107] [node 4] ICS01 socket: 678 reason# EndOfStream
2025-11-27T18:20:41.346083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471820244742978:2107] [node 4] ICP30 unregister session Session# [6:7577471820244743015:2107] VirtualId# [6:7577471820244743012:0]
2025-11-27T18:20:41.346117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471820244743015:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.346191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:41.346247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:41.355549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471820535394945:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:41.389624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:41.389844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471820244743010:2106] [node 5] ICS07 socket disconnect 645 reason# EndOfStream
2025-11-27T18:20:41.389874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471820244743010:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.389950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:41.390002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471820244743010:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:41.390641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471820244742975:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1679) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.390686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471820244743010:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.390738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471820244742975:2106] [node 5] ICP30 unregister session Session# [6:7577471820244743010:2106] VirtualId# [6:7577471820244742996:0]
2025-11-27T18:20:41.390825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:41.390852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471820244742975:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1679) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.390861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1679) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:41.449603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471819514913055:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:41.449802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471819514913056:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:41.469007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471821570781365:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:41.469029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471821570781366:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:41.486995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471820455130339:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:41.487018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471820455130340:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:41.505650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471823123022081:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:41.505687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471823123022083:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:41.505887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471823123022082:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:41.525678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471819722703884:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:41.525715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471819722703885:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:41.525957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471819722703886:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:41.547557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471820152569989:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:41.547576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471820152569990:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:41.547586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471820152569991:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:41.621817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471820152570051:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:41.633778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471819514913118:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:41.633956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471823123022144:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:41.635355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471819514913119:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:41.635186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471821570781428:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:41.635272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471819722703947:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:41.636428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471819514913120:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:41.636203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471820455130402:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:41.636247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471823123022145:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:41.635413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471819722703948:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:41.636340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471821570781429:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:41.636246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471820455130403:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:41.636561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471821570781430:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:41.636758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471820152570053:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:41.636582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471820455130404:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:41.641827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7577471821570781366:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(38913676ns) {Configure:104(38902680ns) {ProcessConfigured:135(86506ns) {StartInitialHandshake:157(84271ns) {DropHandshakes:470(1783ns) {DropIncomingHandshake:442(444ns) DropOutgoingHandshake:457(392ns)} GenerateSessionVirtualId:988(1017ns) PrepareNewSessionHandshake:477(3724ns) {SwitchToState:179(447ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.645873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471821570781444:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54851
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.646245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471820455130414:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33600
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.647428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471821570781445:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54851
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.652045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471823123022152:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15959
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.652231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471820455130419:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33600
2025-11-27T18:20:41.652625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471819722703958:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52239
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.652530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471820455130414:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.653784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471819514913127:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14625
2025-11-27T18:20:41.652693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471823123022153:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15959
2025-11-27T18:20:41.654208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821570781434:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.653342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471823123022152:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.653306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471820455130340:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.654236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471821570781434:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.654640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471819514913128:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14625
2025-11-27T18:20:41.654651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471823123022153:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.655918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471821570781446:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54851
No RDMA context found for GID: 2025-11-27T18:20:41.654677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471823123022082:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.656236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471823123022083:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.656519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821570781436:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:41.656734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819514913128:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:41.656893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471819722703958:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.656557Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471821570781436:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.657072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819514913127:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.656561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471821570781447:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54851
2025-11-27T18:20:41.659723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471820455130421:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33600
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.657076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471819514913133:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14625
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.660147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471820455130422:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33600
2025-11-27T18:20:41.656901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821570781444:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.659742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471819514913134:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14625
No RDMA context found for GID: 2025-11-27T18:20:41.660503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471820455130409:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:41.661118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471823123022163:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15959
2025-11-27T18:20:41.658010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471821570781434:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:41.659755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819514913133:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.659517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821570781445:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.660527Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471820455130409:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.660899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819514913134:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.661139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471821570781436:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:41.661121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471820455130423:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33600
No RDMA context found for GID: 2025-11-27T18:20:41.661162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471820455130421:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:41.661581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471821570781449:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54851
2025-11-27T18:20:41.661352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471820455130419:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.661802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471819514913138:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14625
2025-11-27T18:20:41.661781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821570781446:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.662425Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471820455130414:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:41.662561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819514913138:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.662724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471820455130340:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.661997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471821570781366:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.662806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:41.662810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471823123022164:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15959
2025-11-27T18:20:41.662844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471820455130340:2049] [node 2] ICP22 created new session: [3:7577471820455130427:2049]
2025-11-27T18:20:41.662006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821570781447:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.662978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471820455130409:2111] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:41.663028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471820455130427:2049] [node 2] ICS09 handshake done sender: [3:7577471820455130414:2113] self: [3:7577471820455130420:0] peer: [2:7577471821570781432:0] socket: 624 qp: -1
2025-11-27T18:20:41.662386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471821570781428:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.663101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471820455130427:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:41.663173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:41.664086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471823123022152:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:41.662416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:41.663972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471820455130402:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.662460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471821570781428:2107] [node 4] ICP22 created new session: [2:7577471821570781453:2107]
2025-11-27T18:20:41.664012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.665413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471823123022082:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.662489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:41.664908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471820455130402:2107] [node 4] ICP22 created new session: [3:7577471820455130429:2107]
2025-11-27T18:20:41.662533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471821570781453:2107] [node 4] ICS09 handshake done sender: [2:7577471821570781436:2112] self: [2:7577471821570781435:0] peer: [4:7577471823123022161:0] socket: 621 qp: -1
2025-11-27T18:20:41.665129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471820455130429:2107] [node 4] ICS09 handshake done sender: [3:7577471820455130409:2111] self: [3:7577471820455130406:0] peer: [4:7577471823123022162:0] socket: 622 qp: -1
2025-11-27T18:20:41.665522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:41.662561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471821570781453:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:41.662594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471821570781366:2049] [node 3] ICP22 created new session: [2:7577471821570781454:2049]
2025-11-27T18:20:41.665562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471823123022082:2049] [node 2] ICP22 created new session: [4:7577471823123022166:2049]
2025-11-27T18:20:41.665284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471820455130429:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:41.662610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:41.665311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.665569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471823123022165:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15959
2025-11-27T18:20:41.662767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471821570781454:2049] [node 3] ICS09 handshake done sender: [2:7577471821570781434:2111] self: [2:7577471821570781432:0] peer: [3:7577471820455130420:0] socket: 619 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.665728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471823123022166:2049] [node 2] ICS09 handshake done sender: [4:7577471823123022152:2113] self: [4:7577471823123022161:0] peer: [2:7577471821570781435:0] socket: 627 qp: -1
2025-11-27T18:20:41.662798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471821570781454:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:41.665956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471823123022166:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:41.667068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820152570068:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61800
2025-11-27T18:20:41.665984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:41.662838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:41.667204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471819722703964:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52239
2025-11-27T18:20:41.666926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821570781449:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.667350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471820455130422:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.667377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471823123022163:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.667463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471823123022153:2114] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:41.667557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471821570781428:2107] [node 4] ICP18 handshake (actor [2:7577471821570781447:2118]) is too old
2025-11-27T18:20:41.668338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471823123022083:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16515) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471823123022153:2114] held: no
2025-11-27T18:20:41.668312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471820455130339:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.668836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820152570069:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61800
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.668895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819514913124:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.668931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471819514913124:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.669015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471819722703965:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52239
2025-11-27T18:20:41.669103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471823123022083:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.669172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:41.669207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471823123022083:2050] [node 3] ICP22 created new session: [4:7577471823123022169:2050]
2025-11-27T18:20:41.669370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471823123022169:2050] [node 3] ICS09 handshake done sender: [4:7577471823123022153:2114] self: [4:7577471823123022162:0] peer: [3:7577471820455130406:0] socket: 632 qp: -1
2025-11-27T18:20:41.669638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471823123022169:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:41.669659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:41.669689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471823123022081:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.670179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471821570781365:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.670460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471819514913124:2110] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:41.670468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471820152570069:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.670640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471820152570068:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.670658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819514913125:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.670930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471820152569990:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.671763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471819722703964:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.670686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471819514913125:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.671834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471820152569991:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.671856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471819514913056:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.671810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821570781441:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.671833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471821570781441:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.672376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820152570070:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61800
2025-11-27T18:20:41.672435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471819722703885:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.671906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:41.672412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471820455130416:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.672060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471819514913056:2049] [node 3] ICP22 created new session: [1:7577471819514913146:2049]
2025-11-27T18:20:41.672482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471821570781443:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.672681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471819722703965:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.672434Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471820455130416:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.672438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471819514913146:2049] [node 3] ICS09 handshake done sender: [1:7577471819514913124:2110] self: [1:7577471819514913122:0] peer: [3:7577471820455130433:0] socket: 652 qp: -1
2025-11-27T18:20:41.672501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471821570781443:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.674573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471819722703886:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.672497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471819514913146:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:41.672789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471820455130434:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33600
2025-11-27T18:20:41.672819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:41.676332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471821570781441:2113] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:41.674529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471820455130339:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8505) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471820455130422:2118] held: no
2025-11-27T18:20:41.674551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819514913126:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.674572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471819514913126:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.676587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471821570781461:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54851
2025-11-27T18:20:41.675473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471819514913125:2111] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:41.676691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471819514913055:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.676743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.676902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471819514913055:2048] [node 2] ICP22 created new session: [1:7577471819514913150:2048]
2025-11-27T18:20:41.676898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471820455130418:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.676930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471819514913150:2048] [node 2] ICS09 handshake done sender: [1:7577471819514913125:2111] self: [1:7577471819514913123:0] peer: [2:7577471821570781460:0] socket: 654 qp: -1
2025-11-27T18:20:41.676921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471820455130418:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.677950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471819722703968:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52239
2025-11-27T18:20:41.677957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471819514913150:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:41.677991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471821570781443:2114] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:41.678179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471819722703969:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52239
2025-11-27T18:20:41.677982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:41.678200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471821570781449:2119] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:41.678316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820152570073:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61800
2025-11-27T18:20:41.678280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471821570781430:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-27T18:20:41.678316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.02025-11-27T18:20:41.678643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471820455130422:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.678820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471820455130418:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:41.678635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471821570781430:2109] [node 6] ICP22 created new session: [2:7577471821570781464:2109]
2025-11-27T18:20:41.679235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471819722703968:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.678897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471820455130416:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:41.678668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471821570781464:2109] [node 6] ICS09 handshake done sender: [2:7577471821570781441:2113] self: [2:7577471821570781440:0] peer: [6:7577471820152570071:0] socket: 623 qp: -1
2025-11-27T18:20:41.679506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471820152570070:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.679655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471819722703970:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52239
2025-11-27T18:20:41.679520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471820455130339:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.678689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471821570781464:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:41.679665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471823123022173:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15959
2025-11-27T18:20:41.679732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471820152570068:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:41.680032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471819514913126:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.679842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471819722703964:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:41.679690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:41.680585Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471820152569990:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17366) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471820152570068:2116] held: no
2025-11-27T18:20:41.679865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471819722703947:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.680972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471823123022157:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.679962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471820455130339:2048] [node 1] ICP22 created new session: [3:7577471820455130437:2048]
2025-11-27T18:20:41.680403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471819722703885:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.681001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471823123022157:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.678977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471821570781365:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.681049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471820455130403:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.680626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:41.681706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471823123022163:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:41.679027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:41.682143Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471823123022157:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:41.679045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:41.681111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471820455130404:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.682081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820152570074:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61800
2025-11-27T18:20:41.681008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471819722703885:2049] [node 2] ICP22 created new session: [5:7577471819722703973:2049]
2025-11-27T18:20:41.679649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471821570781365:2048] [node 1] ICP22 created new session: [2:7577471821570781466:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.681130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471820455130437:2048] [node 1] ICS09 handshake done sender: [3:7577471820455130422:2118] self: [3:7577471820455130433:0] peer: [1:7577471819514913122:0] socket: 653 qp: -1
2025-11-27T18:20:41.682554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471819514913118:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.682535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471823123022081:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.679691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471821570781466:2048] [node 1] ICS09 handshake done sender: [2:7577471821570781449:2119] self: [2:7577471821570781460:0] peer: [1:7577471819514913123:0] socket: 658 qp: -1
2025-11-27T18:20:41.681559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471819722703973:2049] [node 2] ICS09 handshake done sender: [5:7577471819722703964:2116] self: [5:7577471819722703966:0] peer: [2:7577471821570781442:0] socket: 667 qp: -1
2025-11-27T18:20:41.682593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:41.681140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:41.679802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471821570781429:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.681594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471819722703973:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:41.682641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471823123022081:2048] [node 1] ICP22 created new session: [4:7577471823123022175:2048]
2025-11-27T18:20:41.681461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:41.681845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:41.679824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471821570781466:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:41.683566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:41.683747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471823123022175:2048] [node 1] ICS09 handshake done sender: [4:7577471823123022163:2117] self: [4:7577471823123022170:0] peer: [1:7577471819514913121:0] socket: 660 qp: -1
2025-11-27T18:20:41.681504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471820455130403:2108] [node 5] ICP22 created new session: [3:7577471820455130438:2108]
2025-11-27T18:20:41.684110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471819722703972:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52239
2025-11-27T18:20:41.679945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:41.683764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471819514913118:2107] [node 4] ICP22 created new session: [1:7577471819514913153:2107]
2025-11-27T18:20:41.684170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471823123022175:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:41.681558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471820455130438:2108] [node 5] ICS09 handshake done sender: [3:7577471820455130418:2115] self: [3:7577471820455130417:0] peer: [5:7577471819722703967:0] socket: 668 qp: -1
2025-11-27T18:20:41.684174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471820152570053:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.684204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:41.680392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:41.684193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471819514913153:2107] [node 4] ICS09 handshake done sender: [1:7577471819514913126:2112] self: [1:7577471819514913121:0] peer: [4:7577471823123022170:0] socket: 656 qp: -1
2025-11-27T18:20:41.680622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471821570781429:2108] [node 5] ICP22 created new session: [2:7577471821570781467:2108]
2025-11-27T18:20:41.681589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471820455130438:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:41.684302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471819514913153:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:41.684337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:41.681544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471821570781467:2108] [node 5] ICS09 handshake done sender: [2:7577471821570781443:2114] self: [2:7577471821570781442:0] peer: [5:7577471819722703966:0] socket: 625 qp: -1
2025-11-27T18:20:41.681757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471820455130404:2109] [node 6] ICP22 created new session: [3:7577471820455130439:2109]
2025-11-27T18:20:41.681771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471820455130437:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:41.681574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471821570781467:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:41.684478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471820152569990:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.682117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:41.682127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:41.685354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471819722703965:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.685073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471823123022145:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.684535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:41.685626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:41.682245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:41.685676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471823123022145:2108] [node 5] ICP22 created new session: [4:7577471823123022177:2108]
2025-11-27T18:20:41.685368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471820152570069:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:41.682458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471820455130439:2109] [node 6] ICS09 handshake done sender: [3:7577471820455130416:2114] self: [3:7577471820455130415:0] peer: [6:7577471820152570072:0] socket: 648 qp: -1
2025-11-27T18:20:41.686255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471823123022177:2108] [node 5] ICS09 handshake done sender: [4:7577471823123022157:2116] self: [4:7577471823123022156:0] peer: [5:7577471819722703971:0] socket: 633 qp: -1
2025-11-27T18:20:41.685460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471820152569990:2049] [node 2] ICP22 created new session: [6:7577471820152570077:2049]
2025-11-27T18:20:41.683433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471820455130439:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:41.683464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:41.685944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471820152570077:2049] [node 2] ICS09 handshake done sender: [6:7577471820152570068:2116] self: [6:7577471820152570071:0] peer: [2:7577471821570781440:0] socket: 666 qp: -1
2025-11-27T18:20:41.685974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471820152570077:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:41.686223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471819722703975:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52239
2025-11-27T18:20:41.686207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471820152569991:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.686309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471823123022177:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:41.686243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:41.686357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:41.686274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471820152569991:2050] [node 3] ICP22 created new session: [6:7577471820152570079:2050]
2025-11-27T18:20:41.686579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471823123022155:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.686278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:41.686618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471823123022155:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.686300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471820152570079:2050] [node 3] ICS09 handshake done sender: [6:7577471820152570069:2117] self: [6:7577471820152570072:0] peer: [3:7577471820455130415:0] socket: 651 qp: -1
2025-11-27T18:20:41.686322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471820152570079:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:41.686339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:41.688239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471819722703886:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.688288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:41.688330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471819722703886:2050] [node 3] ICP22 created new session: [5:7577471819722703977:2050]
2025-11-27T18:20:41.688466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471819722703977:2050] [node 3] ICS09 handshake done sender: [5:7577471819722703965:2117] self: [5:7577471819722703967:0] peer: [3:7577471820455130417:0] socket: 669 qp: -1
2025-11-27T18:20:41.688476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820152570076:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61800
2025-11-27T18:20:41.688499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471819722703977:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:41.688532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.689130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471823123022179:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15959
2025-11-27T18:20:41.689239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471823123022155:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:41.690537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471819722703968:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:41.690666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471823123022144:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.690836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:41.690934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471823123022144:2109] [node 6] ICP22 created new session: [4:7577471823123022181:2109]
2025-11-27T18:20:41.691178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471823123022181:2109] [node 6] ICS09 handshake done sender: [4:7577471823123022155:2115] self: [4:7577471823123022154:0] peer: [6:7577471820152570075:0] socket: 649 qp: -1
2025-11-27T18:20:41.691200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471823123022181:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:41.691354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471819722703970:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.691231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:41.691418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471819722703947:2108] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.691539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:41.691582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471819722703947:2108] [node 4] ICP22 created new session: [5:7577471819722703979:2108]
2025-11-27T18:20:41.691621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820152570082:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61800
2025-11-27T18:20:41.691641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471819722703886:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:41.691696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471819722703979:2108] [node 4] ICS09 handshake done sender: [5:7577471819722703968:2118] self: [5:7577471819722703971:0] peer: [4:7577471823123022156:0] socket: 674 qp: -1
2025-11-27T18:20:41.691736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471819722703979:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:41.691765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:41.691792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471820152570076:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.692567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820152570083:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61800
2025-11-27T18:20:41.692601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471820152569989:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.692723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471820152570070:2118] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:41.693009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819514913130:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.693022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471823123022179:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.693036Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471819514913130:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.693057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471820152570053:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.693280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7577471823123022144:2109] [node 6] ICP18 handshake (actor [4:7577471823123022179:2121]) is too old
2025-11-27T18:20:41.693250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:41.693398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471820152570053:2107] [node 4] ICP22 created new session: [6:7577471820152570085:2107]
2025-11-27T18:20:41.693443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471820152570085:2107] [node 4] ICS09 handshake done sender: [6:7577471820152570070:2118] self: [6:7577471820152570075:0] peer: [4:7577471823123022154:0] socket: 670 qp: -1
2025-11-27T18:20:41.693781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471820152570085:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:41.693840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:41.694094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471819514913130:2115] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:41.694588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471819722703884:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.694667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471819514913119:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.694761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.694797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471819514913119:2109] [node 6] ICP22 created new session: [1:7577471819514913156:2109]
2025-11-27T18:20:41.694842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471819514913156:2109] [node 6] ICS09 handshake done sender: [1:7577471819514913130:2115] self: [1:7577471819514913129:0] peer: [6:7577471820152570084:0] socket: 631 qp: -1
2025-11-27T18:20:41.694868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471819514913156:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:41.694887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820152570087:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61800
2025-11-27T18:20:41.694898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:41.694907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471819514913132:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.694929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471819514913132:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.695035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471820152570082:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.695100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471820152570076:2121] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:41.695795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471820152569989:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.695856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:41.695888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471820152569989:2048] [node 1] ICP22 created new session: [6:7577471820152570088:2048]
2025-11-27T18:20:41.695948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471820152570088:2048] [node 1] ICS09 handshake done sender: [6:7577471820152570076:2121] self: [6:7577471820152570084:0] peer: [1:7577471819514913129:0] socket: 641 qp: -1
2025-11-27T18:20:41.695967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471820152570088:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:41.695990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:41.696094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471819514913132:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.696215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471819722703983:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52239
2025-11-27T18:20:41.696299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471819514913120:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.696348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:41.696638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471819514913120:2108] [node 5] ICP22 created new session: [1:7577471819514913159:2108]
2025-11-27T18:20:41.696676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471819514913159:2108] [node 5] ICS09 handshake done sender: [1:7577471819514913132:2116] self: [1:7577471819514913131:0] peer: [5:7577471819722703982:0] socket: 620 qp: -1
2025-11-27T18:20:41.696705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471819722703970:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:41.696710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471819514913159:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:41.696850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:41.696960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471819722703884:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.697011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:41.697084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471820152570051:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:41.697123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471819722703884:2048] [node 1] ICP22 created new session: [5:7577471819722703984:2048]
2025-11-27T18:20:41.697148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471819722703984:2048] [node 1] ICS09 handshake done sender: [5:7577471819722703970:2120] self: [5:7577471819722703982:0] peer: [1:7577471819514913131:0] socket: 679 qp: -1
2025-11-27T18:20:41.697179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471819722703984:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:41.697717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:41.697991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471820152569989:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:41.698205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471819722703960:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:41.698226Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471819722703960:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:41.699486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471819722703960:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:41.699678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471819722703948:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:41.699659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471820152570091:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61800
2025-11-27T18:20:41.699766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:41.699801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471819722703948:2109] [node 6] ICP22 created new session: [5:7577471819722703987:2109]
2025-11-27T18:20:41.699838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471819722703987:2109] [node 6] ICS09 handshake done sender: [5:7577471819722703960:2114] self: [5:7577471819722703959:0] peer: [6:7577471820152570090:0] socket: 645 qp: -1
2025-11-27T18:20:41.699860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471819722703987:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:41.699872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471820152570082:2122] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:41.699884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:41.700024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471820152570051:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:41.700654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:41.700699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471820152570051:2106] [node 5] ICP22 created new session: [6:7577471820152570092:2106]
2025-11-27T18:20:41.700729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471820152570092:2106] [node 5] ICS09 handshake done sender: [6:7577471820152570082:2122] self: [6:7577471820152570090:0] peer: [5:7577471819722703959:0] socket: 686 qp: -1
2025-11-27T18:20:41.700750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471820152570092:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:41.700777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:41.701220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7577471820152570051:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:41.743500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:41.743766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471821570781466:2048] [node 1] ICS07 socket disconnect 658 reason# EndOfStream
2025-11-27T18:20:41.743809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471821570781466:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.743930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:41.743994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471821570781466:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:41.744574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:41.744735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471820455130437:2048] [node 1] ICS01 socket: 653 reason# EndOfStream
2025-11-27T18:20:41.744770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471820455130339:2048] [node 1] ICP30 unregister session Session# [3:7577471820455130437:2048] VirtualId# [3:7577471820455130433:0]
2025-11-27T18:20:41.744795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471820455130437:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.744854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:41.744893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:41.746229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471821570781365:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8505) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:41.746259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471821570781466:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:41.747198Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:41.747343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471823123022175:2048] [node 1] ICS01 socket: 660 reason# EndOfStream
2025-11-27T18:20:41.747372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471823123022081:2048] [node 1] ICP30 unregister session Session# [4:7577471823123022175:2048] VirtualId# [4:7577471823123022170:0]
2025-11-27T18:20:41.747381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471821570781365:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8505) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.747393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471823123022175:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.747463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471821570781466:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.747459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:41.747492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471821570781365:2048] [node 1] ICP30 unregister session Session# [2:7577471821570781466:2048] VirtualId# [2:7577471821570781460:0]
2025-11-27T18:20:41.747511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:41.747580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:41.747626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471821570781365:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8505) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.747641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8505) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.748108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:41.748248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471819722703984:2048] [node 1] ICS01 socket: 679 reason# EndOfStream
2025-11-27T18:20:41.748282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471819722703884:2048] [node 1] ICP30 unregister session Session# [5:7577471819722703984:2048] VirtualId# [5:7577471819722703982:0]
2025-11-27T18:20:41.748306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471819722703984:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.748355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:41.748395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:41.748883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:41.749015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471820152570088:2048] [node 1] ICS07 socket disconnect 641 reason# EndOfStream
2025-11-27T18:20:41.749038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471820152570088:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.749091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:41.749121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471820152570088:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:41.749715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471820152569989:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8505) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.749754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471820152570088:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.749828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471820152569989:2048] [node 1] ICP30 unregister session Session# [6:7577471820152570088:2048] VirtualId# [6:7577471820152570084:0]
2025-11-27T18:20:41.749926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:41.749951Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471820152569989:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8505) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.749960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8505) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.760043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471820152569989:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:41.805720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:41.805940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471820455130427:2049] [node 2] ICS07 socket disconnect 624 reason# EndOfStream
2025-11-27T18:20:41.805970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471820455130427:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.806048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:41.806101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471820455130427:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:41.808236Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:41.808300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471820455130340:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17366) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:41.808340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471820455130427:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:41.808523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471823123022166:2049] [node 2] ICS07 socket disconnect 627 reason# EndOfStream
2025-11-27T18:20:41.808572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471823123022166:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.808638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:41.808688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471823123022166:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:41.809325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471820455130340:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17366) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.809401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471820455130427:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.809441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471820455130340:2049] [node 2] ICP30 unregister session Session# [3:7577471820455130427:2049] VirtualId# [3:7577471820455130420:0]
2025-11-27T18:20:41.809485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:41.809506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471820455130340:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17366) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.809514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17366) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.809616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471823123022082:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17366) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.809632Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:41.809659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471823123022166:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.809789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471819722703973:2049] [node 2] ICS07 socket disconnect 667 reason# EndOfStream
2025-11-27T18:20:41.809837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471819722703973:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.809694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471823123022082:2049] [node 2] ICP30 unregister session Session# [4:7577471823123022166:2049] VirtualId# [4:7577471823123022161:0]
2025-11-27T18:20:41.809909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:41.809893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:41.809956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471819722703973:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:41.809921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471823123022082:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17366) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.809947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17366) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.810551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471819722703885:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17366) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.810590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471819722703973:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.810647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471819722703885:2049] [node 2] ICP30 unregister session Session# [5:7577471819722703973:2049] VirtualId# [5:7577471819722703966:0]
2025-11-27T18:20:41.810759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:41.810711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:41.810782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471819722703885:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17366) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.810791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17366) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.810908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471820152570077:2049] [node 2] ICS01 socket: 666 reason# EndOfStream
2025-11-27T18:20:41.810954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471820152569990:2049] [node 2] ICP30 unregister session Session# [6:7577471820152570077:2049] VirtualId# [6:7577471820152570071:0]
2025-11-27T18:20:41.811009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471820152570077:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.811090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:41.811147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:41.819440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471823123022082:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:41.820465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471819722703885:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:41.855302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:41.855482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471823123022169:2050] [node 3] ICS01 socket: 632 reason# EndOfStream
2025-11-27T18:20:41.855527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471823123022083:2050] [node 3] ICP30 unregister session Session# [4:7577471823123022169:2050] VirtualId# [4:7577471823123022162:0]
2025-11-27T18:20:41.855557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471823123022169:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.855622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:41.855666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:41.856197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:41.856401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471819722703977:2050] [node 3] ICS07 socket disconnect 669 reason# EndOfStream
2025-11-27T18:20:41.856456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471819722703977:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.856545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:41.856594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471819722703977:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:41.856924Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:41.857077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471820152570079:2050] [node 3] ICS07 socket disconnect 651 reason# EndOfStream
2025-11-27T18:20:41.857107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471820152570079:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.857168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:41.857209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471820152570079:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:41.857356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471819722703886:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16515) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.857936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471819722703977:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.857989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471819722703886:2050] [node 3] ICP30 unregister session Session# [5:7577471819722703977:2050] VirtualId# [5:7577471819722703967:0]
2025-11-27T18:20:41.858109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:41.858139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471819722703886:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16515) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.858149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16515) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.858406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471820152569991:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16515) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.858607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471820152570079:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.858644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471820152569991:2050] [node 3] ICP30 unregister session Session# [6:7577471820152570079:2050] VirtualId# [6:7577471820152570072:0]
2025-11-27T18:20:41.858724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:41.858762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471820152569991:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16515) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.858771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16515) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.867559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471819722703886:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:41.868570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471820152569991:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:41.924336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:41.924509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471819722703979:2108] [node 4] ICS07 socket disconnect 674 reason# EndOfStream
2025-11-27T18:20:41.924535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471819722703979:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.924603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:41.924639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471819722703979:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:41.925351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:41.925409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471819722703947:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22334) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.925539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471820152570085:2107] [node 4] ICS07 socket disconnect 670 reason# EndOfStream
2025-11-27T18:20:41.925537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471819722703979:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.925575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471820152570085:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.925583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471819722703947:2108] [node 4] ICP30 unregister session Session# [5:7577471819722703979:2108] VirtualId# [5:7577471819722703971:0]
2025-11-27T18:20:41.925647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:41.925686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471820152570085:2107] [node 4] ICS15 start handshake
2025-11-27T18:20:41.925747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:41.925802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471819722703947:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22334) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.925824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22334) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.926312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471820152570053:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22334) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.926360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471820152570085:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.926409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471820152570053:2107] [node 4] ICP30 unregister session Session# [6:7577471820152570085:2107] VirtualId# [6:7577471820152570075:0]
2025-11-27T18:20:41.926512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:41.926542Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471820152570053:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22334) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.926555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22334) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.936206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471820152570053:2107] [node 4] ICP33 wake up from error state
2025-11-27T18:20:41.965049Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:41.965218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471820152570092:2106] [node 5] ICS07 socket disconnect 686 reason# EndOfStream
2025-11-27T18:20:41.965247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471820152570092:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:41.965335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:41.965394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471820152570092:2106] [node 5] ICS15 start handshake
2025-11-27T18:20:41.965996Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471820152570051:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4044) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:41.966139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471820152570092:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:41.966175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471820152570051:2106] [node 5] ICP30 unregister session Session# [6:7577471820152570092:2106] VirtualId# [6:7577471820152570090:0]
2025-11-27T18:20:41.966265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-11-27T18:20:41.966294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471820152570051:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4044) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.966305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4044) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:41.976143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471820152570051:2106] [node 5] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:42.051198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471825104598819:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:42.051258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471825104598820:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:42.071178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471826153864365:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:42.071187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471826153864366:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:42.093360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471824685924630:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:42.093488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471824685924631:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:42.115550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471826206544751:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:42.115579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471826206544752:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:42.115609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471826206544753:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:42.136208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471825082187324:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:42.136260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471825082187326:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:42.136475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471825082187325:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:42.156254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471824845941840:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:42.156293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471824845941841:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:42.156330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471824845941842:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:42.244143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471826206544815:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:42.244049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471826153864428:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:42.246124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471824685924693:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:42.246558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471825104598883:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:42.247202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471824845941901:2107] [node 5] ICP01 ready to work
2025-11-27T18:20:42.247048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471825082187387:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:42.246535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471826206544814:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:42.247307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471825104598884:2110] [node 6] ICP01 ready to work
2025-11-27T18:20:42.248194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471826153864430:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:42.248424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471824685924694:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:42.247361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471825082187388:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:42.247376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471824845941902:2106] [node 4] ICP01 ready to work
2025-11-27T18:20:42.248449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471825104598885:2109] [node 5] ICP01 ready to work
2025-11-27T18:20:42.248467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471826153864429:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:42.248611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471824685924695:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.261744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471824845941913:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59681
2025-11-27T18:20:42.261978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824685924704:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61296
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.265353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471825104598899:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:100
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.266284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471826153864442:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39007
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.268190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471826206544824:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57604
2025-11-27T18:20:42.269726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471826206544824:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.269649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471825104598900:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:100
No RDMA context found for GID: 2025-11-27T18:20:42.269676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471825082187400:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13619
2025-11-27T18:20:42.269012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824685924704:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-27T18:20:42.269704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471824845941913:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.271112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471826206544751:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.270644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471824845941917:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59681
2025-11-27T18:20:42.271211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825104598891:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.269649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824685924709:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61296
2025-11-27T18:20:42.271242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471825104598891:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.270639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824685924708:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61296
2025-11-27T18:20:42.272225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471824845941918:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59681
2025-11-27T18:20:42.272213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471826206544825:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57604
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.272207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471826153864443:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39007
2025-11-27T18:20:42.272352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471825082187400:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.271556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825104598900:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.273237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471824845941917:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.273648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824685924708:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.272286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471826153864442:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.272581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471825082187401:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13619
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.272185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471825104598901:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:100
2025-11-27T18:20:42.273323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471824845941840:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.275300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471826153864433:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.273721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824685924709:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.273659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471824845941918:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.272348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471825104598902:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:100
2025-11-27T18:20:42.274247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471824685924630:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.275051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471825082187394:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.273976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471824845941919:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59681
2025-11-27T18:20:42.275330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471826153864433:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.274252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471824845941901:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.275079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471825082187394:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.276837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471826206544825:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.272682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825104598899:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.276098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471826153864443:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.275129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824685924712:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61296
2025-11-27T18:20:42.278342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471825082187401:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.273279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825104598901:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.279992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471826206544753:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.274515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471824845941841:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.276322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471826153864438:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.275201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471824685924631:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.280941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471826206544831:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57604
2025-11-27T18:20:42.277743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824685924697:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.280957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471825082187388:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.274921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471824845941919:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:42.277765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471824685924697:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.273323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471825104598891:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:42.281216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471825082187394:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:42.279974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824685924712:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.275308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471824845941842:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.276347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471826153864438:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.276892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825104598893:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.282141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471826206544821:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.282850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471824845941927:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59681
2025-11-27T18:20:42.276871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471826153864433:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:42.276921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471825104598893:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.280767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824685924701:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.282178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471826206544821:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.283355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471824845941926:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59681
2025-11-27T18:20:42.280790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471824685924701:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.281969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471826153864438:2113] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:42.277761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825104598890:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.284493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471825082187387:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.280962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471824685924697:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:42.284177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471826206544824:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:42.282193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471826153864430:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.277779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471825104598890:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.285105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471826206544751:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.284798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:42.282747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824685924716:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61296
2025-11-27T18:20:42.282819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:42.279958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471825104598883:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.285060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471825082187387:2109] [node 6] ICP22 created new session: [5:7577471825082187407:2109]
2025-11-27T18:20:42.282790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471824685924701:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.285413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471826206544821:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:42.283120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471826153864430:2109] [node 6] ICP22 created new session: [2:7577471826153864450:2109]
2025-11-27T18:20:42.289327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471824845941919:2117] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:42.286275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471825082187407:2109] [node 6] ICS09 handshake done sender: [5:7577471825082187394:2111] self: [5:7577471825082187390:0] peer: [6:7577471824845941923:0] socket: 633 qp: -1
2025-11-27T18:20:42.280007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:42.280060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471825104598883:2107] [node 4] ICP22 created new session: [1:7577471825104598907:2107]
2025-11-27T18:20:42.287006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:42.284121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471826153864450:2109] [node 6] ICS09 handshake done sender: [2:7577471826153864433:2110] self: [2:7577471826153864432:0] peer: [6:7577471824845941924:0] socket: 639 qp: -1
2025-11-27T18:20:42.283014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471824685924693:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.289337Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471824845941918:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.287008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471825082187407:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.289152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471826206544751:2048] [node 1] ICP22 created new session: [4:7577471826206544836:2048]
2025-11-27T18:20:42.291947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471824845941901:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471824845941917:2115] held: no
2025-11-27T18:20:42.287039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:42.283047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.284190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471826153864366:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.280510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471825104598907:2107] [node 4] ICS09 handshake done sender: [1:7577471825104598891:2112] self: [1:7577471825104598887:0] peer: [4:7577471826206544829:0] socket: 625 qp: -1
2025-11-27T18:20:42.283379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471824685924693:2109] [node 6] ICP22 created new session: [3:7577471824685924721:2109]
2025-11-27T18:20:42.289303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471826206544814:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.292439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471824845941842:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.283972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824685924719:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61296
2025-11-27T18:20:42.289384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471826206544836:2048] [node 1] ICS09 handshake done sender: [4:7577471826206544824:2114] self: [4:7577471826206544829:0] peer: [1:7577471825104598887:0] socket: 630 qp: -1
2025-11-27T18:20:42.284442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:42.290070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471826206544836:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:42.285202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471824685924721:2109] [node 6] ICS09 handshake done sender: [3:7577471824685924697:2110] self: [3:7577471824685924696:0] peer: [6:7577471824845941925:0] socket: 652 qp: -1
2025-11-27T18:20:42.292731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:42.284451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471826153864450:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:42.285236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471824685924721:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:42.290097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:42.292765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471825082187408:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13619
2025-11-27T18:20:42.280548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471825104598907:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:42.290605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471826206544834:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57604
2025-11-27T18:20:42.291325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:42.292751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471824845941841:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24472) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471824845941918:2116] held: no
2025-11-27T18:20:42.280835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471825104598890:2111] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:42.285295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:42.284491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:42.293079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471825082187410:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13619
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.291394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471826206544814:2108] [node 5] ICP22 created new session: [4:7577471826206544839:2108]
2025-11-27T18:20:42.292776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471824845941842:2050] [node 3] ICP22 created new session: [6:7577471824845941930:2050]
2025-11-27T18:20:42.285820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471824685924695:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.291706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471826206544839:2108] [node 5] ICS09 handshake done sender: [4:7577471826206544821:2111] self: [4:7577471826206544818:0] peer: [5:7577471825082187406:0] socket: 662 qp: -1
2025-11-27T18:20:42.284766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471826153864366:2049] [node 3] ICP22 created new session: [2:7577471826153864452:2049]
2025-11-27T18:20:42.294828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471825082187401:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.292940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471824845941930:2050] [node 3] ICS09 handshake done sender: [6:7577471824845941919:2117] self: [6:7577471824845941925:0] peer: [3:7577471824685924696:0] socket: 657 qp: -1
2025-11-27T18:20:42.285885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:42.291731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471826206544839:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:42.293159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471824845941930:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:42.281011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:42.285881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471826153864452:2049] [node 3] ICS09 handshake done sender: [2:7577471826153864438:2113] self: [2:7577471826153864437:0] peer: [3:7577471824685924715:0] socket: 647 qp: -1
2025-11-27T18:20:42.293746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:42.286269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471824685924695:2107] [node 4] ICP22 created new session: [3:7577471824685924723:2107]
2025-11-27T18:20:42.281223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825104598902:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.292008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:42.295252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471826206544753:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28911) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471826206544825:2115] held: no
2025-11-27T18:20:42.286789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471826153864452:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:42.288781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471824685924723:2107] [node 4] ICS09 handshake done sender: [3:7577471824685924701:2112] self: [3:7577471824685924700:0] peer: [4:7577471826206544833:0] socket: 651 qp: -1
2025-11-27T18:20:42.296526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471825082187412:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13619
2025-11-27T18:20:42.286832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:42.281934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471825104598820:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.289186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471824685924709:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.296634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471825082187388:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.296529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471824845941841:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.293097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471826153864451:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39007
2025-11-27T18:20:42.282139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:42.289352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471824685924723:2107] [node 4] ICS10 traffic start
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-27T18:20:42.282198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471825104598820:2049] [node 3] ICP22 created new session: [1:7577471825104598912:2049]
2025-11-27T18:20:42.290064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:42.296573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:42.296660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:42.295317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471826153864456:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39007
2025-11-27T18:20:42.292288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471824685924704:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:42.296714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471824845941841:2049] [node 2] ICP22 created new session: [6:7577471824845941932:2049]
2025-11-27T18:20:42.283062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471825104598912:2049] [node 3] ICS09 handshake done sender: [1:7577471825104598890:2111] self: [1:7577471825104598889:0] peer: [3:7577471824685924713:0] socket: 624 qp: -1
2025-11-27T18:20:42.296786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471825082187388:2108] [node 4] ICP22 created new session: [5:7577471825082187414:2108]
2025-11-27T18:20:42.296499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471826153864451:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.292776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471824685924631:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.297080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471824845941932:2049] [node 2] ICS09 handshake done sender: [6:7577471824845941918:2116] self: [6:7577471824845941924:0] peer: [2:7577471826153864432:0] socket: 643 qp: -1
2025-11-27T18:20:42.296991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471825082187410:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.292936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:42.297204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471824845941932:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:42.296982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471826153864456:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.292981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471824685924631:2049] [node 2] ICP22 created new session: [3:7577471824685924727:2049]
2025-11-27T18:20:42.283373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471825104598912:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:42.297215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471825082187414:2108] [node 4] ICS09 handshake done sender: [5:7577471825082187401:2115] self: [5:7577471825082187406:0] peer: [4:7577471826206544818:0] socket: 664 qp: -1
2025-11-27T18:20:42.297246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:42.297234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471826206544838:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57604
2025-11-27T18:20:42.297340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471824845941928:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59681
2025-11-27T18:20:42.293442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471824685924727:2049] [node 2] ICS09 handshake done sender: [3:7577471824685924709:2116] self: [3:7577471824685924715:0] peer: [2:7577471826153864437:0] socket: 648 qp: -1
2025-11-27T18:20:42.297182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471826153864458:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39007
2025-11-27T18:20:42.297396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471824845941929:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59681
2025-11-27T18:20:42.293816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471824685924727:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:42.297481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471826206544825:2115] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:42.297232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471825082187413:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13619
2025-11-27T18:20:42.297361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471826153864365:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.283400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471825104598893:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:42.293848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:42.294550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471824685924630:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.294598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:42.297383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471825082187414:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:42.294814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471824685924630:2048] [node 1] ICP22 created new session: [3:7577471824685924729:2048]
2025-11-27T18:20:42.295175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471824685924729:2048] [node 1] ICS09 handshake done sender: [3:7577471824685924704:2114] self: [3:7577471824685924713:0] peer: [1:7577471825104598889:0] socket: 627 qp: -1
2025-11-27T18:20:42.283407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:42.295926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471824685924729:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:42.297487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:42.298131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471826206544753:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.298275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:42.296149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:42.298635Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471824845941840:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471824845941913:2114] held: no
2025-11-27T18:20:42.298092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471825082187324:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.287627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471825104598884:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.287670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:42.288853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471825104598884:2110] [node 6] ICP22 created new session: [1:7577471825104598915:2110]
2025-11-27T18:20:42.298733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471826206544753:2050] [node 3] ICP22 created new session: [4:7577471826206544843:2050]
2025-11-27T18:20:42.290044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471825104598915:2110] [node 6] ICS09 handshake done sender: [1:7577471825104598893:2114] self: [1:7577471825104598888:0] peer: [6:7577471824845941922:0] socket: 626 qp: -1
2025-11-27T18:20:42.298819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471826206544843:2050] [node 3] ICS09 handshake done sender: [4:7577471826206544825:2115] self: [4:7577471826206544833:0] peer: [3:7577471824685924700:0] socket: 655 qp: -1
2025-11-27T18:20:42.290433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471825104598915:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:42.298830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471825082187412:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.291678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:42.298843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471826206544843:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:42.298541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471825104598917:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:100
2025-11-27T18:20:42.298832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825104598892:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.298874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:42.298846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471825104598892:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.299551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471825082187325:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.299591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7577471824685924631:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:42.299282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825104598898:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.299533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471826206544838:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.299317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471825104598898:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.299662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471826153864446:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.299745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471825104598883:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(18985776ns) {HandleHandshakeStatus:329(18983319ns) {DropIncomingHandshake:442(1326ns) DropOutgoingHandshake:457(1241ns) SwitchToState:179(477ns) ProcessPendingSessionEvents:495(14546658ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13877008ns max# 1248807ns}}}
2025-11-27T18:20:42.299688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471826153864446:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.301059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471825104598892:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:42.301961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471826206544752:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.302011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471825104598883:2107] [node 4] ICP18 handshake (actor [1:7577471825104598902:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.302287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471824845941917:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.302311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471824845941913:2114] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:42.302241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471826153864446:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.302790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471825104598819:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.302279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471826153864461:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39007
2025-11-27T18:20:42.303345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:42.303331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471825082187419:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13619
2025-11-27T18:20:42.302614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471826153864436:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.303790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471825082187418:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13619
2025-11-27T18:20:42.303791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471825104598819:2048] [node 2] ICP22 created new session: [1:7577471825104598920:2048]
2025-11-27T18:20:42.303854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471824845941840:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.304101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471825104598920:2048] [node 2] ICS09 handshake done sender: [1:7577471825104598892:2113] self: [1:7577471825104598886:0] peer: [2:7577471826153864459:0] socket: 679 qp: -1
2025-11-27T18:20:42.302634Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471826153864436:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:42.303899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:42.304134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471825104598920:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.304413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471825104598898:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:42.303484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471826153864458:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.304459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:42.303757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471826153864429:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.304380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471825082187412:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:42.304367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471824845941901:2107] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.303812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:42.304777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471825082187410:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:42.304393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471824845941840:2048] [node 1] ICP22 created new session: [6:7577471824845941937:2048]
2025-11-27T18:20:42.304497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:42.304532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471824845941901:2107] [node 5] ICP22 created new session: [6:7577471824845941938:2107]
2025-11-27T18:20:42.304771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471826206544846:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57604
2025-11-27T18:20:42.303849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471826153864429:2108] [node 5] ICP22 created new session: [2:7577471826153864464:2108]
2025-11-27T18:20:42.304711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471824845941934:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59681
2025-11-27T18:20:42.303922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471826153864451:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:42.304791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471824845941937:2048] [node 1] ICS09 handshake done sender: [6:7577471824845941913:2114] self: [6:7577471824845941922:0] peer: [1:7577471825104598888:0] socket: 628 qp: -1
2025-11-27T18:20:42.304303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471826153864464:2108] [node 5] ICS09 handshake done sender: [2:7577471826153864446:2116] self: [2:7577471826153864445:0] peer: [5:7577471825082187417:0] socket: 631 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.304325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471826153864464:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:42.304796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471824845941938:2107] [node 5] ICS09 handshake done sender: [6:7577471824845941917:2115] self: [6:7577471824845941923:0] peer: [5:7577471825082187390:0] socket: 636 qp: -1
2025-11-27T18:20:42.304355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:42.307024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471825082187325:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.304941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471824845941938:2107] [node 5] ICS10 traffic start
2025-11-27T18:20:42.307187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471826153864436:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:42.307426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471826206544847:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57604
2025-11-27T18:20:42.307012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471825104598885:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.307075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:42.305195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471824845941937:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:42.307291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471826153864365:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.307594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:42.305239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:42.307201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471825082187324:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.306672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:42.307272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471825082187325:2049] [node 2] ICP22 created new session: [5:7577471825082187420:2049]
2025-11-27T18:20:42.307625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471824845941934:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.307065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:42.307653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471826153864365:2048] [node 1] ICP22 created new session: [2:7577471826153864467:2048]
2025-11-27T18:20:42.307448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:42.307099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471825104598885:2109] [node 5] ICP22 created new session: [1:7577471825104598922:2109]
2025-11-27T18:20:42.307462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471825082187413:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.308836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471826153864467:2048] [node 1] ICS09 handshake done sender: [2:7577471826153864451:2117] self: [2:7577471826153864459:0] peer: [1:7577471825104598886:0] socket: 683 qp: -1
2025-11-27T18:20:42.308613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471826206544838:2119] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:42.307490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471825104598922:2109] [node 5] ICS09 handshake done sender: [1:7577471825104598898:2115] self: [1:7577471825104598897:0] peer: [5:7577471825082187416:0] socket: 685 qp: -1
2025-11-27T18:20:42.309507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471826206544752:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.307530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471825104598922:2109] [node 5] ICS10 traffic start
2025-11-27T18:20:42.307475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471825082187324:2048] [node 1] ICP22 created new session: [5:7577471825082187421:2048]
2025-11-27T18:20:42.308946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471826153864428:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.310122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:42.307512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471825082187421:2048] [node 1] ICS09 handshake done sender: [5:7577471825082187410:2118] self: [5:7577471825082187416:0] peer: [1:7577471825104598897:0] socket: 686 qp: -1
2025-11-27T18:20:42.308144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:42.308994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:42.310168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471826206544752:2049] [node 2] ICP22 created new session: [4:7577471826206544848:2049]
2025-11-27T18:20:42.307544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471825082187420:2049] [node 2] ICS09 handshake done sender: [5:7577471825082187412:2119] self: [5:7577471825082187417:0] peer: [2:7577471826153864445:0] socket: 632 qp: -1
2025-11-27T18:20:42.309200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471826153864428:2107] [node 4] ICP22 created new session: [2:7577471826153864468:2107]
2025-11-27T18:20:42.311380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471824845941902:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.309408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471826153864467:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:42.309455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:42.308102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471825082187421:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:42.311649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471826206544848:2049] [node 2] ICS09 handshake done sender: [4:7577471826206544838:2119] self: [4:7577471826206544845:0] peer: [2:7577471826153864434:0] socket: 687 qp: -1
2025-11-27T18:20:42.309855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471826153864468:2107] [node 4] ICS09 handshake done sender: [2:7577471826153864436:2112] self: [2:7577471826153864434:0] peer: [4:7577471826206544845:0] socket: 654 qp: -1
2025-11-27T18:20:42.308133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:42.311669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471826206544832:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.311695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471826206544848:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:42.310168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471826153864468:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:42.311696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471826206544832:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.308168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471825082187420:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:42.311725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:42.311271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:42.308410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:42.314167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471826206544846:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.314409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471826206544832:2117] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:42.314451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471825082187326:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.314628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471826206544815:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.314729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:42.314764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7577471825082187325:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:42.314764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471826206544815:2109] [node 6] ICP22 created new session: [4:7577471826206544852:2109]
2025-11-27T18:20:42.314800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471826206544852:2109] [node 6] ICS09 handshake done sender: [4:7577471826206544832:2117] self: [4:7577471826206544830:0] peer: [6:7577471824845941941:0] socket: 689 qp: -1
2025-11-27T18:20:42.314753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824685924703:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.315013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471826153864365:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-27T18:20:42.314880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471826206544852:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.314787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471824685924703:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.314906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:42.315438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471824845941942:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59681
2025-11-27T18:20:42.316818Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471824845941902:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1249) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471824845941934:2123] held: no
2025-11-27T18:20:42.317664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471824685924703:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.318137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471825082187425:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13619
2025-11-27T18:20:42.318162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471824685924694:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.318260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471824845941934:2123] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:42.318304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:42.318336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471824685924694:2108] [node 5] ICP22 created new session: [3:7577471824685924732:2108]
2025-11-27T18:20:42.318386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471824685924732:2108] [node 5] ICS09 handshake done sender: [3:7577471824685924703:2113] self: [3:7577471824685924702:0] peer: [5:7577471825082187424:0] socket: 635 qp: -1
2025-11-27T18:20:42.318478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471824845941902:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.318623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:42.318411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471824685924732:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:42.318669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471824845941902:2106] [node 4] ICP22 created new session: [6:7577471824845941943:2106]
2025-11-27T18:20:42.318447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:42.319189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471825082187413:2120] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:42.319216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471824845941943:2106] [node 4] ICS09 handshake done sender: [6:7577471824845941934:2123] self: [6:7577471824845941941:0] peer: [4:7577471826206544830:0] socket: 659 qp: -1
2025-11-27T18:20:42.319259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471824845941943:2106] [node 4] ICS10 traffic start
2025-11-27T18:20:42.319485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:42.320668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471825082187326:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28911) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471825082187413:2120] held: no
2025-11-27T18:20:42.321172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471825082187326:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.321204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:42.321230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471825082187326:2050] [node 3] ICP22 created new session: [5:7577471825082187426:2050]
2025-11-27T18:20:42.321278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471825082187426:2050] [node 3] ICS09 handshake done sender: [5:7577471825082187413:2120] self: [5:7577471825082187424:0] peer: [3:7577471824685924702:0] socket: 688 qp: -1
2025-11-27T18:20:42.321302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471825082187426:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:42.321326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:42.361885Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:42.362056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471826153864467:2048] [node 1] ICS07 socket disconnect 683 reason# EndOfStream
2025-11-27T18:20:42.362092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471826153864467:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.362155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:42.362194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471826153864467:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:42.363327Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:42.363548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471824685924729:2048] [node 1] ICS07 socket disconnect 627 reason# EndOfStream
2025-11-27T18:20:42.363583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471824685924729:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.363660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:42.363838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471824685924729:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:42.364807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471826153864365:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:42.364837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471826153864467:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:42.366045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471824685924630:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:42.366086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471824685924729:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:42.367040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:42.367177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471826153864365:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:42.367309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471826206544836:2048] [node 1] ICS07 socket disconnect 630 reason# EndOfStream
2025-11-27T18:20:42.367353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471826206544836:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.367295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471826153864467:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:42.367414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:42.367345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471826153864365:2048] [node 1] ICP30 unregister session Session# [2:7577471826153864467:2048] VirtualId# [2:7577471826153864459:0]
2025-11-27T18:20:42.367476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:42.367434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471824685924630:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:42.367571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471824685924729:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:42.367454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471826206544836:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:42.367525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471826153864365:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.367534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.367603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471824685924630:2048] [node 1] ICP30 unregister session Session# [3:7577471824685924729:2048] VirtualId# [3:7577471824685924713:0]
2025-11-27T18:20:42.367793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:42.367836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471824685924630:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.367859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.368183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471826206544751:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:42.368318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471826206544836:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:42.368360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471826206544751:2048] [node 1] ICP30 unregister session Session# [4:7577471826206544836:2048] VirtualId# [4:7577471826206544829:0]
2025-11-27T18:20:42.368390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:42.368417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471826206544751:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.368427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.368381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:42.368513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471825082187421:2048] [node 1] ICS07 socket disconnect 686 reason# EndOfStream
2025-11-27T18:20:42.368537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471825082187421:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.368592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:42.368629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471825082187421:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:42.369102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471825082187324:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:42.369211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471825082187421:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:42.369282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471825082187324:2048] [node 1] ICP30 unregister session Session# [5:7577471825082187421:2048] VirtualId# [5:7577471825082187416:0]
2025-11-27T18:20:42.369427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:42.369463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471825082187324:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.369472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.369621Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:42.369793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471824845941937:2048] [node 1] ICS01 socket: 628 reason# EndOfStream
2025-11-27T18:20:42.369821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471824845941840:2048] [node 1] ICP30 unregister session Session# [6:7577471824845941937:2048] VirtualId# [6:7577471824845941922:0]
2025-11-27T18:20:42.369841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471824845941937:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.369893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:42.369955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:42.377798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471824685924630:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:42.378529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471826206544751:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:42.379528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471825082187324:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:42.422643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:42.422901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7577471824685924727:2049] [node 2] ICS07 socket disconnect 648 reason# EndOfStream
2025-11-27T18:20:42.422936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471824685924727:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.423028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:42.423081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471824685924727:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:42.425527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471824685924631:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24472) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:20:42.425560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7577471824685924727:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:42.425461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:42.425605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7577471826206544848:2049] [node 2] ICS07 socket disconnect 687 reason# EndOfStream
2025-11-27T18:20:42.425626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471826206544848:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.425684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:42.425713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7577471826206544848:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:42.426601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471824685924631:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24472) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:42.426650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471826206544752:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24472) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:42.426638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471824685924727:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:42.426787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471826206544848:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:42.426696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471824685924631:2049] [node 2] ICP30 unregister session Session# [3:7577471824685924727:2049] VirtualId# [3:7577471824685924715:0]
2025-11-27T18:20:42.426852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:42.426899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471826206544752:2049] [node 2] ICP30 unregister session Session# [4:7577471826206544848:2049] VirtualId# [4:7577471826206544845:0]
2025-11-27T18:20:42.426992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:42.427006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:42.427020Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7577471826206544752:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24472) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.427030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24472) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.427047Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7577471824685924631:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24472) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.427037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471825082187420:2049] [node 2] ICS07 socket disconnect 632 reason# EndOfStream
2025-11-27T18:20:42.427065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24472) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.427058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471825082187420:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.427106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:42.427135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471825082187420:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:42.427790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471825082187325:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24472) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:42.427830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471825082187420:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:42.427880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471825082187325:2049] [node 2] ICP30 unregister session Session# [5:7577471825082187420:2049] VirtualId# [5:7577471825082187417:0]
2025-11-27T18:20:42.427959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:42.428002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:42.428039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471825082187325:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24472) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.428053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24472) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.428151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471824845941932:2049] [node 2] ICS01 socket: 643 reason# EndOfStream
2025-11-27T18:20:42.428196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471824845941841:2049] [node 2] ICP30 unregister session Session# [6:7577471824845941932:2049] VirtualId# [6:7577471824845941924:0]
2025-11-27T18:20:42.428239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471824845941932:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.428319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:42.428383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:42.436894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7577471824685924631:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:42.436896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7577471826206544752:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:42.437885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471825082187325:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:42.494774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:42.494942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471826206544843:2050] [node 3] ICS01 socket: 655 reason# EndOfStream
2025-11-27T18:20:42.494985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471826206544753:2050] [node 3] ICP30 unregister session Session# [4:7577471826206544843:2050] VirtualId# [4:7577471826206544833:0]
2025-11-27T18:20:42.495012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471826206544843:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.495073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:42.495107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:42.495709Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:42.495848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471825082187426:2050] [node 3] ICS01 socket: 688 reason# EndOfStream
2025-11-27T18:20:42.495877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471825082187326:2050] [node 3] ICP30 unregister session Session# [5:7577471825082187426:2050] VirtualId# [5:7577471825082187424:0]
2025-11-27T18:20:42.495899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471825082187426:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.495951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:42.495994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:42.496404Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:42.496571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471824845941930:2050] [node 3] ICS07 socket disconnect 657 reason# EndOfStream
2025-11-27T18:20:42.496603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471824845941930:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.496671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:42.496718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471824845941930:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:42.497201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471824845941842:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28911) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:42.497244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471824845941930:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:42.497298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471824845941842:2050] [node 3] ICP30 unregister session Session# [6:7577471824845941930:2050] VirtualId# [6:7577471824845941925:0]
2025-11-27T18:20:42.497398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:42.497426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471824845941842:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28911) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.497436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28911) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.507571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471824845941842:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:42.536097Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:42.536277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7577471825082187414:2108] [node 4] ICS07 socket disconnect 664 reason# EndOfStream
2025-11-27T18:20:42.536302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471825082187414:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.536362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:42.536395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7577471825082187414:2108] [node 4] ICS15 start handshake
2025-11-27T18:20:42.537001Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471825082187388:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1249) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:42.537084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471825082187414:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:42.537123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471825082187388:2108] [node 4] ICP30 unregister session Session# [5:7577471825082187414:2108] VirtualId# [5:7577471825082187406:0]
2025-11-27T18:20:42.537042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:42.537234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:42.537250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471824845941943:2106] [node 4] ICS01 socket: 659 reason# EndOfStream
2025-11-27T18:20:42.537277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7577471825082187388:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1249) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.537298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471824845941902:2106] [node 4] ICP30 unregister session Session# [6:7577471824845941943:2106] VirtualId# [6:7577471824845941941:0]
2025-11-27T18:20:42.537287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1249) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:42.537325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471824845941943:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.537421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:42.537471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:42.547468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7577471825082187388:2108] [node 4] ICP33 wake up from error state
2025-11-27T18:20:42.621048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:42.621318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471824845941938:2107] [node 5] ICS01 socket: 636 reason# EndOfStream
2025-11-27T18:20:42.621371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471824845941901:2107] [node 5] ICP30 unregister session Session# [6:7577471824845941938:2107] VirtualId# [6:7577471824845941923:0]
2025-11-27T18:20:42.621412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471824845941938:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.621489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:42.621544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:42.705370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471825541189117:2048] [node 2] ICP01 ready to work
2025-11-27T18:20:42.705598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471825541189118:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:42.723623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471824435912577:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:42.723668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471824435912578:2049] [node 3] ICP01 ready to work
2025-11-27T18:20:42.741250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471824476024586:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:42.741266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471824476024587:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:42.759048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471827193602480:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:42.759122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471827193602481:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:42.759256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471827193602482:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:42.777281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471827922976065:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:42.777466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471827922976066:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:42.777620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471827922976067:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:42.795948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471827100408580:2048] [node 1] ICP01 ready to work
2025-11-27T18:20:42.796009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471827100408581:2049] [node 2] ICP01 ready to work
2025-11-27T18:20:42.796056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471827100408582:2050] [node 3] ICP01 ready to work
2025-11-27T18:20:42.870902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471824435912641:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:42.877640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471825541189180:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:42.883078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471824476024649:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:42.884143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471827100408640:2106] [node 5] ICP01 ready to work
2025-11-27T18:20:42.884357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471825541189181:2107] [node 4] ICP01 ready to work
2025-11-27T18:20:42.884195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471827193602543:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:42.884611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471824435912642:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:42.884189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471827922976129:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:42.884156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471824476024650:2108] [node 5] ICP01 ready to work
2025-11-27T18:20:42.884195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7577471827100408641:2105] [node 4] ICP01 ready to work
2025-11-27T18:20:42.884561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471825541189182:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:42.884629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7577471827922976128:2108] [node 4] ICP01 ready to work
2025-11-27T18:20:42.884364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7577471824476024651:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:42.885572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7577471827193602544:2109] [node 6] ICP01 ready to work
2025-11-27T18:20:42.884910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471824435912643:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.898341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471824435912653:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35442
2025-11-27T18:20:42.898717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824476024661:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35921
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.900387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471824435912653:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.900412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824476024661:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.901057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471825541189193:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:594
2025-11-27T18:20:42.901339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471827922976143:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.901560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471824435912657:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35442
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.902006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824476024666:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35921
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.904115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471825541189198:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:594
2025-11-27T18:20:42.904598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471824435912659:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35442
2025-11-27T18:20:42.905484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471827100408655:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34580
2025-11-27T18:20:42.905866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825541189193:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.905854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471827193602558:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19227
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.906096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471825541189197:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:594
2025-11-27T18:20:42.905506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471827100408656:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34580
2025-11-27T18:20:42.906652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471824435912657:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.907270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471827193602558:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.906360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825541189198:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.906445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471827100408655:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.908232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471827193602480:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.906716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471827100408659:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34580
2025-11-27T18:20:42.907196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471827922976143:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.907193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825541189197:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.908325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824476024666:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.908764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471824435912661:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35442
2025-11-27T18:20:42.908217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471825541189199:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:594
2025-11-27T18:20:42.907837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471827922976065:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.906720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471827100408580:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.908792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471824476024587:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.910150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471824435912647:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.907789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471827100408657:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34580
2025-11-27T18:20:42.909082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471827922976144:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-27T18:20:42.908773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825541189187:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.912386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824476024657:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.908935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471827100408657:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.913997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471827193602550:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.910174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471824435912647:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.912406Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471824476024657:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.908792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471825541189187:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.912719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471824435912659:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.909095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471827100408659:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.914037Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471827193602550:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.909346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471827100408582:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.912897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471824435912652:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.912915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471824435912652:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.908907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825541189191:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.909675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471827100408656:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.909739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471827100408581:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.908925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471825541189191:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.914038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471824435912647:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-27T18:20:42.912908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471827100408641:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.909072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825541189189:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.915386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471827922976144:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-27T18:20:42.914629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471824435912652:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.909097Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471825541189189:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.909274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7577471825541189200:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:594
2025-11-27T18:20:42.915680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471824435912643:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.915836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471827100408666:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34580
2025-11-27T18:20:42.916174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471827193602550:2111] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:42.916120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471827922976146:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-27T18:20:42.915847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824476024669:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35921
2025-11-27T18:20:42.910206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825541189199:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.915880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471824435912661:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.916715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471827193602560:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19227
2025-11-27T18:20:42.917124Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471827922976065:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20994) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471827922976143:2115] held: no
2025-11-27T18:20:42.915878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471824476024657:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.912520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471825541189189:2111] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:42.915888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:42.918601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471827100408668:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34580
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.915567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471825541189180:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.918652Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471824476024587:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29322) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471824476024666:2116] held: no
2025-11-27T18:20:42.916128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471824435912578:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.919427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471827193602558:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:42.918905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471827100408655:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:42.918658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471824476024666:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:42.919984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471827193602544:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.915698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471825541189187:2110] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:42.916163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471824435912643:2110] [node 6] ICP22 created new session: [2:7577471824435912666:2110]
2025-11-27T18:20:42.920583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471827100408580:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.920283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471824476024651:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.915850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471825541189191:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:42.916365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.921488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471827100408667:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34580
2025-11-27T18:20:42.915863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:42.920026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:42.921070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:42.916764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471824435912578:2049] [node 3] ICP22 created new session: [2:7577471824435912667:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.916711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471825541189180:2108] [node 5] ICP22 created new session: [1:7577471825541189208:2108]
2025-11-27T18:20:42.921499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:42.921519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471827193602544:2109] [node 6] ICP22 created new session: [4:7577471827193602563:2109]
2025-11-27T18:20:42.917124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471825541189208:2108] [node 5] ICS09 handshake done sender: [1:7577471825541189189:2111] self: [1:7577471825541189184:0] peer: [5:7577471827922976145:0] socket: 647 qp: -1
2025-11-27T18:20:42.916872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471824435912666:2110] [node 6] ICS09 handshake done sender: [2:7577471824435912652:2114] self: [2:7577471824435912648:0] peer: [6:7577471827100408664:0] socket: 654 qp: -1
2025-11-27T18:20:42.917301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471825541189182:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.921494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471824476024651:2109] [node 6] ICP22 created new session: [3:7577471824476024672:2109]
2025-11-27T18:20:42.921977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471827100408669:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34580
2025-11-27T18:20:42.917153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471824435912666:2110] [node 6] ICS10 traffic start
2025-11-27T18:20:42.922045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471827193602480:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.917311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471825541189208:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:42.917154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471824435912667:2049] [node 3] ICS09 handshake done sender: [2:7577471824435912647:2111] self: [2:7577471824435912645:0] peer: [3:7577471824476024668:0] socket: 643 qp: -1
2025-11-27T18:20:42.923311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471824476024672:2109] [node 6] ICS09 handshake done sender: [3:7577471824476024657:2111] self: [3:7577471824476024653:0] peer: [6:7577471827100408663:0] socket: 648 qp: -1
2025-11-27T18:20:42.922058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471827100408656:2116] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:42.922066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471827193602563:2109] [node 6] ICS09 handshake done sender: [4:7577471827193602550:2111] self: [4:7577471827193602546:0] peer: [6:7577471827100408665:0] socket: 646 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.917411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825541189200:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.925768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471827922976148:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-27T18:20:42.917322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:42.922819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471827100408580:2048] [node 1] ICP22 created new session: [6:7577471827100408670:2048]
2025-11-27T18:20:42.922699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:42.923347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471824476024672:2109] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.925834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471827922976143:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:42.923481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:42.917341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471824435912667:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:42.917593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:42.923334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471827100408670:2048] [node 1] ICS09 handshake done sender: [6:7577471827100408655:2115] self: [6:7577471827100408662:0] peer: [1:7577471825541189186:0] socket: 638 qp: -1
2025-11-27T18:20:42.927113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471827922976065:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.925344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471824476024587:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.923974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471827100408670:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:42.917643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:42.923996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471827100408641:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.917683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471825541189182:2109] [node 6] ICP22 created new session: [1:7577471825541189209:2109]
2025-11-27T18:20:42.917786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:42.924006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:42.927137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:42.925370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:42.917778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471825541189181:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.927228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471827922976150:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-27T18:20:42.922848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471827193602563:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:42.925804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471824476024587:2049] [node 2] ICP22 created new session: [3:7577471824476024676:2049]
2025-11-27T18:20:42.924027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:42.926193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471824435912672:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35442
2025-11-27T18:20:42.917815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:42.926043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824476024671:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35921
2025-11-27T18:20:42.927385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471827922976065:2048] [node 1] ICP22 created new session: [5:7577471827922976152:2048]
2025-11-27T18:20:42.922870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471827193602480:2048] [node 1] ICP22 created new session: [4:7577471827193602564:2048]
2025-11-27T18:20:42.918625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471825541189209:2109] [node 6] ICS09 handshake done sender: [1:7577471825541189187:2110] self: [1:7577471825541189186:0] peer: [6:7577471827100408662:0] socket: 637 qp: -1
2025-11-27T18:20:42.928064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471824435912672:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.924064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471827100408641:2105] [node 4] ICP22 created new session: [6:7577471827100408672:2105]
2025-11-27T18:20:42.926077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471824476024676:2049] [node 2] ICS09 handshake done sender: [3:7577471824476024666:2116] self: [3:7577471824476024668:0] peer: [2:7577471824435912645:0] socket: 644 qp: -1
2025-11-27T18:20:42.927637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471827922976152:2048] [node 1] ICS09 handshake done sender: [5:7577471827922976143:2115] self: [5:7577471827922976145:0] peer: [1:7577471825541189184:0] socket: 650 qp: -1
2025-11-27T18:20:42.926953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471824476024676:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:42.927669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471827922976152:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:42.923350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:42.928231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471824435912577:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.927670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471827922976151:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-27T18:20:42.923981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471827193602564:2048] [node 1] ICS09 handshake done sender: [4:7577471827193602558:2115] self: [4:7577471827193602559:0] peer: [1:7577471825541189183:0] socket: 661 qp: -1
2025-11-27T18:20:42.918858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471825541189181:2107] [node 4] ICP22 created new session: [1:7577471825541189212:2107]
2025-11-27T18:20:42.927671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471827922976148:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.924562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471827193602564:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:42.918871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471825541189209:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:42.924947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471827100408657:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-11-27T18:20:42.926988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:42.927842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:42.924602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:42.919385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.921440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471825541189212:2107] [node 4] ICS09 handshake done sender: [1:7577471825541189191:2113] self: [1:7577471825541189183:0] peer: [4:7577471827193602559:0] socket: 652 qp: -1
2025-11-27T18:20:42.925223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471827100408672:2105] [node 4] ICS09 handshake done sender: [6:7577471827100408656:2116] self: [6:7577471827100408665:0] peer: [4:7577471827193602546:0] socket: 657 qp: -1
2025-11-27T18:20:42.928329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471827922976066:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.927461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824476024671:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.926059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471827193602562:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19227
2025-11-27T18:20:42.928530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471827922976150:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.925794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471827100408672:2105] [node 4] ICS10 traffic start
2025-11-27T18:20:42.921466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471825541189212:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:42.928942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7577471824435912643:2110] [node 6] ICP18 handshake (actor [2:7577471824435912661:2119]) is too old
2025-11-27T18:20:42.927881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471827193602562:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.928888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7577471824476024586:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.926080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:42.928924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471827922976067:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.928910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471827193602481:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.922667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:42.928949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471824435912654:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.928944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824476024678:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35921
2025-11-27T18:20:42.929315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471827922976151:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.926163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471827100408659:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:42.928950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471827193602568:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19227
2025-11-27T18:20:42.928948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825541189192:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.929389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824476024679:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35921
2025-11-27T18:20:42.928969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471824435912654:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.927609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471827100408582:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.928970Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471825541189192:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.930387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471827193602568:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.931375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7577471827922976128:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.929621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471824435912650:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.929658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824476024662:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.927652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:42.929507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471825541189190:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.929683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471824476024662:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.932027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7577471827193602482:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.929640Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7577471824435912650:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.927832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471827100408582:2050] [node 3] ICP22 created new session: [6:7577471827100408676:2050]
2025-11-27T18:20:42.932880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471824476024662:2115] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:42.929523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577471825541189190:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.932158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471824435912654:2116] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:42.928114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471827100408676:2050] [node 3] ICS09 handshake done sender: [6:7577471827100408657:2117] self: [6:7577471827100408663:0] peer: [3:7577471824476024653:0] socket: 659 qp: -1
2025-11-27T18:20:42.932633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7577471827193602553:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.933223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471825541189190:2112] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:42.933079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824476024659:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.932785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471824435912650:2112] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:42.928145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471827100408581:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.935049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471825541189192:2114] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:42.932659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7577471827193602553:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.933103Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7577471824476024659:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.933464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471824435912642:2108] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.935265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7577471825541189182:2109] [node 6] ICP18 handshake (actor [1:7577471825541189200:2119]) is too old
2025-11-27T18:20:42.928197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:42.932844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7577471827193602480:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.933497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:42.933299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7577471824476024679:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.928247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471827100408581:2049] [node 2] ICP22 created new session: [6:7577471827100408677:2049]
2025-11-27T18:20:42.937213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471825541189118:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.935698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471827193602553:2113] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:42.937873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471827922976158:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-27T18:20:42.935448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471824435912642:2108] [node 5] ICP22 created new session: [2:7577471824435912676:2108]
2025-11-27T18:20:42.935110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471824476024650:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.936775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7577471827193602543:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.928152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471827100408676:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:42.937293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:42.936628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471824435912641:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.928305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:42.937740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471825541189118:2049] [node 3] ICP22 created new session: [1:7577471825541189218:2049]
2025-11-27T18:20:42.936667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:42.938256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471827922976157:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-27T18:20:42.928840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471827100408677:2049] [node 2] ICS09 handshake done sender: [6:7577471827100408659:2118] self: [6:7577471827100408664:0] peer: [2:7577471824435912648:0] socket: 662 qp: -1
2025-11-27T18:20:42.938580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471825541189218:2049] [node 3] ICS09 handshake done sender: [1:7577471825541189190:2112] self: [1:7577471825541189185:0] peer: [3:7577471824476024680:0] socket: 645 qp: -1
2025-11-27T18:20:42.937713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:42.936701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471824435912641:2107] [node 4] ICP22 created new session: [2:7577471824435912678:2107]
2025-11-27T18:20:42.935267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:42.928883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471827100408677:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:42.938624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471825541189117:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.940447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471827922976067:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471827922976150:2119] held: no
2025-11-27T18:20:42.937897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471827193602543:2108] [node 5] ICP22 created new session: [4:7577471827193602574:2108]
2025-11-27T18:20:42.936701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471824435912676:2108] [node 5] ICS09 handshake done sender: [2:7577471824435912654:2116] self: [2:7577471824435912644:0] peer: [5:7577471827922976154:0] socket: 686 qp: -1
2025-11-27T18:20:42.935329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471824476024650:2108] [node 5] ICP22 created new session: [3:7577471824476024683:2108]
2025-11-27T18:20:42.928915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.936735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471824435912676:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:42.938684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:42.938594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471827193602574:2108] [node 5] ICS09 handshake done sender: [4:7577471827193602553:2113] self: [4:7577471827193602549:0] peer: [5:7577471827922976156:0] socket: 693 qp: -1
2025-11-27T18:20:42.938788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471825541189218:2049] [node 3] ICS10 traffic start
2025-11-27T18:20:42.936742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471824435912678:2107] [node 4] ICS09 handshake done sender: [2:7577471824435912650:2112] self: [2:7577471824435912646:0] peer: [4:7577471827193602570:0] socket: 635 qp: -1
2025-11-27T18:20:42.939918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471827193602574:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:42.940715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7577471827922976066:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29322) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7577471827922976148:2118] held: no
2025-11-27T18:20:42.939938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471825541189117:2048] [node 2] ICP22 created new session: [1:7577471825541189219:2048]
2025-11-27T18:20:42.935959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471824476024683:2108] [node 5] ICS09 handshake done sender: [3:7577471824476024662:2115] self: [3:7577471824476024660:0] peer: [5:7577471827922976155:0] socket: 690 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.939962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:42.932717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471827100408673:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34580
2025-11-27T18:20:42.936771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:42.940132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:42.937683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471824435912678:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:42.944378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7577471827922976139:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:42.935992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471824476024683:2108] [node 5] ICS10 traffic start
2025-11-27T18:20:42.935791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7577471827100408673:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.940386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471827193602572:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19227
2025-11-27T18:20:42.944408Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7577471827922976139:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-27T18:20:42.940142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471825541189219:2048] [node 2] ICS09 handshake done sender: [1:7577471825541189192:2114] self: [1:7577471825541189188:0] peer: [2:7577471824435912673:0] socket: 639 qp: -1
2025-11-27T18:20:42.937712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:42.940464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7577471827100408640:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:42.936018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:42.947437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7577471827922976159:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14456
2025-11-27T18:20:42.940732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7577471824435912578:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(16888038ns) {HandleHandshakeStatus:329(16817556ns) {DropIncomingHandshake:442(15326ns) DropOutgoingHandshake:457(317ns) SwitchToState:179(286ns) ProcessPendingSessionEvents:495(13063643ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12984877ns max# 1747015ns}}}
2025-11-27T18:20:42.940288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471825541189219:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:42.945085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471827193602575:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19227
2025-11-27T18:20:42.937141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471824476024659:2113] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:42.945307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577471824435912677:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35442
2025-11-27T18:20:42.938315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7577471824476024682:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35921
2025-11-27T18:20:42.946465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471827193602482:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471827193602568:2118] held: no
2025-11-27T18:20:42.940328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:42.947771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471824435912577:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20994) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7577471824435912672:2120] held: no
2025-11-27T18:20:42.946584Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7577471825541189181:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(18950489ns) {HandleHandshakeStatus:329(18947945ns) {DropIncomingHandshake:442(9686ns) DropOutgoingHandshake:457(214ns) SwitchToState:179(338ns) ProcessPendingSessionEvents:495(12160838ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11149711ns max# 1777231ns}}}
2025-11-27T18:20:42.938933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7577471824476024649:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.939967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:42.940163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471824476024649:2107] [node 4] ICP22 created new session: [3:7577471824476024687:2107]
2025-11-27T18:20:42.940706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471824476024687:2107] [node 4] ICS09 handshake done sender: [3:7577471824476024659:2113] self: [3:7577471824476024655:0] peer: [4:7577471827193602571:0] socket: 689 qp: -1
2025-11-27T18:20:42.941097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471824476024687:2107] [node 4] ICS10 traffic start
2025-11-27T18:20:42.948713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471827193602562:2117] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:42.942910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7577471824476024671:2118] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:42.942381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:42.949338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471827922976150:2119] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:42.945395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7577471824476024586:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20994) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7577471824476024671:2118] held: no
2025-11-27T18:20:42.946243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7577471824476024651:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16453901ns) {HandleHandshakeStatus:329(16451803ns) {DropIncomingHandshake:442(1311ns) DropOutgoingHandshake:457(995ns) SwitchToState:179(326ns) ProcessPendingSessionEvents:495(11479742ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10647300ns max# 1904088ns}}}
2025-11-27T18:20:42.949402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471827193602544:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(20975545ns) {HandleHandshakeStatus:329(20972995ns) {DropIncomingHandshake:442(513ns) DropOutgoingHandshake:457(296ns) SwitchToState:179(558ns) ProcessPendingSessionEvents:495(15389344ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14754502ns max# 1789580ns}}}
2025-11-27T18:20:42.949438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471827922976148:2118] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:42.949464Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7577471827100408641:2105] [node 4] ICP35 event processing took too much time PendingConnection:279(17123751ns) {HandleHandshakeStatus:329(17122116ns) {DropIncomingHandshake:442(545ns) DropOutgoingHandshake:457(3439ns) SwitchToState:179(221ns) ProcessPendingSessionEvents:495(13458451ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13375618ns max# 1775399ns}}}
2025-11-27T18:20:42.949854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7577471824476024586:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.949420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7577471827193602481:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29322) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7577471827193602562:2117] held: no
2025-11-27T18:20:42.950019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471827922976151:2120] [node 4] ICH04 handshake succeeded
2025-11-27T18:20:42.950223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7577471827193602568:2118] [node 3] ICH04 handshake succeeded
2025-11-27T18:20:42.951130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471824435912672:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:42.950023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:42.950161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471827922976067:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.951913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471827193602482:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.950073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7577471824476024586:2048] [node 1] ICP22 created new session: [3:7577471824476024689:2048]
2025-11-27T18:20:42.950197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:42.951450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471824435912577:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.950598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471827922976067:2050] [node 3] ICP22 created new session: [5:7577471827922976162:2050]
2025-11-27T18:20:42.951903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:42.950874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471827922976066:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.950620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7577471824476024689:2048] [node 1] ICS09 handshake done sender: [3:7577471824476024671:2118] self: [3:7577471824476024680:0] peer: [1:7577471825541189185:0] socket: 688 qp: -1
2025-11-27T18:20:42.953765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471824435912577:2048] [node 1] ICP22 created new session: [2:7577471824435912681:2048]
2025-11-27T18:20:42.950889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7577471827922976128:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.953932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471824435912681:2048] [node 1] ICS09 handshake done sender: [2:7577471824435912672:2120] self: [2:7577471824435912673:0] peer: [1:7577471825541189188:0] socket: 664 qp: -1
2025-11-27T18:20:42.950908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:42.953966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471824435912681:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:42.951014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471827922976162:2050] [node 3] ICS09 handshake done sender: [5:7577471827922976150:2119] self: [5:7577471827922976155:0] peer: [3:7577471824476024660:0] socket: 691 qp: -1
2025-11-27T18:20:42.951042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-27T18:20:42.954016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:42.951073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471827922976128:2108] [node 4] ICP22 created new session: [5:7577471827922976164:2108]
2025-11-27T18:20:42.951146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7577471827922976139:2114] [node 6] ICH04 handshake succeeded
2025-11-27T18:20:42.950958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471827922976066:2049] [node 2] ICP22 created new session: [5:7577471827922976163:2049]
2025-11-27T18:20:42.951555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471827922976162:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:42.951558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471827922976164:2108] [node 4] ICS09 handshake done sender: [5:7577471827922976151:2120] self: [5:7577471827922976156:0] peer: [4:7577471827193602549:0] socket: 694 qp: -1
2025-11-27T18:20:42.951986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:20:42.951993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471827922976164:2108] [node 4] ICS10 traffic start
2025-11-27T18:20:42.952086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471827922976163:2049] [node 2] ICS09 handshake done sender: [5:7577471827922976148:2118] self: [5:7577471827922976154:0] peer: [2:7577471824435912644:0] socket: 631 qp: -1
2025-11-27T18:20:42.952166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-11-27T18:20:42.953789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471827922976163:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:42.954256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7577471827100408681:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34580
2025-11-27T18:20:42.951041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7577471824476024689:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:42.953818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:42.953796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-27T18:20:42.954029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7577471827922976129:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-27T18:20:42.951948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:42.954130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-27T18:20:42.953983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471827193602482:2050] [node 3] ICP22 created new session: [4:7577471827193602578:2050]
2025-11-27T18:20:42.954030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7577471827193602481:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.954168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7577471827922976129:2109] [node 6] ICP22 created new session: [5:7577471827922976168:2109]
2025-11-27T18:20:42.954219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7577471827922976168:2109] [node 6] ICS09 handshake done sender: [5:7577471827922976139:2114] self: [5:7577471827922976136:0] peer: [6:7577471827100408680:0] socket: 665 qp: -1
2025-11-27T18:20:42.954042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471827193602578:2050] [node 3] ICS09 handshake done sender: [4:7577471827193602568:2118] self: [4:7577471827193602571:0] peer: [3:7577471824476024655:0] socket: 660 qp: -1
2025-11-27T18:20:42.955576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7577471827100408673:2123] [node 5] ICH04 handshake succeeded
2025-11-27T18:20:42.954250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7577471827922976168:2109] [node 6] ICS10 traffic start
2025-11-27T18:20:42.954074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471827193602578:2050] [node 3] ICS10 traffic start
2025-11-27T18:20:42.954109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-11-27T18:20:42.954131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:42.954277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-11-27T18:20:42.954172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7577471827193602481:2049] [node 2] ICP22 created new session: [4:7577471827193602580:2049]
2025-11-27T18:20:42.954221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7577471827193602580:2049] [node 2] ICS09 handshake done sender: [4:7577471827193602562:2117] self: [4:7577471827193602570:0] peer: [2:7577471824435912646:0] socket: 687 qp: -1
2025-11-27T18:20:42.957080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471827100408640:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30776) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7577471827100408673:2123] held: no
2025-11-27T18:20:42.954262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7577471827193602577:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19227
2025-11-27T18:20:42.954264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7577471827193602580:2049] [node 2] ICS10 traffic start
2025-11-27T18:20:42.954296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:42.957890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7577471827193602543:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17419507ns) {HandleHandshakeStatus:329(17416335ns) {DropIncomingHandshake:442(642ns) DropOutgoingHandshake:457(2778ns) SwitchToState:179(347ns) ProcessPendingSessionEvents:495(13083493ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12977380ns max# 2027080ns}}}
2025-11-27T18:20:42.957988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7577471827100408640:2106] [node 5] ICP19 incoming handshake succeeded
2025-11-27T18:20:42.958046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-27T18:20:42.958088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7577471827100408640:2106] [node 5] ICP22 created new session: [6:7577471827100408682:2106]
2025-11-27T18:20:42.958135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7577471827100408682:2106] [node 5] ICS09 handshake done sender: [6:7577471827100408673:2123] self: [6:7577471827100408680:0] peer: [5:7577471827922976136:0] socket: 692 qp: -1
2025-11-27T18:20:42.958181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7577471827100408682:2106] [node 5] ICS10 traffic start
2025-11-27T18:20:42.958218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-11-27T18:20:42.996725Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:42.996898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471824435912681:2048] [node 1] ICS01 socket: 664 reason# EndOfStream
2025-11-27T18:20:42.996930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471824435912577:2048] [node 1] ICP30 unregister session Session# [2:7577471824435912681:2048] VirtualId# [2:7577471824435912673:0]
2025-11-27T18:20:42.996953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471824435912681:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.997018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:42.997064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:42.997994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-27T18:20:42.998135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471824476024689:2048] [node 1] ICS01 socket: 688 reason# EndOfStream
2025-11-27T18:20:42.998163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471824476024586:2048] [node 1] ICP30 unregister session Session# [3:7577471824476024689:2048] VirtualId# [3:7577471824476024680:0]
2025-11-27T18:20:42.998184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471824476024689:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:42.998232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:42.998258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:43.000534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-27T18:20:43.000663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471827193602564:2048] [node 1] ICS01 socket: 661 reason# EndOfStream
2025-11-27T18:20:43.000690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471827193602480:2048] [node 1] ICP30 unregister session Session# [4:7577471827193602564:2048] VirtualId# [4:7577471827193602559:0]
2025-11-27T18:20:43.000708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471827193602564:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.000749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:43.000777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:43.001311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-27T18:20:43.001429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471827922976152:2048] [node 1] ICS01 socket: 650 reason# EndOfStream
2025-11-27T18:20:43.001460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471827922976065:2048] [node 1] ICP30 unregister session Session# [5:7577471827922976152:2048] VirtualId# [5:7577471827922976145:0]
2025-11-27T18:20:43.001475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471827922976152:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.001517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:43.001543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:43.002086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-27T18:20:43.002256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471827100408670:2048] [node 1] ICS07 socket disconnect 638 reason# EndOfStream
2025-11-27T18:20:43.002287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471827100408670:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.002367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:43.002405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471827100408670:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:43.002880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471827100408580:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20994) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:43.002980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471827100408670:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:43.003012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471827100408580:2048] [node 1] ICP30 unregister session Session# [6:7577471827100408670:2048] VirtualId# [6:7577471827100408662:0]
2025-11-27T18:20:43.003100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:43.003126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471827100408580:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20994) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:43.003136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20994) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:43.013410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471827100408580:2048] [node 1] ICP33 wake up from error state
2025-11-27T18:20:43.057804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-27T18:20:43.057965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7577471824476024676:2049] [node 2] ICS01 socket: 644 reason# EndOfStream
2025-11-27T18:20:43.058000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7577471824476024587:2049] [node 2] ICP30 unregister session Session# [3:7577471824476024676:2049] VirtualId# [3:7577471824476024668:0]
2025-11-27T18:20:43.058029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7577471824476024676:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.058088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:43.058123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:43.059717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-27T18:20:43.059902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471827193602580:2049] [node 2] ICS01 socket: 687 reason# EndOfStream
2025-11-27T18:20:43.059941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471827193602481:2049] [node 2] ICP30 unregister session Session# [4:7577471827193602580:2049] VirtualId# [4:7577471827193602570:0]
2025-11-27T18:20:43.059968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471827193602580:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.060033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:43.060131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:43.060535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-27T18:20:43.060741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471827922976163:2049] [node 2] ICS01 socket: 631 reason# EndOfStream
2025-11-27T18:20:43.060802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471827922976066:2049] [node 2] ICP30 unregister session Session# [5:7577471827922976163:2049] VirtualId# [5:7577471827922976154:0]
2025-11-27T18:20:43.060847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471827922976163:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.060919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:43.060960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:43.061216Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-27T18:20:43.061378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471827100408677:2049] [node 2] ICS07 socket disconnect 662 reason# EndOfStream
2025-11-27T18:20:43.061406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471827100408677:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.061455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:43.061486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471827100408677:2049] [node 2] ICS15 start handshake
2025-11-27T18:20:43.062068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471827100408581:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29322) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:43.062153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471827100408677:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:43.062185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471827100408581:2049] [node 2] ICP30 unregister session Session# [6:7577471827100408677:2049] VirtualId# [6:7577471827100408664:0]
2025-11-27T18:20:43.062259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:43.062287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471827100408581:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29322) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:43.062299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29322) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:43.071752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471827100408581:2049] [node 2] ICP33 wake up from error state
2025-11-27T18:20:43.111159Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-27T18:20:43.111340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7577471827193602578:2050] [node 3] ICS01 socket: 660 reason# EndOfStream
2025-11-27T18:20:43.111373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7577471827193602482:2050] [node 3] ICP30 unregister session Session# [4:7577471827193602578:2050] VirtualId# [4:7577471827193602571:0]
2025-11-27T18:20:43.111399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7577471827193602578:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.111456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:43.111490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:43.112038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-27T18:20:43.112200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471827922976162:2050] [node 3] ICS01 socket: 691 reason# EndOfStream
2025-11-27T18:20:43.112230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471827922976067:2050] [node 3] ICP30 unregister session Session# [5:7577471827922976162:2050] VirtualId# [5:7577471827922976155:0]
2025-11-27T18:20:43.112253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471827922976162:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.112301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:43.112335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:43.112814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-27T18:20:43.113014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471827100408676:2050] [node 3] ICS07 socket disconnect 659 reason# EndOfStream
2025-11-27T18:20:43.113045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471827100408676:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.113119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-11-27T18:20:43.113156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471827100408676:2050] [node 3] ICS15 start handshake
2025-11-27T18:20:43.113690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471827100408582:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20876) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:43.113750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471827100408676:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:43.113777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471827100408582:2050] [node 3] ICP30 unregister session Session# [6:7577471827100408676:2050] VirtualId# [6:7577471827100408663:0]
2025-11-27T18:20:43.113817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-11-27T18:20:43.113840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471827100408582:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20876) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:43.113854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20876) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:43.124046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471827100408582:2050] [node 3] ICP33 wake up from error state
2025-11-27T18:20:43.163219Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-27T18:20:43.163376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7577471827922976164:2108] [node 4] ICS01 socket: 694 reason# EndOfStream
2025-11-27T18:20:43.163421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7577471827922976128:2108] [node 4] ICP30 unregister session Session# [5:7577471827922976164:2108] VirtualId# [5:7577471827922976156:0]
2025-11-27T18:20:43.163446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7577471827922976164:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.163504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:43.163533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:43.164090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-27T18:20:43.164224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7577471827100408672:2105] [node 4] ICS07 socket disconnect 657 reason# EndOfStream
2025-11-27T18:20:43.164246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471827100408672:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.164302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-11-27T18:20:43.164332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7577471827100408672:2105] [node 4] ICS15 start handshake
2025-11-27T18:20:43.164922Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7577471827100408641:2105] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6987) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:20:43.165001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471827100408672:2105] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:20:43.165037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471827100408641:2105] [node 4] ICP30 unregister session Session# [6:7577471827100408672:2105] VirtualId# [6:7577471827100408665:0]
2025-11-27T18:20:43.165071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-11-27T18:20:43.165095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7577471827100408641:2105] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6987) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:43.165108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6987) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:20:43.175181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7577471827100408641:2105] [node 4] ICP33 wake up from error state
2025-11-27T18:20:43.202371Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-27T18:20:43.202528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7577471827100408682:2106] [node 5] ICS01 socket: 692 reason# EndOfStream
2025-11-27T18:20:43.202579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7577471827100408640:2106] [node 5] ICP30 unregister session Session# [6:7577471827100408682:2106] VirtualId# [6:7577471827100408680:0]
2025-11-27T18:20:43.202614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7577471827100408682:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.202673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-11-27T18:20:43.202707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-27T18:20:43.336839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471828332131888:2048] [node 2] ICP01 ready to work
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:20:43.352864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471829304041991:2048] [node 1] ICP01 ready to work
Subscribe
2025-11-27T18:20:43.357587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471828332131897:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:43.359061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:43.359353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471829304042001:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:43.361619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471828332131897:2056] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:43.361682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471829304042001:2056] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:43.361813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:43.361848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:43.361889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471828332131900:2048]
2025-11-27T18:20:43.361935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471828332131900:2048] [node 2] ICS09 handshake done sender: [1:7577471828332131897:2056] self: [1:7577471828332131898:0] peer: [2:7577471829304042000:0] socket: 17 qp: 5261
2025-11-27T18:20:43.361964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:43.361961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471828332131900:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:43.361987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:43.361993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:43.362020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471829304042004:2048]
2025-11-27T18:20:43.362046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471829304042004:2048] [node 1] ICS09 handshake done sender: [2:7577471829304042001:2056] self: [2:7577471829304042000:0] peer: [1:7577471828332131898:0] socket: 16 qp: 5262
2025-11-27T18:20:43.362068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471829304042004:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:43.362087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:20:43.745939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471828332131900:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-27T18:20:43.746026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471828332131900:2048] VirtualId# [1:7577471828332131898:0]
2025-11-27T18:20:43.746058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471828332131900:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:43.746115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:43.746166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:43.746199Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:43.746428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471829304042004:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:20:43.746505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471829304042004:2048] VirtualId# [2:7577471829304042000:0]
2025-11-27T18:20:43.746551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471829304042004:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:43.746626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:43.746686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:43.749803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471829304042006:2057] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:43.749971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:43.750321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471828332131903:2058] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:43.753049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471829304042006:2057] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:43.753116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471828332131903:2058] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:43.753310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:43.753363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:43.753423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471829304042009:2048]
2025-11-27T18:20:43.753469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471829304042009:2048] [node 1] ICS09 handshake done sender: [2:7577471829304042006:2057] self: [2:7577471829304042007:0] peer: [1:7577471828332131902:0] socket: 17 qp: 5264
2025-11-27T18:20:43.753477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:43.753542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:43.753589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471828332131906:2048]
2025-11-27T18:20:43.753506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471829304042009:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:43.753627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471828332131906:2048] [node 2] ICS09 handshake done sender: [1:7577471828332131903:2058] self: [1:7577471828332131902:0] peer: [2:7577471829304042007:0] socket: 16 qp: 5263
2025-11-27T18:20:43.753546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:43.753661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471828332131906:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:43.753698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:20:44.214249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:20:44.214354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:44.214570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471829304042009:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:20:44.214609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471829304042009:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:44.214685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:44.214733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471829304042009:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:44.214834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:44.215039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-27T18:20:44.215072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471828332131906:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:44.215094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471828332131906:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:44.215126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:44.215200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471828332131906:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:44.215265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471828332131906:2048] VirtualId# [1:7577471828332131902:0]
2025-11-27T18:20:44.215293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:44.218871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471832627099204:2059]) peer tries to resume nonexistent session Self# [2:7577471829304042007:0] Peer# [1:7577471828332131902:0]
2025-11-27T18:20:44.218939Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471832627099204:2059] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:44.219127Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471833599009307:2059] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:44.219812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:44.219874Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:44.219895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471829304042009:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:44.219955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471829304042009:2048] VirtualId# [2:7577471829304042007:0]
2025-11-27T18:20:44.220026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:44.223634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471833599009311:2061] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:44.223832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:44.223822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471832627099208:2061] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:44.224026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471832627099206:2060] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:44.226570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471833599009311:2061] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:44.226643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471832627099206:2060] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:44.226728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:44.226764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:44.226808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471833599009315:2048]
2025-11-27T18:20:44.226852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:44.226850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471833599009315:2048] [node 1] ICS09 handshake done sender: [2:7577471833599009311:2061] self: [2:7577471833599009313:0] peer: [1:7577471832627099205:0] socket: 17 qp: 5267
2025-11-27T18:20:44.226873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471833599009315:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:44.226897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:44.226897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:44.226956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471832627099211:2048]
2025-11-27T18:20:44.226996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471832627099211:2048] [node 2] ICS09 handshake done sender: [1:7577471832627099206:2060] self: [1:7577471832627099205:0] peer: [2:7577471833599009313:0] socket: 16 qp: 5266
2025-11-27T18:20:44.227024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471832627099211:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:44.227056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:20:44.524164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:20:44.524354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471833599009315:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-27T18:20:44.524380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471833599009315:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:44.524448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:44.524490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471833599009315:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:44.524556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:44.524766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471832627099211:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:44.524792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471832627099211:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:44.524812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:44.524839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471832627099211:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:44.526530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471832627099211:2048] [node 2] ICS08 incoming handshake Self# [2:7577471833599009313:0] Peer# [1:7577471832627099205:0] Counter# 43 LastInputSerial# 1732
2025-11-27T18:20:44.527966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471833599009315:2048] [node 1] ICS08 incoming handshake Self# [1:7577471832627099205:0] Peer# [2:7577471833599009313:0] Counter# 1732 LastInputSerial# 43
2025-11-27T18:20:44.528265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471832627099213:2062] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:44.528547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:44.528598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471832627099211:2048] [node 2] ICS09 handshake done sender: [1:7577471832627099213:2062] self: [1:7577471832627099205:0] peer: [2:7577471833599009313:0] socket: 24 qp: 5273
2025-11-27T18:20:44.529026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471832627099211:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:44.529063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:44.531034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471833599009319:2064] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:44.531173Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471833599009319:2064] held: no
2025-11-27T18:20:44.531204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:44.531230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471833599009315:2048] [node 1] ICS09 handshake done sender: [2:7577471833599009319:2064] self: [2:7577471833599009313:0] peer: [1:7577471832627099205:0] socket: 25 qp: 5272
2025-11-27T18:20:44.531702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471833599009315:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:44.531734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:20:44.830270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471832627099211:2048] [node 2] ICS01 socket: 24 reason# Debug
2025-11-27T18:20:44.830357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471832627099211:2048] VirtualId# [1:7577471832627099205:0]
2025-11-27T18:20:44.830392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471832627099211:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:44.830446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:44.830517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:44.830513Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:44.830697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471833599009315:2048] [node 1] ICS01 socket: 25 reason# EndOfStream
2025-11-27T18:20:44.830760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471833599009315:2048] VirtualId# [2:7577471833599009313:0]
2025-11-27T18:20:44.830798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471833599009315:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:44.830861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:44.830913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:44.833908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471833599009322:2066] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:44.834052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:44.834338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471832627099219:2064] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:44.836560Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471833599009322:2066] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:44.836706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:44.836746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:44.836791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471833599009325:2048]
2025-11-27T18:20:44.836771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471832627099219:2064] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:44.836831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471833599009325:2048] [node 1] ICS09 handshake done sender: [2:7577471833599009322:2066] self: [2:7577471833599009323:0] peer: [1:7577471832627099218:0] socket: 17 qp: 5275
2025-11-27T18:20:44.836863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471833599009325:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:44.836896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:44.837088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:44.837133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:44.837175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471832627099222:2048]
2025-11-27T18:20:44.837224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471832627099222:2048] [node 2] ICS09 handshake done sender: [1:7577471832627099219:2064] self: [1:7577471832627099218:0] peer: [2:7577471833599009323:0] socket: 16 qp: 5274
2025-11-27T18:20:44.837250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471832627099222:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:44.837296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:20:45.059363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:20:45.059502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:45.059507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:45.059703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471833599009325:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:20:45.059726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471832627099222:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:45.059754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471832627099222:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:45.059785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471833599009325:2048] VirtualId# [2:7577471833599009323:0]
2025-11-27T18:20:45.059823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471833599009325:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:45.059776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:45.059802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471832627099222:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:45.059854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:45.059927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:45.062138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471837893976623:2068]) peer tries to resume nonexistent session Self# [1:7577471832627099218:0] Peer# [2:7577471833599009323:0]
2025-11-27T18:20:45.062205Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471837893976623:2068] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:45.062411Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471836922066520:2065] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:20:45.064274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:45.064346Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:45.064365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471832627099222:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:45.064418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471832627099222:2048] VirtualId# [1:7577471832627099218:0]
2025-11-27T18:20:45.064464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:45.067115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471837893976624:2069] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:45.067229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:45.067454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471836922066523:2066] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:45.069736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471837893976624:2069] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:45.069844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471836922066523:2066] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:45.069953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:45.069999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:45.070045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471837893976627:2048]
2025-11-27T18:20:45.070100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471837893976627:2048] [node 1] ICS09 handshake done sender: [2:7577471837893976624:2069] self: [2:7577471837893976625:0] peer: [1:7577471836922066522:0] socket: 17 qp: 5278
2025-11-27T18:20:45.070139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471837893976627:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:45.070160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:45.070166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:45.070197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:45.070225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471836922066526:2048]
2025-11-27T18:20:45.070255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471836922066526:2048] [node 2] ICS09 handshake done sender: [1:7577471836922066523:2066] self: [1:7577471836922066522:0] peer: [2:7577471837893976625:0] socket: 16 qp: 5277
2025-11-27T18:20:45.070277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471836922066526:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:45.070296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:20:45.349500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:20:45.349700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471837893976627:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-27T18:20:45.349734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471837893976627:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:45.349809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:45.349850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471837893976627:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:45.349861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:45.350079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471836922066526:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:45.350115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471836922066526:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:45.350146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:45.350206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471836922066526:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:45.352220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471836922066526:2048] [node 2] ICS08 incoming handshake Self# [2:7577471837893976625:0] Peer# [1:7577471836922066522:0] Counter# 52 LastInputSerial# 2236
2025-11-27T18:20:45.353903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471837893976627:2048] [node 1] ICS08 incoming handshake Self# [1:7577471836922066522:0] Peer# [2:7577471837893976625:0] Counter# 2236 LastInputSerial# 52
2025-11-27T18:20:45.354299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471836922066528:2067] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:45.354568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:45.354629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471836922066526:2048] [node 2] ICS09 handshake done sender: [1:7577471836922066528:2067] self: [1:7577471836922066522:0] peer: [2:7577471837893976625:0] socket: 24 qp: 5282
2025-11-27T18:20:45.355124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471836922066526:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:45.355174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:45.357794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471837893976631:2072] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:45.357971Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471837893976631:2072] held: no
2025-11-27T18:20:45.358006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:45.358056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471837893976627:2048] [node 1] ICS09 handshake done sender: [2:7577471837893976631:2072] self: [2:7577471837893976625:0] peer: [1:7577471836922066522:0] socket: 25 qp: 5281
2025-11-27T18:20:45.358650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471837893976627:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:45.358699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-27T18:20:45.862676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471837893976627:2048] [node 1] ICS01 socket: 25 reason# Debug
2025-11-27T18:20:45.862792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471837893976627:2048] VirtualId# [2:7577471837893976625:0]
2025-11-27T18:20:45.862835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471837893976627:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:45.862906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:45.862997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:45.863619Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:45.863973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471836922066526:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
TEvNodeDisconnected
2025-11-27T18:20:45.864013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471836922066526:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:45.864041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:45.864111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471836922066526:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:45.866042Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471837893976634:2074]) peer tries to resume nonexistent session Self# [1:7577471836922066522:0] Peer# [2:7577471837893976625:0]
2025-11-27T18:20:45.866103Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471837893976634:2074] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:45.866263Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471836922066533:2069] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:20:45.867081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:45.867164Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:45.867195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471836922066526:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:45.867253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471836922066526:2048] VirtualId# [1:7577471836922066522:0]
2025-11-27T18:20:45.867299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:45.869789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471837893976635:2075] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:45.869905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:45.870206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471836922066536:2070] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:45.872738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471837893976635:2075] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:45.872847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471836922066536:2070] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:45.872898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:45.872936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:45.872967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471837893976638:2048]
2025-11-27T18:20:45.873000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471837893976638:2048] [node 1] ICS09 handshake done sender: [2:7577471837893976635:2075] self: [2:7577471837893976636:0] peer: [1:7577471836922066535:0] socket: 17 qp: 5285
2025-11-27T18:20:45.873031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471837893976638:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:45.873061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:45.873119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:45.873155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:45.873200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471836922066539:2048]
2025-11-27T18:20:45.873241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471836922066539:2048] [node 2] ICS09 handshake done sender: [1:7577471836922066536:2070] self: [1:7577471836922066535:0] peer: [2:7577471837893976636:0] socket: 16 qp: 5284
2025-11-27T18:20:45.873279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471836922066539:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:45.873300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:20:46.057745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:20:46.057968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471837893976638:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-27T18:20:46.058003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471837893976638:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:46.058068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:46.058115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471837893976638:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:46.058175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:46.058412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471836922066539:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:46.058448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471836922066539:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:46.058523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:46.058572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471836922066539:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:46.060615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471836922066539:2048] [node 2] ICS08 incoming handshake Self# [2:7577471837893976636:0] Peer# [1:7577471836922066535:0] Counter# 25 LastInputSerial# 1090
2025-11-27T18:20:46.062154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471837893976638:2048] [node 1] ICS08 incoming handshake Self# [1:7577471836922066535:0] Peer# [2:7577471837893976636:0] Counter# 1090 LastInputSerial# 25
2025-11-27T18:20:46.062558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471841217033837:2071] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:46.062824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:46.062886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471836922066539:2048] [node 2] ICS09 handshake done sender: [1:7577471841217033837:2071] self: [1:7577471836922066535:0] peer: [2:7577471837893976636:0] socket: 24 qp: 5289
2025-11-27T18:20:46.063355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471836922066539:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:46.063400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:46.065358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471842188943938:2078] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:46.065524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471842188943938:2078] held: no
2025-11-27T18:20:46.065551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:46.065583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471837893976638:2048] [node 1] ICS09 handshake done sender: [2:7577471842188943938:2078] self: [2:7577471837893976636:0] peer: [1:7577471836922066535:0] socket: 25 qp: 5288
2025-11-27T18:20:46.066109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471837893976638:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:46.066147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:20:46.260848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471836922066539:2048] [node 2] ICS01 socket: 24 reason# Debug
2025-11-27T18:20:46.260945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471836922066539:2048] VirtualId# [1:7577471836922066535:0]
2025-11-27T18:20:46.260979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471836922066539:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:46.261022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:46.261073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:46.262969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:46.263149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471837893976638:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-11-27T18:20:46.263175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471837893976638:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:46.263245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:46.263284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471837893976638:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:46.265399Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471841217033842:2073]) peer tries to resume nonexistent session Self# [2:7577471837893976636:0] Peer# [1:7577471836922066535:0]
2025-11-27T18:20:46.265476Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471841217033842:2073] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:46.265611Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471842188943941:2080] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:46.266314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:46.266381Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:46.266409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471837893976638:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:46.266459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471837893976638:2048] VirtualId# [2:7577471837893976636:0]
2025-11-27T18:20:46.266586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 5625
TEvUndelivered Cookie# 5626
TEvUndelivered Cookie# 5627
TEvUndelivered Cookie# 5628
TEvUndelivered Cookie# 5629
TEvUndelivered Cookie# 5630
TEvUndelivered Cookie# 5631
TEvUndelivered Cookie# 5632
TEvUndelivered Cookie# 5633
TEvUndelivered Cookie# 5634
2025-11-27T18:20:46.269278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471841217033843:2074] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:46.269446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:46.269683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471842188943944:2081] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:46.271962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471841217033843:2074] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:46.272026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471842188943944:2081] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:46.272232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:46.272295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:46.272316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:46.272344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471841217033846:2048]
2025-11-27T18:20:46.272347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:46.272376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471842188943947:2048]
2025-11-27T18:20:46.272386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471841217033846:2048] [node 2] ICS09 handshake done sender: [1:7577471841217033843:2074] self: [1:7577471841217033844:0] peer: [2:7577471842188943943:0] socket: 17 qp: 5292
2025-11-27T18:20:46.272404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471842188943947:2048] [node 1] ICS09 handshake done sender: [2:7577471842188943944:2081] self: [2:7577471842188943943:0] peer: [1:7577471841217033844:0] socket: 16 qp: 5291
2025-11-27T18:20:46.272428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471841217033846:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:46.272427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471842188943947:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:46.272472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:46.272474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:20:46.578137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:20:46.578231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-27T18:20:46.578251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471841217033846:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:46.578272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471841217033846:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:46.578295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:46.578341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:46.578476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471841217033846:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:46.578547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471841217033846:2048] VirtualId# [1:7577471841217033844:0]
2025-11-27T18:20:46.578574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:46.578665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:46.578974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471842188943947:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:46.579001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471842188943947:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:46.579025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:46.579059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471842188943947:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:46.581096Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471841217033848:2076]) peer tries to resume nonexistent session Self# [2:7577471842188943943:0] Peer# [1:7577471841217033844:0]
2025-11-27T18:20:46.581224Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471841217033848:2076] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:46.581348Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471842188943949:2082] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:46.582006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:46.582052Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:46.582074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471842188943947:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:46.582122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471842188943947:2048] VirtualId# [2:7577471842188943943:0]
2025-11-27T18:20:46.582190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 6312
TEvUndelivered Cookie# 6313
TEvUndelivered Cookie# 6314
2025-11-27T18:20:46.584377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471842188943951:2083] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:46.586114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:46.586370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471841217033850:2077] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:46.586515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471841217033852:2078] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:46.588976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471842188943951:2083] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:46.589069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471841217033850:2077] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:46.589208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:46.589247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:46.589297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471842188943957:2048]
2025-11-27T18:20:46.589335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471842188943957:2048] [node 1] ICS09 handshake done sender: [2:7577471842188943951:2083] self: [2:7577471842188943955:0] peer: [1:7577471841217033849:0] socket: 23 qp: 5295
2025-11-27T18:20:46.589360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471842188943957:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:46.589369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:46.589385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:46.589411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:46.589439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471841217033855:2048]
2025-11-27T18:20:46.589469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471841217033855:2048] [node 2] ICS09 handshake done sender: [1:7577471841217033850:2077] self: [1:7577471841217033849:0] peer: [2:7577471842188943955:0] socket: 22 qp: 5294
2025-11-27T18:20:46.589493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471841217033855:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:46.589518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-27T18:20:46.988160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471842188943957:2048] [node 1] ICS01 socket: 23 reason# Debug
2025-11-27T18:20:46.988256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471842188943957:2048] VirtualId# [2:7577471842188943955:0]
2025-11-27T18:20:46.988281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471842188943957:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:46.988333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:46.988373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:46.988389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:46.988559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471841217033855:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-27T18:20:46.988583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471841217033855:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:46.988637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:46.988669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471841217033855:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-27T18:20:46.990800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471842188943959:2086]) peer tries to resume nonexistent session Self# [1:7577471841217033849:0] Peer# [2:7577471842188943955:0]
2025-11-27T18:20:46.990844Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471842188943959:2086] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:46.990942Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471841217033857:2079] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:20:46.991586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:46.991636Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:46.991656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471841217033855:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:46.991708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471841217033855:2048] VirtualId# [1:7577471841217033849:0]
2025-11-27T18:20:46.991746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:46.994039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471842188943960:2087] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:46.994142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:46.994338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471841217033860:2080] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:46.996514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471842188943960:2087] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:46.996586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471841217033860:2080] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:46.996725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:46.996812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:46.996849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471842188943963:2048]
2025-11-27T18:20:46.996848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:46.996880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471842188943963:2048] [node 1] ICS09 handshake done sender: [2:7577471842188943960:2087] self: [2:7577471842188943961:0] peer: [1:7577471841217033859:0] socket: 17 qp: 5300
2025-11-27T18:20:46.996877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:46.996903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471841217033863:2048]
2025-11-27T18:20:46.996901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471842188943963:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:46.996943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:46.996943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471841217033863:2048] [node 2] ICS09 handshake done sender: [1:7577471841217033860:2080] self: [1:7577471841217033859:0] peer: [2:7577471842188943961:0] socket: 16 qp: 5299
2025-11-27T18:20:46.996973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471841217033863:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:46.997006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:20:47.586396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:20:47.586673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471842188943963:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-27T18:20:47.586699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471842188943963:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:47.586753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:47.586796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471842188943963:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:47.587059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:47.587356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471841217033863:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:47.587387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471841217033863:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:47.587417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:47.587463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471841217033863:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:47.589665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471841217033863:2048] [node 2] ICS08 incoming handshake Self# [2:7577471842188943961:0] Peer# [1:7577471841217033859:0] Counter# 145 LastInputSerial# 5927
2025-11-27T18:20:47.591103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471842188943963:2048] [node 1] ICS08 incoming handshake Self# [1:7577471841217033859:0] Peer# [2:7577471842188943961:0] Counter# 5927 LastInputSerial# 145
2025-11-27T18:20:47.591456Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471845512001161:2081] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:47.591721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:47.591779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471841217033863:2048] [node 2] ICS09 handshake done sender: [1:7577471845512001161:2081] self: [1:7577471841217033859:0] peer: [2:7577471842188943961:0] socket: 24 qp: 5304
2025-11-27T18:20:47.592292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471841217033863:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:47.592337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:47.594346Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471846483911263:2090] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:47.594488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471846483911263:2090] held: no
2025-11-27T18:20:47.594512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:47.594540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471842188943963:2048] [node 1] ICS09 handshake done sender: [2:7577471846483911263:2090] self: [2:7577471842188943961:0] peer: [1:7577471841217033859:0] socket: 25 qp: 5303
2025-11-27T18:20:47.595033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471842188943963:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:47.595070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:20:47.921392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:20:47.921502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:47.921705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471841217033863:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-27T18:20:47.921735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471841217033863:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:47.921758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:47.921790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471841217033863:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:47.922618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:47.922772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471842188943963:2048] [node 1] ICS01 socket: 25 reason# EndOfStream
2025-11-27T18:20:47.922821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471842188943963:2048] VirtualId# [2:7577471842188943961:0]
2025-11-27T18:20:47.922840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471842188943963:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:47.922883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:47.922913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:47.926288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471846483911266:2092]) peer tries to resume nonexistent session Self# [1:7577471841217033859:0] Peer# [2:7577471842188943961:0]
2025-11-27T18:20:47.926372Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471846483911266:2092] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:47.926504Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471845512001166:2083] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:20:47.927271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:47.927342Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:47.927372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471841217033863:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:47.927429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471841217033863:2048] VirtualId# [1:7577471841217033859:0]
2025-11-27T18:20:47.927481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:47.931396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471845512001170:2085] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:47.931820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471846483911270:2094] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:47.931969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:47.932245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471845512001169:2084] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:47.935335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471846483911270:2094] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:47.935441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471845512001169:2084] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:47.935611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:47.935665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:47.935703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471846483911273:2048]
2025-11-27T18:20:47.935740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471846483911273:2048] [node 1] ICS09 handshake done sender: [2:7577471846483911270:2094] self: [2:7577471846483911271:0] peer: [1:7577471845512001168:0] socket: 21 qp: 5308
2025-11-27T18:20:47.935762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:47.935766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471846483911273:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:47.935809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:47.935792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:47.935839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471845512001174:2048]
2025-11-27T18:20:47.935868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471845512001174:2048] [node 2] ICS09 handshake done sender: [1:7577471845512001169:2084] self: [1:7577471845512001168:0] peer: [2:7577471846483911271:0] socket: 20 qp: 5309
2025-11-27T18:20:47.935898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471845512001174:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:47.935931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:20:48.267541Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:20:48.267702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471845512001174:2048] [node 2] ICS07 socket disconnect 20 reason# Debug
2025-11-27T18:20:48.267727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471845512001174:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:48.267772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:48.267803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471845512001174:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:48.267891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:48.268127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:48.268160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471846483911273:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:48.268183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471846483911273:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:48.268220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:48.268272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471846483911273:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:48.268289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471846483911273:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:48.270059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471846483911273:2048] [node 1] ICS08 incoming handshake Self# [1:7577471845512001168:0] Peer# [2:7577471846483911271:0] Counter# 3101 LastInputSerial# 83
2025-11-27T18:20:48.271402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471849806968472:2086] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:48.271657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:48.271694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471845512001174:2048] [node 2] ICS09 handshake done sender: [1:7577471849806968472:2086] self: [1:7577471845512001168:0] peer: [2:7577471846483911271:0] socket: 16 qp: 5310
2025-11-27T18:20:48.272114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471845512001174:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:48.272148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:48.272224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471845512001174:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:48.272246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471845512001174:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:20:48.272282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:48.272316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471845512001174:2048] [node 2] ICS08 incoming handshake Self# [2:7577471846483911271:0] Peer# [1:7577471845512001168:0] Counter# 83 LastInputSerial# 3101
2025-11-27T18:20:48.272364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:48.272478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471845512001174:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:48.272519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471845512001174:2048] VirtualId# [1:7577471845512001168:0]
2025-11-27T18:20:48.272537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:48.273505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471850778878572:2097] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:48.273715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:48.273772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471846483911273:2048] [node 1] ICS09 handshake done sender: [2:7577471850778878572:2097] self: [2:7577471846483911271:0] peer: [1:7577471845512001168:0] socket: 17 qp: 5311
2025-11-27T18:20:48.274392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471846483911273:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:48.274441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:48.274764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:48.274801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471846483911273:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:48.274825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471846483911273:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:48.274853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:48.274928Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:48.275096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471846483911273:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:48.275122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471846483911273:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:48.278068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471849806968477:2088]) peer tries to resume nonexistent session Self# [2:7577471846483911271:0] Peer# [1:7577471845512001168:0]
2025-11-27T18:20:48.278108Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471849806968477:2088] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:48.278280Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471850778878576:2099] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:48.279672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:48.279744Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:48.279774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471846483911273:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:48.279834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471846483911273:2048] VirtualId# [2:7577471846483911271:0]
2025-11-27T18:20:48.279917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 10353
TEvUndelivered Cookie# 10354
2025-11-27T18:20:48.282615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471849806968478:2089] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:48.282702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:48.283476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471850778878580:2100] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:48.285613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471849806968478:2089] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:48.285686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471850778878580:2100] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:48.285771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:48.285805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:48.285834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471849806968481:2048]
2025-11-27T18:20:48.285868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471849806968481:2048] [node 2] ICS09 handshake done sender: [1:7577471849806968478:2089] self: [1:7577471849806968479:0] peer: [2:7577471850778878579:0] socket: 17 qp: 5316
2025-11-27T18:20:48.285896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471849806968481:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:48.285926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:48.285948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:48.285985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:48.286024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471850778878583:2048]
2025-11-27T18:20:48.286054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471850778878583:2048] [node 1] ICS09 handshake done sender: [2:7577471850778878580:2100] self: [2:7577471850778878579:0] peer: [1:7577471849806968479:0] socket: 16 qp: 5315
2025-11-27T18:20:48.286076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471850778878583:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:48.286096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:20:48.800286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471849806968481:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-27T18:20:48.800383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471849806968481:2048] VirtualId# [1:7577471849806968479:0]
2025-11-27T18:20:48.800413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471849806968481:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:48.800463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:48.800503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:48.800524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:48.800757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471850778878583:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:20:48.800872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471850778878583:2048] VirtualId# [2:7577471850778878579:0]
2025-11-27T18:20:48.800926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471850778878583:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:48.801011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:48.801065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:48.803633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471850778878585:2101] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:48.803744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:48.803977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471849806968484:2091] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:48.806303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471850778878585:2101] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:48.806374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471849806968484:2091] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:48.806469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:48.806502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:48.806533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471850778878588:2048]
2025-11-27T18:20:48.806565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471850778878588:2048] [node 1] ICS09 handshake done sender: [2:7577471850778878585:2101] self: [2:7577471850778878586:0] peer: [1:7577471849806968483:0] socket: 16 qp: 5318
2025-11-27T18:20:48.806592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471850778878588:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:48.806629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:48.806682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:48.806714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:48.806742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471849806968487:2048]
2025-11-27T18:20:48.806773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471849806968487:2048] [node 2] ICS09 handshake done sender: [1:7577471849806968484:2091] self: [1:7577471849806968483:0] peer: [2:7577471850778878586:0] socket: 17 qp: 5317
2025-11-27T18:20:48.806793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471849806968487:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:48.806815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:20:48.962181Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:20:48.962408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471850778878588:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-27T18:20:48.962432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471850778878588:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:48.962467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:48.962496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471850778878588:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:48.962684Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:48.962844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471849806968487:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:20:48.962866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471849806968487:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:48.962912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:48.962942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471849806968487:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:48.964785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471849806968487:2048] [node 2] ICS08 incoming handshake Self# [2:7577471850778878586:0] Peer# [1:7577471849806968483:0] Counter# 18 LastInputSerial# 748
2025-11-27T18:20:48.966363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471850778878588:2048] [node 1] ICS08 incoming handshake Self# [1:7577471849806968483:0] Peer# [2:7577471850778878586:0] Counter# 748 LastInputSerial# 18
2025-11-27T18:20:48.966685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471849806968489:2092] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:48.966903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:48.966948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471849806968487:2048] [node 2] ICS09 handshake done sender: [1:7577471849806968489:2092] self: [1:7577471849806968483:0] peer: [2:7577471850778878586:0] socket: 24 qp: 5322
2025-11-27T18:20:48.967534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471849806968487:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:48.967582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:48.969843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471850778878592:2104] held: no
2025-11-27T18:20:48.969898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471850778878592:2104] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:48.970016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:48.970051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471850778878588:2048] [node 1] ICS09 handshake done sender: [2:7577471850778878592:2104] self: [2:7577471850778878586:0] peer: [1:7577471849806968483:0] socket: 25 qp: 5321
2025-11-27T18:20:48.971172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471850778878588:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:48.971209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:20:49.397293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:20:49.397487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471849806968487:2048] [node 2] ICS07 socket disconnect 24 reason# Debug
2025-11-27T18:20:49.397514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471849806968487:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:49.397551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:49.397580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471849806968487:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:49.397730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:49.397763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471850778878588:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:49.397783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471850778878588:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:49.397807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:49.397855Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:49.397979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471850778878588:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:49.397996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471850778878588:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:49.400331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471850778878588:2048] [node 1] ICS08 incoming handshake Self# [1:7577471849806968483:0] Peer# [2:7577471850778878586:0] Counter# 6696 LastInputSerial# 261
2025-11-27T18:20:49.400790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471854101935790:2094] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:49.400979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:49.401017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471849806968487:2048] [node 2] ICS09 handshake done sender: [1:7577471854101935790:2094] self: [1:7577471849806968483:0] peer: [2:7577471850778878586:0] socket: 16 qp: 5323
2025-11-27T18:20:49.401402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471849806968487:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:49.401433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:49.401781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471849806968487:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:49.401806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471849806968487:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:20:49.401844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:49.401875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471849806968487:2048] [node 2] ICS08 incoming handshake Self# [2:7577471850778878586:0] Peer# [1:7577471849806968483:0] Counter# 261 LastInputSerial# 6696
2025-11-27T18:20:49.401903Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:49.402013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471849806968487:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:49.402056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471849806968487:2048] VirtualId# [1:7577471849806968483:0]
2025-11-27T18:20:49.402079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:49.403396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471855073845892:2107] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:49.403593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:49.403632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471850778878588:2048] [node 1] ICS09 handshake done sender: [2:7577471855073845892:2107] self: [2:7577471850778878586:0] peer: [1:7577471849806968483:0] socket: 17 qp: 5324
2025-11-27T18:20:49.404083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471850778878588:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:49.404114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:49.404324Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:49.404347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471850778878588:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:49.404367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471850778878588:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:49.404382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:49.404427Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:49.404534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471850778878588:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:49.404551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471850778878588:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:49.406821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471854101935796:2097]) peer tries to resume nonexistent session Self# [2:7577471850778878586:0] Peer# [1:7577471849806968483:0]
2025-11-27T18:20:49.406863Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471854101935796:2097] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:49.406966Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471855073845897:2109] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:49.407495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:49.407542Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:49.407562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471850778878588:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:49.407596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471850778878588:2048] VirtualId# [2:7577471850778878586:0]
2025-11-27T18:20:49.407673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 13133
TEvUndelivered Cookie# 13134
TEvUndelivered Cookie# 13135
TEvUndelivered Cookie# 13136
TEvUndelivered Cookie# 13137
TEvUndelivered Cookie# 13138
2025-11-27T18:20:49.409782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471854101935797:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:49.409863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:49.410558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471855073845900:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:49.412675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471854101935797:2098] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:49.412724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471855073845900:2110] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:49.412876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:49.412928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:49.412990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:49.412991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471854101935800:2048]
2025-11-27T18:20:49.413019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:49.413043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471854101935800:2048] [node 2] ICS09 handshake done sender: [1:7577471854101935797:2098] self: [1:7577471854101935798:0] peer: [2:7577471855073845899:0] socket: 17 qp: 5329
2025-11-27T18:20:49.413047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471855073845903:2048]
2025-11-27T18:20:49.413080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471855073845903:2048] [node 1] ICS09 handshake done sender: [2:7577471855073845900:2110] self: [2:7577471855073845899:0] peer: [1:7577471854101935798:0] socket: 16 qp: 5328
2025-11-27T18:20:49.413081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471854101935800:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:49.413102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471855073845903:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:49.413127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:49.413123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:20:49.580062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:20:49.580236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471854101935800:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-27T18:20:49.580263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471854101935800:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:49.580309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:49.580330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471854101935800:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:49.580409Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:49.580680Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:49.580708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471855073845903:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:49.580729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471855073845903:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:49.580757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:49.580805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471855073845903:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:49.580822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471855073845903:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:49.582560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471855073845903:2048] [node 1] ICS08 incoming handshake Self# [1:7577471854101935798:0] Peer# [2:7577471855073845899:0] Counter# 754 LastInputSerial# 17
2025-11-27T18:20:49.583455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471854101935802:2100] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:49.583760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:49.583797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471854101935800:2048] [node 2] ICS09 handshake done sender: [1:7577471854101935802:2100] self: [1:7577471854101935798:0] peer: [2:7577471855073845899:0] socket: 22 qp: 5330
2025-11-27T18:20:49.584227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471854101935800:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:49.584279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:49.584473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471854101935800:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:49.584495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471854101935800:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:20:49.584530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:49.584564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471854101935800:2048] [node 2] ICS08 incoming handshake Self# [2:7577471855073845899:0] Peer# [1:7577471854101935798:0] Counter# 17 LastInputSerial# 754
2025-11-27T18:20:49.584596Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:49.584814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471854101935800:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:49.584860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471854101935800:2048] VirtualId# [1:7577471854101935798:0]
2025-11-27T18:20:49.584885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:49.586102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471855073845906:2112] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:49.586251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:49.586288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471855073845903:2048] [node 1] ICS09 handshake done sender: [2:7577471855073845906:2112] self: [2:7577471855073845899:0] peer: [1:7577471854101935798:0] socket: 23 qp: 5331
2025-11-27T18:20:49.586712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471855073845903:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:49.586744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:49.586975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:49.586999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471855073845903:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:49.587019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471855073845903:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:49.587040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:49.587104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:49.587212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471855073845903:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:49.587229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471855073845903:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:49.589939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471854101935808:2103]) peer tries to resume nonexistent session Self# [2:7577471855073845899:0] Peer# [1:7577471854101935798:0]
2025-11-27T18:20:49.589982Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471854101935808:2103] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:49.590087Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471855073845911:2114] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:49.590708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:49.590755Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:49.590772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471855073845903:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:49.590810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471855073845903:2048] VirtualId# [2:7577471855073845899:0]
2025-11-27T18:20:49.590907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 13314
TEvUndelivered Cookie# 13315
TEvUndelivered Cookie# 13316
TEvUndelivered Cookie# 13317
TEvUndelivered Cookie# 13318
2025-11-27T18:20:49.593123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471854101935809:2104] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:49.593203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:49.594022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471855073845914:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:49.595942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471854101935809:2104] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:49.596015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471855073845914:2115] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:49.596109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:49.596156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:49.596197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471854101935812:2048]
2025-11-27T18:20:49.596229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471854101935812:2048] [node 2] ICS09 handshake done sender: [1:7577471854101935809:2104] self: [1:7577471854101935810:0] peer: [2:7577471855073845913:0] socket: 17 qp: 5337
2025-11-27T18:20:49.596257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471854101935812:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:49.596295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:49.596274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:49.596305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:49.596333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471855073845917:2048]
2025-11-27T18:20:49.596357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471855073845917:2048] [node 1] ICS09 handshake done sender: [2:7577471855073845914:2115] self: [2:7577471855073845913:0] peer: [1:7577471854101935810:0] socket: 16 qp: 5336
2025-11-27T18:20:49.596375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471855073845917:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:49.596395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:20:49.790178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:20:49.790302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:49.790544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471854101935812:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:20:49.790603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471854101935812:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:49.790648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:49.790696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471854101935812:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:49.791462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:49.791753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471855073845917:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:49.791777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471855073845917:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:49.791802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:49.791834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471855073845917:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:49.794223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471855073845917:2048] [node 1] ICS08 incoming handshake Self# [1:7577471854101935810:0] Peer# [2:7577471855073845913:0] Counter# 1091 LastInputSerial# 27
2025-11-27T18:20:49.794665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471854101935814:2106] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:49.794925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:49.794979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471854101935812:2048] [node 2] ICS09 handshake done sender: [1:7577471854101935814:2106] self: [1:7577471854101935810:0] peer: [2:7577471855073845913:0] socket: 22 qp: 5338
2025-11-27T18:20:49.795466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471854101935812:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:49.795584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:49.795988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471854101935812:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:49.796018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471854101935812:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:20:49.796061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:49.796121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471854101935812:2048] [node 2] ICS08 incoming handshake Self# [2:7577471855073845913:0] Peer# [1:7577471854101935810:0] Counter# 27 LastInputSerial# 1091
2025-11-27T18:20:49.796169Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:49.796371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471854101935812:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:49.796430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471854101935812:2048] VirtualId# [1:7577471854101935810:0]
2025-11-27T18:20:49.796468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:49.798353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471855073845920:2117] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:49.798506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:49.798544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471855073845917:2048] [node 1] ICS09 handshake done sender: [2:7577471855073845920:2117] self: [2:7577471855073845913:0] peer: [1:7577471854101935810:0] socket: 23 qp: 5339
2025-11-27T18:20:49.798956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471855073845917:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:49.798987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:49.799205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:49.799229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471855073845917:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:49.799249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471855073845917:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:49.799267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:49.799317Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:49.799422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471855073845917:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:49.799439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471855073845917:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:49.803443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471854101935820:2109]) peer tries to resume nonexistent session Self# [2:7577471855073845913:0] Peer# [1:7577471854101935810:0]
2025-11-27T18:20:49.803505Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471854101935820:2109] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:49.803654Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471855073845925:2119] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:49.804398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:49.804449Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:49.804476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471855073845917:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:49.804532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471855073845917:2048] VirtualId# [2:7577471855073845913:0]
2025-11-27T18:20:49.804644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 13593
TEvUndelivered Cookie# 13594
TEvUndelivered Cookie# 13595
TEvUndelivered Cookie# 13596
TEvUndelivered Cookie# 13597
TEvUndelivered Cookie# 13598
2025-11-27T18:20:49.807248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471854101935821:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:49.807364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:49.808303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471855073845928:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:49.810648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471854101935821:2110] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:49.810745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471855073845928:2120] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:49.810833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:49.810894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:49.810951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471854101935824:2048]
2025-11-27T18:20:49.811014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471854101935824:2048] [node 2] ICS09 handshake done sender: [1:7577471854101935821:2110] self: [1:7577471854101935822:0] peer: [2:7577471855073845927:0] socket: 17 qp: 5345
2025-11-27T18:20:49.811048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471854101935824:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:49.811039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:49.811087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:49.811097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:49.811127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471855073845931:2048]
2025-11-27T18:20:49.811167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471855073845931:2048] [node 1] ICS09 handshake done sender: [2:7577471855073845928:2120] self: [2:7577471855073845927:0] peer: [1:7577471854101935822:0] socket: 16 qp: 5344
2025-11-27T18:20:49.811196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471855073845931:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:49.811216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:20:49.922705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471854101935824:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-27T18:20:49.922805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471854101935824:2048] VirtualId# [1:7577471854101935822:0]
2025-11-27T18:20:49.922842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471854101935824:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:49.922906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:49.922961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:49.922975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:49.923179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471855073845931:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:49.923204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471855073845931:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:49.923259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:49.923292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471855073845931:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:49.926469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471854101935826:2112]) peer tries to resume nonexistent session Self# [2:7577471855073845927:0] Peer# [1:7577471854101935822:0]
2025-11-27T18:20:49.926605Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471854101935826:2112] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:49.926808Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471855073845933:2121] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:49.927702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:49.927771Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:49.927799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471855073845931:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:49.927868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471855073845931:2048] VirtualId# [2:7577471855073845927:0]
2025-11-27T18:20:49.927931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:49.930660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471855073845935:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:49.932097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:49.932405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471854101935828:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:49.932603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471854101935830:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:49.935264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471855073845935:2122] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:49.935392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471854101935828:2113] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:49.935474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:49.935525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:49.935578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471855073845941:2048]
2025-11-27T18:20:49.935622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471855073845941:2048] [node 1] ICS09 handshake done sender: [2:7577471855073845935:2122] self: [2:7577471855073845939:0] peer: [1:7577471854101935827:0] socket: 23 qp: 5348
2025-11-27T18:20:49.935648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471855073845941:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:49.935649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:49.935686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:49.935718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:49.935751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471854101935833:2048]
2025-11-27T18:20:49.935779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471854101935833:2048] [node 2] ICS09 handshake done sender: [1:7577471854101935828:2113] self: [1:7577471854101935827:0] peer: [2:7577471855073845939:0] socket: 22 qp: 5347
2025-11-27T18:20:49.935803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471854101935833:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:49.935839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-27T18:20:50.082439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471855073845941:2048] [node 1] ICS01 socket: 23 reason# Debug
2025-11-27T18:20:50.082545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471855073845941:2048] VirtualId# [2:7577471855073845939:0]
2025-11-27T18:20:50.082581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471855073845941:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:50.082652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:50.082715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:50.082701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:50.082907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471854101935833:2048] [node 2] ICS01 socket: 22 reason# EndOfStream
2025-11-27T18:20:50.082963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471854101935833:2048] VirtualId# [1:7577471854101935827:0]
2025-11-27T18:20:50.082985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471854101935833:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:50.083015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:50.083072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:50.087569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471859368813239:2125] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:50.087683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:50.087926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471858396903132:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:50.091013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471859368813239:2125] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:50.091105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471858396903132:2115] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:50.091220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:50.091271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:50.091331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471859368813242:2048]
2025-11-27T18:20:50.091373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471859368813242:2048] [node 1] ICS09 handshake done sender: [2:7577471859368813239:2125] self: [2:7577471859368813240:0] peer: [1:7577471858396903131:0] socket: 17 qp: 5352
2025-11-27T18:20:50.091401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471859368813242:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:50.091403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:50.091430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:50.091459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:50.091507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471858396903135:2048]
2025-11-27T18:20:50.091548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471858396903135:2048] [node 2] ICS09 handshake done sender: [1:7577471858396903132:2115] self: [1:7577471858396903131:0] peer: [2:7577471859368813240:0] socket: 16 qp: 5351
2025-11-27T18:20:50.091579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471858396903135:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:50.091614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:20:50.296581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:20:50.296825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471858396903135:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-27T18:20:50.296864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471858396903135:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:50.296940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:50.296988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471858396903135:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:50.298473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:50.298519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471859368813242:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:50.298546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471859368813242:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:50.298586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:50.299063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:50.299213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471859368813242:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:50.299233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471859368813242:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:50.301118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471859368813242:2048] [node 1] ICS08 incoming handshake Self# [1:7577471858396903131:0] Peer# [2:7577471859368813240:0] Counter# 1255 LastInputSerial# 30
2025-11-27T18:20:50.301807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471858396903137:2116] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:50.302119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:50.302172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471858396903135:2048] [node 2] ICS09 handshake done sender: [1:7577471858396903137:2116] self: [1:7577471858396903131:0] peer: [2:7577471859368813240:0] socket: 22 qp: 5353
2025-11-27T18:20:50.302740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471858396903135:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:50.302792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:50.303339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471858396903135:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:50.303376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471858396903135:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:20:50.303425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:50.303468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471858396903135:2048] [node 2] ICS08 incoming handshake Self# [2:7577471859368813240:0] Peer# [1:7577471858396903131:0] Counter# 30 LastInputSerial# 1255
2025-11-27T18:20:50.303513Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:50.303717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471858396903135:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:50.303782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471858396903135:2048] VirtualId# [1:7577471858396903131:0]
2025-11-27T18:20:50.303820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:50.305632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471859368813245:2128] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:50.305933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:50.305992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471859368813242:2048] [node 1] ICS09 handshake done sender: [2:7577471859368813245:2128] self: [2:7577471859368813240:0] peer: [1:7577471858396903131:0] socket: 23 qp: 5354
2025-11-27T18:20:50.306672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471859368813242:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:50.306727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:50.307045Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:50.307083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471859368813242:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:50.307110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471859368813242:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:50.307133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:50.307237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:50.307411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471859368813242:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:50.307438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471859368813242:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:50.310525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471858396903143:2119]) peer tries to resume nonexistent session Self# [2:7577471859368813240:0] Peer# [1:7577471858396903131:0]
2025-11-27T18:20:50.310594Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471858396903143:2119] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:50.310739Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471859368813250:2130] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:50.311475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:50.311533Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:50.311554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471859368813242:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:50.311605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471859368813242:2048] VirtualId# [2:7577471859368813240:0]
2025-11-27T18:20:50.311709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 14093
TEvUndelivered Cookie# 14094
TEvUndelivered Cookie# 14095
TEvUndelivered Cookie# 14096
TEvUndelivered Cookie# 14097
TEvUndelivered Cookie# 14098
2025-11-27T18:20:50.314574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471858396903144:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:50.314709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:50.315719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471859368813253:2131] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:50.321419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471858396903144:2120] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:50.321550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471859368813253:2131] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:50.321688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:50.321741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:50.321813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471858396903147:2048]
2025-11-27T18:20:50.321862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471858396903147:2048] [node 2] ICS09 handshake done sender: [1:7577471858396903144:2120] self: [1:7577471858396903145:0] peer: [2:7577471859368813252:0] socket: 17 qp: 5359
2025-11-27T18:20:50.321846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:50.321890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:50.321897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471858396903147:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:50.321937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471859368813256:2048]
2025-11-27T18:20:50.321937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:50.321984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471859368813256:2048] [node 1] ICS09 handshake done sender: [2:7577471859368813253:2131] self: [2:7577471859368813252:0] peer: [1:7577471858396903145:0] socket: 16 qp: 5358
2025-11-27T18:20:50.322018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471859368813256:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:50.322055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:20:50.823818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:20:50.823943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-27T18:20:50.823965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471858396903147:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:50.823989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471858396903147:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:50.823957Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:50.824019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:50.824087Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:50.824169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471859368813256:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:20:50.824246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471859368813256:2048] VirtualId# [2:7577471859368813252:0]
2025-11-27T18:20:50.824277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471859368813256:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:50.824285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471858396903147:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:50.824314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471858396903147:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:50.824334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:50.824379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:50.827291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471859368813258:2132]) peer tries to resume nonexistent session Self# [1:7577471858396903145:0] Peer# [2:7577471859368813252:0]
2025-11-27T18:20:50.827344Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471859368813258:2132] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:50.827491Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471858396903149:2122] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:20:50.828118Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:50.828171Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:50.828203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471858396903147:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:50.828258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471858396903147:2048] VirtualId# [1:7577471858396903145:0]
2025-11-27T18:20:50.828299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:50.830885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471859368813259:2133] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:50.831064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:50.831369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471858396903152:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:50.833866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471859368813259:2133] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:50.833963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471858396903152:2123] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:50.834150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:50.834210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:50.834276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471859368813262:2048]
2025-11-27T18:20:50.834332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471859368813262:2048] [node 1] ICS09 handshake done sender: [2:7577471859368813259:2133] self: [2:7577471859368813260:0] peer: [1:7577471858396903151:0] socket: 17 qp: 5362
2025-11-27T18:20:50.834306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:50.834374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471859368813262:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:50.834355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:50.834401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471858396903155:2048]
2025-11-27T18:20:50.834418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:50.834441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471858396903155:2048] [node 2] ICS09 handshake done sender: [1:7577471858396903152:2123] self: [1:7577471858396903151:0] peer: [2:7577471859368813260:0] socket: 16 qp: 5361
2025-11-27T18:20:50.834493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471858396903155:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:50.834533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:20:51.074470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:20:51.074678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:51.074639Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:51.074734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471859368813262:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:51.074771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471859368813262:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:51.074823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:51.074890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:51.074937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471858396903155:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:51.074984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471858396903155:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:51.075069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:51.075096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471859368813262:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:51.075125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471859368813262:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:51.075122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471858396903155:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:51.079455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471858396903155:2048] [node 2] ICS08 incoming handshake Self# [2:7577471859368813260:0] Peer# [1:7577471858396903151:0] Counter# 37 LastInputSerial# 1549
2025-11-27T18:20:51.079899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471862691870453:2124] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:51.080206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:51.080262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471858396903155:2048] [node 2] ICS09 handshake done sender: [1:7577471862691870453:2124] self: [1:7577471858396903151:0] peer: [2:7577471859368813260:0] socket: 22 qp: 5364
2025-11-27T18:20:51.079432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471859368813262:2048] [node 1] ICS08 incoming handshake Self# [1:7577471858396903151:0] Peer# [2:7577471859368813260:0] Counter# 1549 LastInputSerial# 37
2025-11-27T18:20:51.080761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471858396903155:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:51.080820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:51.083655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471863663780562:2136] held: no
2025-11-27T18:20:51.083738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471863663780562:2136] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:51.083964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:51.084025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471859368813262:2048] [node 1] ICS09 handshake done sender: [2:7577471863663780562:2136] self: [2:7577471859368813260:0] peer: [1:7577471858396903151:0] socket: 25 qp: 5365
2025-11-27T18:20:51.084521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471859368813262:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:51.084580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:20:51.492896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:20:51.492969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:51.493128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471859368813262:2048] [node 1] ICS01 socket: 25 reason# EndOfStream
2025-11-27T18:20:51.493185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471859368813262:2048] VirtualId# [2:7577471859368813260:0]
2025-11-27T18:20:51.493210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471859368813262:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:51.493232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:51.493307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:51.493710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
TEvNodeDisconnected
2025-11-27T18:20:51.493970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471858396903155:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-27T18:20:51.493996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471858396903155:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:51.494015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:51.494054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471858396903155:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:51.496068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471863663780565:2138]) peer tries to resume nonexistent session Self# [1:7577471858396903151:0] Peer# [2:7577471859368813260:0]
2025-11-27T18:20:51.496115Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471863663780565:2138] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:51.496228Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471862691870458:2126] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:20:51.496940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:51.496996Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:51.497015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471858396903155:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:51.497055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471858396903155:2048] VirtualId# [1:7577471858396903151:0]
2025-11-27T18:20:51.497091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:51.499883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471863663780566:2139] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:51.500009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:51.500312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471862691870461:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:51.502905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471863663780566:2139] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:51.503070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:51.503115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:51.503120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471862691870461:2127] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:51.503163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471863663780569:2048]
2025-11-27T18:20:51.503209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471863663780569:2048] [node 1] ICS09 handshake done sender: [2:7577471863663780566:2139] self: [2:7577471863663780567:0] peer: [1:7577471862691870460:0] socket: 17 qp: 5369
2025-11-27T18:20:51.503246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471863663780569:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:51.503286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:51.503436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:51.503497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:51.503538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471862691870464:2048]
2025-11-27T18:20:51.503573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471862691870464:2048] [node 2] ICS09 handshake done sender: [1:7577471862691870461:2127] self: [1:7577471862691870460:0] peer: [2:7577471863663780567:0] socket: 16 qp: 5368
2025-11-27T18:20:51.503601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471862691870464:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:51.503630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:20:51.596049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:20:51.596215Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:51.596250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471863663780569:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:51.596280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471863663780569:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:51.596329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:51.596332Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:51.596405Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:51.596601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471862691870464:2048] [node 2] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:20:51.596625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471863663780569:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:51.596658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471863663780569:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:51.596711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471862691870464:2048] VirtualId# [1:7577471862691870460:0]
2025-11-27T18:20:51.596766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471862691870464:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:51.596864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:51.596918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:51.600385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471862691870466:2128]) peer tries to resume nonexistent session Self# [2:7577471863663780567:0] Peer# [1:7577471862691870460:0]
2025-11-27T18:20:51.600503Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471862691870466:2128] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:51.600730Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471863663780571:2141] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:51.601633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:51.601711Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:51.601745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471863663780569:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:51.601818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471863663780569:2048] VirtualId# [2:7577471863663780567:0]
2025-11-27T18:20:51.601954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 16594
TEvUndelivered Cookie# 16595
TEvUndelivered Cookie# 16596
TEvUndelivered Cookie# 16597
TEvUndelivered Cookie# 16598
TEvUndelivered Cookie# 16599
TEvUndelivered Cookie# 16600
2025-11-27T18:20:51.606209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471863663780575:2143] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:51.606458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:51.606501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471862691870470:2130] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:51.606730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471862691870468:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:51.609644Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471863663780575:2143] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:51.609766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471862691870468:2129] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:51.609849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:51.609901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:51.609959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471863663780579:2048]
2025-11-27T18:20:51.610015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471863663780579:2048] [node 1] ICS09 handshake done sender: [2:7577471863663780575:2143] self: [2:7577471863663780577:0] peer: [1:7577471862691870467:0] socket: 17 qp: 5372
2025-11-27T18:20:51.610066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471863663780579:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:51.610112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:51.610099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:51.610160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:51.610213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471862691870473:2048]
2025-11-27T18:20:51.610258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471862691870473:2048] [node 2] ICS09 handshake done sender: [1:7577471862691870468:2129] self: [1:7577471862691870467:0] peer: [2:7577471863663780577:0] socket: 16 qp: 5371
2025-11-27T18:20:51.610292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471862691870473:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:51.610327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:20:52.020151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:20:52.020292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:52.020535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471862691870473:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:52.020577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471862691870473:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:52.020615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:52.020659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471862691870473:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:52.021408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:52.021688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471863663780579:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:20:52.021709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471863663780579:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:52.021732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:52.021764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471863663780579:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:52.023607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471863663780579:2048] [node 1] ICS08 incoming handshake Self# [1:7577471862691870467:0] Peer# [2:7577471863663780577:0] Counter# 2982 LastInputSerial# 76
2025-11-27T18:20:52.024285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471866986837771:2131] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:52.024524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:52.024580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471862691870473:2048] [node 2] ICS09 handshake done sender: [1:7577471866986837771:2131] self: [1:7577471862691870467:0] peer: [2:7577471863663780577:0] socket: 20 qp: 5375
2025-11-27T18:20:52.025175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471862691870473:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:52.025225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:52.025666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471862691870473:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:52.025701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471862691870473:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:20:52.025750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:52.025804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471862691870473:2048] [node 2] ICS08 incoming handshake Self# [2:7577471863663780577:0] Peer# [1:7577471862691870467:0] Counter# 76 LastInputSerial# 2982
2025-11-27T18:20:52.025850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:52.026027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471862691870473:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:52.026093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471862691870473:2048] VirtualId# [1:7577471862691870467:0]
2025-11-27T18:20:52.026131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:52.027733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471867958747878:2146] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:52.028010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:52.028079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471863663780579:2048] [node 1] ICS09 handshake done sender: [2:7577471867958747878:2146] self: [2:7577471863663780577:0] peer: [1:7577471862691870467:0] socket: 21 qp: 5376
2025-11-27T18:20:52.028740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471863663780579:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:52.028776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:52.029048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:52.029074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471863663780579:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:52.029096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471863663780579:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:52.029118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:52.029173Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:52.029329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471863663780579:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:52.029348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471863663780579:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:52.032498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471866986837777:2134]) peer tries to resume nonexistent session Self# [2:7577471863663780577:0] Peer# [1:7577471862691870467:0]
2025-11-27T18:20:52.032564Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471866986837777:2134] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:52.032719Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471867958747883:2148] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:52.033457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:52.033537Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:52.033566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471863663780579:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:52.033622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471863663780579:2048] VirtualId# [2:7577471863663780577:0]
2025-11-27T18:20:52.033760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 17354
TEvUndelivered Cookie# 17355
TEvUndelivered Cookie# 17356
TEvUndelivered Cookie# 17357
TEvUndelivered Cookie# 17358
TEvUndelivered Cookie# 17359
TEvUndelivered Cookie# 17360
2025-11-27T18:20:52.036941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471866986837778:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:52.037068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:52.038064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471867958747886:2149] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:52.040505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471866986837778:2135] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:52.040666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471867958747886:2149] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:52.040690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:52.040738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:52.040785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471866986837781:2048]
2025-11-27T18:20:52.040838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471866986837781:2048] [node 2] ICS09 handshake done sender: [1:7577471866986837778:2135] self: [1:7577471866986837779:0] peer: [2:7577471867958747885:0] socket: 17 qp: 5381
2025-11-27T18:20:52.040889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471866986837781:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:52.040927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:52.040935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:52.040979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:52.041024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471867958747889:2048]
2025-11-27T18:20:52.041069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471867958747889:2048] [node 1] ICS09 handshake done sender: [2:7577471867958747886:2149] self: [2:7577471867958747885:0] peer: [1:7577471866986837779:0] socket: 16 qp: 5380
2025-11-27T18:20:52.041104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471867958747889:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:52.041152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:20:52.498265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471866986837781:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-27T18:20:52.498372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471866986837781:2048] VirtualId# [1:7577471866986837779:0]
2025-11-27T18:20:52.498408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471866986837781:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:52.498465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:52.498523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:52.500066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:52.500488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471867958747889:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:52.500524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471867958747889:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:52.500564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:52.500616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471867958747889:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:52.503272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471866986837783:2137]) peer tries to resume nonexistent session Self# [2:7577471867958747885:0] Peer# [1:7577471866986837779:0]
2025-11-27T18:20:52.503333Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471866986837783:2137] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:52.503514Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471867958747891:2150] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:52.504393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:52.504467Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:52.504501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471867958747889:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:52.504577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471867958747889:2048] VirtualId# [2:7577471867958747885:0]
2025-11-27T18:20:52.504727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 18292
TEvUndelivered Cookie# 18293
TEvUndelivered Cookie# 18294
TEvUndelivered Cookie# 18295
TEvUndelivered Cookie# 18296
TEvUndelivered Cookie# 18297
TEvUndelivered Cookie# 18298
TEvUndelivered Cookie# 18299
TEvUndelivered Cookie# 18300
2025-11-27T18:20:52.508247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471866986837784:2138] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:52.508402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:52.508742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471867958747894:2151] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:52.511140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471866986837784:2138] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:52.511253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471867958747894:2151] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:52.511292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:52.511328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:52.511358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471866986837787:2048]
2025-11-27T18:20:52.511386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471866986837787:2048] [node 2] ICS09 handshake done sender: [1:7577471866986837784:2138] self: [1:7577471866986837785:0] peer: [2:7577471867958747893:0] socket: 17 qp: 5384
2025-11-27T18:20:52.511407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471866986837787:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:52.511440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:52.511503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:52.511544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:52.511590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471867958747897:2048]
2025-11-27T18:20:52.511634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471867958747897:2048] [node 1] ICS09 handshake done sender: [2:7577471867958747894:2151] self: [2:7577471867958747893:0] peer: [1:7577471866986837785:0] socket: 16 qp: 5383
2025-11-27T18:20:52.511671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471867958747897:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:52.511706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-27T18:20:53.078419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471867958747897:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-27T18:20:53.078543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471867958747897:2048] VirtualId# [2:7577471867958747893:0]
2025-11-27T18:20:53.078589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471867958747897:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:53.078663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:53.078787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:53.078754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:53.079041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471866986837787:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:20:53.079169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471866986837787:2048] VirtualId# [1:7577471866986837785:0]
2025-11-27T18:20:53.079211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471866986837787:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:53.079252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:53.079319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 19446
TEvUndelivered Cookie# 19447
TEvUndelivered Cookie# 19448
TEvUndelivered Cookie# 19449
TEvUndelivered Cookie# 19450
2025-11-27T18:20:53.083670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471872253715195:2152] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:53.083840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:53.084130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471871281805086:2140] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:53.087324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471872253715195:2152] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:53.087428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471871281805086:2140] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:53.087535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:53.087582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:53.087632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471872253715198:2048]
2025-11-27T18:20:53.087676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471872253715198:2048] [node 1] ICS09 handshake done sender: [2:7577471872253715195:2152] self: [2:7577471872253715196:0] peer: [1:7577471871281805085:0] socket: 17 qp: 5386
2025-11-27T18:20:53.087710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471872253715198:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:53.087753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:53.087827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:53.087880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:53.087912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471871281805089:2048]
2025-11-27T18:20:53.087952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471871281805089:2048] [node 2] ICS09 handshake done sender: [1:7577471871281805086:2140] self: [1:7577471871281805085:0] peer: [2:7577471872253715196:0] socket: 16 qp: 5385
2025-11-27T18:20:53.087982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471871281805089:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:53.088015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-27T18:20:53.369941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471872253715198:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-11-27T18:20:53.370048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471872253715198:2048] VirtualId# [2:7577471872253715196:0]
2025-11-27T18:20:53.370083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471872253715198:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:53.370154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:53.370291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:53.370246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:53.370543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471871281805089:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:53.370589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471871281805089:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:53.370664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:53.370712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471871281805089:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 19911
TEvUndelivered Cookie# 19912
TEvUndelivered Cookie# 19913
TEvUndelivered Cookie# 19914
TEvUndelivered Cookie# 19915
TEvUndelivered Cookie# 19916
TEvUndelivered Cookie# 19917
TEvUndelivered Cookie# 19918
2025-11-27T18:20:53.374078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471872253715202:2155]) peer tries to resume nonexistent session Self# [1:7577471871281805085:0] Peer# [2:7577471872253715196:0]
2025-11-27T18:20:53.374173Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471872253715202:2155] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:53.375308Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471871281805091:2141] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:20:53.376164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:53.376231Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:53.376261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471871281805089:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:53.376316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471871281805089:2048] VirtualId# [1:7577471871281805085:0]
2025-11-27T18:20:53.376370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:53.378020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471871281805093:2142] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:53.379844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471872253715204:2156] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:53.379944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:53.380205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471871281805095:2143] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:53.383158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471872253715204:2156] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:53.383234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471871281805095:2143] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:53.383356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:53.383432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:53.383487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471872253715207:2048]
2025-11-27T18:20:53.383465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:53.383508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:53.383534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471872253715207:2048] [node 1] ICS09 handshake done sender: [2:7577471872253715204:2156] self: [2:7577471872253715205:0] peer: [1:7577471871281805094:0] socket: 17 qp: 5391
2025-11-27T18:20:53.383537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471871281805099:2048]
2025-11-27T18:20:53.383576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471872253715207:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:53.383583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471871281805099:2048] [node 2] ICS09 handshake done sender: [1:7577471871281805095:2143] self: [1:7577471871281805094:0] peer: [2:7577471872253715205:0] socket: 16 qp: 5392
2025-11-27T18:20:53.383604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471871281805099:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:53.383633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:53.383630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:20:53.570660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:20:53.570797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:53.570796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:53.571047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471872253715207:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:20:53.571052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-27T18:20:53.571080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471872253715207:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:53.571090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471871281805099:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:53.571110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471871281805099:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:53.571155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:53.571148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:53.571197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471872253715207:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:53.571233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471871281805099:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:53.571351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471871281805099:2048] VirtualId# [1:7577471871281805094:0]
2025-11-27T18:20:53.571398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:53.575175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471871281805101:2144]) peer tries to resume nonexistent session Self# [2:7577471872253715205:0] Peer# [1:7577471871281805094:0]
2025-11-27T18:20:53.575244Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471871281805101:2144] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:53.575409Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471872253715209:2158] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:53.576249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:53.576317Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:53.576345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471872253715207:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:53.576416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471872253715207:2048] VirtualId# [2:7577471872253715205:0]
2025-11-27T18:20:53.576489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:53.580978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471872253715213:2160] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:53.581205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:53.581199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471871281805105:2146] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:53.581529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471871281805103:2145] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:53.584335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471872253715213:2160] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:53.584417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471871281805103:2145] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:53.584536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:53.584593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:53.584660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471872253715217:2048]
2025-11-27T18:20:53.584710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471872253715217:2048] [node 1] ICS09 handshake done sender: [2:7577471872253715213:2160] self: [2:7577471872253715215:0] peer: [1:7577471871281805102:0] socket: 17 qp: 5395
2025-11-27T18:20:53.584745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471872253715217:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:53.584683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:53.584735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:53.584785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:53.584766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471871281805108:2048]
2025-11-27T18:20:53.584793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471871281805108:2048] [node 2] ICS09 handshake done sender: [1:7577471871281805103:2145] self: [1:7577471871281805102:0] peer: [2:7577471872253715215:0] socket: 16 qp: 5394
2025-11-27T18:20:53.584812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471871281805108:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:53.584833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:20:54.050802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:20:54.050944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-27T18:20:54.050975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471871281805108:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:54.050927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:54.051003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471871281805108:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:54.051042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:54.051110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:54.051150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471872253715217:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:20:54.051225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471872253715217:2048] VirtualId# [2:7577471872253715215:0]
2025-11-27T18:20:54.051253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471872253715217:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:54.051312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:54.051317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471871281805108:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:54.051343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471871281805108:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:54.051359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:54.053553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471876548682515:2162]) peer tries to resume nonexistent session Self# [1:7577471871281805102:0] Peer# [2:7577471872253715215:0]
2025-11-27T18:20:54.053603Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471876548682515:2162] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:54.053755Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471875576772406:2147] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:20:54.055188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:54.055244Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:54.055266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471871281805108:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:54.055320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471871281805108:2048] VirtualId# [1:7577471871281805102:0]
2025-11-27T18:20:54.055370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:54.057922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471876548682516:2163] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:54.058033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:54.058314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471875576772409:2148] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:54.060994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471876548682516:2163] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:54.061086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471875576772409:2148] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:54.061280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:54.061328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:54.061395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471876548682519:2048]
2025-11-27T18:20:54.061446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471876548682519:2048] [node 1] ICS09 handshake done sender: [2:7577471876548682516:2163] self: [2:7577471876548682517:0] peer: [1:7577471875576772408:0] socket: 17 qp: 5400
2025-11-27T18:20:54.061440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:54.061483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471876548682519:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:54.061483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:54.061525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:54.061534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471875576772412:2048]
2025-11-27T18:20:54.061580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471875576772412:2048] [node 2] ICS09 handshake done sender: [1:7577471875576772409:2148] self: [1:7577471875576772408:0] peer: [2:7577471876548682517:0] socket: 16 qp: 5399
2025-11-27T18:20:54.061622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471875576772412:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:54.061657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:20:54.171333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:20:54.171449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:54.171601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471876548682519:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:20:54.171674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471876548682519:2048] VirtualId# [2:7577471876548682517:0]
2025-11-27T18:20:54.171729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471876548682519:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:54.171764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:54.171814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:20:54.171922Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:54.172273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471875576772412:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:54.172301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471875576772412:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:54.172340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:54.172401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471875576772412:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-27T18:20:54.175440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471876548682521:2165]) peer tries to resume nonexistent session Self# [1:7577471875576772408:0] Peer# [2:7577471876548682517:0]
2025-11-27T18:20:54.175493Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471876548682521:2165] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:54.175645Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471875576772414:2149] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:20:54.176515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:54.176574Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:54.176610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471875576772412:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:54.176674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471875576772412:2048] VirtualId# [1:7577471875576772408:0]
2025-11-27T18:20:54.176712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:54.179696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471876548682522:2166] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:54.179812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:54.180092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471875576772417:2150] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:54.182438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471876548682522:2166] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:54.182501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471875576772417:2150] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:54.182608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:54.182647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:54.182701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471876548682525:2048]
2025-11-27T18:20:54.182733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:54.182744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471876548682525:2048] [node 1] ICS09 handshake done sender: [2:7577471876548682522:2166] self: [2:7577471876548682523:0] peer: [1:7577471875576772416:0] socket: 17 qp: 5403
2025-11-27T18:20:54.182769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471876548682525:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:54.182766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:54.182794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:54.182810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471875576772420:2048]
2025-11-27T18:20:54.182868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471875576772420:2048] [node 2] ICS09 handshake done sender: [1:7577471875576772417:2150] self: [1:7577471875576772416:0] peer: [2:7577471876548682523:0] socket: 16 qp: 5402
2025-11-27T18:20:54.182910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471875576772420:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:54.182935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:20:54.639485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:20:54.639630Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:54.639600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:54.639662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471876548682525:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:54.639691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471876548682525:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:54.639727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:54.639784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471875576772420:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:54.639792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:54.639807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471875576772420:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:54.639860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:54.639906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471875576772420:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:54.640002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471876548682525:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:54.640035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471876548682525:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:54.643245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471875576772420:2048] [node 2] ICS08 incoming handshake Self# [2:7577471876548682523:0] Peer# [1:7577471875576772416:0] Counter# 92 LastInputSerial# 3716
2025-11-27T18:20:54.643254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471876548682525:2048] [node 1] ICS08 incoming handshake Self# [1:7577471875576772416:0] Peer# [2:7577471876548682523:0] Counter# 3716 LastInputSerial# 92
2025-11-27T18:20:54.643701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471875576772422:2151] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:54.643954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:54.644017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471875576772420:2048] [node 2] ICS09 handshake done sender: [1:7577471875576772422:2151] self: [1:7577471875576772416:0] peer: [2:7577471876548682523:0] socket: 22 qp: 5404
2025-11-27T18:20:54.644435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471875576772420:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:54.644474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:54.645730Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471876548682529:2169] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:54.646637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:54.646699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471876548682525:2048] [node 1] ICS09 handshake done sender: [2:7577471876548682529:2169] self: [2:7577471876548682523:0] peer: [1:7577471875576772416:0] socket: 24 qp: 5406
2025-11-27T18:20:54.647212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471876548682525:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:54.647258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:54.647457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471829304041991:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:20:55.014604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471875576772420:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-27T18:20:55.014702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471875576772420:2048] VirtualId# [1:7577471875576772416:0]
2025-11-27T18:20:55.014752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471875576772420:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:55.014817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:55.014883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:55.015762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:55.016100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471876548682525:2048] [node 1] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-27T18:20:55.016135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471876548682525:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:55.016163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:55.016210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471876548682525:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:55.018509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471879871739723:2153]) peer tries to resume nonexistent session Self# [2:7577471876548682523:0] Peer# [1:7577471875576772416:0]
2025-11-27T18:20:55.018594Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471879871739723:2153] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:55.018720Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471880843649828:2171] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:55.019611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:55.019686Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:55.019722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471876548682525:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:55.019782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471876548682525:2048] VirtualId# [2:7577471876548682523:0]
2025-11-27T18:20:55.019937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 23242
TEvUndelivered Cookie# 23243
TEvUndelivered Cookie# 23244
TEvUndelivered Cookie# 23245
TEvUndelivered Cookie# 23246
TEvUndelivered Cookie# 23247
TEvUndelivered Cookie# 23248
TEvUndelivered Cookie# 23249
TEvUndelivered Cookie# 23250
2025-11-27T18:20:55.023053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471879871739724:2154] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:55.023237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:55.023514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471880843649831:2172] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:55.026284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471879871739724:2154] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:55.026356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471880843649831:2172] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:55.026545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:55.026588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:55.026606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:55.026622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:55.026665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471880843649834:2048]
2025-11-27T18:20:55.026670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471879871739727:2048]
2025-11-27T18:20:55.026695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471880843649834:2048] [node 1] ICS09 handshake done sender: [2:7577471880843649831:2172] self: [2:7577471880843649830:0] peer: [1:7577471879871739725:0] socket: 16 qp: 5409
2025-11-27T18:20:55.026727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471880843649834:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:55.026725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471879871739727:2048] [node 2] ICS09 handshake done sender: [1:7577471879871739724:2154] self: [1:7577471879871739725:0] peer: [2:7577471880843649830:0] socket: 17 qp: 5410
2025-11-27T18:20:55.026756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:55.026771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471879871739727:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:55.026808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:20:55.492219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:20:55.492347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:55.492392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:55.492426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471880843649834:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:55.492454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471880843649834:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:55.492491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:55.492533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471879871739727:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:20:55.492563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471879871739727:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:55.492563Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:55.492623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:55.492657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471879871739727:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:55.492771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471880843649834:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:55.492798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471880843649834:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:55.496173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471879871739727:2048] [node 2] ICS08 incoming handshake Self# [2:7577471880843649830:0] Peer# [1:7577471879871739725:0] Counter# 97 LastInputSerial# 3917
2025-11-27T18:20:55.496189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471880843649834:2048] [node 1] ICS08 incoming handshake Self# [1:7577471879871739725:0] Peer# [2:7577471880843649830:0] Counter# 3917 LastInputSerial# 97
2025-11-27T18:20:55.496481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471879871739729:2156] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:55.496663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:55.496703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471879871739727:2048] [node 2] ICS09 handshake done sender: [1:7577471879871739729:2156] self: [1:7577471879871739725:0] peer: [2:7577471880843649830:0] socket: 22 qp: 5411
2025-11-27T18:20:55.497142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471879871739727:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:55.497176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:55.499704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471880843649837:2174] held: no
2025-11-27T18:20:55.499775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471880843649837:2174] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:55.499917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:55.499966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471880843649834:2048] [node 1] ICS09 handshake done sender: [2:7577471880843649837:2174] self: [2:7577471880843649830:0] peer: [1:7577471879871739725:0] socket: 23 qp: 5412
2025-11-27T18:20:55.500577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471880843649834:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:55.500626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:20:55.954844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471879871739727:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-27T18:20:55.954915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471879871739727:2048] VirtualId# [1:7577471879871739725:0]
2025-11-27T18:20:55.954948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471879871739727:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:55.954986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:55.955013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:55.955073Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:55.955332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471880843649834:2048] [node 1] ICS01 socket: 23 reason# EndOfStream
2025-11-27T18:20:55.955407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471880843649834:2048] VirtualId# [2:7577471880843649830:0]
2025-11-27T18:20:55.955449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471880843649834:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:55.955519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:55.955574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:55.962220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471880843649841:2176] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:55.962379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:55.962634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471879871739735:2158] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:55.965401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471880843649841:2176] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:55.965464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471879871739735:2158] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:55.965606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:55.965668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:55.965710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:55.965736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471880843649844:2048]
2025-11-27T18:20:55.965759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:55.965786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471879871739738:2048]
2025-11-27T18:20:55.965785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471880843649844:2048] [node 1] ICS09 handshake done sender: [2:7577471880843649841:2176] self: [2:7577471880843649842:0] peer: [1:7577471879871739734:0] socket: 17 qp: 5416
2025-11-27T18:20:55.965819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471879871739738:2048] [node 2] ICS09 handshake done sender: [1:7577471879871739735:2158] self: [1:7577471879871739734:0] peer: [2:7577471880843649842:0] socket: 16 qp: 5415
2025-11-27T18:20:55.965819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471880843649844:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:55.965837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471879871739738:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:55.965856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:55.965853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:20:56.288975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471879871739738:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-27T18:20:56.289071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471879871739738:2048] VirtualId# [1:7577471879871739734:0]
2025-11-27T18:20:56.289111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471879871739738:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:56.289185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:56.289245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:56.289826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:56.290196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471880843649844:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:20:56.290236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471880843649844:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:56.290274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:56.290346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471880843649844:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:56.292527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471884166707036:2159]) peer tries to resume nonexistent session Self# [2:7577471880843649842:0] Peer# [1:7577471879871739734:0]
2025-11-27T18:20:56.292593Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471884166707036:2159] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:56.292709Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471885138617142:2178] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:56.293539Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:56.293592Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:56.293610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471880843649844:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:56.293659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471880843649844:2048] VirtualId# [2:7577471880843649842:0]
2025-11-27T18:20:56.293744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 26188
TEvUndelivered Cookie# 26189
TEvUndelivered Cookie# 26190
TEvUndelivered Cookie# 26191
TEvUndelivered Cookie# 26192
TEvUndelivered Cookie# 26193
2025-11-27T18:20:56.296973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471884166707037:2160] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:56.297132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:56.297414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471885138617145:2179] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:56.300079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471884166707037:2160] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:56.300176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471885138617145:2179] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:56.300302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:56.300340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:56.300377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471884166707040:2048]
2025-11-27T18:20:56.300415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471884166707040:2048] [node 2] ICS09 handshake done sender: [1:7577471884166707037:2160] self: [1:7577471884166707038:0] peer: [2:7577471885138617144:0] socket: 17 qp: 5419
2025-11-27T18:20:56.300479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:56.300451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471884166707040:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:56.300518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:56.300544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471885138617148:2048]
2025-11-27T18:20:56.300478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:56.300574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471885138617148:2048] [node 1] ICS09 handshake done sender: [2:7577471885138617145:2179] self: [2:7577471885138617144:0] peer: [1:7577471884166707038:0] socket: 16 qp: 5418
2025-11-27T18:20:56.300601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471885138617148:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:56.300621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:20:56.548693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:20:56.548829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-27T18:20:56.548864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471884166707040:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:56.548893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471884166707040:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:56.548825Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:56.548928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:56.548991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:56.549055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471885138617148:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:20:56.549141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471885138617148:2048] VirtualId# [2:7577471885138617144:0]
2025-11-27T18:20:56.549182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471885138617148:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:56.549184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471884166707040:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:56.549210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471884166707040:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:56.549271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:56.549330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:56.551681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471885138617150:2180]) peer tries to resume nonexistent session Self# [1:7577471884166707038:0] Peer# [2:7577471885138617144:0]
2025-11-27T18:20:56.551761Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471885138617150:2180] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:56.551941Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471884166707042:2162] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:20:56.552851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:56.552925Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:56.552955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471884166707040:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:56.553020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471884166707040:2048] VirtualId# [1:7577471884166707038:0]
2025-11-27T18:20:56.553076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:56.555814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471885138617151:2181] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:56.555956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:56.556241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471884166707045:2163] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:56.558708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471885138617151:2181] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:56.558810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471884166707045:2163] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:56.558874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:56.558942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:56.559012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471885138617154:2048]
2025-11-27T18:20:56.559062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471885138617154:2048] [node 1] ICS09 handshake done sender: [2:7577471885138617151:2181] self: [2:7577471885138617152:0] peer: [1:7577471884166707044:0] socket: 17 qp: 5422
2025-11-27T18:20:56.559053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:56.559103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471885138617154:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:56.559118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:56.559144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:56.559174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471884166707048:2048]
2025-11-27T18:20:56.559216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471884166707048:2048] [node 2] ICS09 handshake done sender: [1:7577471884166707045:2163] self: [1:7577471884166707044:0] peer: [2:7577471885138617152:0] socket: 16 qp: 5421
2025-11-27T18:20:56.559254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471884166707048:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:56.559291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:20:56.807221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:20:56.807343Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:56.807508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471884166707048:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:56.807533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471884166707048:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:56.807557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:56.807586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471884166707048:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:56.808641Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:56.809003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471885138617154:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:20:56.809047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471885138617154:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:56.809078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:56.809123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471885138617154:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:56.811189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471885138617154:2048] [node 1] ICS08 incoming handshake Self# [1:7577471884166707044:0] Peer# [2:7577471885138617152:0] Counter# 1899 LastInputSerial# 44
2025-11-27T18:20:56.811532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471884166707050:2164] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:56.811731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:56.811770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471884166707048:2048] [node 2] ICS09 handshake done sender: [1:7577471884166707050:2164] self: [1:7577471884166707044:0] peer: [2:7577471885138617152:0] socket: 22 qp: 5423
2025-11-27T18:20:56.812138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471884166707048:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:56.812172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:56.812643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471884166707048:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:56.812671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471884166707048:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:20:56.812708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:56.812747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471884166707048:2048] [node 2] ICS08 incoming handshake Self# [2:7577471885138617152:0] Peer# [1:7577471884166707044:0] Counter# 44 LastInputSerial# 1899
2025-11-27T18:20:56.812785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:56.812940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471884166707048:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:56.812984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471884166707048:2048] VirtualId# [1:7577471884166707044:0]
2025-11-27T18:20:56.813013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:56.815000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471885138617157:2184] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:56.815207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:56.815247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471885138617154:2048] [node 1] ICS09 handshake done sender: [2:7577471885138617157:2184] self: [2:7577471885138617152:0] peer: [1:7577471884166707044:0] socket: 23 qp: 5424
2025-11-27T18:20:56.815723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471885138617154:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:56.815756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:56.815973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:56.815999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471885138617154:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:56.816019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471885138617154:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:56.816047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:56.816107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:56.816220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471885138617154:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:56.816237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471885138617154:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:56.818852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471884166707056:2167]) peer tries to resume nonexistent session Self# [2:7577471885138617152:0] Peer# [1:7577471884166707044:0]
2025-11-27T18:20:56.818899Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471884166707056:2167] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:56.819061Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471885138617162:2186] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:56.819752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:56.819801Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:56.819821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471885138617154:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:56.819862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471885138617154:2048] VirtualId# [2:7577471885138617152:0]
2025-11-27T18:20:56.819952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 27078
TEvUndelivered Cookie# 27079
TEvUndelivered Cookie# 27080
TEvUndelivered Cookie# 27081
TEvUndelivered Cookie# 27082
TEvUndelivered Cookie# 27083
2025-11-27T18:20:56.822498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471884166707057:2168] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:56.822618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:56.823582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471885138617165:2187] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:56.826285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471884166707057:2168] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:56.826358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471885138617165:2187] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:56.826470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:56.826528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:56.826567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:56.826587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471884166707060:2048]
2025-11-27T18:20:56.826594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:56.826623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471885138617168:2048]
2025-11-27T18:20:56.826649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471885138617168:2048] [node 1] ICS09 handshake done sender: [2:7577471885138617165:2187] self: [2:7577471885138617164:0] peer: [1:7577471884166707058:0] socket: 16 qp: 5428
2025-11-27T18:20:56.826638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471884166707060:2048] [node 2] ICS09 handshake done sender: [1:7577471884166707057:2168] self: [1:7577471884166707058:0] peer: [2:7577471885138617164:0] socket: 17 qp: 5429
2025-11-27T18:20:56.826673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471885138617168:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:56.826675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471884166707060:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:56.826714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:56.826703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:20:56.942708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:20:56.942986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471885138617168:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-27T18:20:56.943012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471885138617168:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:56.943050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:56.943078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471885138617168:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:56.943316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:56.943571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471884166707060:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:20:56.943601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471884166707060:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:56.943663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:56.943715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471884166707060:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:56.945870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471884166707060:2048] [node 2] ICS08 incoming handshake Self# [2:7577471885138617164:0] Peer# [1:7577471884166707058:0] Counter# 6 LastInputSerial# 282
2025-11-27T18:20:56.947302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471885138617168:2048] [node 1] ICS08 incoming handshake Self# [1:7577471884166707058:0] Peer# [2:7577471885138617164:0] Counter# 282 LastInputSerial# 6
2025-11-27T18:20:56.947679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471884166707062:2170] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:56.947902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:56.947960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471884166707060:2048] [node 2] ICS09 handshake done sender: [1:7577471884166707062:2170] self: [1:7577471884166707058:0] peer: [2:7577471885138617164:0] socket: 24 qp: 5433
2025-11-27T18:20:56.948442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471884166707060:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:56.948485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:56.950967Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471885138617172:2189] held: no
2025-11-27T18:20:56.951050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471885138617172:2189] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:56.951295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:56.951355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471885138617168:2048] [node 1] ICS09 handshake done sender: [2:7577471885138617172:2189] self: [2:7577471885138617164:0] peer: [1:7577471884166707058:0] socket: 25 qp: 5432
2025-11-27T18:20:56.952157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471885138617168:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:56.952212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:20:57.524868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:20:57.525120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471884166707060:2048] [node 2] ICS07 socket disconnect 24 reason# Debug
2025-11-27T18:20:57.525158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471884166707060:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:57.525210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:57.525252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471884166707060:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:57.527011Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:57.527396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471885138617168:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-11-27T18:20:57.527430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471885138617168:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:57.527466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:57.527512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471885138617168:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:57.529884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471885138617168:2048] [node 1] ICS08 incoming handshake Self# [1:7577471884166707058:0] Peer# [2:7577471885138617164:0] Counter# 8906 LastInputSerial# 356
2025-11-27T18:20:57.530379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471888461674363:2172] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:57.530637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:57.530696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471884166707060:2048] [node 2] ICS09 handshake done sender: [1:7577471888461674363:2172] self: [1:7577471884166707058:0] peer: [2:7577471885138617164:0] socket: 16 qp: 5434
2025-11-27T18:20:57.531232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471884166707060:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:57.531281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:57.531713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471884166707060:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:57.531752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471884166707060:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:20:57.531802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:57.531850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471884166707060:2048] [node 2] ICS08 incoming handshake Self# [2:7577471885138617164:0] Peer# [1:7577471884166707058:0] Counter# 356 LastInputSerial# 8906
2025-11-27T18:20:57.531897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:57.532103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471884166707060:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:57.532164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471884166707060:2048] VirtualId# [1:7577471884166707058:0]
2025-11-27T18:20:57.532217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:57.534187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471889433584472:2192] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:57.534436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:57.534494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471885138617168:2048] [node 1] ICS09 handshake done sender: [2:7577471889433584472:2192] self: [2:7577471885138617164:0] peer: [1:7577471884166707058:0] socket: 17 qp: 5435
2025-11-27T18:20:57.535182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471885138617168:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:57.535229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:57.535484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:57.535515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471885138617168:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:57.535548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471885138617168:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:57.535575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:57.535651Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:57.535821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471885138617168:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:57.535845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471885138617168:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:57.538610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471888461674369:2175]) peer tries to resume nonexistent session Self# [2:7577471885138617164:0] Peer# [1:7577471884166707058:0]
2025-11-27T18:20:57.538654Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471888461674369:2175] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:57.538804Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471889433584477:2194] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:57.539491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:57.539558Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:57.539589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471885138617168:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:57.539642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471885138617168:2048] VirtualId# [2:7577471885138617164:0]
2025-11-27T18:20:57.539780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 29235
TEvUndelivered Cookie# 29236
TEvUndelivered Cookie# 29237
TEvUndelivered Cookie# 29238
TEvUndelivered Cookie# 29239
TEvUndelivered Cookie# 29240
TEvUndelivered Cookie# 29241
TEvUndelivered Cookie# 29242
2025-11-27T18:20:57.542597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471888461674370:2176] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:57.542681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:57.543475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471889433584480:2195] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:57.545419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471888461674370:2176] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:57.545551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:57.545538Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471889433584480:2195] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:57.545581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:57.545611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471888461674373:2048]
2025-11-27T18:20:57.545637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471888461674373:2048] [node 2] ICS09 handshake done sender: [1:7577471888461674370:2176] self: [1:7577471888461674371:0] peer: [2:7577471889433584479:0] socket: 17 qp: 5440
2025-11-27T18:20:57.545661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471888461674373:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:57.545690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:57.545799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:57.545838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:57.545883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471889433584483:2048]
2025-11-27T18:20:57.545924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471889433584483:2048] [node 1] ICS09 handshake done sender: [2:7577471889433584480:2195] self: [2:7577471889433584479:0] peer: [1:7577471888461674371:0] socket: 16 qp: 5439
2025-11-27T18:20:57.545959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471889433584483:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:57.545992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:20:57.653774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471888461674373:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-27T18:20:57.653888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471888461674373:2048] VirtualId# [1:7577471888461674371:0]
2025-11-27T18:20:57.653927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471888461674373:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:57.653995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:57.654043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:57.654043Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:57.654310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471889433584483:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:20:57.654399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471889433584483:2048] VirtualId# [2:7577471889433584479:0]
2025-11-27T18:20:57.654445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471889433584483:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:57.654532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:57.654608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:57.658489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471889433584485:2196] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:57.658661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:57.658966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471888461674376:2178] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:57.661827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471889433584485:2196] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:57.661947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471888461674376:2178] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:57.662019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:57.662066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:57.662127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471889433584488:2048]
2025-11-27T18:20:57.662190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471889433584488:2048] [node 1] ICS09 handshake done sender: [2:7577471889433584485:2196] self: [2:7577471889433584486:0] peer: [1:7577471888461674375:0] socket: 17 qp: 5442
2025-11-27T18:20:57.662241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471889433584488:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:57.662255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:57.662283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:57.662303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:57.662355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471888461674379:2048]
2025-11-27T18:20:57.662404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471888461674379:2048] [node 2] ICS09 handshake done sender: [1:7577471888461674376:2178] self: [1:7577471888461674375:0] peer: [2:7577471889433584486:0] socket: 16 qp: 5441
2025-11-27T18:20:57.662441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471888461674379:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:57.662482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:20:58.128769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471888461674379:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-27T18:20:58.128854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471888461674379:2048] VirtualId# [1:7577471888461674375:0]
2025-11-27T18:20:58.128878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471888461674379:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:58.128922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:58.129007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:58.128997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:58.129269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471889433584488:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:20:58.129333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471889433584488:2048] VirtualId# [2:7577471889433584486:0]
2025-11-27T18:20:58.129369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471889433584488:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:58.129430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:58.129492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:58.132539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471893728551786:2198] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:58.132693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:58.132949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471892756641678:2179] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:58.135023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471893728551786:2198] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:58.135097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471892756641678:2179] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:58.135206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:58.135258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:58.135315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471893728551789:2048]
2025-11-27T18:20:58.135329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:58.135361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471893728551789:2048] [node 1] ICS09 handshake done sender: [2:7577471893728551786:2198] self: [2:7577471893728551787:0] peer: [1:7577471892756641677:0] socket: 17 qp: 5444
2025-11-27T18:20:58.135396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471893728551789:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:58.135364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:58.135397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471892756641681:2048]
2025-11-27T18:20:58.135438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:58.135439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471892756641681:2048] [node 2] ICS09 handshake done sender: [1:7577471892756641678:2179] self: [1:7577471892756641677:0] peer: [2:7577471893728551787:0] socket: 16 qp: 5443
2025-11-27T18:20:58.135464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471892756641681:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:58.135490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:20:58.586891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:20:58.587081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471892756641681:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-27T18:20:58.587118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471892756641681:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:58.587178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:58.587223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471892756641681:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:58.587206Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:58.587367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471893728551789:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:20:58.587438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471893728551789:2048] VirtualId# [2:7577471893728551787:0]
2025-11-27T18:20:58.587467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471893728551789:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:58.587519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:58.587556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:58.590344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471893728551791:2200]) peer tries to resume nonexistent session Self# [1:7577471892756641677:0] Peer# [2:7577471893728551787:0]
2025-11-27T18:20:58.590389Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471893728551791:2200] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:58.590498Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471892756641683:2180] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:20:58.591240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:58.591295Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:58.591314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471892756641681:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:58.591374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471892756641681:2048] VirtualId# [1:7577471892756641677:0]
2025-11-27T18:20:58.591457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:58.593773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471893728551792:2201] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:58.593884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:58.594105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471892756641686:2181] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:58.596578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471893728551792:2201] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:58.596636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471892756641686:2181] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:58.596739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:58.596773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:58.596817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471893728551795:2048]
2025-11-27T18:20:58.596856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:58.596860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471893728551795:2048] [node 1] ICS09 handshake done sender: [2:7577471893728551792:2201] self: [2:7577471893728551793:0] peer: [1:7577471892756641685:0] socket: 17 qp: 5447
2025-11-27T18:20:58.596888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:58.596901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471893728551795:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:58.596918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471892756641689:2048]
2025-11-27T18:20:58.596930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:58.596950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471892756641689:2048] [node 2] ICS09 handshake done sender: [1:7577471892756641686:2181] self: [1:7577471892756641685:0] peer: [2:7577471893728551793:0] socket: 16 qp: 5446
2025-11-27T18:20:58.596979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471892756641689:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:58.597008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:20:58.898038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:20:58.898305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471892756641689:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-27T18:20:58.898344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471892756641689:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:58.898461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:58.898530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471892756641689:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:58.898505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:58.898758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471893728551795:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:20:58.898796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471893728551795:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:58.898828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:58.898898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471893728551795:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:58.900932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471893728551795:2048] [node 1] ICS08 incoming handshake Self# [1:7577471892756641685:0] Peer# [2:7577471893728551793:0] Counter# 2217 LastInputSerial# 59
2025-11-27T18:20:58.902308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471892756641691:2182] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:58.902583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:58.902620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471892756641689:2048] [node 2] ICS09 handshake done sender: [1:7577471892756641691:2182] self: [1:7577471892756641685:0] peer: [2:7577471893728551793:0] socket: 22 qp: 5448
2025-11-27T18:20:58.902991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471892756641689:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:58.903032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:58.903109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471892756641689:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:58.903140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471892756641689:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:20:58.903181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:58.903214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471892756641689:2048] [node 2] ICS08 incoming handshake Self# [2:7577471893728551793:0] Peer# [1:7577471892756641685:0] Counter# 59 LastInputSerial# 2217
2025-11-27T18:20:58.903270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:58.903398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471892756641689:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:58.903444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471892756641689:2048] VirtualId# [1:7577471892756641685:0]
2025-11-27T18:20:58.903468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:58.904977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471893728551798:2204] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:58.905188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:58.905247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471893728551795:2048] [node 1] ICS09 handshake done sender: [2:7577471893728551798:2204] self: [2:7577471893728551793:0] peer: [1:7577471892756641685:0] socket: 23 qp: 5449
2025-11-27T18:20:58.905827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471893728551795:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:58.905882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:58.906167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:58.906204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471893728551795:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:58.906231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471893728551795:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:58.906253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:58.906342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:58.906532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471893728551795:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:58.906559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471893728551795:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:58.909920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471892756641697:2185]) peer tries to resume nonexistent session Self# [2:7577471893728551793:0] Peer# [1:7577471892756641685:0]
2025-11-27T18:20:58.909965Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471892756641697:2185] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:58.910117Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471893728551803:2206] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:58.910798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:58.910878Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:58.910904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471893728551795:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:58.910962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471893728551795:2048] VirtualId# [2:7577471893728551793:0]
2025-11-27T18:20:58.911027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:20:58.914273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471892756641698:2186] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:58.914386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:58.915326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471893728551806:2207] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:58.917873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471892756641698:2186] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:58.918041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:58.918074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:58.918046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471893728551806:2207] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:58.918107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471892756641701:2048]
2025-11-27T18:20:58.918137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471892756641701:2048] [node 2] ICS09 handshake done sender: [1:7577471892756641698:2186] self: [1:7577471892756641699:0] peer: [2:7577471893728551805:0] socket: 17 qp: 5454
2025-11-27T18:20:58.918163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471892756641701:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:58.918191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:58.918337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:58.918388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:58.918436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471893728551809:2048]
2025-11-27T18:20:58.918483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471893728551809:2048] [node 1] ICS09 handshake done sender: [2:7577471893728551806:2207] self: [2:7577471893728551805:0] peer: [1:7577471892756641699:0] socket: 16 qp: 5453
2025-11-27T18:20:58.918521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471893728551809:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:58.918556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:20:59.408926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:20:59.409125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471892756641701:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-27T18:20:59.409151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471892756641701:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:59.409194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:59.409228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471892756641701:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:59.410598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:59.411000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471893728551809:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:59.411036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471893728551809:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:59.411071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:59.411123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471893728551809:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:59.412787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471893728551809:2048] [node 1] ICS08 incoming handshake Self# [1:7577471892756641699:0] Peer# [2:7577471893728551805:0] Counter# 5356 LastInputSerial# 197
2025-11-27T18:20:59.413166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471897051608999:2188] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:59.413402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:59.413443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471892756641701:2048] [node 2] ICS09 handshake done sender: [1:7577471897051608999:2188] self: [1:7577471892756641699:0] peer: [2:7577471893728551805:0] socket: 22 qp: 5455
2025-11-27T18:20:59.413872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471892756641701:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:59.413919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:59.414274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471892756641701:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:59.414299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471892756641701:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:20:59.414333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:59.414377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471892756641701:2048] [node 2] ICS08 incoming handshake Self# [2:7577471893728551805:0] Peer# [1:7577471892756641699:0] Counter# 197 LastInputSerial# 5356
2025-11-27T18:20:59.414418Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:59.414545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471892756641701:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:59.414588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471892756641701:2048] VirtualId# [1:7577471892756641699:0]
2025-11-27T18:20:59.414615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:59.416213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471898023519108:2209] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:59.416499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:59.416565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471893728551809:2048] [node 1] ICS09 handshake done sender: [2:7577471898023519108:2209] self: [2:7577471893728551805:0] peer: [1:7577471892756641699:0] socket: 23 qp: 5456
2025-11-27T18:20:59.417131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471893728551809:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:59.417172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:59.417421Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:59.417446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471893728551809:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:59.417466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471893728551809:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:59.417521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:59.417611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:59.417764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471893728551809:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:59.417782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471893728551809:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:59.420413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471897051609005:2191]) peer tries to resume nonexistent session Self# [2:7577471893728551805:0] Peer# [1:7577471892756641699:0]
2025-11-27T18:20:59.420465Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471897051609005:2191] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:59.420589Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471898023519113:2211] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:59.421629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:59.421717Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:59.421750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471893728551809:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:59.421803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471893728551809:2048] VirtualId# [2:7577471893728551805:0]
2025-11-27T18:20:59.421914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 33453
TEvUndelivered Cookie# 33454
TEvUndelivered Cookie# 33455
TEvUndelivered Cookie# 33456
TEvUndelivered Cookie# 33457
TEvUndelivered Cookie# 33458
2025-11-27T18:20:59.424372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471897051609006:2192] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:59.424473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:59.425223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471898023519116:2212] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:59.427149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471897051609006:2192] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:59.427206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471898023519116:2212] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:59.427273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:59.427305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:59.427343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471897051609009:2048]
2025-11-27T18:20:59.427378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471897051609009:2048] [node 2] ICS09 handshake done sender: [1:7577471897051609006:2192] self: [1:7577471897051609007:0] peer: [2:7577471898023519115:0] socket: 17 qp: 5461
2025-11-27T18:20:59.427408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471897051609009:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:59.427435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:59.427431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:59.427460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:59.427514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471898023519119:2048]
2025-11-27T18:20:59.427564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471898023519119:2048] [node 1] ICS09 handshake done sender: [2:7577471898023519116:2212] self: [2:7577471898023519115:0] peer: [1:7577471897051609007:0] socket: 16 qp: 5460
2025-11-27T18:20:59.427595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471898023519119:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:59.427628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:20:59.745242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:20:59.745461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471897051609009:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-27T18:20:59.745485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471897051609009:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:20:59.745526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:59.745552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471897051609009:2048] [node 2] ICS15 start handshake
2025-11-27T18:20:59.746129Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:59.746408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471898023519119:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:20:59.746430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471898023519119:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:20:59.746451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:59.746484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471898023519119:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:59.748232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471898023519119:2048] [node 1] ICS08 incoming handshake Self# [1:7577471897051609007:0] Peer# [2:7577471898023519115:0] Counter# 2545 LastInputSerial# 64
2025-11-27T18:20:59.748657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471897051609011:2194] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:59.748837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:20:59.748872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471897051609009:2048] [node 2] ICS09 handshake done sender: [1:7577471897051609011:2194] self: [1:7577471897051609007:0] peer: [2:7577471898023519115:0] socket: 22 qp: 5462
2025-11-27T18:20:59.749225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471897051609009:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:59.749267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:59.749536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471897051609009:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:20:59.749558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471897051609009:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:20:59.749610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:20:59.749651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471897051609009:2048] [node 2] ICS08 incoming handshake Self# [2:7577471898023519115:0] Peer# [1:7577471897051609007:0] Counter# 64 LastInputSerial# 2545
2025-11-27T18:20:59.749682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:20:59.749809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471897051609009:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:20:59.749853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471897051609009:2048] VirtualId# [1:7577471897051609007:0]
2025-11-27T18:20:59.749878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:20:59.751241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471898023519122:2214] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:59.751423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:20:59.751459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471898023519119:2048] [node 1] ICS09 handshake done sender: [2:7577471898023519122:2214] self: [2:7577471898023519115:0] peer: [1:7577471897051609007:0] socket: 23 qp: 5463
2025-11-27T18:20:59.751913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471898023519119:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:59.751943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:20:59.752183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:20:59.752208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471898023519119:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:20:59.752226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471898023519119:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:20:59.752244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:20:59.752294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:20:59.752420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471898023519119:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:20:59.752437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471898023519119:2048] [node 1] ICS15 start handshake
2025-11-27T18:20:59.754720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471897051609017:2197]) peer tries to resume nonexistent session Self# [2:7577471898023519115:0] Peer# [1:7577471897051609007:0]
2025-11-27T18:20:59.754764Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471897051609017:2197] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:20:59.754894Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471898023519127:2216] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:20:59.755440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:20:59.755510Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:20:59.755534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471898023519119:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:20:59.755586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471898023519119:2048] VirtualId# [2:7577471898023519115:0]
2025-11-27T18:20:59.755678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 34073
TEvUndelivered Cookie# 34074
TEvUndelivered Cookie# 34075
TEvUndelivered Cookie# 34076
TEvUndelivered Cookie# 34077
TEvUndelivered Cookie# 34078
2025-11-27T18:20:59.758143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471897051609018:2198] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:59.758247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:20:59.759015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471898023519130:2217] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:20:59.760940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471897051609018:2198] [node 2] ICH04 handshake succeeded
2025-11-27T18:20:59.761004Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471898023519130:2217] [node 1] ICH04 handshake succeeded
2025-11-27T18:20:59.761094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:20:59.761131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:20:59.761175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471897051609021:2048]
2025-11-27T18:20:59.761192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:20:59.761208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471897051609021:2048] [node 2] ICS09 handshake done sender: [1:7577471897051609018:2198] self: [1:7577471897051609019:0] peer: [2:7577471898023519129:0] socket: 17 qp: 5468
2025-11-27T18:20:59.761220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:20:59.761230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471897051609021:2048] [node 2] ICS10 traffic start
2025-11-27T18:20:59.761252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471898023519133:2048]
2025-11-27T18:20:59.761273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:20:59.761298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471898023519133:2048] [node 1] ICS09 handshake done sender: [2:7577471898023519130:2217] self: [2:7577471898023519129:0] peer: [1:7577471897051609019:0] socket: 16 qp: 5467
2025-11-27T18:20:59.761319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471898023519133:2048] [node 1] ICS10 traffic start
2025-11-27T18:20:59.761340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:21:00.167357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:21:00.167485Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:00.167510Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:00.167707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471897051609021:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:00.167738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471897051609021:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:00.167787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471898023519133:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:00.167812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471898023519133:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:00.167815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:00.167837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:00.167868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471897051609021:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:00.167888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471898023519133:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:00.171808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471897051609021:2048] [node 2] ICS08 incoming handshake Self# [2:7577471898023519129:0] Peer# [1:7577471897051609019:0] Counter# 88 LastInputSerial# 3576
2025-11-27T18:21:00.171810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471898023519133:2048] [node 1] ICS08 incoming handshake Self# [1:7577471897051609019:0] Peer# [2:7577471898023519129:0] Counter# 3576 LastInputSerial# 88
2025-11-27T18:21:00.172147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471901346576319:2200] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:00.172349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:00.172392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471897051609021:2048] [node 2] ICS09 handshake done sender: [1:7577471901346576319:2200] self: [1:7577471897051609019:0] peer: [2:7577471898023519129:0] socket: 23 qp: 5470
2025-11-27T18:21:00.172749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471897051609021:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:00.172782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:00.174714Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471902318486433:2219] held: no
2025-11-27T18:21:00.174770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471902318486433:2219] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:00.174907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:00.174945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471898023519133:2048] [node 1] ICS09 handshake done sender: [2:7577471902318486433:2219] self: [2:7577471898023519129:0] peer: [1:7577471897051609019:0] socket: 25 qp: 5472
2025-11-27T18:21:00.175484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471898023519133:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:00.175525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:21:00.729478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:21:00.729593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:00.729695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:00.729797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471897051609021:2048] [node 2] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-27T18:21:00.729824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471897051609021:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:00.729843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:00.729879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471897051609021:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:00.729918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:21:00.729942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471898023519133:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:21:00.729966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471898023519133:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:00.730015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:00.730107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471898023519133:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:00.730132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471898023519133:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:00.732282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471898023519133:2048] [node 1] ICS08 incoming handshake Self# [1:7577471897051609019:0] Peer# [2:7577471898023519129:0] Counter# 9789 LastInputSerial# 243
2025-11-27T18:21:00.733917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471897051609021:2048] [node 2] ICS08 incoming handshake Self# [2:7577471898023519129:0] Peer# [1:7577471897051609019:0] Counter# 243 LastInputSerial# 9789
2025-11-27T18:21:00.734023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471901346576324:2202] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:00.734224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:00.734265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471897051609021:2048] [node 2] ICS09 handshake done sender: [1:7577471901346576324:2202] self: [1:7577471897051609019:0] peer: [2:7577471898023519129:0] socket: 16 qp: 5473
2025-11-27T18:21:00.734707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471897051609021:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:00.734743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:00.736793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471902318486437:2222] held: no
2025-11-27T18:21:00.736847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471902318486437:2222] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:00.736954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:00.737007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471898023519133:2048] [node 1] ICS09 handshake done sender: [2:7577471902318486437:2222] self: [2:7577471898023519129:0] peer: [1:7577471897051609019:0] socket: 17 qp: 5474
2025-11-27T18:21:00.737518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471898023519133:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:00.737556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:21:00.963905Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:21:00.964187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471898023519133:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-27T18:21:00.964219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471898023519133:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:00.964274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:00.964306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471898023519133:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:00.964383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:00.964617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471897051609021:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:00.964650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471897051609021:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:00.964696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:00.964727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471897051609021:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:00.966332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471897051609021:2048] [node 2] ICS08 incoming handshake Self# [2:7577471898023519129:0] Peer# [1:7577471897051609019:0] Counter# 305 LastInputSerial# 12385
2025-11-27T18:21:00.967585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471898023519133:2048] [node 1] ICS08 incoming handshake Self# [1:7577471897051609019:0] Peer# [2:7577471898023519129:0] Counter# 12385 LastInputSerial# 305
2025-11-27T18:21:00.967969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471901346576329:2204] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:00.968255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:00.968317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471897051609021:2048] [node 2] ICS09 handshake done sender: [1:7577471901346576329:2204] self: [1:7577471897051609019:0] peer: [2:7577471898023519129:0] socket: 23 qp: 5480
2025-11-27T18:21:00.968805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471897051609021:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:00.968854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:00.970861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471902318486443:2225] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:00.971006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471902318486443:2225] held: no
2025-11-27T18:21:00.971038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:00.971073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471898023519133:2048] [node 1] ICS09 handshake done sender: [2:7577471902318486443:2225] self: [2:7577471898023519129:0] peer: [1:7577471897051609019:0] socket: 25 qp: 5479
2025-11-27T18:21:00.971624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471898023519133:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:00.971672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:21:01.273933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:21:01.274087Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-27T18:21:01.274128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471897051609021:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:21:01.274088Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:01.274159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471897051609021:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:01.274198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:01.274294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:01.274370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471898023519133:2048] [node 1] ICS01 socket: 25 reason# EndOfStream
2025-11-27T18:21:01.274455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471898023519133:2048] VirtualId# [2:7577471898023519129:0]
2025-11-27T18:21:01.274500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471898023519133:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:01.274553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471897051609021:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:01.274582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471897051609021:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:01.274574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:01.274642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:01.278442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471906613453742:2227]) peer tries to resume nonexistent session Self# [1:7577471897051609019:0] Peer# [2:7577471898023519129:0]
2025-11-27T18:21:01.278529Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471906613453742:2227] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:01.278738Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471905641543630:2206] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:01.279617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:01.279704Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:01.279739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471897051609021:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:01.279806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471897051609021:2048] VirtualId# [1:7577471897051609019:0]
2025-11-27T18:21:01.279871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:01.283099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471906613453743:2228] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:01.283270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:01.283591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471905641543633:2207] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:01.286406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471906613453743:2228] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:01.286512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471905641543633:2207] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:01.286599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:01.286650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:01.286716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471906613453746:2048]
2025-11-27T18:21:01.286779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471906613453746:2048] [node 1] ICS09 handshake done sender: [2:7577471906613453743:2228] self: [2:7577471906613453744:0] peer: [1:7577471905641543632:0] socket: 17 qp: 5483
2025-11-27T18:21:01.286812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:01.286831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471906613453746:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:01.286862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:01.286877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:01.286919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471905641543636:2048]
2025-11-27T18:21:01.286971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471905641543636:2048] [node 2] ICS09 handshake done sender: [1:7577471905641543633:2207] self: [1:7577471905641543632:0] peer: [2:7577471906613453744:0] socket: 16 qp: 5482
2025-11-27T18:21:01.287006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471905641543636:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:01.287047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:21:01.543861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:21:01.543996Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] socket: Connection reset by peer
2025-11-27T18:21:01.544022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:01.544196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471906613453746:2048] [node 1] ICS07 socket disconnect 17 reason# ECONNRESET
2025-11-27T18:21:01.544223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471906613453746:2048] [node 1] ICS25 shutdown socket, reason# ECONNRESET
2025-11-27T18:21:01.544253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:01.544294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471906613453746:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:01.544306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471905641543636:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:01.544348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471905641543636:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:01.544393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:01.544464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471905641543636:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:01.546625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471905641543636:2048] [node 2] ICS08 incoming handshake Self# [2:7577471906613453744:0] Peer# [1:7577471905641543632:0] Counter# 46 LastInputSerial# 1874
2025-11-27T18:21:01.548115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471906613453746:2048] [node 1] ICS08 incoming handshake Self# [1:7577471905641543632:0] Peer# [2:7577471906613453744:0] Counter# 1874 LastInputSerial# 46
2025-11-27T18:21:01.548542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471905641543638:2208] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:01.548788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:01.548854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471905641543636:2048] [node 2] ICS09 handshake done sender: [1:7577471905641543638:2208] self: [1:7577471905641543632:0] peer: [2:7577471906613453744:0] socket: 24 qp: 5487
2025-11-27T18:21:01.549423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471905641543636:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:01.549477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:01.551507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471906613453750:2231] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:01.551672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471906613453750:2231] held: no
2025-11-27T18:21:01.551700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:01.551731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471906613453746:2048] [node 1] ICS09 handshake done sender: [2:7577471906613453750:2231] self: [2:7577471906613453744:0] peer: [1:7577471905641543632:0] socket: 25 qp: 5486
2025-11-27T18:21:01.552658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471906613453746:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:01.552718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:01.679523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471905641543636:2048] [node 2] ICS01 socket: 24 reason# Debug
2025-11-27T18:21:01.679631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471905641543636:2048] VirtualId# [1:7577471905641543632:0]
2025-11-27T18:21:01.679677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471905641543636:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:01.679738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:01.679801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:01.679802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:01.680104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471906613453746:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-11-27T18:21:01.680136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471906613453746:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:01.680213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:01.680263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471906613453746:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:01.683073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471905641543643:2210]) peer tries to resume nonexistent session Self# [2:7577471906613453744:0] Peer# [1:7577471905641543632:0]
2025-11-27T18:21:01.683251Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471905641543643:2210] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:01.683390Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471906613453753:2233] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:01.684341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:01.684391Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:01.684411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471906613453746:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:01.684452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471906613453746:2048] VirtualId# [2:7577471906613453744:0]
2025-11-27T18:21:01.684505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:01.687272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471906613453755:2234] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:01.689092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:01.689492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471905641543645:2211] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:01.689720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471905641543647:2212] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:01.692399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471906613453755:2234] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:01.692475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471905641543645:2211] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:01.692634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:01.692703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:01.692779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471906613453761:2048]
2025-11-27T18:21:01.692800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:01.692820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471906613453761:2048] [node 1] ICS09 handshake done sender: [2:7577471906613453755:2234] self: [2:7577471906613453759:0] peer: [1:7577471905641543644:0] socket: 21 qp: 5490
2025-11-27T18:21:01.692867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:01.692853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471906613453761:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:01.692925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471905641543650:2048]
2025-11-27T18:21:01.692881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:01.692977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471905641543650:2048] [node 2] ICS09 handshake done sender: [1:7577471905641543645:2211] self: [1:7577471905641543644:0] peer: [2:7577471906613453759:0] socket: 20 qp: 5489
2025-11-27T18:21:01.693016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471905641543650:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:01.693051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-27T18:21:02.019119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471906613453761:2048] [node 1] ICS01 socket: 21 reason# Debug
2025-11-27T18:21:02.019223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471906613453761:2048] VirtualId# [2:7577471906613453759:0]
2025-11-27T18:21:02.019258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471906613453761:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:02.019321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:02.019380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:02.020110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:02.020410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471905641543650:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-27T18:21:02.020435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471905641543650:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:02.020461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:02.020493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471905641543650:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:02.022452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471910908421059:2237]) peer tries to resume nonexistent session Self# [1:7577471905641543644:0] Peer# [2:7577471906613453759:0]
2025-11-27T18:21:02.022514Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471910908421059:2237] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:02.022661Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471909936510948:2213] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:02.023527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:02.023596Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:02.023627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471905641543650:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:02.023760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471905641543650:2048] VirtualId# [1:7577471905641543644:0]
2025-11-27T18:21:02.023831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:02.026820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471910908421060:2238] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:02.026995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:02.027271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471909936510951:2214] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:02.029693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471910908421060:2238] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:02.029804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471909936510951:2214] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:02.029948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:02.030010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:02.030074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471910908421063:2048]
2025-11-27T18:21:02.030106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:02.030119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471910908421063:2048] [node 1] ICS09 handshake done sender: [2:7577471910908421060:2238] self: [2:7577471910908421061:0] peer: [1:7577471909936510950:0] socket: 17 qp: 5495
2025-11-27T18:21:02.030161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471910908421063:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:02.030148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:02.030203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471909936510954:2048]
2025-11-27T18:21:02.030212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:02.030252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471909936510954:2048] [node 2] ICS09 handshake done sender: [1:7577471909936510951:2214] self: [1:7577471909936510950:0] peer: [2:7577471910908421061:0] socket: 16 qp: 5494
2025-11-27T18:21:02.030285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471909936510954:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:02.030312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:02.341405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471909936510954:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-27T18:21:02.341517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471909936510954:2048] VirtualId# [1:7577471909936510950:0]
2025-11-27T18:21:02.341556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471909936510954:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:02.341632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:02.341684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:02.341718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:02.341950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471910908421063:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:02.341992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471910908421063:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:02.342074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:02.342134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471910908421063:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:02.345944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471909936510956:2215]) peer tries to resume nonexistent session Self# [2:7577471910908421061:0] Peer# [1:7577471909936510950:0]
2025-11-27T18:21:02.346170Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471909936510956:2215] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:02.346352Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471910908421065:2240] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:02.348103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:02.348164Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:02.348186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471910908421063:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:02.348256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471910908421063:2048] VirtualId# [2:7577471910908421061:0]
2025-11-27T18:21:02.348327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:02.351053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471910908421067:2241] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:02.352659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:02.352963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471909936510958:2216] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:02.353205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471909936510960:2217] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:02.355489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471910908421067:2241] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:02.355628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:02.355624Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471909936510958:2216] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:02.355667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:02.355699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471910908421073:2048]
2025-11-27T18:21:02.355732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471910908421073:2048] [node 1] ICS09 handshake done sender: [2:7577471910908421067:2241] self: [2:7577471910908421071:0] peer: [1:7577471909936510957:0] socket: 23 qp: 5498
2025-11-27T18:21:02.355757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471910908421073:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:02.355782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:02.355908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:02.355979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:02.356039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471909936510963:2048]
2025-11-27T18:21:02.356093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471909936510963:2048] [node 2] ICS09 handshake done sender: [1:7577471909936510958:2216] self: [1:7577471909936510957:0] peer: [2:7577471910908421071:0] socket: 22 qp: 5497
2025-11-27T18:21:02.356134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471909936510963:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:02.356181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:21:02.675342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:21:02.675542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471910908421073:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-11-27T18:21:02.675571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471910908421073:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:02.675617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:02.675645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471910908421073:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:02.676904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:02.677231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471909936510963:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-27T18:21:02.677289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471909936510963:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:02.677370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:02.677429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471909936510963:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:02.679704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471909936510963:2048] [node 2] ICS08 incoming handshake Self# [2:7577471910908421071:0] Peer# [1:7577471909936510957:0] Counter# 53 LastInputSerial# 2215
2025-11-27T18:21:02.681104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471910908421073:2048] [node 1] ICS08 incoming handshake Self# [1:7577471909936510957:0] Peer# [2:7577471910908421071:0] Counter# 2215 LastInputSerial# 53
2025-11-27T18:21:02.681535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471909936510965:2218] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:02.681787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:02.681850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471909936510963:2048] [node 2] ICS09 handshake done sender: [1:7577471909936510965:2218] self: [1:7577471909936510957:0] peer: [2:7577471910908421071:0] socket: 24 qp: 5504
2025-11-27T18:21:02.682336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471909936510963:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:02.682388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:02.684602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471910908421077:2245] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:02.684770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471910908421077:2245] held: no
2025-11-27T18:21:02.684798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:02.684828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471910908421073:2048] [node 1] ICS09 handshake done sender: [2:7577471910908421077:2245] self: [2:7577471910908421071:0] peer: [1:7577471909936510957:0] socket: 25 qp: 5503
2025-11-27T18:21:02.685280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471910908421073:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:02.685315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:21:03.255488Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:21:03.255667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471910908421073:2048] [node 1] ICS07 socket disconnect 25 reason# Debug
2025-11-27T18:21:03.255692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471910908421073:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:03.255722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:03.255743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471910908421073:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:03.259708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:03.259981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471909936510963:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-27T18:21:03.260016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471909936510963:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:03.260036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:03.260071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471909936510963:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:03.261838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471909936510963:2048] [node 2] ICS08 incoming handshake Self# [2:7577471910908421071:0] Peer# [1:7577471909936510957:0] Counter# 192 LastInputSerial# 7961
2025-11-27T18:21:03.263121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471910908421073:2048] [node 1] ICS08 incoming handshake Self# [1:7577471909936510957:0] Peer# [2:7577471910908421071:0] Counter# 7961 LastInputSerial# 192
2025-11-27T18:21:03.263452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471914231478266:2220] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:03.263669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:03.263709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471909936510963:2048] [node 2] ICS09 handshake done sender: [1:7577471914231478266:2220] self: [1:7577471909936510957:0] peer: [2:7577471910908421071:0] socket: 20 qp: 5508
2025-11-27T18:21:03.264061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471909936510963:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:03.264102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:03.266276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471915203388378:2248] held: no
2025-11-27T18:21:03.266342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471915203388378:2248] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:03.266486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:03.266540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471910908421073:2048] [node 1] ICS09 handshake done sender: [2:7577471915203388378:2248] self: [2:7577471910908421071:0] peer: [1:7577471909936510957:0] socket: 21 qp: 5507
2025-11-27T18:21:03.267125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471910908421073:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:03.267185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:21:03.436860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:21:03.437190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471910908421073:2048] [node 1] ICS07 socket disconnect 21 reason# Debug
2025-11-27T18:21:03.437229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471910908421073:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:03.437376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:03.437425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471910908421073:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:03.437472Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:03.437782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471909936510963:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-27T18:21:03.437814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471909936510963:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:03.437872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:03.437915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471909936510963:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:03.440052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471909936510963:2048] [node 2] ICS08 incoming handshake Self# [2:7577471910908421071:0] Peer# [1:7577471909936510957:0] Counter# 236 LastInputSerial# 9715
2025-11-27T18:21:03.442352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471910908421073:2048] [node 1] ICS08 incoming handshake Self# [1:7577471909936510957:0] Peer# [2:7577471910908421071:0] Counter# 9715 LastInputSerial# 236
2025-11-27T18:21:03.442959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471914231478271:2222] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:03.443217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:03.443264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471909936510963:2048] [node 2] ICS09 handshake done sender: [1:7577471914231478271:2222] self: [1:7577471909936510957:0] peer: [2:7577471910908421071:0] socket: 24 qp: 5512
2025-11-27T18:21:03.443622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471909936510963:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:03.443659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:03.446303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471915203388383:2251] held: no
2025-11-27T18:21:03.446387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471915203388383:2251] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:03.446539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:03.446589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471910908421073:2048] [node 1] ICS09 handshake done sender: [2:7577471915203388383:2251] self: [2:7577471910908421071:0] peer: [1:7577471909936510957:0] socket: 25 qp: 5511
2025-11-27T18:21:03.447149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471910908421073:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:03.447200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:21:03.633963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:21:03.634103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-27T18:21:03.634139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471909936510963:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:21:03.634120Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:03.634167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471909936510963:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:03.634201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:03.634280Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:03.634413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471910908421073:2048] [node 1] ICS01 socket: 25 reason# EndOfStream
2025-11-27T18:21:03.634501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471910908421073:2048] VirtualId# [2:7577471910908421071:0]
2025-11-27T18:21:03.634519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471909936510963:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:03.634539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471910908421073:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:03.634552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471909936510963:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:03.634614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:03.634685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:03.637380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471915203388386:2253]) peer tries to resume nonexistent session Self# [1:7577471909936510957:0] Peer# [2:7577471910908421071:0]
2025-11-27T18:21:03.637450Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471915203388386:2253] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:03.637649Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471914231478276:2224] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:03.638518Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:03.638590Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:03.638623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471909936510963:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:03.638684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471909936510963:2048] VirtualId# [1:7577471909936510957:0]
2025-11-27T18:21:03.638733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:03.641713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471915203388387:2254] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:03.641904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:03.642225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471914231478279:2225] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:03.644959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471915203388387:2254] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:03.645060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471914231478279:2225] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:03.645169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:03.645217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:03.645316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471915203388390:2048]
2025-11-27T18:21:03.645344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:03.645369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471915203388390:2048] [node 1] ICS09 handshake done sender: [2:7577471915203388387:2254] self: [2:7577471915203388388:0] peer: [1:7577471914231478278:0] socket: 17 qp: 5515
2025-11-27T18:21:03.645391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:03.645409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471915203388390:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:03.645450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471914231478282:2048]
2025-11-27T18:21:03.645455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:03.645498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471914231478282:2048] [node 2] ICS09 handshake done sender: [1:7577471914231478279:2225] self: [1:7577471914231478278:0] peer: [2:7577471915203388388:0] socket: 16 qp: 5514
2025-11-27T18:21:03.645530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471914231478282:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:03.645568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:21:04.029376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:21:04.029522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:21:04.029572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471915203388390:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:21:04.029608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-11-27T18:21:04.029600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471915203388390:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:04.029644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:04.029721Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:04.029951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471914231478282:2048] [node 2] ICS07 socket disconnect 16 reason# ECONNRESET
2025-11-27T18:21:04.029987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471914231478282:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-11-27T18:21:04.029961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471915203388390:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:04.029993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471915203388390:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:04.030023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:04.030100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471914231478282:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:04.034084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471914231478282:2048] [node 2] ICS08 incoming handshake Self# [2:7577471915203388388:0] Peer# [1:7577471914231478278:0] Counter# 67 LastInputSerial# 2867
2025-11-27T18:21:04.034638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471918526445580:2226] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:04.035105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:04.035170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471914231478282:2048] [node 2] ICS09 handshake done sender: [1:7577471918526445580:2226] self: [1:7577471914231478278:0] peer: [2:7577471915203388388:0] socket: 23 qp: 5516
2025-11-27T18:21:04.035888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471914231478282:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:04.035933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:04.034060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471915203388390:2048] [node 1] ICS08 incoming handshake Self# [1:7577471914231478278:0] Peer# [2:7577471915203388388:0] Counter# 2867 LastInputSerial# 67
2025-11-27T18:21:04.038441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471919498355690:2257] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:04.038625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471919498355690:2257] held: no
2025-11-27T18:21:04.038663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:04.038705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471915203388390:2048] [node 1] ICS09 handshake done sender: [2:7577471919498355690:2257] self: [2:7577471915203388388:0] peer: [1:7577471914231478278:0] socket: 24 qp: 5519
2025-11-27T18:21:04.039136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471915203388390:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:04.039173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:21:04.540737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:21:04.540880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:04.541157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471914231478282:2048] [node 2] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-27T18:21:04.541198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471914231478282:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:04.541317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:04.541363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471914231478282:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:04.543124Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:04.543492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471915203388390:2048] [node 1] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-27T18:21:04.543532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471915203388390:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:04.543567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:04.543620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471915203388390:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:04.545603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471915203388390:2048] [node 1] ICS08 incoming handshake Self# [1:7577471914231478278:0] Peer# [2:7577471915203388388:0] Counter# 7749 LastInputSerial# 187
2025-11-27T18:21:04.546073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471918526445585:2228] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:04.546335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:04.546406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471914231478282:2048] [node 2] ICS09 handshake done sender: [1:7577471918526445585:2228] self: [1:7577471914231478278:0] peer: [2:7577471915203388388:0] socket: 16 qp: 5520
2025-11-27T18:21:04.547521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471914231478282:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:04.547572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:04.548031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471914231478282:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:21:04.548066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471914231478282:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:21:04.548115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:04.548164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471914231478282:2048] [node 2] ICS08 incoming handshake Self# [2:7577471915203388388:0] Peer# [1:7577471914231478278:0] Counter# 187 LastInputSerial# 7749
2025-11-27T18:21:04.548218Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:04.548414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471914231478282:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:21:04.548472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471914231478282:2048] VirtualId# [1:7577471914231478278:0]
2025-11-27T18:21:04.548508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:04.549899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471919498355694:2260] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:04.550094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:04.550152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471915203388390:2048] [node 1] ICS09 handshake done sender: [2:7577471919498355694:2260] self: [2:7577471915203388388:0] peer: [1:7577471914231478278:0] socket: 17 qp: 5521
2025-11-27T18:21:04.551095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471915203388390:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:04.551147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:04.551390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:21:04.551426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471915203388390:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:21:04.551457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471915203388390:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:04.551484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:04.551561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:04.551755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471915203388390:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:04.551782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471915203388390:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:04.555230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471918526445591:2231]) peer tries to resume nonexistent session Self# [2:7577471915203388388:0] Peer# [1:7577471914231478278:0]
2025-11-27T18:21:04.555299Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471918526445591:2231] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:04.555464Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471919498355699:2262] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:04.556171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:04.556246Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:04.556277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471915203388390:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:04.556332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471915203388390:2048] VirtualId# [2:7577471915203388388:0]
2025-11-27T18:21:04.556494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 44626
TEvUndelivered Cookie# 44627
TEvUndelivered Cookie# 44628
TEvUndelivered Cookie# 44629
TEvUndelivered Cookie# 44630
TEvUndelivered Cookie# 44631
TEvUndelivered Cookie# 44632
TEvUndelivered Cookie# 44633
TEvUndelivered Cookie# 44634
2025-11-27T18:21:04.559320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471918526445592:2232] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:04.559455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:04.560397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471919498355702:2263] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:04.562773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471918526445592:2232] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:04.562857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471919498355702:2263] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:04.562982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:04.563031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:04.563099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471918526445595:2048]
2025-11-27T18:21:04.563156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471918526445595:2048] [node 2] ICS09 handshake done sender: [1:7577471918526445592:2232] self: [1:7577471918526445593:0] peer: [2:7577471919498355701:0] socket: 17 qp: 5526
2025-11-27T18:21:04.563145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:04.563195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471918526445595:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:04.563186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:04.563227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471919498355705:2048]
2025-11-27T18:21:04.563240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:04.563273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471919498355705:2048] [node 1] ICS09 handshake done sender: [2:7577471919498355702:2263] self: [2:7577471919498355701:0] peer: [1:7577471918526445593:0] socket: 16 qp: 5525
2025-11-27T18:21:04.563316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471919498355705:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:04.563355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:04.705625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471918526445595:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-27T18:21:04.705712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471918526445595:2048] VirtualId# [1:7577471918526445593:0]
2025-11-27T18:21:04.705750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471918526445595:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:04.705808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:04.705848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:04.707301Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:04.707768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471919498355705:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:04.707811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471919498355705:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:04.707848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:04.707899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471919498355705:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:04.710388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471918526445597:2234]) peer tries to resume nonexistent session Self# [2:7577471919498355701:0] Peer# [1:7577471918526445593:0]
2025-11-27T18:21:04.710474Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471918526445597:2234] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:04.710664Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471919498355707:2264] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:04.711582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:04.711660Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:04.711697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471919498355705:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:04.711776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471919498355705:2048] VirtualId# [2:7577471919498355701:0]
2025-11-27T18:21:04.711936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 44735
TEvUndelivered Cookie# 44736
TEvUndelivered Cookie# 44737
TEvUndelivered Cookie# 44738
TEvUndelivered Cookie# 44739
TEvUndelivered Cookie# 44740
TEvUndelivered Cookie# 44741
TEvUndelivered Cookie# 44742
TEvUndelivered Cookie# 44743
TEvUndelivered Cookie# 44744
2025-11-27T18:21:04.715554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471918526445598:2235] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:04.715745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:04.716155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471919498355710:2265] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:04.718905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471918526445598:2235] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:04.719024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471919498355710:2265] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:04.719168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:04.719236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:04.719302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471918526445601:2048]
2025-11-27T18:21:04.719354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471918526445601:2048] [node 2] ICS09 handshake done sender: [1:7577471918526445598:2235] self: [1:7577471918526445599:0] peer: [2:7577471919498355709:0] socket: 17 qp: 5529
2025-11-27T18:21:04.719406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471918526445601:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:04.719455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:04.719447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:04.719500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:04.719560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471919498355713:2048]
2025-11-27T18:21:04.719615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471919498355713:2048] [node 1] ICS09 handshake done sender: [2:7577471919498355710:2265] self: [2:7577471919498355709:0] peer: [1:7577471918526445599:0] socket: 16 qp: 5528
2025-11-27T18:21:04.719654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471919498355713:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:04.719711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:21:05.140757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:21:05.140874Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:05.141036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471919498355713:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:21:05.141103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471919498355713:2048] VirtualId# [2:7577471919498355709:0]
2025-11-27T18:21:05.141140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471919498355713:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:05.141188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:05.141238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:21:05.141327Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:05.141694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471918526445601:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:05.141723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471918526445601:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:05.141765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
TEvNodeDisconnected
2025-11-27T18:21:05.141819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471918526445601:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:05.144093Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471922821412899:2237] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:05.143887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471923793323011:2266]) peer tries to resume nonexistent session Self# [1:7577471918526445599:0] Peer# [2:7577471919498355709:0]
2025-11-27T18:21:05.143943Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471923793323011:2266] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:05.144906Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:05.144989Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:05.145024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471918526445601:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:05.145088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471918526445601:2048] VirtualId# [1:7577471918526445599:0]
2025-11-27T18:21:05.145137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:05.148237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471923793323012:2267] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:05.148429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:05.148813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471922821412902:2238] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:05.151481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471923793323012:2267] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:05.151574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471922821412902:2238] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:05.151741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:05.151796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:05.151854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471923793323015:2048]
2025-11-27T18:21:05.151908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471923793323015:2048] [node 1] ICS09 handshake done sender: [2:7577471923793323012:2267] self: [2:7577471923793323013:0] peer: [1:7577471922821412901:0] socket: 17 qp: 5532
2025-11-27T18:21:05.152011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471923793323015:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:05.152045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:05.152072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:05.152088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:05.152143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471922821412905:2048]
2025-11-27T18:21:05.152186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471922821412905:2048] [node 2] ICS09 handshake done sender: [1:7577471922821412902:2238] self: [1:7577471922821412901:0] peer: [2:7577471923793323013:0] socket: 16 qp: 5531
2025-11-27T18:21:05.152228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471922821412905:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:05.152264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:05.401353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471922821412905:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-27T18:21:05.401431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471922821412905:2048] VirtualId# [1:7577471922821412901:0]
2025-11-27T18:21:05.401460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471922821412905:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:05.401518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:05.401585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:05.401607Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:05.401813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471923793323015:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:05.401841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471923793323015:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:05.401895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:05.401930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471923793323015:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:05.404754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471922821412907:2239]) peer tries to resume nonexistent session Self# [2:7577471923793323013:0] Peer# [1:7577471922821412901:0]
2025-11-27T18:21:05.404844Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471922821412907:2239] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:05.404994Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471923793323017:2269] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:05.405842Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:05.405917Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:05.405936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471923793323015:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:05.405987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471923793323015:2048] VirtualId# [2:7577471923793323013:0]
2025-11-27T18:21:05.406047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:05.409715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471923793323021:2271] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:05.409891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:05.409929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471922821412911:2241] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:05.410135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471922821412909:2240] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:05.412690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471923793323021:2271] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:05.412783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471922821412909:2240] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:05.412919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:05.413045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:05.413101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471923793323025:2048]
2025-11-27T18:21:05.413134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471923793323025:2048] [node 1] ICS09 handshake done sender: [2:7577471923793323021:2271] self: [2:7577471923793323023:0] peer: [1:7577471922821412908:0] socket: 17 qp: 5535
2025-11-27T18:21:05.413095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:05.413160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471923793323025:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:05.413153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:05.413208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471922821412914:2048]
2025-11-27T18:21:05.413215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:05.413275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471922821412914:2048] [node 2] ICS09 handshake done sender: [1:7577471922821412909:2240] self: [1:7577471922821412908:0] peer: [2:7577471923793323023:0] socket: 16 qp: 5534
2025-11-27T18:21:05.413312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471922821412914:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:05.413341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-27T18:21:05.969007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471923793323025:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-11-27T18:21:05.969120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471923793323025:2048] VirtualId# [2:7577471923793323023:0]
2025-11-27T18:21:05.969156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471923793323025:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:05.969225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:05.969332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:21:05.970198Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:05.970523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471922821412914:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:05.970553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471922821412914:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:05.970579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:05.970623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471922821412914:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-27T18:21:05.973316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471923793323027:2273]) peer tries to resume nonexistent session Self# [1:7577471922821412908:0] Peer# [2:7577471923793323023:0]
2025-11-27T18:21:05.973399Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471923793323027:2273] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:05.973548Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471922821412916:2242] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:05.974472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:05.974530Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:05.974551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471922821412914:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:05.974611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471922821412914:2048] VirtualId# [1:7577471922821412908:0]
2025-11-27T18:21:05.974659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:05.977831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471923793323028:2274] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:05.978001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:05.978369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471922821412919:2243] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:05.981192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471923793323028:2274] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:05.981300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471922821412919:2243] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:05.981459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:05.981521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:05.981587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471923793323031:2048]
2025-11-27T18:21:05.981633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471923793323031:2048] [node 1] ICS09 handshake done sender: [2:7577471923793323028:2274] self: [2:7577471923793323029:0] peer: [1:7577471922821412918:0] socket: 17 qp: 5540
2025-11-27T18:21:05.981621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:05.981684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471923793323031:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:05.981686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:05.981743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471922821412922:2048]
2025-11-27T18:21:05.981736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:05.981792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471922821412922:2048] [node 2] ICS09 handshake done sender: [1:7577471922821412919:2243] self: [1:7577471922821412918:0] peer: [2:7577471923793323029:0] socket: 16 qp: 5539
2025-11-27T18:21:05.981827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471922821412922:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:05.981868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:06.255318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471922821412922:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-27T18:21:06.255399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471922821412922:2048] VirtualId# [1:7577471922821412918:0]
2025-11-27T18:21:06.255426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471922821412922:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:06.255469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:06.255517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:06.255531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:06.255743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471923793323031:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:06.255772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471923793323031:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:06.255848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:06.255909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471923793323031:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:06.259392Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471927116380220:2244]) peer tries to resume nonexistent session Self# [2:7577471923793323029:0] Peer# [1:7577471922821412918:0]
2025-11-27T18:21:06.259528Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471927116380220:2244] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:06.259691Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471928088290329:2276] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:06.260875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:06.260945Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:06.260964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471923793323031:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:06.261026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471923793323031:2048] VirtualId# [2:7577471923793323029:0]
2025-11-27T18:21:06.261087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:06.265137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471928088290331:2277] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:06.265437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:06.265480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471927116380224:2246] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:06.265700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471927116380222:2245] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:06.268608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471928088290331:2277] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:06.268712Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471927116380222:2245] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:06.268799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:06.268857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:06.268908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471928088290337:2048]
2025-11-27T18:21:06.268948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471928088290337:2048] [node 1] ICS09 handshake done sender: [2:7577471928088290331:2277] self: [2:7577471928088290335:0] peer: [1:7577471927116380221:0] socket: 23 qp: 5543
2025-11-27T18:21:06.268959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:06.268977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471928088290337:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:06.269009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:06.269026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:06.269068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471927116380227:2048]
2025-11-27T18:21:06.269115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471927116380227:2048] [node 2] ICS09 handshake done sender: [1:7577471927116380222:2245] self: [1:7577471927116380221:0] peer: [2:7577471928088290335:0] socket: 22 qp: 5542
2025-11-27T18:21:06.269146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471927116380227:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:06.269188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:21:06.403263Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:21:06.403430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471928088290337:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-11-27T18:21:06.403458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471928088290337:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:06.403513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:06.403556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471928088290337:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:06.404030Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:06.407619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471927116380227:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-27T18:21:06.407661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471927116380227:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:06.407694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:06.407737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471927116380227:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:06.409369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471927116380227:2048] [node 2] ICS08 incoming handshake Self# [2:7577471928088290335:0] Peer# [1:7577471927116380221:0] Counter# 12 LastInputSerial# 488
2025-11-27T18:21:06.410539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471928088290337:2048] [node 1] ICS08 incoming handshake Self# [1:7577471927116380221:0] Peer# [2:7577471928088290335:0] Counter# 488 LastInputSerial# 12
2025-11-27T18:21:06.410881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471927116380229:2247] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:06.411054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:06.411096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471927116380227:2048] [node 2] ICS09 handshake done sender: [1:7577471927116380229:2247] self: [1:7577471927116380221:0] peer: [2:7577471928088290335:0] socket: 24 qp: 5549
2025-11-27T18:21:06.411429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471927116380227:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:06.411470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:06.413409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471928088290341:2281] held: no
2025-11-27T18:21:06.413465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471928088290341:2281] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:06.413570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:06.413606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471928088290337:2048] [node 1] ICS09 handshake done sender: [2:7577471928088290341:2281] self: [2:7577471928088290335:0] peer: [1:7577471927116380221:0] socket: 25 qp: 5548
2025-11-27T18:21:06.414130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471928088290337:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:06.414167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:21:06.834944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:21:06.835119Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:21:06.835151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471928088290337:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:21:06.835181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471928088290337:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:06.835216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:06.835125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:06.835273Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:06.835422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471927116380227:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-27T18:21:06.835460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471927116380227:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:06.835495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471928088290337:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:06.835520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471928088290337:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:06.835527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:06.835594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471927116380227:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:06.839822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471928088290337:2048] [node 1] ICS08 incoming handshake Self# [1:7577471927116380221:0] Peer# [2:7577471928088290335:0] Counter# 4940 LastInputSerial# 124
2025-11-27T18:21:06.839823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471927116380227:2048] [node 2] ICS08 incoming handshake Self# [2:7577471928088290335:0] Peer# [1:7577471927116380221:0] Counter# 124 LastInputSerial# 4940
2025-11-27T18:21:06.840272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471927116380234:2249] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:06.840519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:06.840580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471927116380227:2048] [node 2] ICS09 handshake done sender: [1:7577471927116380234:2249] self: [1:7577471927116380221:0] peer: [2:7577471928088290335:0] socket: 17 qp: 5551
2025-11-27T18:21:06.841716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471927116380227:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:06.841771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:06.843870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471928088290346:2284] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:06.844074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471928088290346:2284] held: no
2025-11-27T18:21:06.844108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:06.844147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471928088290337:2048] [node 1] ICS09 handshake done sender: [2:7577471928088290346:2284] self: [2:7577471928088290335:0] peer: [1:7577471927116380221:0] socket: 21 qp: 5553
2025-11-27T18:21:06.845233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471928088290337:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:06.845324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:21:07.287471Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:21:07.287850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471928088290337:2048] [node 1] ICS07 socket disconnect 21 reason# Debug
2025-11-27T18:21:07.287891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471928088290337:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:07.287958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:07.288020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471928088290337:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:07.288003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:07.288217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471927116380227:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:21:07.288280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471927116380227:2048] VirtualId# [1:7577471927116380221:0]
2025-11-27T18:21:07.288308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471927116380227:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:07.288397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:07.288435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:07.291088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471931411347535:2251]) peer tries to resume nonexistent session Self# [2:7577471928088290335:0] Peer# [1:7577471927116380221:0]
2025-11-27T18:21:07.291152Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471931411347535:2251] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:07.291374Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471932383257645:2286] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:07.292278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:07.292387Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:07.292426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471928088290337:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:07.292492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471928088290337:2048] VirtualId# [2:7577471928088290335:0]
2025-11-27T18:21:07.292674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 49908
TEvUndelivered Cookie# 49909
TEvUndelivered Cookie# 49910
TEvUndelivered Cookie# 49911
TEvUndelivered Cookie# 49912
TEvUndelivered Cookie# 49913
TEvUndelivered Cookie# 49914
TEvUndelivered Cookie# 49915
TEvUndelivered Cookie# 49916
2025-11-27T18:21:07.296842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471932383257649:2288] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:07.297047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:07.297156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471931411347539:2253] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:07.297350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471931411347537:2252] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:07.299703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471932383257649:2288] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:07.299829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471931411347537:2252] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:07.299904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:07.299944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:07.300049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471932383257653:2048]
2025-11-27T18:21:07.300093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471932383257653:2048] [node 1] ICS09 handshake done sender: [2:7577471932383257649:2288] self: [2:7577471932383257651:0] peer: [1:7577471931411347536:0] socket: 16 qp: 5556
2025-11-27T18:21:07.300119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471932383257653:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:07.300149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:07.300136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:07.300191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:07.300238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471931411347542:2048]
2025-11-27T18:21:07.300286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471931411347542:2048] [node 2] ICS09 handshake done sender: [1:7577471931411347537:2252] self: [1:7577471931411347536:0] peer: [2:7577471932383257651:0] socket: 17 qp: 5555
2025-11-27T18:21:07.300319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471931411347542:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:07.300355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:21:07.458412Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:21:07.458781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471931411347542:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-27T18:21:07.458820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471931411347542:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:07.458872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:07.458908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471931411347542:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:07.461410Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:07.461666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471932383257653:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:07.461714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471932383257653:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:07.461808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:07.461857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471932383257653:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:07.463694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471932383257653:2048] [node 1] ICS08 incoming handshake Self# [1:7577471931411347536:0] Peer# [2:7577471932383257651:0] Counter# 716 LastInputSerial# 18
2025-11-27T18:21:07.464194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471931411347544:2254] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:07.464537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:07.464593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471931411347542:2048] [node 2] ICS09 handshake done sender: [1:7577471931411347544:2254] self: [1:7577471931411347536:0] peer: [2:7577471932383257651:0] socket: 20 qp: 5559
2025-11-27T18:21:07.464992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471931411347542:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:07.465041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:07.465601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471931411347542:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:21:07.465640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471931411347542:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:21:07.465694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:07.465746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471931411347542:2048] [node 2] ICS08 incoming handshake Self# [2:7577471932383257651:0] Peer# [1:7577471931411347536:0] Counter# 18 LastInputSerial# 716
2025-11-27T18:21:07.465930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:07.466159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471931411347542:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:07.467520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471932383257656:2291] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:07.467821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:07.467882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471932383257653:2048] [node 1] ICS09 handshake done sender: [2:7577471932383257656:2291] self: [2:7577471932383257651:0] peer: [1:7577471931411347536:0] socket: 21 qp: 5560
2025-11-27T18:21:07.468983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471932383257653:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:07.469042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:07.469364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:07.469574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471932383257653:2048] [node 1] ICS01 socket: 21 reason# EndOfStream
2025-11-27T18:21:07.469641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471932383257653:2048] VirtualId# [2:7577471932383257651:0]
2025-11-27T18:21:07.469676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471932383257653:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:07.469730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:07.469778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:07.475330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471931411347550:2257] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:07.475438Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:07.475469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471931411347542:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:07.475537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471931411347542:2048] VirtualId# [1:7577471931411347536:0]
2025-11-27T18:21:07.475590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:07.475626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:07.477069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471932383257662:2293] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:07.479460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471931411347550:2257] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:07.479606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:07.479563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471932383257662:2293] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:07.479658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:07.479693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471931411347553:2048]
2025-11-27T18:21:07.479739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471931411347553:2048] [node 2] ICS09 handshake done sender: [1:7577471931411347550:2257] self: [1:7577471931411347551:0] peer: [2:7577471932383257661:0] socket: 17 qp: 5564
2025-11-27T18:21:07.479784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471931411347553:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:07.479836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:07.479856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:07.479890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:07.479917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471932383257665:2048]
2025-11-27T18:21:07.479943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471932383257665:2048] [node 1] ICS09 handshake done sender: [2:7577471932383257662:2293] self: [2:7577471932383257661:0] peer: [1:7577471931411347551:0] socket: 16 qp: 5563
2025-11-27T18:21:07.479975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471932383257665:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:07.480003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:21:07.602722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:21:07.602839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:07.604546Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:07.604834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471932383257665:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:07.604868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471932383257665:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:07.604932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:07.604977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471932383257665:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:07.608053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471931411347553:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:07.608094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471931411347553:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:07.608126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:07.608183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471931411347553:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:07.610000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471931411347553:2048] [node 2] ICS08 incoming handshake Self# [2:7577471932383257661:0] Peer# [1:7577471931411347551:0] Counter# 9 LastInputSerial# 354
2025-11-27T18:21:07.612174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471931411347555:2259] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:07.611648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471932383257665:2048] [node 1] ICS08 incoming handshake Self# [1:7577471931411347551:0] Peer# [2:7577471932383257661:0] Counter# 354 LastInputSerial# 9
2025-11-27T18:21:07.612511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:07.612574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471931411347553:2048] [node 2] ICS09 handshake done sender: [1:7577471931411347555:2259] self: [1:7577471931411347551:0] peer: [2:7577471932383257661:0] socket: 24 qp: 5568
2025-11-27T18:21:07.613049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471931411347553:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:07.613098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:07.615282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471932383257669:2295] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:07.615496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471932383257669:2295] held: no
2025-11-27T18:21:07.615540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:07.615582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471932383257665:2048] [node 1] ICS09 handshake done sender: [2:7577471932383257669:2295] self: [2:7577471932383257661:0] peer: [1:7577471931411347551:0] socket: 25 qp: 5567
2025-11-27T18:21:07.616179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471932383257665:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:07.616234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:21:08.168871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:21:08.168986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:08.169229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471932383257665:2048] [node 1] ICS01 socket: 25 reason# EndOfStream
2025-11-27T18:21:08.169329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471932383257665:2048] VirtualId# [2:7577471932383257661:0]
2025-11-27T18:21:08.169373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471932383257665:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:08.169403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:08.169454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:21:08.169991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
TEvNodeDisconnected
2025-11-27T18:21:08.170430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471931411347553:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-11-27T18:21:08.170473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471931411347553:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:08.170499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:08.170552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471931411347553:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:08.172853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471936678224968:2297]) peer tries to resume nonexistent session Self# [1:7577471931411347551:0] Peer# [2:7577471932383257661:0]
2025-11-27T18:21:08.172941Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471936678224968:2297] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:08.173115Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471935706314856:2261] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:08.173901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:08.173980Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:08.174009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471931411347553:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:08.174067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471931411347553:2048] VirtualId# [1:7577471931411347551:0]
2025-11-27T18:21:08.174127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:08.176906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471936678224969:2298] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:08.177094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:08.177520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471935706314859:2262] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:08.180527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471936678224969:2298] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:08.180625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471935706314859:2262] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:08.180762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:08.180816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:08.180886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:08.180922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:08.180882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471936678224972:2048]
2025-11-27T18:21:08.180935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471936678224972:2048] [node 1] ICS09 handshake done sender: [2:7577471936678224969:2298] self: [2:7577471936678224970:0] peer: [1:7577471935706314858:0] socket: 17 qp: 5571
2025-11-27T18:21:08.180970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471935706314862:2048]
2025-11-27T18:21:08.180968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471936678224972:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:08.181004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:08.181016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471935706314862:2048] [node 2] ICS09 handshake done sender: [1:7577471935706314859:2262] self: [1:7577471935706314858:0] peer: [2:7577471936678224970:0] socket: 16 qp: 5570
2025-11-27T18:21:08.181049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471935706314862:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:08.181086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:08.661981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471935706314862:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-27T18:21:08.662083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471935706314862:2048] VirtualId# [1:7577471935706314858:0]
2025-11-27T18:21:08.662119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471935706314862:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:08.662178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:08.662241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:08.662565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:08.662995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471936678224972:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:08.663047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471936678224972:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:08.663086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:08.663140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471936678224972:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:08.665273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471935706314864:2263]) peer tries to resume nonexistent session Self# [2:7577471936678224970:0] Peer# [1:7577471935706314858:0]
2025-11-27T18:21:08.665341Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471935706314864:2263] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:08.665513Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471936678224974:2300] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:08.666371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:08.666443Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:08.666473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471936678224972:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:08.666537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471936678224972:2048] VirtualId# [2:7577471936678224970:0]
2025-11-27T18:21:08.666685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 52557
TEvUndelivered Cookie# 52558
TEvUndelivered Cookie# 52559
TEvUndelivered Cookie# 52560
TEvUndelivered Cookie# 52561
TEvUndelivered Cookie# 52562
TEvUndelivered Cookie# 52563
TEvUndelivered Cookie# 52564
TEvUndelivered Cookie# 52565
TEvUndelivered Cookie# 52566
2025-11-27T18:21:08.670186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471935706314865:2264] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:08.670316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:08.670555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471936678224977:2301] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:08.672736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471935706314865:2264] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:08.672884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471936678224977:2301] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:08.672903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:08.672947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:08.672992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471935706314868:2048]
2025-11-27T18:21:08.673038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471935706314868:2048] [node 2] ICS09 handshake done sender: [1:7577471935706314865:2264] self: [1:7577471935706314866:0] peer: [2:7577471936678224976:0] socket: 17 qp: 5574
2025-11-27T18:21:08.673102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471935706314868:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:08.673147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:08.673143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:08.673183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:08.673227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471936678224980:2048]
2025-11-27T18:21:08.673283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471936678224980:2048] [node 1] ICS09 handshake done sender: [2:7577471936678224977:2301] self: [2:7577471936678224976:0] peer: [1:7577471935706314866:0] socket: 16 qp: 5573
2025-11-27T18:21:08.673312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471936678224980:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:08.673347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:21:08.863271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:21:08.863408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:21:08.863442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471936678224980:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:21:08.863475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471936678224980:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:08.863519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:08.863528Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:08.863585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:08.863833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471936678224980:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:08.863855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471935706314868:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:08.863860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471936678224980:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:08.863894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471935706314868:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:08.863995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:08.864052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471935706314868:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:08.866440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471935706314868:2048] [node 2] ICS08 incoming handshake Self# [2:7577471936678224976:0] Peer# [1:7577471935706314866:0] Counter# 21 LastInputSerial# 870
2025-11-27T18:21:08.867856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471936678224980:2048] [node 1] ICS08 incoming handshake Self# [1:7577471935706314866:0] Peer# [2:7577471936678224976:0] Counter# 870 LastInputSerial# 21
2025-11-27T18:21:08.868330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471935706314870:2266] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:08.868614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:08.868674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471935706314868:2048] [node 2] ICS09 handshake done sender: [1:7577471935706314870:2266] self: [1:7577471935706314866:0] peer: [2:7577471936678224976:0] socket: 24 qp: 5578
2025-11-27T18:21:08.869146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471935706314868:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:08.869191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:08.871537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471936678224984:2303] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:08.871761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471936678224984:2303] held: no
2025-11-27T18:21:08.871811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:08.871852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471936678224980:2048] [node 1] ICS09 handshake done sender: [2:7577471936678224984:2303] self: [2:7577471936678224976:0] peer: [1:7577471935706314866:0] socket: 25 qp: 5577
2025-11-27T18:21:08.872292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471936678224980:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:08.872352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:09.125804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471935706314868:2048] [node 2] ICS01 socket: 24 reason# Debug
2025-11-27T18:21:09.125878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471935706314868:2048] VirtualId# [1:7577471935706314866:0]
2025-11-27T18:21:09.125901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471935706314868:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:09.125941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:09.125990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:09.126032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:09.126320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471936678224980:2048] [node 1] ICS01 socket: 25 reason# EndOfStream
2025-11-27T18:21:09.126394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471936678224980:2048] VirtualId# [2:7577471936678224976:0]
2025-11-27T18:21:09.126433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471936678224980:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:09.126517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:09.126567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:09.128971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471940973192283:2305] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:09.129111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:09.129428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471940001282172:2268] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:09.131975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471940973192283:2305] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:09.132062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471940001282172:2268] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:09.132154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:09.132190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:09.132233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471940973192286:2048]
2025-11-27T18:21:09.132290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471940973192286:2048] [node 1] ICS09 handshake done sender: [2:7577471940973192283:2305] self: [2:7577471940973192284:0] peer: [1:7577471940001282171:0] socket: 17 qp: 5580
2025-11-27T18:21:09.132331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471940973192286:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:09.132315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:09.132362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:09.132360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:09.132404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471940001282175:2048]
2025-11-27T18:21:09.132447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471940001282175:2048] [node 2] ICS09 handshake done sender: [1:7577471940001282172:2268] self: [1:7577471940001282171:0] peer: [2:7577471940973192284:0] socket: 16 qp: 5579
2025-11-27T18:21:09.132480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471940001282175:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:09.132511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:21:09.520300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:21:09.520684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471940001282175:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-27T18:21:09.520722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471940001282175:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:09.520775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:09.520813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471940001282175:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:09.520816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:09.521069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471940973192286:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:21:09.521166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471940973192286:2048] VirtualId# [2:7577471940973192284:0]
2025-11-27T18:21:09.521205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471940973192286:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:09.521298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:09.521358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:09.524012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471940973192288:2307]) peer tries to resume nonexistent session Self# [1:7577471940001282171:0] Peer# [2:7577471940973192284:0]
2025-11-27T18:21:09.524063Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471940973192288:2307] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:09.524256Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471940001282177:2269] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:09.525318Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:09.525402Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:09.525433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471940001282175:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:09.525592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471940001282175:2048] VirtualId# [1:7577471940001282171:0]
2025-11-27T18:21:09.525697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:09.528241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471940973192289:2308] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:09.528374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:09.528646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471940001282180:2270] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:09.531013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471940973192289:2308] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:09.531147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471940001282180:2270] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:09.531174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:09.531207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:09.531237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471940973192292:2048]
2025-11-27T18:21:09.531286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471940973192292:2048] [node 1] ICS09 handshake done sender: [2:7577471940973192289:2308] self: [2:7577471940973192290:0] peer: [1:7577471940001282179:0] socket: 17 qp: 5583
2025-11-27T18:21:09.531320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471940973192292:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:09.531386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:09.531403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:09.531436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:09.531469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471940001282183:2048]
2025-11-27T18:21:09.531511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471940001282183:2048] [node 2] ICS09 handshake done sender: [1:7577471940001282180:2270] self: [1:7577471940001282179:0] peer: [2:7577471940973192290:0] socket: 16 qp: 5582
2025-11-27T18:21:09.531547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471940001282183:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:09.531571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:09.661610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471940001282183:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-27T18:21:09.661700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471940001282183:2048] VirtualId# [1:7577471940001282179:0]
2025-11-27T18:21:09.661726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471940001282183:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:09.661779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:09.661816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:09.661968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:09.662409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471940973192292:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:09.662454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471940973192292:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:09.662489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:09.662546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471940973192292:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:09.664962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471940001282185:2271]) peer tries to resume nonexistent session Self# [2:7577471940973192290:0] Peer# [1:7577471940001282179:0]
2025-11-27T18:21:09.665035Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471940001282185:2271] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:09.665196Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471940973192294:2310] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:09.666138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:09.666209Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:09.666242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471940973192292:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:09.666308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471940973192292:2048] VirtualId# [2:7577471940973192290:0]
2025-11-27T18:21:09.666460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 54212
TEvUndelivered Cookie# 54213
TEvUndelivered Cookie# 54214
TEvUndelivered Cookie# 54215
TEvUndelivered Cookie# 54216
TEvUndelivered Cookie# 54217
TEvUndelivered Cookie# 54218
TEvUndelivered Cookie# 54219
TEvUndelivered Cookie# 54220
2025-11-27T18:21:09.669414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471940001282186:2272] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:09.669550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:09.669839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471940973192297:2311] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:09.672412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471940001282186:2272] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:09.672501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471940973192297:2311] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:09.672635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:09.672687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:09.672741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471940001282189:2048]
2025-11-27T18:21:09.672802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471940001282189:2048] [node 2] ICS09 handshake done sender: [1:7577471940001282186:2272] self: [1:7577471940001282187:0] peer: [2:7577471940973192296:0] socket: 17 qp: 5586
2025-11-27T18:21:09.672845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471940001282189:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:09.672846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:09.672885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:09.672890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:09.672933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471940973192300:2048]
2025-11-27T18:21:09.672972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471940973192300:2048] [node 1] ICS09 handshake done sender: [2:7577471940973192297:2311] self: [2:7577471940973192296:0] peer: [1:7577471940001282187:0] socket: 16 qp: 5585
2025-11-27T18:21:09.673005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471940973192300:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:09.673042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:21:09.971433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:21:09.971603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-27T18:21:09.971561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:09.971638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471940001282189:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:21:09.971665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471940001282189:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:09.971701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:09.971782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:09.971783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471940973192300:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:21:09.971870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471940973192300:2048] VirtualId# [2:7577471940973192296:0]
2025-11-27T18:21:09.971909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471940973192300:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:09.972000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:09.972048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471940001282189:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:09.972057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:21:09.972074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471940001282189:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-27T18:21:09.974633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471940973192302:2312]) peer tries to resume nonexistent session Self# [1:7577471940001282187:0] Peer# [2:7577471940973192296:0]
2025-11-27T18:21:09.974707Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471940973192302:2312] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:09.974884Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471940001282191:2274] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:09.975939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:09.976043Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:09.976070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471940001282189:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:09.976142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471940001282189:2048] VirtualId# [1:7577471940001282187:0]
2025-11-27T18:21:09.976191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:09.979137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471940973192303:2313] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:09.979302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:09.979644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471940001282194:2275] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:09.982421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471940973192303:2313] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:09.982546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471940001282194:2275] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:09.982597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:09.982648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:09.982696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471940973192306:2048]
2025-11-27T18:21:09.982781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471940973192306:2048] [node 1] ICS09 handshake done sender: [2:7577471940973192303:2313] self: [2:7577471940973192304:0] peer: [1:7577471940001282193:0] socket: 17 qp: 5589
2025-11-27T18:21:09.982825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471940973192306:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:09.982823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:09.982863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:09.982868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:09.982906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471940001282197:2048]
2025-11-27T18:21:09.982933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471940001282197:2048] [node 2] ICS09 handshake done sender: [1:7577471940001282194:2275] self: [1:7577471940001282193:0] peer: [2:7577471940973192304:0] socket: 16 qp: 5588
2025-11-27T18:21:09.982952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471940001282197:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:09.982975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-27T18:21:10.525986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471940973192306:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-11-27T18:21:10.526095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471940973192306:2048] VirtualId# [2:7577471940973192304:0]
2025-11-27T18:21:10.526127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471940973192306:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:10.526192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:10.526274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 55721
TEvUndelivered Cookie# 55722
TEvUndelivered Cookie# 55723
2025-11-27T18:21:10.528019Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:10.528315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471940001282197:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:10.528342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471940001282197:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:10.528409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:10.528447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471940001282197:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:10.532208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471944296249496:2277] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:10.532227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471945268159607:2316]) peer tries to resume nonexistent session Self# [1:7577471940001282193:0] Peer# [2:7577471940973192304:0]
2025-11-27T18:21:10.532279Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471945268159607:2316] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:10.532332Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:10.532366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471940001282197:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:10.532433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471940001282197:2048] VirtualId# [1:7577471940001282193:0]
2025-11-27T18:21:10.532499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:10.533462Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471944296249495:2276] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:10.534327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [1:7577471944296249496:2277] held: yes
2025-11-27T18:21:10.534675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471945268159605:2315] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:10.537229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471944296249496:2277] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:10.537341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471945268159605:2315] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:10.537453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:10.537499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:10.537561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471944296249500:2048]
2025-11-27T18:21:10.537596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:10.537611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471944296249500:2048] [node 2] ICS09 handshake done sender: [1:7577471944296249496:2277] self: [1:7577471944296249498:0] peer: [2:7577471945268159604:0] socket: 21 qp: 5591
2025-11-27T18:21:10.537635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:10.537648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471944296249500:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:10.537682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471945268159609:2048]
2025-11-27T18:21:10.537690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:10.537727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471945268159609:2048] [node 1] ICS09 handshake done sender: [2:7577471945268159605:2315] self: [2:7577471945268159604:0] peer: [1:7577471944296249498:0] socket: 17 qp: 5590
2025-11-27T18:21:10.537762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471945268159609:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:10.537794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:21:10.713898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:21:10.714065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:10.714333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471945268159609:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:21:10.714449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471945268159609:2048] VirtualId# [2:7577471945268159604:0]
2025-11-27T18:21:10.714487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471945268159609:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:10.714533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:10.714594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:21:10.714872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:10.715170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471944296249500:2048] [node 2] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-27T18:21:10.715203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471944296249500:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:10.715243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:10.715300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471944296249500:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-27T18:21:10.719423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471945268159611:2317]) peer tries to resume nonexistent session Self# [1:7577471944296249498:0] Peer# [2:7577471945268159604:0]
2025-11-27T18:21:10.719480Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471945268159611:2317] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:10.719697Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471944296249502:2279] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:10.720605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:10.720687Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:10.720721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471944296249500:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:10.720787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471944296249500:2048] VirtualId# [1:7577471944296249498:0]
2025-11-27T18:21:10.720844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:10.723996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471945268159612:2318] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:10.724119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:10.724664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471944296249505:2280] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:10.733622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471945268159612:2318] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:10.733825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471944296249505:2280] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:10.733905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:10.733965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:10.734021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471945268159615:2048]
2025-11-27T18:21:10.734066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471945268159615:2048] [node 1] ICS09 handshake done sender: [2:7577471945268159612:2318] self: [2:7577471945268159613:0] peer: [1:7577471944296249504:0] socket: 17 qp: 5594
2025-11-27T18:21:10.734114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471945268159615:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:10.734170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:10.734216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:10.734289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:10.734340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471944296249508:2048]
2025-11-27T18:21:10.734384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471944296249508:2048] [node 2] ICS09 handshake done sender: [1:7577471944296249505:2280] self: [1:7577471944296249504:0] peer: [2:7577471945268159613:0] socket: 16 qp: 5595
2025-11-27T18:21:10.734418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471944296249508:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:10.734455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:11.130082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471944296249508:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-27T18:21:11.130188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471944296249508:2048] VirtualId# [1:7577471944296249504:0]
2025-11-27T18:21:11.130226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471944296249508:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:11.130300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:11.130373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:11.134237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:11.134520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471945268159615:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:21:11.134645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471945268159615:2048] VirtualId# [2:7577471945268159613:0]
2025-11-27T18:21:11.134682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471945268159615:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:11.134746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:11.134795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:11.140206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471948591216806:2281] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:11.140387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:11.141093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471949563126914:2320] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:11.149563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471948591216806:2281] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:11.149843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:11.149908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:11.149961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471948591216809:2048]
2025-11-27T18:21:11.150007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471948591216809:2048] [node 2] ICS09 handshake done sender: [1:7577471948591216806:2281] self: [1:7577471948591216807:0] peer: [2:7577471949563126913:0] socket: 17 qp: 5596
2025-11-27T18:21:11.150045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471948591216809:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:11.150082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:11.153214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471949563126914:2320] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:11.157531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:11.157598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:11.157656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471949563126917:2048]
2025-11-27T18:21:11.157702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471949563126917:2048] [node 1] ICS09 handshake done sender: [2:7577471949563126914:2320] self: [2:7577471949563126913:0] peer: [1:7577471948591216807:0] socket: 16 qp: 5597
2025-11-27T18:21:11.157740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471949563126917:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:11.157778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:11.508181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471948591216809:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-27T18:21:11.508296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471948591216809:2048] VirtualId# [1:7577471948591216807:0]
2025-11-27T18:21:11.508331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471948591216809:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:11.508391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:11.508457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:11.509364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:11.509769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471949563126917:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:11.509800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471949563126917:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:11.509832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:11.509875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471949563126917:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:11.512250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471948591216811:2283]) peer tries to resume nonexistent session Self# [2:7577471949563126913:0] Peer# [1:7577471948591216807:0]
2025-11-27T18:21:11.512333Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471948591216811:2283] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:11.512915Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471949563126919:2321] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:11.513711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:11.513796Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:11.513828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471949563126917:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:11.513909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471949563126917:2048] VirtualId# [2:7577471949563126913:0]
2025-11-27T18:21:11.514064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 57240
TEvUndelivered Cookie# 57241
TEvUndelivered Cookie# 57242
TEvUndelivered Cookie# 57243
TEvUndelivered Cookie# 57244
TEvUndelivered Cookie# 57245
TEvUndelivered Cookie# 57246
TEvUndelivered Cookie# 57247
2025-11-27T18:21:11.519324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471948591216812:2284] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:11.519509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:11.519846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471949563126922:2322] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:11.522680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471948591216812:2284] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:11.522816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471949563126922:2322] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:11.523030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:11.523089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:11.523144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471949563126925:2048]
2025-11-27T18:21:11.523198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471949563126925:2048] [node 1] ICS09 handshake done sender: [2:7577471949563126922:2322] self: [2:7577471949563126921:0] peer: [1:7577471948591216813:0] socket: 16 qp: 5600
2025-11-27T18:21:11.523232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471949563126925:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:11.523267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:11.523714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:11.523758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:11.523796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471948591216815:2048]
2025-11-27T18:21:11.523839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471948591216815:2048] [node 2] ICS09 handshake done sender: [1:7577471948591216812:2284] self: [1:7577471948591216813:0] peer: [2:7577471949563126921:0] socket: 17 qp: 5599
2025-11-27T18:21:11.523873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471948591216815:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:11.523906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:11.967544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471948591216815:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-27T18:21:11.968222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471948591216815:2048] VirtualId# [1:7577471948591216813:0]
2025-11-27T18:21:11.968271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471948591216815:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:11.968335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:11.968393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:11.969852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:11.970126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471949563126925:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:11.970161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471949563126925:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:11.970258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:11.970316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471949563126925:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:11.973846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471948591216820:2287]) peer tries to resume nonexistent session Self# [2:7577471949563126921:0] Peer# [1:7577471948591216813:0]
2025-11-27T18:21:11.973914Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471948591216820:2287] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:11.977641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471949563126929:2324] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:11.977799Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471949563126927:2323] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:11.978659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:11.978746Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:11.978780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471949563126925:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:11.978934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471949563126925:2048] VirtualId# [2:7577471949563126921:0]
2025-11-27T18:21:11.979012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:11.980651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:11.983829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471948591216818:2286] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:11.986009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471948591216822:2288] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:11.993924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471949563126929:2324] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:11.994539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471948591216818:2286] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:11.994239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:11.994296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:11.994344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471949563126935:2048]
2025-11-27T18:21:11.994387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471949563126935:2048] [node 1] ICS09 handshake done sender: [2:7577471949563126929:2324] self: [2:7577471949563126932:0] peer: [1:7577471948591216817:0] socket: 23 qp: 5604
2025-11-27T18:21:11.994422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471949563126935:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:11.994458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:12.005578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:12.005674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:12.005732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471952886184120:2048]
2025-11-27T18:21:12.005780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471952886184120:2048] [node 2] ICS09 handshake done sender: [1:7577471948591216818:2286] self: [1:7577471948591216817:0] peer: [2:7577471949563126932:0] socket: 21 qp: 5602
2025-11-27T18:21:12.005818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471952886184120:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:12.005857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:21:12.516719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:21:12.517102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471949563126935:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-11-27T18:21:12.517142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471949563126935:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:12.517192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:12.517242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471949563126935:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:12.519769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:12.520178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471952886184120:2048] [node 2] ICS01 socket: 21 reason# EndOfStream
2025-11-27T18:21:12.520339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471952886184120:2048] VirtualId# [1:7577471948591216817:0]
2025-11-27T18:21:12.520382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471952886184120:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:12.520422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:12.520483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:12.523441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471952886184122:2289]) peer tries to resume nonexistent session Self# [2:7577471949563126932:0] Peer# [1:7577471948591216817:0]
2025-11-27T18:21:12.523535Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471952886184122:2289] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:12.523734Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471953858094233:2327] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:12.524591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:12.524670Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:12.524698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471949563126935:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:12.524886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471949563126935:2048] VirtualId# [2:7577471949563126932:0]
2025-11-27T18:21:12.525034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 58941
TEvUndelivered Cookie# 58942
TEvUndelivered Cookie# 58943
TEvUndelivered Cookie# 58944
TEvUndelivered Cookie# 58945
TEvUndelivered Cookie# 58946
TEvUndelivered Cookie# 58947
2025-11-27T18:21:12.529895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471953858094237:2329] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:12.530173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:12.530200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471952886184126:2291] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:12.530482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471952886184124:2290] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:12.534214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471952886184124:2290] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:12.534042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471953858094237:2329] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:12.534337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:12.534423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:12.534493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471953858094241:2048]
2025-11-27T18:21:12.534553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471953858094241:2048] [node 1] ICS09 handshake done sender: [2:7577471953858094237:2329] self: [2:7577471953858094239:0] peer: [1:7577471952886184123:0] socket: 17 qp: 5609
2025-11-27T18:21:12.534615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471953858094241:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:12.534677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:12.534523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:12.534597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:12.534647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471952886184129:2048]
2025-11-27T18:21:12.534695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471952886184129:2048] [node 2] ICS09 handshake done sender: [1:7577471952886184124:2290] self: [1:7577471952886184123:0] peer: [2:7577471953858094239:0] socket: 16 qp: 5608
2025-11-27T18:21:12.534727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471952886184129:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:12.534764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:21:12.745746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:21:12.746199Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:12.746450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471953858094241:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:21:12.746556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471953858094241:2048] VirtualId# [2:7577471953858094239:0]
2025-11-27T18:21:12.746606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471953858094241:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:12.746672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:12.746718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:21:12.746033Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:12.746317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471952886184129:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:12.746346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471952886184129:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:12.746378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:12.746420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471952886184129:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-27T18:21:12.753150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471953858094243:2331]) peer tries to resume nonexistent session Self# [1:7577471952886184123:0] Peer# [2:7577471953858094239:0]
2025-11-27T18:21:12.753244Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471953858094243:2331] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:12.754065Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471952886184131:2292] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:12.755079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:12.755169Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:12.755202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471952886184129:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:12.755316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471952886184129:2048] VirtualId# [1:7577471952886184123:0]
2025-11-27T18:21:12.755388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:12.758243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471952886184133:2293] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:12.760049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471953858094247:2333] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:12.760160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:12.760473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471952886184135:2294] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:12.763549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471953858094247:2333] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:12.763980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:12.764026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:12.764096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471953858094250:2048]
2025-11-27T18:21:12.764151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471953858094250:2048] [node 1] ICS09 handshake done sender: [2:7577471953858094247:2333] self: [2:7577471953858094248:0] peer: [1:7577471952886184134:0] socket: 17 qp: 5616
2025-11-27T18:21:12.764194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471953858094250:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:12.764238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:12.767314Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471952886184135:2294] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:12.767682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:12.767763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:12.767831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471952886184139:2048]
2025-11-27T18:21:12.767888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471952886184139:2048] [node 2] ICS09 handshake done sender: [1:7577471952886184135:2294] self: [1:7577471952886184134:0] peer: [2:7577471953858094248:0] socket: 16 qp: 5617
2025-11-27T18:21:12.767950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471952886184139:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:12.767994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-27T18:21:13.009952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-27T18:21:13.010276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471952886184139:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-27T18:21:13.010309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471952886184139:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:13.010387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:13.010445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471952886184139:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:13.010684Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:21:13.010723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471953858094250:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:21:13.010748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471953858094250:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:13.010776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:13.011051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:13.011282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471953858094250:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:13.011305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471953858094250:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:13.013209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471953858094250:2048] [node 1] ICS08 incoming handshake Self# [1:7577471952886184134:0] Peer# [2:7577471953858094248:0] Counter# 1591 LastInputSerial# 40
2025-11-27T18:21:13.014283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471957181151437:2295] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:13.014768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:13.014854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471952886184139:2048] [node 2] ICS09 handshake done sender: [1:7577471957181151437:2295] self: [1:7577471952886184134:0] peer: [2:7577471953858094248:0] socket: 21 qp: 5618
2025-11-27T18:21:13.015329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471952886184139:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:13.015386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:13.015730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471952886184139:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:21:13.015766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471952886184139:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:21:13.015821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:13.015880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471952886184139:2048] [node 2] ICS08 incoming handshake Self# [2:7577471953858094248:0] Peer# [1:7577471952886184134:0] Counter# 40 LastInputSerial# 1591
2025-11-27T18:21:13.015930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:13.016365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471952886184139:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:21:13.016429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471952886184139:2048] VirtualId# [1:7577471952886184134:0]
2025-11-27T18:21:13.016471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:13.018314Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471958153061549:2336] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:13.018579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:13.018640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471953858094250:2048] [node 1] ICS09 handshake done sender: [2:7577471958153061549:2336] self: [2:7577471953858094248:0] peer: [1:7577471952886184134:0] socket: 23 qp: 5619
2025-11-27T18:21:13.019537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471953858094250:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:13.019590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:13.019921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:21:13.019964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471953858094250:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:21:13.019995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471953858094250:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:13.020020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:13.020100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:13.020295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471953858094250:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:13.020318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471953858094250:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:13.023594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471957181151442:2297]) peer tries to resume nonexistent session Self# [2:7577471953858094248:0] Peer# [1:7577471952886184134:0]
2025-11-27T18:21:13.023667Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471957181151442:2297] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:13.023820Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471958153061553:2338] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:13.024795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:13.024883Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:13.024918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471953858094250:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:13.024992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471953858094250:2048] VirtualId# [2:7577471953858094248:0]
2025-11-27T18:21:13.025122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 59592
TEvUndelivered Cookie# 59593
TEvUndelivered Cookie# 59594
TEvUndelivered Cookie# 59595
TEvUndelivered Cookie# 59596
2025-11-27T18:21:13.028183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471957181151443:2298] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:13.028327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:13.029422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471958153061557:2339] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:13.032307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471957181151443:2298] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:13.032413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471958153061557:2339] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:13.032672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:13.032710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:13.032744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471958153061560:2048]
2025-11-27T18:21:13.032770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471958153061560:2048] [node 1] ICS09 handshake done sender: [2:7577471958153061557:2339] self: [2:7577471958153061556:0] peer: [1:7577471957181151444:0] socket: 16 qp: 5623
2025-11-27T18:21:13.032537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:13.032789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471958153061560:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:13.032599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:13.032817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:13.032657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471957181151446:2048]
2025-11-27T18:21:13.032700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471957181151446:2048] [node 2] ICS09 handshake done sender: [1:7577471957181151443:2298] self: [1:7577471957181151444:0] peer: [2:7577471958153061556:0] socket: 17 qp: 5624
2025-11-27T18:21:13.032739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471957181151446:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:13.032791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:21:13.509993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:21:13.510275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471958153061560:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-27T18:21:13.510312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471958153061560:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:13.510384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:13.510427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471958153061560:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:13.511049Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:13.511321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471957181151446:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:13.511345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471957181151446:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:13.511368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:13.511404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471957181151446:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:13.513156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471957181151446:2048] [node 2] ICS08 incoming handshake Self# [2:7577471958153061556:0] Peer# [1:7577471957181151444:0] Counter# 84 LastInputSerial# 3508
2025-11-27T18:21:13.514949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471958153061560:2048] [node 1] ICS08 incoming handshake Self# [1:7577471957181151444:0] Peer# [2:7577471958153061556:0] Counter# 3508 LastInputSerial# 84
2025-11-27T18:21:13.515316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471957181151448:2300] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:13.515563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:13.515618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471957181151446:2048] [node 2] ICS09 handshake done sender: [1:7577471957181151448:2300] self: [1:7577471957181151444:0] peer: [2:7577471958153061556:0] socket: 24 qp: 5627
2025-11-27T18:21:13.516831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471957181151446:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:13.516883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:13.519032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471958153061564:2341] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:13.519239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471958153061564:2341] held: no
2025-11-27T18:21:13.519277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:13.519327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471958153061560:2048] [node 1] ICS09 handshake done sender: [2:7577471958153061564:2341] self: [2:7577471958153061556:0] peer: [1:7577471957181151444:0] socket: 25 qp: 5628
2025-11-27T18:21:13.519790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471958153061560:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:13.519849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-27T18:21:14.012701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7577471828332131888:2048] [node 2] ICP34 closed connection by debug command
2025-11-27T18:21:14.012852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-11-27T18:21:14.012881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471957181151446:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:21:14.012906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471957181151446:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:14.012887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:14.012939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:14.013037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:14.013230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471958153061560:2048] [node 1] ICS01 socket: 25 reason# EndOfStream
2025-11-27T18:21:14.013274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471957181151446:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:14.013297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471957181151446:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:14.013336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471958153061560:2048] VirtualId# [2:7577471958153061556:0]
2025-11-27T18:21:14.013393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471958153061560:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:14.013494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:14.013560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:14.016027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471962448028863:2343]) peer tries to resume nonexistent session Self# [1:7577471957181151444:0] Peer# [2:7577471958153061556:0]
2025-11-27T18:21:14.016095Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471962448028863:2343] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:14.016257Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471961476118749:2302] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:14.017114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:14.017186Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:14.017217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471957181151446:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:14.017320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471957181151446:2048] VirtualId# [1:7577471957181151444:0]
2025-11-27T18:21:14.017386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:14.020286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471962448028864:2344] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:14.020461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:14.020763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471961476118752:2303] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:14.023281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471962448028864:2344] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:14.023376Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471961476118752:2303] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:14.023508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:14.023560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:14.023616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471962448028867:2048]
2025-11-27T18:21:14.023654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471962448028867:2048] [node 1] ICS09 handshake done sender: [2:7577471962448028864:2344] self: [2:7577471962448028865:0] peer: [1:7577471961476118751:0] socket: 17 qp: 5631
2025-11-27T18:21:14.023683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471962448028867:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:14.023710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:14.023693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:14.023741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:14.023796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471961476118755:2048]
2025-11-27T18:21:14.023852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471961476118755:2048] [node 2] ICS09 handshake done sender: [1:7577471961476118752:2303] self: [1:7577471961476118751:0] peer: [2:7577471962448028865:0] socket: 16 qp: 5630
2025-11-27T18:21:14.023889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471961476118755:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:14.023921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:21:14.160646Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:21:14.161033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471962448028867:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-27T18:21:14.161071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471962448028867:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:14.161123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:14.161161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471962448028867:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:14.163391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:14.163587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471961476118755:2048] [node 2] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:21:14.163666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471961476118755:2048] VirtualId# [1:7577471961476118751:0]
2025-11-27T18:21:14.163710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471961476118755:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:14.163792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:14.163847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:14.166292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471961476118757:2304]) peer tries to resume nonexistent session Self# [2:7577471962448028865:0] Peer# [1:7577471961476118751:0]
2025-11-27T18:21:14.166387Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471961476118757:2304] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:14.166615Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471962448028869:2346] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:14.167498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:14.167601Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:14.167641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471962448028867:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:14.167725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471962448028867:2048] VirtualId# [2:7577471962448028865:0]
2025-11-27T18:21:14.167899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 61738
TEvUndelivered Cookie# 61739
TEvUndelivered Cookie# 61740
TEvUndelivered Cookie# 61741
TEvUndelivered Cookie# 61742
TEvUndelivered Cookie# 61743
TEvUndelivered Cookie# 61744
TEvUndelivered Cookie# 61745
TEvUndelivered Cookie# 61746
2025-11-27T18:21:14.172071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471962448028873:2348] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:14.172325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:14.172342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471961476118761:2306] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:14.172672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471961476118759:2305] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:14.175054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471962448028873:2348] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:14.175116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471961476118759:2305] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:14.175263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:14.175312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:14.175387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471962448028877:2048]
2025-11-27T18:21:14.175443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471962448028877:2048] [node 1] ICS09 handshake done sender: [2:7577471962448028873:2348] self: [2:7577471962448028875:0] peer: [1:7577471961476118758:0] socket: 17 qp: 5634
2025-11-27T18:21:14.175421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:14.175489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471962448028877:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:14.175520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:14.175570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471961476118764:2048]
2025-11-27T18:21:14.175618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471961476118764:2048] [node 2] ICS09 handshake done sender: [1:7577471961476118759:2305] self: [1:7577471961476118758:0] peer: [2:7577471962448028875:0] socket: 16 qp: 5633
2025-11-27T18:21:14.175541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:14.175651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471961476118764:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:14.175695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:21:14.456382Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:21:14.456716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471962448028877:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-27T18:21:14.456760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471962448028877:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:14.456817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:14.456869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471962448028877:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:14.456943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:14.457272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471961476118764:2048] [node 2] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:21:14.457390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471961476118764:2048] VirtualId# [1:7577471961476118758:0]
2025-11-27T18:21:14.457439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471961476118764:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:14.457526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:14.457574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:14.460461Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471961476118766:2307]) peer tries to resume nonexistent session Self# [2:7577471962448028875:0] Peer# [1:7577471961476118758:0]
2025-11-27T18:21:14.460525Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471961476118766:2307] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:14.460683Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471962448028879:2350] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:14.462027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:14.462109Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:14.462136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471962448028877:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:14.462205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471962448028877:2048] VirtualId# [2:7577471962448028875:0]
2025-11-27T18:21:14.462371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 62169
TEvUndelivered Cookie# 62170
TEvUndelivered Cookie# 62171
TEvUndelivered Cookie# 62172
TEvUndelivered Cookie# 62173
TEvUndelivered Cookie# 62174
TEvUndelivered Cookie# 62175
TEvUndelivered Cookie# 62176
2025-11-27T18:21:14.466415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471962448028883:2352] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:14.466631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:14.466595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471961476118770:2309] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:14.466901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471961476118768:2308] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:14.469445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471962448028883:2352] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:14.469587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471961476118768:2308] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:14.469632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:14.469688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:14.469738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471962448028887:2048]
2025-11-27T18:21:14.469793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471962448028887:2048] [node 1] ICS09 handshake done sender: [2:7577471962448028883:2352] self: [2:7577471962448028885:0] peer: [1:7577471961476118767:0] socket: 17 qp: 5639
2025-11-27T18:21:14.469830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471962448028887:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:14.469869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:14.469919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:14.469979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:14.470032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471961476118773:2048]
2025-11-27T18:21:14.470082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471961476118773:2048] [node 2] ICS09 handshake done sender: [1:7577471961476118768:2308] self: [1:7577471961476118767:0] peer: [2:7577471962448028885:0] socket: 16 qp: 5638
2025-11-27T18:21:14.470118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471961476118773:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:14.470153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:21:14.887430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:21:14.887519Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:14.887509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:14.887710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471962448028887:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:14.887737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471962448028887:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:14.887767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:14.887805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471962448028887:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:14.887953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471961476118773:2048] [node 2] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:21:14.888061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471961476118773:2048] VirtualId# [1:7577471961476118767:0]
2025-11-27T18:21:14.888110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471961476118773:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:14.888151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:14.888213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:14.891738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471961476118775:2310]) peer tries to resume nonexistent session Self# [2:7577471962448028885:0] Peer# [1:7577471961476118767:0]
2025-11-27T18:21:14.891808Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471961476118775:2310] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:14.891964Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471962448028889:2354] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:14.892638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:14.892708Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:14.892735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471962448028887:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:14.892816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471962448028887:2048] VirtualId# [2:7577471962448028885:0]
2025-11-27T18:21:14.892882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-11-27T18:21:14.896313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471962448028893:2356] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:14.896537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:14.896523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471961476118779:2312] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:14.896785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471961476118777:2311] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:14.899539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471962448028893:2356] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:14.899657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471961476118777:2311] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:14.899750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:14.899806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:14.899866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471962448028897:2048]
2025-11-27T18:21:14.899943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471962448028897:2048] [node 1] ICS09 handshake done sender: [2:7577471962448028893:2356] self: [2:7577471962448028895:0] peer: [1:7577471961476118776:0] socket: 17 qp: 5644
2025-11-27T18:21:14.899959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:14.899981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471962448028897:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:14.900027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:14.900031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:14.900085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471961476118782:2048]
2025-11-27T18:21:14.900134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471961476118782:2048] [node 2] ICS09 handshake done sender: [1:7577471961476118777:2311] self: [1:7577471961476118776:0] peer: [2:7577471962448028895:0] socket: 16 qp: 5643
2025-11-27T18:21:14.900164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471961476118782:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:14.900196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:21:15.008266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:21:15.008431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:21:15.008457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471962448028897:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:21:15.008427Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:15.008478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471962448028897:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:15.008511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:15.008577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:15.008722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471961476118782:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:15.008768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471962448028897:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:15.008774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471961476118782:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:15.008793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471962448028897:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:15.008855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:15.008917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471961476118782:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:15.012517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471962448028897:2048] [node 1] ICS08 incoming handshake Self# [1:7577471961476118776:0] Peer# [2:7577471962448028895:0] Counter# 136 LastInputSerial# 4
2025-11-27T18:21:15.012526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471961476118782:2048] [node 2] ICS08 incoming handshake Self# [2:7577471962448028895:0] Peer# [1:7577471961476118776:0] Counter# 4 LastInputSerial# 136
2025-11-27T18:21:15.012943Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471965771086080:2313] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:15.013195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:15.013276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471961476118782:2048] [node 2] ICS09 handshake done sender: [1:7577471965771086080:2313] self: [1:7577471961476118776:0] peer: [2:7577471962448028895:0] socket: 21 qp: 5648
2025-11-27T18:21:15.013680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471961476118782:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:15.013729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:15.015912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471966742996197:2359] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:15.016131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471966742996197:2359] held: no
2025-11-27T18:21:15.016170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:15.016220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471962448028897:2048] [node 1] ICS09 handshake done sender: [2:7577471966742996197:2359] self: [2:7577471962448028895:0] peer: [1:7577471961476118776:0] socket: 25 qp: 5650
2025-11-27T18:21:15.016705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471962448028897:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:15.016745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:21:15.481991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:21:15.482122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:21:15.482151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471962448028897:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:21:15.482138Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:15.482171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471962448028897:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:15.482191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:15.482244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:15.482405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471962448028897:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:15.482439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471961476118782:2048] [node 2] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-27T18:21:15.482483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471961476118782:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:15.482422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471962448028897:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:15.482568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:15.482606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471961476118782:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:15.485468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471961476118782:2048] [node 2] ICS08 incoming handshake Self# [2:7577471962448028895:0] Peer# [1:7577471961476118776:0] Counter# 141 LastInputSerial# 5206
2025-11-27T18:21:15.485515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471962448028897:2048] [node 1] ICS08 incoming handshake Self# [1:7577471961476118776:0] Peer# [2:7577471962448028895:0] Counter# 5206 LastInputSerial# 141
2025-11-27T18:21:15.485899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471965771086085:2315] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:15.486135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:15.486191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471961476118782:2048] [node 2] ICS09 handshake done sender: [1:7577471965771086085:2315] self: [1:7577471961476118776:0] peer: [2:7577471962448028895:0] socket: 20 qp: 5653
2025-11-27T18:21:15.486936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471961476118782:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:15.486979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:15.487973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471966742996202:2362] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:15.488934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:15.489001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471962448028897:2048] [node 1] ICS09 handshake done sender: [2:7577471966742996202:2362] self: [2:7577471962448028895:0] peer: [1:7577471961476118776:0] socket: 22 qp: 5654
2025-11-27T18:21:15.489474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471962448028897:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:15.489513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:15.489629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7577471829304041991:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-27T18:21:16.026940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-27T18:21:16.027293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471962448028897:2048] [node 1] ICS07 socket disconnect 22 reason# Debug
2025-11-27T18:21:16.027327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471962448028897:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:16.027392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:16.027435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471962448028897:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:16.027439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:16.027649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471961476118782:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-27T18:21:16.027677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471961476118782:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:16.027739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:16.027798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471961476118782:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:16.030045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471961476118782:2048] [node 2] ICS08 incoming handshake Self# [2:7577471962448028895:0] Peer# [1:7577471961476118776:0] Counter# 294 LastInputSerial# 11408
2025-11-27T18:21:16.031363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471962448028897:2048] [node 1] ICS08 incoming handshake Self# [1:7577471961476118776:0] Peer# [2:7577471962448028895:0] Counter# 11408 LastInputSerial# 294
2025-11-27T18:21:16.031706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471970066053386:2317] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:16.031927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:16.031969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471961476118782:2048] [node 2] ICS09 handshake done sender: [1:7577471970066053386:2317] self: [1:7577471961476118776:0] peer: [2:7577471962448028895:0] socket: 21 qp: 5658
2025-11-27T18:21:16.032337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471961476118782:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:16.032384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:16.034366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7577471971037963503:2365] held: no
2025-11-27T18:21:16.034433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471971037963503:2365] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:16.034557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:16.034591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471962448028897:2048] [node 1] ICS09 handshake done sender: [2:7577471971037963503:2365] self: [2:7577471962448028895:0] peer: [1:7577471961476118776:0] socket: 25 qp: 5657
2025-11-27T18:21:16.034981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471962448028897:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:16.035011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-27T18:21:16.202855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471962448028897:2048] [node 1] ICS01 socket: 25 reason# Debug
2025-11-27T18:21:16.202948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471962448028897:2048] VirtualId# [2:7577471962448028895:0]
2025-11-27T18:21:16.202998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471962448028897:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:16.203063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:16.203146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:21:16.203132Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:16.203364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471961476118782:2048] [node 2] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-27T18:21:16.203397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471961476118782:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:16.203473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:16.203525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471961476118782:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 66244
TEvUndelivered Cookie# 66245
TEvUndelivered Cookie# 66246
2025-11-27T18:21:16.206068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471971037963506:2367]) peer tries to resume nonexistent session Self# [1:7577471961476118776:0] Peer# [2:7577471962448028895:0]
2025-11-27T18:21:16.206177Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471971037963506:2367] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:16.207263Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471970066053391:2319] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:16.208007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:16.208075Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:16.208105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471961476118782:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:16.208175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471961476118782:2048] VirtualId# [1:7577471961476118776:0]
2025-11-27T18:21:16.208228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:16.209222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471970066053393:2320] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:16.210932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471971037963510:2369] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:16.211089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:16.211370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471970066053395:2321] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:16.213978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471971037963510:2369] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:16.214051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471970066053395:2321] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:16.214174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:16.214243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:16.214262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:16.214310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:16.214342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471971037963513:2048]
2025-11-27T18:21:16.214364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471970066053399:2048]
2025-11-27T18:21:16.214399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471971037963513:2048] [node 1] ICS09 handshake done sender: [2:7577471971037963510:2369] self: [2:7577471971037963511:0] peer: [1:7577471970066053394:0] socket: 17 qp: 5664
2025-11-27T18:21:16.214424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471970066053399:2048] [node 2] ICS09 handshake done sender: [1:7577471970066053395:2321] self: [1:7577471970066053394:0] peer: [2:7577471971037963511:0] socket: 16 qp: 5663
2025-11-27T18:21:16.214467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471970066053399:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:16.214464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471971037963513:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:16.214510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:16.214510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-27T18:21:16.735720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7577471829304041991:2048] [node 1] ICP34 closed connection by debug command
2025-11-27T18:21:16.735853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:16.735928Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:21:16.735963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471971037963513:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:21:16.735991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471971037963513:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:16.736038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:16.736113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:16.736101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471970066053399:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:16.736128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471970066053399:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:16.736191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:16.736227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471970066053399:2048] [node 2] ICS15 start handshake
2025-11-27T18:21:16.736389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471971037963513:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:16.736418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471971037963513:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:16.738317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7577471971037963513:2048] [node 1] ICS08 incoming handshake Self# [1:7577471970066053394:0] Peer# [2:7577471971037963511:0] Counter# 4794 LastInputSerial# 121
2025-11-27T18:21:16.739155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471970066053401:2322] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:16.739579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:16.739619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471970066053399:2048] [node 2] ICS09 handshake done sender: [1:7577471970066053401:2322] self: [1:7577471970066053394:0] peer: [2:7577471971037963511:0] socket: 20 qp: 5665
2025-11-27T18:21:16.740031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471970066053399:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:16.740082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:16.740300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7577471970066053399:2048] [node 2] ICS13 reestablish connection
2025-11-27T18:21:16.740323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471970066053399:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-27T18:21:16.740385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:16.740442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7577471970066053399:2048] [node 2] ICS08 incoming handshake Self# [2:7577471971037963511:0] Peer# [1:7577471970066053394:0] Counter# 121 LastInputSerial# 4794
2025-11-27T18:21:16.740477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:16.740733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471970066053399:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-27T18:21:16.740786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471970066053399:2048] VirtualId# [1:7577471970066053394:0]
2025-11-27T18:21:16.740818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:16.742238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471971037963516:2372] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:16.742417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:16.742481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471971037963513:2048] [node 1] ICS09 handshake done sender: [2:7577471971037963516:2372] self: [2:7577471971037963511:0] peer: [1:7577471970066053394:0] socket: 22 qp: 5666
2025-11-27T18:21:16.743036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471971037963513:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:16.743084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:16.743347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-11-27T18:21:16.743382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7577471971037963513:2048] [node 1] ICS13 reestablish connection
2025-11-27T18:21:16.743412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471971037963513:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-27T18:21:16.743437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:16.743512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:16.743742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471971037963513:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-27T18:21:16.743768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471971037963513:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:16.746683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471970066053407:2325]) peer tries to resume nonexistent session Self# [2:7577471971037963511:0] Peer# [1:7577471970066053394:0]
2025-11-27T18:21:16.746733Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471970066053407:2325] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:16.746887Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471971037963521:2374] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:16.747436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:16.747487Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:16.747509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471971037963513:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:16.747560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471971037963513:2048] VirtualId# [2:7577471971037963511:0]
2025-11-27T18:21:16.747695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 67446
TEvUndelivered Cookie# 67447
TEvUndelivered Cookie# 67448
TEvUndelivered Cookie# 67449
TEvUndelivered Cookie# 67450
TEvUndelivered Cookie# 67451
TEvUndelivered Cookie# 67452
TEvUndelivered Cookie# 67453
2025-11-27T18:21:16.750117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471970066053408:2326] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:16.750213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:16.750920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471971037963524:2375] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:16.752781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471970066053408:2326] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:16.752853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471971037963524:2375] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:16.752941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:16.752980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:16.753019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471970066053411:2048]
2025-11-27T18:21:16.753054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471970066053411:2048] [node 2] ICS09 handshake done sender: [1:7577471970066053408:2326] self: [1:7577471970066053409:0] peer: [2:7577471971037963523:0] socket: 17 qp: 5672
2025-11-27T18:21:16.753076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471970066053411:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:16.753112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:16.753079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:16.753117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:16.753169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471971037963527:2048]
2025-11-27T18:21:16.753196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471971037963527:2048] [node 1] ICS09 handshake done sender: [2:7577471971037963524:2375] self: [2:7577471971037963523:0] peer: [1:7577471970066053409:0] socket: 16 qp: 5671
2025-11-27T18:21:16.753229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471971037963527:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:16.753279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-27T18:21:17.134060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471970066053411:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-27T18:21:17.134144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471970066053411:2048] VirtualId# [1:7577471970066053409:0]
2025-11-27T18:21:17.134174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471970066053411:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:17.134230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:17.134271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:17.135527Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:17.135965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471971037963527:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-27T18:21:17.136002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471971037963527:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:17.136034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:17.136099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471971037963527:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:17.138385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7577471828332131888:2048] [node 2] ICP12 (actor [1:7577471974361020709:2328]) peer tries to resume nonexistent session Self# [2:7577471971037963523:0] Peer# [1:7577471970066053409:0]
2025-11-27T18:21:17.138459Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7577471974361020709:2328] [node 2] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:17.138614Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7577471975332930825:2376] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake
2025-11-27T18:21:17.139424Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19790) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:17.139523Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7577471829304041991:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:17.139557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471971037963527:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:17.139633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471971037963527:2048] VirtualId# [2:7577471971037963523:0]
2025-11-27T18:21:17.139790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 68274
TEvUndelivered Cookie# 68275
TEvUndelivered Cookie# 68276
TEvUndelivered Cookie# 68277
TEvUndelivered Cookie# 68278
TEvUndelivered Cookie# 68279
TEvUndelivered Cookie# 68280
TEvUndelivered Cookie# 68281
TEvUndelivered Cookie# 68282
TEvUndelivered Cookie# 68283
2025-11-27T18:21:17.142600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471974361020710:2329] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:17.142732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471828332131888:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:17.143022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471975332930828:2377] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:17.145948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471974361020710:2329] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:17.146062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471975332930828:2377] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:17.146096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471828332131888:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:17.146129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:17.146161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471974361020713:2048]
2025-11-27T18:21:17.146195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471974361020713:2048] [node 2] ICS09 handshake done sender: [1:7577471974361020710:2329] self: [1:7577471974361020711:0] peer: [2:7577471975332930827:0] socket: 17 qp: 5675
2025-11-27T18:21:17.146219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471974361020713:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:17.146261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:17.146392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471829304041991:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:17.146433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:17.146487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471975332930831:2048]
2025-11-27T18:21:17.146541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471975332930831:2048] [node 1] ICS09 handshake done sender: [2:7577471975332930828:2377] self: [2:7577471975332930827:0] peer: [1:7577471974361020711:0] socket: 16 qp: 5674
2025-11-27T18:21:17.146579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471975332930831:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:17.146622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-27T18:21:17.262219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471975332930831:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-27T18:21:17.262323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471975332930831:2048] VirtualId# [2:7577471975332930827:0]
2025-11-27T18:21:17.262358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471975332930831:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-27T18:21:17.262416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:17.262475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:21:17.262817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:17.263224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7577471974361020713:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:17.263267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471974361020713:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:17.263308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:17.263361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7577471974361020713:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-27T18:21:17.265875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7577471829304041991:2048] [node 1] ICP12 (actor [2:7577471975332930833:2378]) peer tries to resume nonexistent session Self# [1:7577471974361020711:0] Peer# [2:7577471975332930827:0]
2025-11-27T18:21:17.265921Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7577471975332930833:2378] [node 1] ICH08 Continuation request rejected by proxy
2025-11-27T18:21:17.266082Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7577471974361020715:2331] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake
2025-11-27T18:21:17.266740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7577471828332131888:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:22361) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-27T18:21:17.266797Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7577471828332131888:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-27T18:21:17.266822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471974361020713:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-27T18:21:17.266883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471828332131888:2048] [node 2] ICP30 unregister session Session# [1:7577471974361020713:2048] VirtualId# [1:7577471974361020711:0]
2025-11-27T18:21:17.266931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:17.269812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471975332930834:2379] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:17.269992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577471829304041991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:17.270260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471974361020718:2332] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:17.272941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471975332930834:2379] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:17.273027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471974361020718:2332] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:17.273158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577471829304041991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:17.273204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:17.273272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471829304041991:2048] [node 1] ICP22 created new session: [2:7577471975332930837:2048]
2025-11-27T18:21:17.273315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471975332930837:2048] [node 1] ICS09 handshake done sender: [2:7577471975332930834:2379] self: [2:7577471975332930835:0] peer: [1:7577471974361020717:0] socket: 17 qp: 5678
2025-11-27T18:21:17.273315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577471828332131888:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:17.273342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471975332930837:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:17.273352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:17.273368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-11-27T18:21:17.273385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471828332131888:2048] [node 2] ICP22 created new session: [1:7577471974361020721:2048]
2025-11-27T18:21:17.273449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471974361020721:2048] [node 2] ICS09 handshake done sender: [1:7577471974361020718:2332] self: [1:7577471974361020717:0] peer: [2:7577471975332930835:0] socket: 16 qp: 5677
2025-11-27T18:21:17.273477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471974361020721:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:17.273500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
2025-11-27T18:21:17.787818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:17.788141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577471975332930837:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-27T18:21:17.788180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471975332930837:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:17.788285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:17.788362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471975332930837:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:17.791510Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:19790) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:21:17.791551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577471975332930837:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:17.792144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577471829304041991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# ::1(::1:19790) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:21:17.792235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471975332930837:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:21:17.792351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471829304041991:2048] [node 1] ICP30 unregister session Session# [2:7577471975332930837:2048] VirtualId# [2:7577471975332930835:0]
2025-11-27T18:21:17.792428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:21:17.792473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577471829304041991:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# ::1(::1:19790) Couldn't connect to any resolved address LastSessionDieTime# 2025-11-27T18:21:17.262043Z
2025-11-27T18:21:17.792503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# ::1(::1:19790) Couldn't connect to any resolved address LastSessionDieTime# 2025-11-27T18:21:17.262043Z
TEvNodeDisconnected
2025-11-27T18:21:17.844411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471974366372147:2048] [node 2] ICP01 ready to work
2025-11-27T18:21:17.862959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471977795453196:2048] [node 1] ICP01 ready to work
Subscribe
2025-11-27T18:21:17.868421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471974366372156:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:17.870753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471974366372147:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:17.871122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471977795453206:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:17.873894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471974366372156:2056] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:17.873966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471977795453206:2056] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:17.874202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471974366372147:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:17.874269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:17.874320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471974366372147:2048] [node 2] ICP22 created new session: [1:7577471974366372159:2048]
2025-11-27T18:21:17.874385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471977795453196:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:17.874358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471974366372159:2048] [node 2] ICS09 handshake done sender: [1:7577471974366372156:2056] self: [1:7577471974366372157:0] peer: [2:7577471977795453205:0] socket: 17 qp: 5680
2025-11-27T18:21:17.874418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:17.874386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471974366372159:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:17.874454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471977795453196:2048] [node 1] ICP22 created new session: [2:7577471977795453209:2048]
2025-11-27T18:21:17.874414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:17.874488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471977795453209:2048] [node 1] ICS09 handshake done sender: [2:7577471977795453206:2056] self: [2:7577471977795453205:0] peer: [1:7577471974366372157:0] socket: 16 qp: 5679
2025-11-27T18:21:17.874515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471977795453209:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:17.874539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
TEvNodeConnected
2025-11-27T18:21:17.962997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:622: Proxy [1:7577471974366372147:2048] [node 2] ICP31 proxy http called
2025-11-27T18:21:17.988453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:17.988677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471977795453209:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:21:17.988763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471977795453196:2048] [node 1] ICP30 unregister session Session# [2:7577471977795453209:2048] VirtualId# [2:7577471977795453205:0]
2025-11-27T18:21:17.988792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471977795453209:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:17.988892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:17.988954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
[[bad]]assertion failed at ydb/library/actors/interconnect/ut/interconnect_ut.cpp:228, virtual void NTestSuiteInterconnect::TTestCaseSetupRdmaSession::Execute_(NUnitTest::TTestContext &): (tokens.size() == 2) [[rst]]
[[alt1]]TBackTrace::Capture()+28 (0xED8D9C)
NUnitTest::NPrivate::RaiseError(char const*, TBasicString<char, std::__y1::char_traits<char>> const&, bool)+604 (0xFE710C)
NTestSuiteInterconnect::TTestCaseSetupRdmaSession::Execute_(NUnitTest::TTestContext&)+3553 (0xB4DA71)
std::__y1::__function::__func<NTestSuiteInterconnect::TCurrentTest::Execute()::'lambda'(), std::__y1::allocator<NTestSuiteInterconnect::TCurrentTest::Execute()::'lambda'()>, void ()>::operator()()+280 (0xB528E8)
TColoredProcessor::Run(std::__y1::function<void ()>, TBasicString<char, std::__y1::char_traits<char>> const&, char const*, bool)+538 (0x101F80A)
NUnitTest::TTestBase::Run(std::__y1::function<void ()>, TBasicString<char, std::__y1::char_traits<char>> const&, char const*, bool)+504 (0xFEDDE8)
NTestSuiteInterconnect::TCurrentTest::Execute()+1229 (0xB51C3D)
NUnitTest::TTestFactory::Execute()+2176 (0xFEF5A0)
NUnitTest::RunMain(int, char**)+5805 (0x10195CD)
??+0 (0x7F97B601AD90)
__libc_start_main+128 (0x7F97B601AE40)
_start+41 (0xAB4029)
[[rst]]
2025-11-27T18:21:18.043091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577471980276005273:2048] [node 2] ICP01 ready to work
2025-11-27T18:21:18.063344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577471982599216017:2048] [node 1] ICP01 ready to work
2025-11-27T18:21:18.161183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471980276005282:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:18.162730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471980276005273:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:18.163080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471982599216027:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:18.165418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471980276005282:2056] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:18.165498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471982599216027:2056] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:18.165672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471980276005273:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:18.165713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:18.165786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471980276005273:2048] [node 2] ICP22 created new session: [1:7577471980276005285:2048]
2025-11-27T18:21:18.165842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471980276005285:2048] [node 2] ICS09 handshake done sender: [1:7577471980276005282:2056] self: [1:7577471980276005283:0] peer: [2:7577471982599216026:0] socket: 17 qp: 5682
2025-11-27T18:21:18.165865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471982599216017:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:18.165870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471980276005285:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:18.165898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:18.165899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:18.165931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471982599216017:2048] [node 1] ICP22 created new session: [2:7577471982599216030:2048]
2025-11-27T18:21:18.165967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471982599216030:2048] [node 1] ICS09 handshake done sender: [2:7577471982599216027:2056] self: [2:7577471982599216026:0] peer: [1:7577471980276005283:0] socket: 16 qp: 5681
2025-11-27T18:21:18.165992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471982599216030:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:18.166016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
RECEIVED TEvTest
2025-11-27T18:21:18.806074Z :INTERCONNECT CRIT: interconnect_tcp_session.cpp:517: Session [2:7577471982599216030:2048] [node 1] ICS31 serialized event Type# 0x7ff00000 is too large
2025-11-27T18:21:18.806112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471982599216030:2048] [node 1] ICS01 socket: 16 reason# EventTooLarge
2025-11-27T18:21:18.806184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471982599216017:2048] [node 1] ICP30 unregister session Session# [2:7577471982599216030:2048] VirtualId# [2:7577471982599216026:0]
2025-11-27T18:21:18.806217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471982599216030:2048] [node 1] ICS25 shutdown socket, reason# EventTooLarge
2025-11-27T18:21:18.806266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:18.806353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-27T18:21:18.806508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7577471980276005285:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-11-27T18:21:18.806561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7577471980276005273:2048] [node 2] ICP30 unregister session Session# [1:7577471980276005285:2048] VirtualId# [1:7577471980276005283:0]
2025-11-27T18:21:18.806586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7577471980276005285:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:18.806641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-11-27T18:21:18.806680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-11-27T18:21:18.869561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvUndelivered
2025-11-27T18:21:18.911610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577471980276005287:2058] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:18.911786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7577471980276005273:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:18.912062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577471982599216033:2057] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:18.914354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577471980276005287:2058] [node 2] ICH04 handshake succeeded
2025-11-27T18:21:18.914427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577471982599216033:2057] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:18.914518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7577471980276005273:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-27T18:21:18.914558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:18.914604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577471980276005273:2048] [node 2] ICP22 created new session: [1:7577471980276005290:2048]
2025-11-27T18:21:18.914659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577471980276005290:2048] [node 2] ICS09 handshake done sender: [1:7577471980276005287:2058] self: [1:7577471980276005288:0] peer: [2:7577471982599216032:0] socket: 17 qp: 5684
2025-11-27T18:21:18.914680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577471980276005290:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:18.914753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7577471982599216017:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-27T18:21:18.914716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:18.914793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:18.914841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577471982599216017:2048] [node 1] ICP22 created new session: [2:7577471982599216036:2048]
2025-11-27T18:21:18.914877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577471982599216036:2048] [node 1] ICS09 handshake done sender: [2:7577471982599216033:2057] self: [2:7577471982599216032:0] peer: [1:7577471980276005288:0] socket: 16 qp: 5683
2025-11-27T18:21:18.914908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577471982599216036:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:18.914940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
RECEIVED TEvTest
2025-11-27T18:21:18.943601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:18.943814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577471982599216036:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-27T18:21:18.943875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577471982599216017:2048] [node 1] ICP30 unregister session Session# [2:7577471982599216036:2048] VirtualId# [2:7577471982599216032:0]
2025-11-27T18:21:18.943910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577471982599216036:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:18.943977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:18.944022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
written 2560000 bytes
written 3437884 bytes
written 4133621 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
2025-11-27T18:21:37.550075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7577472061271556027:2048] [node 2] ICP01 ready to work
2025-11-27T18:21:37.567346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7577472061091808112:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:21:37.569639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577472061091808121:2056] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32090
2025-11-27T18:21:37.572203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7577472061091808121:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:37.573584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7577472061091808112:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-27T18:21:37.573874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7577472061271556037:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-27T18:21:37.573915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7577472061271556037:2056] [node 2] ICRDMA Non success qp response from remote side
2025-11-27T18:21:37.574665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7577472061271556037:2056] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-27T18:21:37.574841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7577472061091808123:2057] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32090
2025-11-27T18:21:37.574999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7577472061271556027:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-27T18:21:37.575080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-27T18:21:37.575161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7577472061271556027:2048] [node 2] ICP22 created new session: [1:7577472061271556040:2048]
2025-11-27T18:21:37.575206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7577472061271556040:2048] [node 2] ICS09 handshake done sender: [1:7577472061271556037:2056] self: [1:7577472061271556036:0] peer: [2:7577472061091808122:0] socket: 24 qp: -1
2025-11-27T18:21:37.575236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7577472061271556040:2048] [node 2] ICS10 traffic start
2025-11-27T18:21:37.575343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-11-27T18:21:37.575482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7577472061091808121:2056] [node 1] ICH04 handshake succeeded
2025-11-27T18:21:37.575702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7577472061091808112:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-27T18:21:37.575807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-27T18:21:37.575880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7577472061091808112:2048] [node 1] ICP22 created new session: [2:7577472061091808124:2048]
2025-11-27T18:21:37.575939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7577472061091808124:2048] [node 1] ICS09 handshake done sender: [2:7577472061091808121:2056] self: [2:7577472061091808122:0] peer: [1:7577472061271556036:0] socket: 25 qp: -1
2025-11-27T18:21:37.575989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7577472061091808124:2048] [node 1] ICS10 traffic start
2025-11-27T18:21:37.576046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Updated MaxRTT# 0.009736s
Updated MaxRTT# 0.021187s
Updated MaxRTT# 0.021348s
Updated MaxRTT# 0.021355s
Updated MaxRTT# 0.023521s
Updated MaxRTT# 0.032013s
Updated MaxRTT# 0.034979s
Updated MaxRTT# 0.034986s
2025-11-27T18:21:47.597820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-27T18:21:47.598014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7577472061091808124:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-11-27T18:21:47.598064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7577472061091808124:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-27T18:21:47.598189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-11-27T18:21:47.598310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577472061091808124:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:47.602362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577472061091808112:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19157) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-27T18:21:47.602399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7577472061091808124:2048] [node 1] ICS15 start handshake
2025-11-27T18:21:47.603089Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7577472061091808112:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19157) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-27T18:21:47.603140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7577472061091808124:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-27T18:21:47.603181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7577472061091808112:2048] [node 1] ICP30 unregister session Session# [2:7577472061091808124:2048] VirtualId# [2:7577472061091808122:0]
2025-11-27T18:21:47.603243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-11-27T18:21:47.603285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7577472061091808112:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19157) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:21:47.603317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19157) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-27T18:21:47.613599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7577472061091808112:2048] [node 1] ICP33 wake up from error state
